diff --git a/common/task_sets.py b/common/task_sets.py index 7f5d4fb..70b36a2 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -47,6 +47,7 @@ ssh_set = [security.DisableSSHPasswordAuthentication, apt_set = [apt.DisableDaemonAutostart, apt.AptSources, + apt.AptUpdate, apt.AptUpgrade, apt.PurgeUnusedPackages, apt.AptClean, diff --git a/common/tasks/apt.py b/common/tasks/apt.py index 6e6b1ce..0fcf5b6 100644 --- a/common/tasks/apt.py +++ b/common/tasks/apt.py @@ -39,10 +39,25 @@ class DisableDaemonAutostart(Task): stat.S_IROTH | stat.S_IXOTH) +class AptUpdate(Task): + description = 'Updating the package cache' + phase = phases.system_modification + predecessors = [locale.GenerateLocale, AptSources] + successors = [network.RemoveDNSInfo] + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', + '--fix-broken', + '--assume-yes', + 'install']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '--assume-yes', 'upgrade']) + + class AptUpgrade(Task): description = 'Upgrading packages and fixing broken dependencies' phase = phases.system_modification - predecessors = [locale.GenerateLocale, AptSources, DisableDaemonAutostart] + predecessors = [AptUpdate, DisableDaemonAutostart] successors = [network.RemoveDNSInfo] def run(self, info): diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index cec6c35..9226074 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -31,12 +31,17 @@ } }, "plugins": { - "backports": { - "packages": [ "cloud-init" ] - }, - "user_packages": { - "repo": [ "sudo" ], - "local": [] + "packages": { + "sources": { + "backports": [ + "deb {apt_mirror} {release}-backports main", + "deb-src {apt_mirror} {release}-backports main" + ] + }, + "remote": [ + "sudo", + { "name": "cloud-init", "target": "{release}-backports" } + ] }, "sed": { "file": "etc/cloud/cloud.cfg", diff --git a/manifests/ec2-ebs-debian-official.manifest.json b/manifests/ec2-ebs-debian-official.manifest.json index 29ea513..78515b6 100644 --- a/manifests/ec2-ebs-debian-official.manifest.json +++ b/manifests/ec2-ebs-debian-official.manifest.json @@ -31,12 +31,17 @@ } }, "plugins": { - "backports": { - "packages": [ "cloud-init" ] - }, - "user_packages": { - "repo": [ "sudo" ], - "local": [] + "packages": { + "sources": { + "backports": [ + "deb {apt_mirror} {release}-backports main", + "deb-src {apt_mirror} {release}-backports main" + ] + }, + "remote": [ + "sudo", + { "name": "cloud-init", "target": "{release}-backports" } + ] }, "sed": { "file": "etc/cloud/cloud.cfg", diff --git a/plugins/backports/__init__.py b/plugins/backports/__init__.py deleted file mode 100644 index 834c2d8..0000000 --- a/plugins/backports/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ - - -def tasks(tasklist, manifest): - import tasks - tasklist.add(tasks.AptSourcesBackports, - tasks.AddBackportsPackages) - - -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) diff --git a/plugins/backports/manifest-schema.json b/plugins/backports/manifest-schema.json deleted file mode 100644 index deafab1..0000000 --- a/plugins/backports/manifest-schema.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Backports setup and package install", - "type": "object", - "properties": { - "plugins": { - "type": "object", - "properties": { - "backports": { - "type": "object", - "properties": { - "packages": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": ["packages"] - } - }, - "required": ["backports"] - } - }, - "required": ["plugins"] -} diff --git a/plugins/backports/tasks.py b/plugins/backports/tasks.py deleted file mode 100644 index 887a4ce..0000000 --- a/plugins/backports/tasks.py +++ /dev/null @@ -1,38 +0,0 @@ -from base import Task -from common import phases -from common.tasks.packages import ImagePackages -from common.tasks.initd import InstallInitScripts -from common.tasks.apt import AptUpgrade -from common.tasks.apt import AptSources -import os - - -class AptSourcesBackports(Task): - description = 'Adding backports to sources.list' - phase = phases.system_modification - predecessors = [AptSources] - successors = [AptUpgrade] - - def run(self, info): - sources_path = os.path.join(info.root, 'etc/apt/sources.list') - with open(sources_path, 'a') as apt_sources: - apt_sources.write(('deb {apt_mirror} {release}-backports main\n' - 'deb-src {apt_mirror} {release}-backports main\n' - .format(apt_mirror='http://http.debian.net/debian', - release=info.manifest.system['release']))) - - -class AddBackportsPackages(Task): - description = 'Adding backport packages to the image' - phase = phases.system_modification - predecessors = [AptUpgrade] - - def run(self, info): - if 'packages' not in info.manifest.plugins['backports']: - return - - from common.tools import log_check_call - for pkg in info.manifest.plugins['backports']['packages']: - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install', - '--assume-yes', '--target-release', info.manifest.system['release'] + '-backports', - pkg]) diff --git a/plugins/packages/__init__.py b/plugins/packages/__init__.py new file mode 100644 index 0000000..01962c6 --- /dev/null +++ b/plugins/packages/__init__.py @@ -0,0 +1,16 @@ + + +def validate_manifest(data, schema_validate): + from os import path + schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) + schema_validate(data, schema_path) + + +def tasks(tasklist, manifest): + from tasks import AptSources, InstallRemotePackages, InstallLocalPackages + if 'sources' in manifest.plugins['packages']: + tasklist.add(AptSources) + if 'remote' in manifest.plugins['packages']: + tasklist.add(InstallRemotePackages) + if 'local' in manifest.plugins['packages']: + tasklist.add(InstallLocalPackages) diff --git a/plugins/packages/manifest-schema.json b/plugins/packages/manifest-schema.json new file mode 100644 index 0000000..12b2499 --- /dev/null +++ b/plugins/packages/manifest-schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Packages plugin manifest", + "type": "object", + "properties": { + "plugins": { + "type": "object", + "properties": { + "packages": { + "type": "object", + "properties": { + "sources": { + "type": "object", + "patternProperties": { + "^\\w+$": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } + }, + "additionalProperties": false, + "minItems": 1 + }, + "remote": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": {"type": "string"}, + "target": {"type": "string"} + }, + "required": ["name"] + }, + { "type": "string" } + ] + }, + "minItems": 1 + }, + "local": { + "type": "array", + "items": { "$ref": "#/definitions/absolute_path" }, + "minItems": 1 + } + }, + "anyOf": [ + {"required": ["sources"]}, + {"required": ["remote"]}, + {"required": ["local"]} + ] + } + } + } + }, + "definitions": { + "absolute_path": { + "type": "string", + "pattern": "^/[^\\0]+$" + } + } +} diff --git a/plugins/packages/tasks.py b/plugins/packages/tasks.py new file mode 100644 index 0000000..7839217 --- /dev/null +++ b/plugins/packages/tasks.py @@ -0,0 +1,63 @@ +from base import Task +from common import phases +from common.tasks import network +from common.tasks import apt +import os.path + + +class AptSources(Task): + description = 'Adding additional aptitude sources' + phase = phases.system_modification + predecessors = [apt.AptSources] + successors = [apt.AptUpdate] + + def run(self, info): + manifest_vars = {'release': info.manifest.system['release'], + 'architecture': info.manifest.system['architecture'], + 'apt_mirror': 'http://http.debian.net/debian'} + for name in info.manifest.plugins['packages']['sources'].iterkeys(): + list_path = os.path.join(info.root, 'etc/apt/sources.list.d/', name + '.list') + with open(list_path, 'a') as source_list: + for line in info.manifest.plugins['packages']['sources'][name]: + source_list.write('{line}\n'.format(line=line.format(**manifest_vars))) + + +class InstallRemotePackages(Task): + description = 'Installing remote packages' + phase = phases.system_modification + predecessors = [apt.AptUpdate, apt.DisableDaemonAutostart] + successors = [network.RemoveDNSInfo] + + def run(self, info): + manifest_vars = {'release': info.manifest.system['release'], + 'architecture': info.manifest.system['architecture']} + from common.tools import log_check_call + for package in info.manifest.plugins['packages']['remote']: + target_release = [] + if isinstance(package, basestring): + name = package + else: + name = package['name'].format(**manifest_vars) + if hasattr(package, 'target'): + target = package['target'].format(**manifest_vars) + target_release = ['--targe-release', target] + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install', + '--assume-yes'] + target_release + [name]) + + +class InstallLocalPackages(Task): + description = 'Installing local packages' + phase = phases.system_modification + + def run(self, info): + from shutil import copy + from os import remove + from common.tools import log_check_call + + for package_src in info.manifest.plugins['packages']['local']: + pkg_name = os.path.basename(package_src) + package_dst = os.path.join('/tmp', pkg_name) + copy(package_src, os.path.join(info.root, package_dst)) + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/bin/dpkg', '--install', package_dst]) + remove(package_dst) diff --git a/plugins/user_packages/README.md b/plugins/user_packages/README.md deleted file mode 100644 index 5f75766..0000000 --- a/plugins/user_packages/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# User package plugin - -This plugin gives the possibility to the user to install Debian packages in the virtual image. - -Plugin is defined in the manifest file, plugin section with: - - "user_packages": { - "repo": [ "apache2" ], - "local": [ "/tmp/mypackage.deb" ] - } - -The *repo* element refers to packages available in Debian repository (apt-get). - -The *local* element will copy the specified .deb files and install them in the image with a dpkg command. Packages are installed in the order of their declaration. diff --git a/plugins/user_packages/__init__.py b/plugins/user_packages/__init__.py deleted file mode 100644 index 3596486..0000000 --- a/plugins/user_packages/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ - - -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) - - -def tasks(tasklist, manifest): - from user_packages import AddUserPackages, AddLocalUserPackages - if 'repo' in manifest.plugins['user_packages']: - tasklist.add(AddUserPackages) - if 'local' in manifest.plugins['user_packages']: - tasklist.add(AddLocalUserPackages) diff --git a/plugins/user_packages/manifest-schema.json b/plugins/user_packages/manifest-schema.json deleted file mode 100644 index 865701c..0000000 --- a/plugins/user_packages/manifest-schema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Image commands plugin manifest", - "type": "object", - "properties": { - "plugins": { - "type": "object", - "properties": { - "user_packages": { - "type": "object", - "properties": { - "repo": { - "type": "array", - "items": {"type": "string"}, - "minItems": 1 - }, - "local": { - "type": "array", - "items": {"type": "string"}, - "minItems": 1 - } - }, - "anyOf": [ - {"required": ["repo"]}, - {"required": ["local"]} - ] - } - }, - "required": ["user_packages"] - } - }, - "required": ["plugins"] -} diff --git a/plugins/user_packages/user_packages.py b/plugins/user_packages/user_packages.py deleted file mode 100644 index 6ab39a8..0000000 --- a/plugins/user_packages/user_packages.py +++ /dev/null @@ -1,41 +0,0 @@ -from base import Task -from common import phases -import os -from common.tasks.packages import ImagePackages -from common.tasks.host import CheckPackages -from common.tasks.filesystem import MountRoot - - -class AddUserPackages(Task): - description = 'Adding user defined packages to the image packages' - phase = phases.preparation - predecessors = [ImagePackages] - successors = [CheckPackages] - - def run(self, info): - for pkg in info.manifest.plugins['user_packages']['repo']: - info.img_packages[0].add(pkg) - - -class AddLocalUserPackages(Task): - description = 'Adding user local packages to the image packages' - phase = phases.system_modification - predecessors = [MountRoot] - - def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - - from shutil import copy - from common.tools import log_check_call - - for pkg in info.manifest.plugins['user_packages']['local']: - script_src = os.path.normpath(pkg) - script_dst = os.path.join(info.root, 'tmp/'+os.path.basename(script_src)) - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg', '--install', '/tmp/'+os.path.basename(script_src)])