From 1c93094833676af4472f9df10fd78e0c03c2424a Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 29 Dec 2013 16:09:47 +0100 Subject: [PATCH] Integrated package plugin with base system New phase introduced "package installation" (fixes #114) Apt source lines are now parsed, this allows to verify the target release of added packages. All packages (except locales) are now installed *after* bootstrapping (fixes #123) Added env argument to log_(check_)call HostDependencies have been refactored a little --- base/bootstrapinfo.py | 16 ++++- base/manifest-schema.json | 44 +++++++++++++ base/pkg/__init__.py | 4 ++ base/pkg/exceptions.py | 8 +++ base/pkg/packagelist.py | 40 ++++++++++++ base/pkg/sourceslist.py | 63 +++++++++++++++++++ common/phases.py | 4 +- common/task_sets.py | 14 +++-- common/tasks/apt.py | 41 +++++------- common/tasks/bootstrap.py | 9 ++- common/tasks/filesystem.py | 3 +- common/tasks/host.py | 26 ++++++-- common/tasks/locale.py | 12 +++- common/tasks/packages.py | 60 +++++++++++------- common/tools.py | 16 +++-- ...bs-debian-official-amd64-hvm.manifest.json | 22 +++---- ...bs-debian-official-amd64-pvm.manifest.json | 22 +++---- ...ebs-debian-official-i386-pvm.manifest.json | 22 +++---- plugins/admin_user/tasks.py | 8 +-- plugins/packages/__init__.py | 17 ----- plugins/packages/manifest-schema.json | 62 ------------------ plugins/packages/tasks.py | 63 ------------------- plugins/unattended_upgrades/tasks.py | 6 +- providers/ec2/__init__.py | 4 +- providers/ec2/tasks/host.py | 11 ++++ providers/ec2/tasks/packages.py | 35 +++-------- providers/virtualbox/__init__.py | 2 +- providers/virtualbox/tasks/guest_additions.py | 15 ++--- providers/virtualbox/tasks/packages.py | 18 ++---- 29 files changed, 358 insertions(+), 309 deletions(-) create mode 100644 base/pkg/__init__.py create mode 100644 base/pkg/exceptions.py create mode 100644 base/pkg/packagelist.py create mode 100644 base/pkg/sourceslist.py delete mode 100644 plugins/packages/__init__.py delete mode 100644 plugins/packages/manifest-schema.json delete mode 100644 plugins/packages/tasks.py diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index 8ccc75f..ddb24e5 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -3,12 +3,24 @@ class BootstrapInformation(object): def __init__(self, manifest=None, debug=False): self.manifest = manifest - from fs import load_volume - self.volume = load_volume(self.manifest.volume) self.debug = debug + import random self.run_id = '{id:08x}'.format(id=random.randrange(16 ** 8)) + import os.path self.workspace = os.path.join(manifest.bootstrapper['workspace'], self.run_id) + from fs import load_volume + self.volume = load_volume(self.manifest.volume) + + from pkg.source import SourceLists + self.source_lists = SourceLists(self.manifest) + from pkg.package import PackageList + self.packages = PackageList(self.source_lists, self.manifest) + self.include_packages = set() + self.exclude_packages = set() + + self.host_dependencies = set() + self.initd = {'install': {}, 'disable': []} diff --git a/base/manifest-schema.json b/base/manifest-schema.json index d9eb7c9..d8491c4 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -26,6 +26,46 @@ }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, + "packages": { + "type": "object", + "properties": { + "sources": { + "type": "object", + "patternProperties": { + "^\\w+$": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } + }, + "additionalProperties": false, + "minItems": 1 + }, + "remote": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": {"type": "string"}, + "target": {"type": "string"} + }, + "required": ["name"] + }, + { "type": "string" } + ] + }, + "minItems": 1 + }, + "local": { + "type": "array", + "items": { "$ref": "#/definitions/absolute_path" }, + "minItems": 1 + } + }, + "additionalProperties": false + }, "volume": { "type": "object", "properties": { @@ -56,6 +96,10 @@ "type": "string", "pattern": "^[^\\0]+$" }, + "absolute_path": { + "type": "string", + "pattern": "^/[^\\0]+$" + }, "no_partitions": { "type": "object", "properties": { diff --git a/base/pkg/__init__.py b/base/pkg/__init__.py new file mode 100644 index 0000000..ccd082a --- /dev/null +++ b/base/pkg/__init__.py @@ -0,0 +1,4 @@ + + +def load_packages(manifest): + pass diff --git a/base/pkg/exceptions.py b/base/pkg/exceptions.py new file mode 100644 index 0000000..9437f9c --- /dev/null +++ b/base/pkg/exceptions.py @@ -0,0 +1,8 @@ + + +class PackageError(Exception): + pass + + +class SourceError(Exception): + pass diff --git a/base/pkg/packagelist.py b/base/pkg/packagelist.py new file mode 100644 index 0000000..3dbaaab --- /dev/null +++ b/base/pkg/packagelist.py @@ -0,0 +1,40 @@ +from exceptions import PackageError + + +class PackageList(object): + + def __init__(self, sources_list, manifest): + self.sources_list = sources_list + self.default_target = manifest.system['release'] + self.remote = {} + self.local = set() + if 'remote' in manifest.packages: + manifest_vars = {'release': manifest.system['release'], + 'architecture': manifest.system['architecture']} + for package in manifest.packages['remote']: + target = None + if isinstance(package, dict): + name = package['name'].format(**manifest_vars) + if 'target' in package: + target = package['target'].format(**manifest_vars) + else: + name = package.format(**manifest_vars) + self.add(name, target) + if 'local' in manifest.packages: + for package_path in manifest.packages['local']: + self.local.add(package_path) + + def add(self, name, target=None): + if target is None: + target = self.default_target + if name in self.remote: + if self.remote[name] != target: + msg = ('The package {name} was already added to the package list, ' + 'but with another target release ({target})').format(name=name, target=self.remote[name]) + raise PackageError(msg) + return + + if not self.sources_list.target_exists(target): + msg = ('The target release {target} was not found in the sources list').format(target=target) + raise PackageError(msg) + self.remote[name] = target diff --git a/base/pkg/sourceslist.py b/base/pkg/sourceslist.py new file mode 100644 index 0000000..077ebbb --- /dev/null +++ b/base/pkg/sourceslist.py @@ -0,0 +1,63 @@ + + +class SourceLists(object): + + def __init__(self, manifest): + self.sources = {} + self.manifest_vars = {'release': manifest.system['release'], + 'architecture': manifest.system['architecture'], + 'apt_mirror': 'http://http.debian.net/debian'} + if 'sources' in manifest.packages: + for name, lines in manifest.packages['sources'].iteritems(): + for line in lines: + self.add_source(name, '{line}\n'.format(line=line.format(**self.manifest_vars))) + + def add_source(self, name, line): + name = name.format(**self.manifest_vars) + if name not in self.sources: + self.sources[name] = [] + self.sources[name].append(Source(line)) + + def target_exists(self, target): + for lines in self.sources.itervalues(): + if target in (source.distribution for source in lines): + return True + return False + + +class Source(object): + + def __init__(self, line): + import re + regexp = re.compile('^(?Pdeb|deb-src)\s+' + '(\[\s*(?P.+\S)?\s*\]\s+)?' + '(?P\S+)\s+' + '(?P\S+)' + '(\s+(?P.+\S))?\s*$') + match = regexp.match(line).groupdict() + if match is None: + from exceptions import SourceError + raise SourceError('Unable to parse source line `{line}\''.format(line=line)) + self.type = match['type'] + self.options = [] + if match['options'] is not None: + self.options = re.sub(' +', ' ', match['options']).split(' ') + self.uri = match['uri'] + self.distribution = match['distribution'] + self.components = [] + if match['components'] is not None: + self.components = re.sub(' +', ' ', match['components']).split(' ') + + def __str__(self): + options = '' + if len(self.options) > 0: + options = ' [{options}]'.format(options=' '.join(self.options)) + + components = '' + if len(self.components) > 0: + components = ' {components}'.format(components=' '.join(self.components)) + + return ('{type}{options} {uri}' + ' {distribution}{components}').format(type=self.type, options=options, + uri=self.uri, distribution=self.distribution, + components=components) diff --git a/common/phases.py b/common/phases.py index 3b44eb2..d6cf8a1 100644 --- a/common/phases.py +++ b/common/phases.py @@ -5,7 +5,8 @@ volume_creation = Phase('Volume creation', 'Creating the volume to bootstrap ont volume_preparation = Phase('Volume preparation', 'Formatting the bootstrap volume') volume_mounting = Phase('Volume mounting', 'Mounting bootstrap volume') os_installation = Phase('OS installation', 'Installing the operating system') -system_modification = Phase('System modification', 'Installing software, modifying configuration files etc.') +package_installation = Phase('Package installation', 'Installing software') +system_modification = Phase('System modification', 'Modifying configuration files, adding resources, etc.') system_cleaning = Phase('System cleaning', 'Removing sensitive data, temporary files and other leftovers') volume_unmounting = Phase('Volume unmounting', 'Unmounting the bootstrap volume') image_registration = Phase('Image registration', 'Uploading/Registering with the provider') @@ -16,6 +17,7 @@ order = [preparation, volume_preparation, volume_mounting, os_installation, + package_installation, system_modification, system_cleaning, volume_unmounting, diff --git a/common/task_sets.py b/common/task_sets.py index 70b36a2..4402b00 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -11,9 +11,8 @@ from common.tasks import security from common.tasks import locale base_set = [workspace.CreateWorkspace, - packages.HostPackages, - packages.ImagePackages, - host.CheckPackages, + host.HostDependencies, + host.CheckHostDependencies, bootstrap.Bootstrap, workspace.DeleteWorkspace, ] @@ -45,16 +44,19 @@ ssh_set = [security.DisableSSHPasswordAuthentication, cleanup.ShredHostkeys, ] -apt_set = [apt.DisableDaemonAutostart, - apt.AptSources, +apt_set = [apt.WriteSources, + apt.DisableDaemonAutostart, apt.AptUpdate, apt.AptUpgrade, + packages.InstallRemotePackages, + packages.InstallLocalPackages, apt.PurgeUnusedPackages, apt.AptClean, apt.EnableDaemonAutostart, ] -locale_set = [locale.GenerateLocale, +locale_set = [locale.LocaleBootstrapPackage, + locale.GenerateLocale, locale.SetTimezone, ] diff --git a/common/tasks/apt.py b/common/tasks/apt.py index 0fcf5b6..bf1397a 100644 --- a/common/tasks/apt.py +++ b/common/tasks/apt.py @@ -1,31 +1,28 @@ from base import Task from common import phases from common.tools import log_check_call -import network import locale import os -class AptSources(Task): - description = 'Adding aptitude sources' - phase = phases.system_modification +class WriteSources(Task): + description = 'Writing aptitude sources to disk' + phase = phases.package_installation def run(self, info): - sources_path = os.path.join(info.root, 'etc/apt/sources.list') - with open(sources_path, 'w') as apt_sources: - apt_sources.write(('deb {apt_mirror} {release} main\n' - 'deb-src {apt_mirror} {release} main\n' - .format(apt_mirror='http://http.debian.net/debian', - release=info.manifest.system['release']))) - apt_sources.write(('deb {apt_mirror} {release}/updates main\n' - 'deb-src {apt_mirror} {release}/updates main\n' - .format(apt_mirror='http://security.debian.org/', - release=info.manifest.system['release']))) + for name, sources in info.source_lists.sources.iteritems(): + if name == 'main': + list_path = os.path.join(info.root, 'etc/apt/sources.list') + else: + list_path = os.path.join(info.root, 'etc/apt/sources.list.d/', name + '.list') + with open(list_path, 'w') as source_list: + for source in sources: + source_list.write('{line}\n'.format(line=str(source))) class DisableDaemonAutostart(Task): description = 'Disabling daemon autostart' - phase = phases.system_modification + phase = phases.package_installation def run(self, info): rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d') @@ -41,27 +38,19 @@ class DisableDaemonAutostart(Task): class AptUpdate(Task): description = 'Updating the package cache' - phase = phases.system_modification - predecessors = [locale.GenerateLocale, AptSources] - successors = [network.RemoveDNSInfo] + phase = phases.package_installation + predecessors = [locale.GenerateLocale, WriteSources] def run(self, info): log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', - '--fix-broken', - '--assume-yes', - 'install']) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '--assume-yes', 'upgrade']) class AptUpgrade(Task): description = 'Upgrading packages and fixing broken dependencies' - phase = phases.system_modification + phase = phases.package_installation predecessors = [AptUpdate, DisableDaemonAutostart] - successors = [network.RemoveDNSInfo] def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '--fix-broken', '--assume-yes', diff --git a/common/tasks/bootstrap.py b/common/tasks/bootstrap.py index 14315c6..a45f7e7 100644 --- a/common/tasks/bootstrap.py +++ b/common/tasks/bootstrap.py @@ -8,11 +8,10 @@ log = logging.getLogger(__name__) def get_bootstrap_args(info): executable = ['/usr/sbin/debootstrap'] options = ['--arch=' + info.manifest.system['architecture']] - include, exclude = info.img_packages - if len(include) > 0: - options.append('--include=' + ','.join(include)) - if len(exclude) > 0: - options.append('--exclude=' + ','.join(exclude)) + if len(info.include_packages) > 0: + options.append('--include=' + ','.join(info.include_packages)) + if len(info.exclude_packages) > 0: + options.append('--exclude=' + ','.join(info.exclude_packages)) arguments = [info.manifest.system['release'], info.root, info.manifest.bootstrapper['mirror']] return executable, options, arguments diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 457e656..ee5d55a 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -32,8 +32,7 @@ class AddXFSProgs(Task): phase = phases.preparation def run(self, info): - include, exclude = info.img_packages - include.add('xfsprogs') + info.packages.add('xfsprogs') class CreateMountDir(Task): diff --git a/common/tasks/host.py b/common/tasks/host.py index c5b6e34..f211df5 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -1,18 +1,36 @@ from base import Task from common import phases from common.exceptions import TaskError -import packages -class CheckPackages(Task): +class HostDependencies(Task): + description = 'Determining required host dependencies' + phase = phases.preparation + + def run(self, info): + info.host_dependencies.add('debootstrap') + + from common.fs.loopbackvolume import LoopbackVolume + if isinstance(info.volume, LoopbackVolume): + info.host_dependencies.add('qemu-utils') + + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): + info.host_dependencies.add('xfsprogs') + + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + info.host_dependencies.update(['parted', 'kpartx']) + + +class CheckHostDependencies(Task): description = 'Checking installed host packages' phase = phases.preparation - predecessors = [packages.HostPackages, packages.ImagePackages] + predecessors = [HostDependencies] def run(self, info): from common.tools import log_check_call from subprocess import CalledProcessError - for package in info.host_packages: + for package in info.host_dependencies: try: log_check_call(['/usr/bin/dpkg-query', '-s', package]) except CalledProcessError: diff --git a/common/tasks/locale.py b/common/tasks/locale.py index 7c7371a..5456a24 100644 --- a/common/tasks/locale.py +++ b/common/tasks/locale.py @@ -3,9 +3,19 @@ from common import phases import os.path +class LocaleBootstrapPackage(Task): + description = 'Adding locale package to bootstrap installation' + phase = phases.preparation + + def run(self, info): + # We could bootstrap without locales, but things just suck without them + # eg. error messages when running apt + info.include_packages.add('locales') + + class GenerateLocale(Task): description = 'Generating the selected locale' - phase = phases.system_modification + phase = phases.package_installation def run(self, info): from common.tools import sed_i diff --git a/common/tasks/packages.py b/common/tasks/packages.py index 41a3774..f1cf696 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -1,33 +1,51 @@ from base import Task from common import phases +from common.tasks import apt -class HostPackages(Task): - description = 'Determining required host packages' - phase = phases.preparation +class InstallRemotePackages(Task): + description = 'Installing remote packages' + phase = phases.package_installation + predecessors = [apt.AptUpgrade] def run(self, info): - info.host_packages = set() - info.host_packages.add('debootstrap') + if len(info.packages.remote) == 0: + return + import os + from common.tools import log_check_call - from common.fs.loopbackvolume import LoopbackVolume - if isinstance(info.volume, LoopbackVolume): - info.host_packages.add('qemu-utils') + packages = [] + for name, target in info.packages.remote.iteritems(): + packages.append('{name}/{target}'.format(name=name, target=target)) - if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): - info.host_packages.add('xfsprogs') - - from base.fs.partitionmaps.none import NoPartitions - if not isinstance(info.volume.partition_map, NoPartitions): - info.host_packages.update(['parted', 'kpartx']) + env = os.environ.copy() + env['DEBIAN_FRONTEND'] = 'noninteractive' + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install', + '--assume-yes'] + packages, + env=env) -class ImagePackages(Task): - description = 'Determining required image packages' - phase = phases.preparation +class InstallLocalPackages(Task): + description = 'Installing local packages' + phase = phases.package_installation + predecessors = [apt.AptUpgrade] + successors = [InstallRemotePackages] def run(self, info): - info.img_packages = set(), set() - include, exclude = info.img_packages - # We could bootstrap without locales, but things just suck without them, error messages etc. - include.add('locales') + if len(info.packages.local) == 0: + return + from shutil import copy + from common.tools import log_check_call + import os + + for package_src in info.packages.local: + pkg_name = os.path.basename(package_src) + package_dst = os.path.join('/tmp', pkg_name) + copy(package_src, os.path.join(info.root, package_dst)) + + env = os.environ.copy() + env['DEBIAN_FRONTEND'] = 'noninteractive' + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/bin/dpkg', '--install', package_dst], + env=env) + os.remove(package_dst) diff --git a/common/tools.py b/common/tools.py index 47a14b0..31d76c2 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,14 +1,14 @@ -def log_check_call(command, stdin=None): - status, stdout, stderr = log_call(command, stdin) +def log_check_call(command, stdin=None, env=None): + status, stdout, stderr = log_call(command, stdin, env) if status != 0: from subprocess import CalledProcessError raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr)) return stdout -def log_call(command, stdin=None): +def log_call(command, stdin=None, env=None): import subprocess import select @@ -18,13 +18,19 @@ def log_call(command, stdin=None): log = logging.getLogger(__name__ + command_log) log.debug('Executing: {command}'.format(command=' '.join(command))) + popen_args = {'args': command, + 'env': env, + 'stdin': subprocess.PIPE, + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE, } if stdin is not None: - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen_args['stdin'] = subprocess.PIPE + process = subprocess.Popen(**popen_args) process.stdin.write(stdin + "\n") process.stdin.flush() process.stdin.close() else: - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen(**popen_args) stdout = [] stderr = [] while True: diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index 45c726c..7bd2311 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -30,19 +30,19 @@ } } }, - "plugins": { - "packages": { - "sources": { - "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" - ] - }, - "remote": [ - "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + "packages": { + "sources": { + "backports": [ + "deb {apt_mirror} {release}-backports main", + "deb-src {apt_mirror} {release}-backports main" ] }, + "remote": [ + "sudo", + { "name": "cloud-init", "target": "{release}-backports" } + ] + }, + "plugins": { "cloud_init": { "username": "admin", //"metadata_sources": "Ec2", diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index 9631562..47bcece 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -30,19 +30,19 @@ } } }, - "plugins": { - "packages": { - "sources": { - "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" - ] - }, - "remote": [ - "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + "packages": { + "sources": { + "backports": [ + "deb {apt_mirror} {release}-backports main", + "deb-src {apt_mirror} {release}-backports main" ] }, + "remote": [ + "sudo", + { "name": "cloud-init", "target": "{release}-backports" } + ] + }, + "plugins": { "cloud_init": { "username": "admin", //"metadata_sources": "Ec2", diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index e44cfe6..40c45f4 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -30,19 +30,19 @@ } } }, - "plugins": { - "packages": { - "sources": { - "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" - ] - }, - "remote": [ - "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + "packages": { + "sources": { + "backports": [ + "deb {apt_mirror} {release}-backports main", + "deb-src {apt_mirror} {release}-backports main" ] }, + "remote": [ + "sudo", + { "name": "cloud-init", "target": "{release}-backports" } + ] + }, + "plugins": { "cloud_init": { "username": "admin", //"metadata_sources": "Ec2", diff --git a/plugins/admin_user/tasks.py b/plugins/admin_user/tasks.py index b5d34a8..5f55690 100644 --- a/plugins/admin_user/tasks.py +++ b/plugins/admin_user/tasks.py @@ -1,20 +1,15 @@ from base import Task from common import phases -from common.tasks.packages import ImagePackages -from common.tasks.host import CheckPackages from common.tasks.initd import InstallInitScripts -from plugins.packages.tasks import InstallRemotePackages import os class AddSudoPackage(Task): description = 'Adding ``sudo\'\' to the image packages' phase = phases.preparation - predecessors = [ImagePackages] - successors = [CheckPackages] def run(self, info): - info.img_packages[0].add('sudo') + info.packages.add('sudo') class CreateAdminUser(Task): @@ -58,7 +53,6 @@ class AdminUserCredentials(Task): class DisableRootLogin(Task): description = 'Disabling SSH login for root' phase = phases.system_modification - predecessors = [InstallRemotePackages] def run(self, info): from subprocess import CalledProcessError diff --git a/plugins/packages/__init__.py b/plugins/packages/__init__.py deleted file mode 100644 index db6e5cc..0000000 --- a/plugins/packages/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ - - -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) - - -def resolve_tasks(tasklist, manifest): - from tasks import AptSources, InstallRemotePackages, InstallLocalPackages - packages = manifest.plugins['packages'] - if 'sources' in packages: - tasklist.add(AptSources) - if 'remote' in packages: - tasklist.add(InstallRemotePackages) - if 'local' in packages: - tasklist.add(InstallLocalPackages) diff --git a/plugins/packages/manifest-schema.json b/plugins/packages/manifest-schema.json deleted file mode 100644 index 12b2499..0000000 --- a/plugins/packages/manifest-schema.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Packages plugin manifest", - "type": "object", - "properties": { - "plugins": { - "type": "object", - "properties": { - "packages": { - "type": "object", - "properties": { - "sources": { - "type": "object", - "patternProperties": { - "^\\w+$": { - "type": "array", - "items": {"type": "string"}, - "minItems": 1 - } - }, - "additionalProperties": false, - "minItems": 1 - }, - "remote": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "object", - "properties": { - "name": {"type": "string"}, - "target": {"type": "string"} - }, - "required": ["name"] - }, - { "type": "string" } - ] - }, - "minItems": 1 - }, - "local": { - "type": "array", - "items": { "$ref": "#/definitions/absolute_path" }, - "minItems": 1 - } - }, - "anyOf": [ - {"required": ["sources"]}, - {"required": ["remote"]}, - {"required": ["local"]} - ] - } - } - } - }, - "definitions": { - "absolute_path": { - "type": "string", - "pattern": "^/[^\\0]+$" - } - } -} diff --git a/plugins/packages/tasks.py b/plugins/packages/tasks.py deleted file mode 100644 index b372545..0000000 --- a/plugins/packages/tasks.py +++ /dev/null @@ -1,63 +0,0 @@ -from base import Task -from common import phases -from common.tasks import network -from common.tasks import apt -import os.path - - -class AptSources(Task): - description = 'Adding additional aptitude sources' - phase = phases.system_modification - predecessors = [apt.AptSources] - successors = [apt.AptUpdate] - - def run(self, info): - manifest_vars = {'release': info.manifest.system['release'], - 'architecture': info.manifest.system['architecture'], - 'apt_mirror': 'http://http.debian.net/debian'} - for name, lines in info.manifest.plugins['packages']['sources'].iteritems(): - list_path = os.path.join(info.root, 'etc/apt/sources.list.d/', name + '.list') - with open(list_path, 'a') as source_list: - for line in lines: - source_list.write('{line}\n'.format(line=line.format(**manifest_vars))) - - -class InstallRemotePackages(Task): - description = 'Installing remote packages' - phase = phases.system_modification - predecessors = [apt.AptUpdate, apt.DisableDaemonAutostart] - successors = [network.RemoveDNSInfo] - - def run(self, info): - manifest_vars = {'release': info.manifest.system['release'], - 'architecture': info.manifest.system['architecture']} - from common.tools import log_check_call - for package in info.manifest.plugins['packages']['remote']: - target_release = [] - if isinstance(package, basestring): - name = package - else: - name = package['name'].format(**manifest_vars) - if hasattr(package, 'target'): - target = package['target'].format(**manifest_vars) - target_release = ['--targe-release', target] - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install', - '--assume-yes'] + target_release + [name]) - - -class InstallLocalPackages(Task): - description = 'Installing local packages' - phase = phases.system_modification - - def run(self, info): - from shutil import copy - from os import remove - from common.tools import log_check_call - - for package_src in info.manifest.plugins['packages']['local']: - pkg_name = os.path.basename(package_src) - package_dst = os.path.join('/tmp', pkg_name) - copy(package_src, os.path.join(info.root, package_dst)) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg', '--install', package_dst]) - remove(package_dst) diff --git a/plugins/unattended_upgrades/tasks.py b/plugins/unattended_upgrades/tasks.py index 6dedf55..d66e644 100644 --- a/plugins/unattended_upgrades/tasks.py +++ b/plugins/unattended_upgrades/tasks.py @@ -1,17 +1,13 @@ from base import Task from common import phases -from common.tasks.packages import ImagePackages -from common.tasks.host import CheckPackages class AddUnattendedUpgradesPackage(Task): description = 'Adding ``unattended-upgrades\'\' to the image packages' phase = phases.preparation - predecessors = [ImagePackages] - successors = [CheckPackages] def run(self, info): - info.img_packages[0].add('unattended-upgrades') + info.packages.add('unattended-upgrades') class EnablePeriodicUpgrades(Task): diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 9dae360..c4c7dee 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -43,8 +43,8 @@ def resolve_tasks(tasklist, manifest): from common.task_sets import partitioning_set tasklist.add(*partitioning_set) - tasklist.add(packages.HostPackages, - packages.ImagePackages, + tasklist.add(packages.HostDependencies, + packages.DefaultPackages, connection.GetCredentials, host.GetInfo, ami.AMIName, diff --git a/providers/ec2/tasks/host.py b/providers/ec2/tasks/host.py index c452e0f..851431b 100644 --- a/providers/ec2/tasks/host.py +++ b/providers/ec2/tasks/host.py @@ -1,5 +1,16 @@ from base import Task from common import phases +from common.tasks import host + + +class HostDependencies(Task): + description = 'Adding more required host packages' + phase = phases.preparation + successors = [host.CheckHostDependencies] + + def run(self, info): + if info.manifest.volume['backing'] == 's3': + info.host_dependencies.add('euca2ools') class GetInfo(Task): diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index 080de06..13b11a4 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -1,39 +1,24 @@ from base import Task from common import phases -from common.tasks import packages -from common.tasks.host import CheckPackages -class HostPackages(Task): - description = 'Adding more required host packages' +class DefaultPackages(Task): + description = 'Adding image packages required for EC2' phase = phases.preparation - predecessors = [packages.HostPackages] - successors = [CheckPackages] def run(self, info): - if info.manifest.volume['backing'] == 's3': - info.host_packages.add('euca2ools') + info.packages.add('openssh-server') + info.packages.add('file') # Needed for the init scripts + info.packages.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2 + info.packages.add('grub-pc') - -class ImagePackages(Task): - description = 'Adding more required image packages' - phase = phases.preparation - predecessors = [packages.ImagePackages] - - def run(self, info): - manifest = info.manifest - include, exclude = info.img_packages - include.add('openssh-server') - include.add('file') # Needed for the init scripts - include.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2 - include.add('grub-pc') - - exclude.add('isc-dhcp-client') - exclude.add('isc-dhcp-common') + info.exclude_packages.add('isc-dhcp-client') + info.exclude_packages.add('isc-dhcp-common') # In squeeze, we need a special kernel flavor for xen kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64', 'i386': 'linux-image-xen-686', }, 'wheezy': {'amd64': 'linux-image-amd64', 'i386': 'linux-image-686', }, } - include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + kernel_package = kernels.get(info.manifest.system['release']).get(info.manifest.system['architecture']) + info.packages.add(kernel_package) diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 6681fcd..5bb1db6 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -33,7 +33,7 @@ def resolve_tasks(tasklist, manifest): from common.task_sets import partitioning_set tasklist.add(*partitioning_set) - tasklist.add(packages.ImagePackages, + tasklist.add(packages.DefaultPackages, loopback.Create, diff --git a/providers/virtualbox/tasks/guest_additions.py b/providers/virtualbox/tasks/guest_additions.py index 3d1707d..16119e6 100644 --- a/providers/virtualbox/tasks/guest_additions.py +++ b/providers/virtualbox/tasks/guest_additions.py @@ -1,7 +1,6 @@ from base import Task from common import phases -from common.tasks.packages import ImagePackages -from common.tasks.host import CheckPackages +from common.tasks.packages import InstallRemotePackages from common.tasks.filesystem import FStab from common.exceptions import TaskError @@ -21,20 +20,18 @@ class CheckGuestAdditionsPath(Task): class AddGuestAdditionsPackages(Task): description = 'Adding packages to support Guest Additions installation' phase = phases.preparation - predecessors = [ImagePackages] - successors = [CheckPackages] def run(self, info): - info.img_packages[0].update(['bzip2', - 'build-essential', - 'dkms', - ]) + info.packages.add('bzip2') + info.packages.add('build-essential') + info.packages.add('dkms') + # info.packages.add('linux-headers-3.2.0-4-amd64') # linux-headers-$(uname -r) class InstallGuestAdditions(Task): description = 'Installing the VirtualBox Guest Additions' phase = phases.system_modification - predecessors = [FStab] + predecessors = [FStab, InstallRemotePackages] def run(self, info): import os diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 8d16285..ca4cb6d 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -3,20 +3,14 @@ from common import phases from common.tasks import packages -class ImagePackages(Task): - description = 'Determining required image packages' +class DefaultPackages(Task): + description = 'Adding image packages required for virtualbox' phase = phases.preparation - predecessors = [packages.ImagePackages] def run(self, info): - manifest = info.manifest - include, exclude = info.img_packages # Add some basic packages we are going to need - include.add('grub2') + info.packages.add('grub2') - # In squeeze, we need a special kernel flavor for xen - kernels = {'squeeze': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, - 'wheezy': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, } - include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + kernels = {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', } + info.packages.add(kernels.get(info.manifest.system['architecture']))