From 45521b23771799c85313fa91832346a70b235bdd Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Jan 2014 12:46:59 +0100 Subject: [PATCH] Merge remote and local package array in manifest This allows us to specify the order of installation for packages Fixes #127 --- base/manifest-schema.json | 23 ++---- base/phase.py | 2 +- base/pkg/packagelist.py | 50 ++++++++--- common/task_sets.py | 11 ++- common/tasks/apt.py | 49 ++++------- common/tasks/packages.py | 82 +++++++++++-------- plugins/prebootstrapped/tasks.py | 4 +- providers/virtualbox/tasks/guest_additions.py | 6 +- 8 files changed, 122 insertions(+), 105 deletions(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index fe587f3..52461d5 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -53,27 +53,20 @@ "additionalProperties": false, "minItems": 1 }, - "remote": { + "trusted-keys": { + "type": "array", + "items": { "$ref": "#/definitions/absolute_path" }, + "minItems": 1 + }, + "install": { "type": "array", "items": { "anyOf": [ - { - "type": "object", - "properties": { - "name": {"type": "string"}, - "target": {"type": "string"} - }, - "required": ["name"] - }, - { "type": "string" } + { "pattern": "^[^/]+(/[^/]+)?$" }, + { "$ref": "#/definitions/absolute_path" } ] }, "minItems": 1 - }, - "local": { - "type": "array", - "items": { "$ref": "#/definitions/absolute_path" }, - "minItems": 1 } }, "additionalProperties": false diff --git a/base/phase.py b/base/phase.py index 1ed1233..9acf825 100644 --- a/base/phase.py +++ b/base/phase.py @@ -7,7 +7,7 @@ class Phase(object): def pos(self): from common.phases import order - return (i for i, phase in enumerate(order) if phase is self).next() + return next(i for i, phase in enumerate(order) if phase is self) def __cmp__(self, other): return self.pos() - other.pos() diff --git a/base/pkg/packagelist.py b/base/pkg/packagelist.py index 1037374..762bb4a 100644 --- a/base/pkg/packagelist.py +++ b/base/pkg/packagelist.py @@ -3,29 +3,55 @@ from exceptions import PackageError class PackageList(object): + class Remote(object): + def __init__(self, name, target): + self.name = name + self.target = target + + def __str__(self): + if self.target is None: + return self.name + else: + return '{name}/{target}'.format(name=self.name, target=self.target) + + class Local(object): + def __init__(self, path): + self.path = path + + def __str__(self): + return self.path + def __init__(self, manifest_vars, source_lists): self.manifest_vars = manifest_vars self.source_lists = source_lists - self.remote = {} - self.local = set() + self.default_target = '{system.release}'.format(**self.manifest_vars) + self.install = [] + self.remote = lambda: filter(lambda x: isinstance(x, self.Remote), self.install) def add(self, name, target=None): - if target is None: - target = '{system.release}' name = name.format(**self.manifest_vars) - target = target.format(**self.manifest_vars) - if name in self.remote: - if self.remote[name] != target: + if target is not None: + target = target.format(**self.manifest_vars) + package = next((pkg for pkg in self.remote() if pkg.name == name), None) + if package is not None: + same_target = package.target != target + same_target = same_target or package.target is None and target == self.default_target + same_target = same_target or package.target == self.default_target and target is None + if not same_target: msg = ('The package {name} was already added to the package list, ' - 'but with another target release ({target})').format(name=name, target=self.remote[name]) + 'but with another target release ({target})').format(name=name, target=package.target) raise PackageError(msg) return - if not self.source_lists.target_exists(target): - msg = ('The target release {target} was not found in the sources list').format(target=target) + check_target = target + if check_target is None: + check_target = self.default_target + if not self.source_lists.target_exists(check_target): + msg = ('The target release {target} was not found in the sources list').format(target=check_target) raise PackageError(msg) - self.remote[name] = target + + self.install.append(self.Remote(name, target)) def add_local(self, package_path): package_path = package_path.format(**self.manifest_vars) - self.local.add(package_path) + self.install.append(self.Local(package_path)) diff --git a/common/task_sets.py b/common/task_sets.py index dac378c..d0afec4 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -52,18 +52,17 @@ def get_apt_set(manifest): apt.DisableDaemonAutostart, apt.AptUpdate, apt.AptUpgrade, - packages.InstallRemotePackages, - packages.InstallLocalPackages, + packages.InstallPackages, apt.PurgeUnusedPackages, apt.AptClean, apt.EnableDaemonAutostart, ] if 'sources' in manifest.packages: base.append(apt.AddManifestSources) - if 'remote' in manifest.packages: - base.append(apt.AddRemoteManifestPackages) - if 'local' in manifest.packages: - base.append(apt.AddLocalManifestPackages) + if 'trusted-keys' in manifest.packages: + base.append(apt.InstallTrustedKeys) + if 'install' in manifest.packages: + base.append(packages.AddManifestPackages) return base diff --git a/common/tasks/apt.py b/common/tasks/apt.py index 97bef76..ca3c545 100644 --- a/common/tasks/apt.py +++ b/common/tasks/apt.py @@ -11,10 +11,9 @@ class AddManifestSources(Task): @classmethod def run(cls, info): - if 'sources' in info.manifest.packages: - for name, lines in info.manifest.packages['sources'].iteritems(): - for line in lines: - info.source_lists.add(name, line) + for name, lines in info.manifest.packages['sources'].iteritems(): + for line in lines: + info.source_lists.add(name, line) class AddDefaultSources(Task): @@ -24,45 +23,29 @@ class AddDefaultSources(Task): @classmethod def run(cls, info): - if info.source_lists.target_exists('{system.release}'): - import logging - msg = ('{system.release} target already exists').format(**info.manifest_vars) - logging.getLogger(__name__).info(msg) - else: - info.source_lists.add('main', 'deb {apt_mirror} {system.release} main') - info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} main') - info.source_lists.add('main', 'deb {apt_mirror} {system.release}-updates main') - info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates main') + info.source_lists.add('main', 'deb {apt_mirror} {system.release} main') + info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} main') + info.source_lists.add('main', 'deb {apt_mirror} {system.release}-updates main') + info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates main') -class AddRemoteManifestPackages(Task): - description = 'Adding remote packages from the manifest' - phase = phases.preparation - predecessors = [AddDefaultSources] +class InstallTrustedKeys(Task): + description = 'Installing trusted keys' + phase = phases.package_installation @classmethod def run(cls, info): - for package in info.manifest.packages['remote']: - if isinstance(package, dict): - info.packages.add(package['name'], package.get('target', None)) - else: - info.packages.add(package, None) - - -class AddLocalManifestPackages(Task): - description = 'Adding local packages from the manifest' - phase = phases.preparation - predecessors = [AddDefaultSources] - - @classmethod - def run(cls, info): - for package_path in info.manifest.packages['local']: - info.packages.add_local(package_path) + from shutil import copy + for key_path in info.manifest.packages['trusted-keys']: + key_name = os.path.basename(key_path) + destination = os.path.join(info.root, 'etc/apt/trusted.gpg.d', key_name) + copy(key_path, destination) class WriteSources(Task): description = 'Writing aptitude sources to disk' phase = phases.package_installation + predecessors = [InstallTrustedKeys] @classmethod def run(cls, info): diff --git a/common/tasks/packages.py b/common/tasks/packages.py index a266571..a21db2c 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -3,26 +3,43 @@ from common import phases from common.tasks import apt -class InstallRemotePackages(Task): - description = 'Installing remote packages' +class AddManifestPackages(Task): + description = 'Adding packages from the manifest' + phase = phases.preparation + predecessors = [apt.AddDefaultSources] + + @classmethod + def run(cls, info): + import re + remote = re.compile('^(?P[^/]+)(/(?P[^/]+))?$') + for package in info.manifest.packages['install']: + match = remote.match(package).groupdict() + if match is not None: + info.packages.add(match['name'], match['target']) + else: + info.packages.add_local(package) + + +class InstallPackages(Task): + description = 'Installing packages' phase = phases.package_installation predecessors = [apt.AptUpgrade] @classmethod def run(cls, info): - if len(info.packages.remote) == 0: - return + batch = [] + actions = {info.packages.Remote: cls.install_remote, + info.packages.Local: cls.install_local} + for i, package in enumerate(info.packages.install): + batch.append(package) + next_package = info.packages.install[i + 1] if i + 1 < len(info.packages.install) else None + if next_package is None or package.__class__ is not next_package.__class__: + actions[package.__class__](info, batch) + batch = [] + + @classmethod + def install_remote(cls, info, remote_packages): import os - - packages = [] - for name, target in info.packages.remote.iteritems(): - packages.append('{name}/{target}'.format(name=name, target=target)) - - import logging - msg = ('The following packages will be installed (package/target-release):' - '\n{packages}\n').format(packages='\n'.join(packages)) - logging.getLogger(__name__).debug(msg) - from common.tools import log_check_call from subprocess import CalledProcessError try: @@ -32,9 +49,10 @@ class InstallRemotePackages(Task): '/usr/bin/apt-get', 'install', '--no-install-recommends', '--assume-yes'] - + packages, + + map(str, remote_packages), env=env) except CalledProcessError as e: + import logging disk_stat = os.statvfs(info.root) root_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024 disk_stat = os.statvfs(os.path.join(info.root, 'boot')) @@ -53,31 +71,29 @@ class InstallRemotePackages(Task): logging.getLogger(__name__).warn(msg) raise e - -class InstallLocalPackages(Task): - description = 'Installing local packages' - phase = phases.package_installation - predecessors = [apt.AptUpgrade] - successors = [InstallRemotePackages] - @classmethod - def run(cls, info): - if len(info.packages.local) == 0: - return + def install_local(cls, info, local_packages): from shutil import copy from common.tools import log_check_call import os - for package_src in info.packages.local: + absolute_package_paths = [] + chrooted_package_paths = [] + for package_src in local_packages: pkg_name = os.path.basename(package_src) package_rel_dst = os.path.join('tmp', pkg_name) package_dst = os.path.join(info.root, package_rel_dst) copy(package_src, package_dst) - + absolute_package_paths.append(package_dst) package_path = os.path.join('/', package_rel_dst) - env = os.environ.copy() - env['DEBIAN_FRONTEND'] = 'noninteractive' - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg', '--install', package_path], - env=env) - os.remove(package_dst) + chrooted_package_paths.append(package_path) + + env = os.environ.copy() + env['DEBIAN_FRONTEND'] = 'noninteractive' + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/bin/dpkg', '--install'] + + chrooted_package_paths, + env=env) + + for path in absolute_package_paths: + os.remove(path) diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index c3ec9a6..6e39d05 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -15,7 +15,7 @@ log = logging.getLogger(__name__) class Snapshot(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.package_installation - predecessors = [packages.InstallRemotePackages, guest_additions.InstallGuestAdditions] + predecessors = [packages.InstallPackages, guest_additions.InstallGuestAdditions] @classmethod def run(cls, info): @@ -49,7 +49,7 @@ class CreateFromSnapshot(Task): class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.package_installation - predecessors = [packages.InstallRemotePackages, guest_additions.InstallGuestAdditions] + predecessors = [packages.InstallPackages, guest_additions.InstallGuestAdditions] @classmethod def run(cls, info): diff --git a/providers/virtualbox/tasks/guest_additions.py b/providers/virtualbox/tasks/guest_additions.py index 75eb028..37dd537 100644 --- a/providers/virtualbox/tasks/guest_additions.py +++ b/providers/virtualbox/tasks/guest_additions.py @@ -1,6 +1,6 @@ from base import Task from common import phases -from common.tasks.packages import InstallRemotePackages +from common.tasks.packages import InstallPackages from common.exceptions import TaskError @@ -20,7 +20,7 @@ class CheckGuestAdditionsPath(Task): class AddGuestAdditionsPackages(Task): description = 'Adding packages to support Guest Additions installation' phase = phases.package_installation - successors = [InstallRemotePackages] + successors = [InstallPackages] @classmethod def run(cls, info): @@ -38,7 +38,7 @@ class AddGuestAdditionsPackages(Task): class InstallGuestAdditions(Task): description = 'Installing the VirtualBox Guest Additions' phase = phases.package_installation - predecessors = [InstallRemotePackages] + predecessors = [InstallPackages] @classmethod def run(cls, info):