From 8d32339fda473168917b750d57100eb78dd2a1fb Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 29 Dec 2013 16:48:55 +0100 Subject: [PATCH] Introduced a global manifest_vars dictionary All manifest entries can now be accessed from the manifest itself --- base/bootstrapinfo.py | 37 +++++++++++++++++-- base/manifest.py | 2 + base/pkg/packagelist.py | 27 +++++++------- base/pkg/sourceslist.py | 11 +++--- common/tasks/loopback.py | 12 +----- ...bs-debian-official-amd64-hvm.manifest.json | 10 ++--- ...bs-debian-official-amd64-pvm.manifest.json | 10 ++--- ...ebs-debian-official-i386-pvm.manifest.json | 10 ++--- manifests/ec2-ebs-partitioned.manifest.json | 4 +- manifests/ec2-ebs-single.manifest.json | 4 +- manifests/ec2-s3.manifest.json | 4 +- manifests/virtualbox.manifest.json | 4 +- providers/ec2/tasks/ami.py | 16 +------- 13 files changed, 80 insertions(+), 71 deletions(-) diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index ddb24e5..4650a2d 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -14,10 +14,39 @@ class BootstrapInformation(object): from fs import load_volume self.volume = load_volume(self.manifest.volume) - from pkg.source import SourceLists - self.source_lists = SourceLists(self.manifest) - from pkg.package import PackageList - self.packages = PackageList(self.source_lists, self.manifest) + class DictClass(dict): + def __getattr__(self, name): + return self[name] + + def __setattr__(self, name, value): + self[name] = value + + def set_manifest_vars(obj, data): + for key, value in data.iteritems(): + if isinstance(value, dict): + obj[key] = DictClass() + set_manifest_vars(obj[key], value) + continue + if not isinstance(value, list): + obj[key] = value + + self.manifest_vars = {} + self.manifest_vars['apt_mirror'] = 'http://http.debian.net/debian' + set_manifest_vars(self.manifest_vars, self.manifest.data) + + from datetime import datetime + now = datetime.now() + time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', + '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', + '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] + for key in time_vars: + self.manifest_vars[key] = now.strftime(key) + + from pkg.sourceslist import SourceLists + self.source_lists = SourceLists(self.manifest.packages, self.manifest_vars) + from pkg.packagelist import PackageList + self.packages = PackageList(self.manifest.packages, self.manifest_vars, + default_target=manifest.system['release'], source_lists=self.source_lists) self.include_packages = set() self.exclude_packages = set() diff --git a/base/manifest.py b/base/manifest.py index 0f65149..a452a96 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -45,12 +45,14 @@ class Manifest(object): raise ManifestError(e.message, self, e.path) def parse(self, data): + self.data = data self.provider = data['provider'] self.bootstrapper = data['bootstrapper'] if 'mirror' not in self.bootstrapper: self.bootstrapper['mirror'] = 'http://http.debian.net/debian' self.volume = data['volume'] self.system = data['system'] + self.packages = data['packages'] self.plugins = data['plugins'] if 'plugins' in data else {} def load_plugins(self, data): diff --git a/base/pkg/packagelist.py b/base/pkg/packagelist.py index 3dbaaab..7296521 100644 --- a/base/pkg/packagelist.py +++ b/base/pkg/packagelist.py @@ -3,30 +3,31 @@ from exceptions import PackageError class PackageList(object): - def __init__(self, sources_list, manifest): - self.sources_list = sources_list - self.default_target = manifest.system['release'] + def __init__(self, data, manifest_vars, default_target, source_lists): + self.manifest_vars = manifest_vars + self.source_lists = source_lists + self.default_target = default_target self.remote = {} self.local = set() - if 'remote' in manifest.packages: - manifest_vars = {'release': manifest.system['release'], - 'architecture': manifest.system['architecture']} - for package in manifest.packages['remote']: + if 'remote' in data: + for package in data['remote']: target = None if isinstance(package, dict): - name = package['name'].format(**manifest_vars) + name = package['name'].format(**self.manifest_vars) if 'target' in package: - target = package['target'].format(**manifest_vars) + target = package['target'].format(**self.manifest_vars) else: - name = package.format(**manifest_vars) + name = package.format(**self.manifest_vars) self.add(name, target) - if 'local' in manifest.packages: - for package_path in manifest.packages['local']: + if 'local' in data: + for package_path in data['local']: self.local.add(package_path) def add(self, name, target=None): if target is None: target = self.default_target + name = name.format(**self.manifest_vars) + target = target.format(**self.manifest_vars) if name in self.remote: if self.remote[name] != target: msg = ('The package {name} was already added to the package list, ' @@ -34,7 +35,7 @@ class PackageList(object): raise PackageError(msg) return - if not self.sources_list.target_exists(target): + if not self.source_lists.target_exists(target): msg = ('The target release {target} was not found in the sources list').format(target=target) raise PackageError(msg) self.remote[name] = target diff --git a/base/pkg/sourceslist.py b/base/pkg/sourceslist.py index 077ebbb..c2b5df8 100644 --- a/base/pkg/sourceslist.py +++ b/base/pkg/sourceslist.py @@ -2,18 +2,17 @@ class SourceLists(object): - def __init__(self, manifest): + def __init__(self, data, manifest_vars): self.sources = {} - self.manifest_vars = {'release': manifest.system['release'], - 'architecture': manifest.system['architecture'], - 'apt_mirror': 'http://http.debian.net/debian'} - if 'sources' in manifest.packages: - for name, lines in manifest.packages['sources'].iteritems(): + self.manifest_vars = manifest_vars + if 'sources' in data: + for name, lines in data['sources'].iteritems(): for line in lines: self.add_source(name, '{line}\n'.format(line=line.format(**self.manifest_vars))) def add_source(self, name, line): name = name.format(**self.manifest_vars) + line = line.format(**self.manifest_vars) if name not in self.sources: self.sources[name] = [] self.sources[name].append(Source(line)) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 4d3b536..7af87b9 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -19,17 +19,7 @@ class MoveImage(Task): phase = phases.image_registration def run(self, info): - manifest_vars = {'release': info.manifest.system['release'], - 'architecture': info.manifest.system['architecture']} - from datetime import datetime - now = datetime.now() - time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', - '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', - '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] - for var in time_vars: - manifest_vars[var] = now.strftime(var) - - image_name = info.manifest.image['name'].format(**manifest_vars) + image_name = info.manifest.image['name'].format(**info.manifest_vars) filename = '{image_name}.{ext}'.format(image_name=image_name, ext=info.volume.extension) import os.path diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index 7bd2311..5a09085 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", - "description": "Debian {release} {architecture}" + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {system.release} {system.architecture}" }, "system": { "release": "wheezy", @@ -33,13 +33,13 @@ "packages": { "sources": { "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" + "deb {apt_mirror} {system.release}-backports main", + "deb-src {apt_mirror} {system.release}-backports main" ] }, "remote": [ "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + { "name": "cloud-init", "target": "{system.release}-backports" } ] }, "plugins": { diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index 47bcece..560a135 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", - "description": "Debian {release} {architecture}" + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {system.release} {system.architecture}" }, "system": { "release": "wheezy", @@ -33,13 +33,13 @@ "packages": { "sources": { "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" + "deb {apt_mirror} {system.release}-backports main", + "deb-src {apt_mirror} {system.release}-backports main" ] }, "remote": [ "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + { "name": "cloud-init", "target": "{system.release}-backports" } ] }, "plugins": { diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index 40c45f4..894c102 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", - "description": "Debian {release} {architecture}" + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {system.release} {system.architecture}" }, "system": { "release": "wheezy", @@ -33,13 +33,13 @@ "packages": { "sources": { "backports": [ - "deb {apt_mirror} {release}-backports main", - "deb-src {apt_mirror} {release}-backports main" + "deb {apt_mirror} {system.release}-backports main", + "deb-src {apt_mirror} {system.release}-backports main" ] }, "remote": [ "sudo", - { "name": "cloud-init", "target": "{release}-backports" } + { "name": "cloud-init", "target": "{system.release}-backports" } ] }, "plugins": { diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json index 41666dc..abefea6 100644 --- a/manifests/ec2-ebs-partitioned.manifest.json +++ b/manifests/ec2-ebs-partitioned.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {system.release} {system.architecture} AMI ({virtualization})" }, "system": { "release": "wheezy", diff --git a/manifests/ec2-ebs-single.manifest.json b/manifests/ec2-ebs-single.manifest.json index b8c6456..3b70683 100644 --- a/manifests/ec2-ebs-single.manifest.json +++ b/manifests/ec2-ebs-single.manifest.json @@ -10,8 +10,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {system.release} {system.architecture} AMI ({virtualization})" }, "system": { "release": "wheezy", diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json index 93100d0..6dbb7c9 100644 --- a/manifests/ec2-s3.manifest.json +++ b/manifests/ec2-s3.manifest.json @@ -13,8 +13,8 @@ "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI", + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {system.release} {system.architecture} AMI", "bucket": "debian-amis" }, "system": { diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index a795c49..30eb166 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -5,8 +5,8 @@ // "guest_additions": "/root/images/VBoxGuestAdditions.iso" }, "image": { - "name": "debian-{release}-{architecture}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture}" + "name": "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}", + "description": "Debian {system.release} {system.architecture}" }, "system": { "release": "wheezy", diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index e975684..801eb69 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -16,20 +16,8 @@ class AMIName(Task): predecessors = [Connect] def run(self, info): - image_vars = {'release': info.manifest.system['release'], - 'architecture': info.manifest.system['architecture'], - 'virtualization': info.manifest.virtualization, - 'backing': info.manifest.volume['backing']} - from datetime import datetime - now = datetime.now() - time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', - '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', - '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] - for var in time_vars: - image_vars[var] = now.strftime(var) - - ami_name = info.manifest.image['name'].format(**image_vars) - ami_description = info.manifest.image['description'].format(**image_vars) + ami_name = info.manifest.image['name'].format(**info.manifest_vars) + ami_description = info.manifest.image['description'].format(**info.manifest_vars) images = info.connection.get_all_images() for image in images: