From 46dbe9e8c6dce3917e282bc7b53c1a389ce268b4 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 23 Feb 2014 20:14:23 +0100 Subject: [PATCH] Move AKI info into JSON file. Fixes #13 Also introduced config_get() to retrieve a specific value from these JSON files. Comments are now supported in those files as well --- base/manifest.py | 5 ++- {base => common}/minify_json.py | 0 common/tasks/network.py | 10 ++--- common/tools.py | 14 ++++++ providers/ec2/tasks/ami-akis.json | 34 ++++++++++++++ providers/ec2/tasks/ami.py | 55 ++--------------------- providers/ec2/tasks/packages-kernels.json | 1 + providers/ec2/tasks/packages.py | 11 +++-- 8 files changed, 66 insertions(+), 64 deletions(-) rename {base => common}/minify_json.py (100%) create mode 100644 providers/ec2/tasks/ami-akis.json diff --git a/base/manifest.py b/base/manifest.py index a51f826..632c28e 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -1,3 +1,4 @@ +from common.tools import load_json import logging log = logging.getLogger(__name__) @@ -10,7 +11,7 @@ class Manifest(object): self.parse() def load(self): - self.data = self.load_json(self.path) + self.data = load_json(self.path) provider_modname = 'providers.{provider}'.format(provider=self.data['provider']) log.debug('Loading provider `{modname}\''.format(modname=provider_modname)) self.modules = {'provider': __import__(provider_modname, fromlist=['providers']), @@ -55,7 +56,7 @@ class Manifest(object): def schema_validator(self, data, schema_path): import jsonschema - schema = self.load_json(schema_path) + schema = load_json(schema_path) try: jsonschema.validate(data, schema) except jsonschema.ValidationError as e: diff --git a/base/minify_json.py b/common/minify_json.py similarity index 100% rename from base/minify_json.py rename to common/minify_json.py diff --git a/common/tasks/network.py b/common/tasks/network.py index e499929..3ebdf44 100644 --- a/common/tasks/network.py +++ b/common/tasks/network.py @@ -29,10 +29,10 @@ class ConfigureNetworkIF(Task): @classmethod def run(cls, info): + network_config_path = os.path.join(os.path.dirname(__file__), 'network-configuration.json') + from common.tools import config_get + if_config = config_get(network_config_path, [info.manifest.system['release']]) + interfaces_path = os.path.join(info.root, 'etc/network/interfaces') - if_config = [] - with open('common/tasks/network-configuration.json') as stream: - import json - if_config = json.loads(stream.read()) with open(interfaces_path, 'a') as interfaces: - interfaces.write('\n'.join(if_config.get(info.manifest.system['release'])) + '\n') + interfaces.write('\n'.join(if_config) + '\n') diff --git a/common/tools.py b/common/tools.py index 2833b94..dc56637 100644 --- a/common/tools.py +++ b/common/tools.py @@ -56,3 +56,17 @@ def sed_i(file_path, pattern, subst): import re for line in fileinput.input(files=file_path, inplace=True): print re.sub(pattern, subst, line), + + +def load_json(path): + import json + from minify_json import json_minify + with open(path) as stream: + return json.loads(json_minify(stream.read(), False)) + + +def config_get(path, config_path): + config = load_json(path) + for key in config_path: + config = config.get(key) + return config diff --git a/providers/ec2/tasks/ami-akis.json b/providers/ec2/tasks/ami-akis.json new file mode 100644 index 0000000..79e1b66 --- /dev/null +++ b/providers/ec2/tasks/ami-akis.json @@ -0,0 +1,34 @@ +// This is a mapping of EC2 regions to processor architectures to Amazon Kernel Images +// Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs +{ +"ap-northeast-1": // Asia Pacific (Tokyo) Region + {"i386": "aki-136bf512", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-176bf516"}, // pv-grub-hd0_1.04-x86_64.gz +"ap-southeast-1": // Asia Pacific (Singapore) Region + {"i386": "aki-ae3973fc", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-503e7402"}, // pv-grub-hd0_1.04-x86_64.gz +"ap-southeast-2": // Asia Pacific (Sydney) Region + {"i386": "aki-cd62fff7", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-c362fff9"}, // pv-grub-hd0_1.04-x86_64.gz +"eu-west-1": // EU (Ireland) Region + {"i386": "aki-68a3451f", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-52a34525"}, // pv-grub-hd0_1.04-x86_64.gz +"sa-east-1": // South America (Sao Paulo) Region + {"i386": "aki-5b53f446", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-5553f448"}, // pv-grub-hd0_1.04-x86_64.gz +"us-east-1": // US East (Northern Virginia) Region + {"i386": "aki-8f9dcae6", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-919dcaf8"}, // pv-grub-hd0_1.04-x86_64.gz +"us-gov-west-1": // AWS GovCloud (US) + {"i386": "aki-1fe98d3c", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-1de98d3e"}, // pv-grub-hd0_1.04-x86_64.gz +"us-west-1": // US West (Northern California) Region + {"i386": "aki-8e0531cb", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-880531cd"}, // pv-grub-hd0_1.04-x86_64.gz +"us-west-2": // US West (Oregon) Region + {"i386": "aki-f08f11c0", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-fc8f11cc"}, // pv-grub-hd0_1.04-x86_64.gz +"cn-north-1":// China North (Beijing) Region + {"i386": "aki-908f1da9", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-9e8f1da7"} // pv-grub-hd0_1.04-x86_64.gz +} diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index c5f5df4..28de349 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -92,48 +92,6 @@ class RegisterAMI(Task): phase = phases.image_registration predecessors = [Snapshot, UploadImage] - # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs - kernel_mapping = {'ap-northeast-1': # Asia Pacific (Tokyo) Region - {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-176bf516' # pv-grub-hd0_1.04-x86_64.gz - }, - 'ap-southeast-1': # Asia Pacific (Singapore) Region - {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-503e7402' # pv-grub-hd0_1.04-x86_64.gz - }, - 'ap-southeast-2': # Asia Pacific (Sydney) Region - {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-c362fff9' # pv-grub-hd0_1.04-x86_64.gz - }, - 'eu-west-1': # EU (Ireland) Region - {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-52a34525' # pv-grub-hd0_1.04-x86_64.gz - }, - 'sa-east-1': # South America (Sao Paulo) Region - {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-5553f448' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-east-1': # US East (Northern Virginia) Region - {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-919dcaf8' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-gov-west-1': # AWS GovCloud (US) - {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-1de98d3e' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-west-1': # US West (Northern California) Region - {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-880531cd' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-west-2': # US West (Oregon) Region - {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-fc8f11cc' # pv-grub-hd0_1.04-x86_64.gz - }, - 'cn-north-1': # China North (Beijing) Region - {'i386': 'aki-908f1da9', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-9e8f1da7' # pv-grub-hd0_1.04-x86_64.gz - } - } @classmethod def run(cls, info): registration_params = {'name': info.ami_name, @@ -142,17 +100,11 @@ class RegisterAMI(Task): 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) if info.manifest.volume['backing'] == 's3': - grub_boot_device = 'hd0' registration_params['image_location'] = info.manifest.manifest_location else: root_dev_name = {'pvm': '/dev/sda', 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization']) registration_params['root_device_name'] = root_dev_name - from base.fs.partitionmaps.none import NoPartitions - if isinstance(info.volume.partition_map, NoPartitions): - grub_boot_device = 'hd0' - else: - grub_boot_device = 'hd00' from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping @@ -165,8 +117,9 @@ class RegisterAMI(Task): registration_params['virtualization_type'] = 'hvm' else: registration_params['virtualization_type'] = 'paravirtual' - registration_params['kernel_id'] = (cls.kernel_mapping - .get(info.host['region']) - .get(info.manifest.system['architecture'])) + akis_path = os.path.join(os.path.dirname(__file__), 'akis.json') + from common.tools import config_get + registration_params['kernel_id'] = config_get(akis_path, [info.host['region'], + info.manifest.system['architecture']]) info.image = info.connection.register_image(**registration_params) diff --git a/providers/ec2/tasks/packages-kernels.json b/providers/ec2/tasks/packages-kernels.json index 3c9e39f..6bf1e1c 100644 --- a/providers/ec2/tasks/packages-kernels.json +++ b/providers/ec2/tasks/packages-kernels.json @@ -1,4 +1,5 @@ { + // In squeeze, we need a special kernel flavor for xen "squeeze": { "amd64": "linux-image-xen-amd64", "i386" : "linux-image-xen-686" }, diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index 242457a..ebd49ef 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -17,10 +17,9 @@ class DefaultPackages(Task): info.exclude_packages.add('isc-dhcp-client') info.exclude_packages.add('isc-dhcp-common') - # In squeeze, we need a special kernel flavor for xen - kernels = {} - with open('providers/ec2/tasks/packages-kernels.json') as stream: - import json - kernels = json.loads(stream.read()) - kernel_package = kernels.get(info.manifest.system['release']).get(info.manifest.system['architecture']) + import os.path + kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.json') + from common.tools import config_get + kernel_package = config_get(kernel_packages_path, [info.manifest.system['release'], + info.manifest.system['architecture']]) info.packages.add(kernel_package)