From 648369587aa9c8d2d904c50db6962dca6ae76a0d Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Thu, 6 Feb 2014 14:01:31 +0000 Subject: [PATCH 1/9] Remove deprecated hd00 pvgrub AKIs (hd0 pvgrubs AKIs check hd0 and hd0,0). Add in cn-north-1 pvgrub AKIs. --- providers/ec2/tasks/ami.py | 63 +++++++++++++++----------------------- 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 000c0db..865e636 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -89,59 +89,44 @@ class RegisterAMI(Task): # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs kernel_mapping = {'ap-northeast-1': # Asia Pacific (Tokyo) Region - {'hd0': {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-176bf516'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-196bf518', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-1f6bf51e'} # pv-grub-hd00_1.04-x86_64.gz - }, + {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-176bf516' # pv-grub-hd0_1.04-x86_64.gz + }, 'ap-southeast-1': # Asia Pacific (Singapore) Region - {'hd0': {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-503e7402'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-563e7404', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-5e3e740c'} # pv-grub-hd00_1.04-x86_64.gz - }, + {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-503e7402' # pv-grub-hd0_1.04-x86_64.gz + }, 'ap-southeast-2': # Asia Pacific (Sydney) Region - {'hd0': {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-c362fff9'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-c162fffb', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-3b1d8001'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-c362fff9' # pv-grub-hd0_1.04-x86_64.gz }, 'eu-west-1': # EU (Ireland) Region - {'hd0': {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-52a34525'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-5ea34529', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-58a3452f'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-52a34525' # pv-grub-hd0_1.04-x86_64.gz }, 'sa-east-1': # South America (Sao Paulo) Region - {'hd0': {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-5553f448'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-5753f44a', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-5153f44c'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-5553f448' # pv-grub-hd0_1.04-x86_64.gz }, 'us-east-1': # US East (Northern Virginia) Region - {'hd0': {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-919dcaf8'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-659ccb0c', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-499ccb20'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-919dcaf8' # pv-grub-hd0_1.04-x86_64.gz }, 'us-gov-west-1': # AWS GovCloud (US) - {'hd0': {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-1de98d3e'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-63e98d40', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-61e98d42'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-1de98d3e' # pv-grub-hd0_1.04-x86_64.gz }, 'us-west-1': # US West (Northern California) Region - {'hd0': {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-880531cd'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-960531d3', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-920531d7'} # pv-grub-hd00_1.04-x86_64.gz + {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-880531cd' # pv-grub-hd0_1.04-x86_64.gz }, 'us-west-2': # US West (Oregon) Region - {'hd0': {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-fc8f11cc'}, # pv-grub-hd0_1.04-x86_64.gz - 'hd00': {'i386': 'aki-e28f11d2', # pv-grub-hd00_1.04-i386.gz - 'amd64': 'aki-e68f11d6'} # pv-grub-hd00_1.04-x86_64.gz - } + {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-fc8f11cc' # pv-grub-hd0_1.04-x86_64.gz + }, + 'cn-north-1': # China North (Beijing) Region + {'i386': 'aki-908f1da9', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-9e8f1da7' # pv-grub-hd0_1.04-x86_64.gz } @classmethod From 6168717ba9cd38bd2fe54f45919fe2a6e87742c7 Mon Sep 17 00:00:00 2001 From: James Brombegrer Date: Thu, 6 Feb 2014 15:02:13 +0000 Subject: [PATCH 2/9] Permit Jessie as a release name. --- base/manifest-schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index a5d09c3..b9adbbe 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -25,7 +25,7 @@ "system": { "type": "object", "properties": { - "release": { "enum": ["wheezy", "testing", "unstable"] }, + "release": { "enum": ["wheezy", "jessie", "testing", "unstable"] }, "sections": { "type": "array", "minItems": 1 From 047920b1208c5702237a31f0e790d0df3b2233a9 Mon Sep 17 00:00:00 2001 From: James Brombegrer Date: Fri, 7 Feb 2014 14:07:45 +0000 Subject: [PATCH 3/9] The s3 endpoint in China is under a different DNS TLD: amazonaws.com.cn. --- providers/ec2/tasks/ami.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 865e636..bf8a987 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -58,6 +58,8 @@ class UploadImage(Task): manifest_file = os.path.join(info.bundle_path, info.ami_name + '.manifest.xml') if info.host['region'] == 'us-east-1': s3_url = 'https://s3.amazonaws.com/' + elif info.host['region'] == 'cn-north-1': + s3_url = 'https://s3.cn-north-1.amazonaws.com.cn' else: s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region']) log_check_call(['/usr/bin/euca-upload-bundle', From abe70d6df96ab70bdc3946197561317b0a5231bc Mon Sep 17 00:00:00 2001 From: James Brombegrer Date: Fri, 7 Feb 2014 15:21:07 +0000 Subject: [PATCH 4/9] Missing closing brace. --- providers/ec2/tasks/ami.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index bf8a987..fc463de 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -130,7 +130,7 @@ class RegisterAMI(Task): {'i386': 'aki-908f1da9', # pv-grub-hd0_1.04-i386.gz 'amd64': 'aki-9e8f1da7' # pv-grub-hd0_1.04-x86_64.gz } - + } @classmethod def run(cls, info): registration_params = {'name': info.ami_name, From b64ad495ab199548253b02a29fe06e2579c745aa Mon Sep 17 00:00:00 2001 From: James Brombegrer Date: Fri, 7 Feb 2014 17:42:45 +0000 Subject: [PATCH 5/9] Only delete files form the image if they exist. Add in an HVM manifest file. --- common/tasks/host.py | 4 +- common/tasks/network.py | 8 +++- ...fficial-amd64-hvm-cn-north-1.manifest.json | 44 +++++++++++++++++++ providers/ec2/tasks/ami.py | 1 - 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json diff --git a/common/tasks/host.py b/common/tasks/host.py index f156bf4..7bf79dd 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -35,7 +35,9 @@ class CheckHostDependencies(Task): missing_packages = [] for package in info.host_dependencies: try: - log_check_call(['/usr/bin/dpkg-query', '-s', package]) + import os.path + if os.path.isfile('/usr/bin/dpkg-query'): + log_check_call(['/usr/bin/dpkg-query', '-s', package]) except CalledProcessError: missing_packages.append(package) if len(missing_packages) > 0: diff --git a/common/tasks/network.py b/common/tasks/network.py index 32bfd01..8418a84 100644 --- a/common/tasks/network.py +++ b/common/tasks/network.py @@ -10,7 +10,9 @@ class RemoveDNSInfo(Task): @classmethod def run(cls, info): from os import remove - remove(os.path.join(info.root, 'etc/resolv.conf')) + import os.path + if os.path.isfile(os.path.join(info.root, 'etc/resolv.conf')): + remove(os.path.join(info.root, 'etc/resolv.conf')) class RemoveHostname(Task): @@ -20,7 +22,9 @@ class RemoveHostname(Task): @classmethod def run(cls, info): from os import remove - remove(os.path.join(info.root, 'etc/hostname')) + import os.path + if os.path.isfile(os.path.join(info.root, 'etc/hostname')): + remove(os.path.join(info.root, 'etc/hostname')) class ConfigureNetworkIF(Task): diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json new file mode 100644 index 0000000..afd55ba --- /dev/null +++ b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json @@ -0,0 +1,44 @@ +{ + "provider": "ec2", + "virtualization": "hvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {system.release} {system.architecture}" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "bootloader": "extlinux", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "packages": { + "mirror": "http://ftp.cn.debian.org/debian" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": "8GiB", + "filesystem": "ext4" + } + } + }, + "plugins": { + "cloud_init": { + "username": "admin", + //"metadata_sources": "Ec2", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] + } + } +} diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index fc463de..582c59e 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -163,7 +163,6 @@ class RegisterAMI(Task): registration_params['virtualization_type'] = 'paravirtual' registration_params['kernel_id'] = (cls.kernel_mapping .get(info.host['region']) - .get(grub_boot_device) .get(info.manifest.system['architecture'])) info.image = info.connection.register_image(**registration_params) From c708e5ab42fe9a2837a358d42bd2ada3b198888d Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Sat, 8 Feb 2014 16:16:02 +0000 Subject: [PATCH 6/9] Add image_location param when registering an S3 backde AMI. --- providers/ec2/tasks/ami.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 582c59e..9e0ff0b 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -62,6 +62,7 @@ class UploadImage(Task): s3_url = 'https://s3.cn-north-1.amazonaws.com.cn' else: s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region']) + info.manifest.manifest_location = info.manifest.image['bucket'] + '/' + info.ami_name + '.manifest.xml' log_check_call(['/usr/bin/euca-upload-bundle', '--bucket', info.manifest.image['bucket'], '--manifest', manifest_file, @@ -140,6 +141,7 @@ class RegisterAMI(Task): if info.manifest.volume['backing'] == 's3': grub_boot_device = 'hd0' + registration_params['image_location'] = info.manifest.manifest_location else: root_dev_name = {'pvm': '/dev/sda', 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization']) From 1ef9d07118256507e785598149e1fcaf10cec592 Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Sat, 8 Feb 2014 16:28:52 +0000 Subject: [PATCH 7/9] Permit referring to Jessie as 'jessie', not just as 'testing' (as when Jessie goes stable in a year, we dont want to be then updating this build script). --- common/tasks/network-configuration.json | 3 +++ providers/ec2/tasks/packages-kernels.json | 3 +++ 2 files changed, 6 insertions(+) diff --git a/common/tasks/network-configuration.json b/common/tasks/network-configuration.json index 44e540b..5a3c533 100644 --- a/common/tasks/network-configuration.json +++ b/common/tasks/network-configuration.json @@ -7,6 +7,9 @@ "wheezy": [ "auto eth0", "iface eth0 inet dhcp" ], + "jessie": [ + "auto eth0", + "iface eth0 inet dhcp" ], "testing": [ "auto eth0", "iface eth0 inet dhcp" ], diff --git a/providers/ec2/tasks/packages-kernels.json b/providers/ec2/tasks/packages-kernels.json index 74a3df7..3c9e39f 100644 --- a/providers/ec2/tasks/packages-kernels.json +++ b/providers/ec2/tasks/packages-kernels.json @@ -5,6 +5,9 @@ "wheezy": { "amd64": "linux-image-amd64", "i386" : "linux-image-686" }, + "jessie": { + "amd64": "linux-image-amd64", + "i386" : "linux-image-686" }, "testing": { "amd64": "linux-image-amd64", "i386" : "linux-image-686" }, From 628957c3042dd25b02eb88a1f2dca2224dd54aff Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Sat, 8 Feb 2014 18:37:10 +0000 Subject: [PATCH 8/9] Add PVM manifest file for China (amd64). --- ...fficial-amd64-pvm-cn-north-1.manifest.json | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json new file mode 100644 index 0000000..18dd022 --- /dev/null +++ b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json @@ -0,0 +1,44 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs", + "description": "Debian {system.release} {system.architecture}" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "bootloader": "pvgrub", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "packages": { + "mirror": "http://ftp.cn.debian.org/debian" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": "8GiB", + "filesystem": "ext4" + } + } + }, + "plugins": { + "cloud_init": { + "username": "admin", + //"metadata_sources": "Ec2", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] + } + } +} From d11aee0e3e6360ccf309ba18f7f547636fcd887a Mon Sep 17 00:00:00 2001 From: James Bromberger Date: Sat, 8 Feb 2014 18:46:23 +0000 Subject: [PATCH 9/9] Add S3 backed AMI manifest for China (Beijing). --- ...fficial-amd64-pvm-cn-north-1.manifest.json | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json diff --git a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json new file mode 100644 index 0000000..57cf316 --- /dev/null +++ b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json @@ -0,0 +1,47 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null, + // "certificate": null, + // "private-key": null, + // "user-id": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}", + "description": "Debian {system.release} {system.architecture} AMI", + "bucket": "debian-amis-cn-north-1" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "bootloader": "pvgrub", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "packages": { + "mirror": "http://ftp.cn.debian.org/debian" + }, + "volume": { + "backing": "s3", + "partitions": { + "type": "none", + "root": { + "size": "1GiB", + "filesystem": "ext4" + } + } + }, + "plugins": { + "cloud_init": { + "username": "admin", + "disable_modules": [ "landscape", "byobu", "ssh-import-id" ] + } + } +}