From 6eb56721dca0f6be07f474fd23e702c607e7f940 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 17:47:39 +0100 Subject: [PATCH] AWS provider fully working again (S3 & EBS) EBS booted AMIs now also support partitions --- providers/ec2/__init__.py | 17 +-- providers/ec2/assets/grub.d/40_custom | 2 +- providers/ec2/tasks/ami.py | 142 ++++++++++++++++++-------- providers/ec2/tasks/boot.py | 13 +++ providers/ec2/tasks/filesystem.py | 27 +++++ 5 files changed, 152 insertions(+), 49 deletions(-) create mode 100644 providers/ec2/tasks/filesystem.py diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 9741c7a..98cef05 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -8,7 +8,8 @@ from common.tasks import volume as volume_tasks from tasks import ebs from common.tasks import partitioning from common.tasks import loopback -from common.tasks import filesystem +from common.tasks import filesystem as common_filesystem +from tasks import filesystem from common.tasks import bootstrap from tasks import boot from common.tasks import boot as common_boot @@ -66,15 +67,16 @@ def tasks(tasklist, manifest): backing_specific_tasks = {'ebs': [ebs.Create, ebs.Attach, + common_filesystem.FStab, ebs.Snapshot], 's3': [loopback.Create, volume_tasks.Attach, + filesystem.S3FStab, ami.BundleImage, ami.UploadImage, ami.RemoveBundle]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) - tasklist.add(filesystem.Format, - filesystem.FStab, + tasklist.add(common_filesystem.Format, volume_tasks.Detach, volume_tasks.Delete) @@ -103,10 +105,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(volume_tasks.Attach, volume_tasks.Detach) counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) - counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir) + counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials) - counter_task(filesystem.MountRoot, filesystem.UnmountRoot) - counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(common_filesystem.MountRoot, common_filesystem.UnmountRoot) + counter_task(common_filesystem.MountBoot, common_filesystem.UnmountBoot) counter_task(volume_tasks.Attach, volume_tasks.Detach) counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) + counter_task(ami.BundleImage, ami.RemoveBundle) diff --git a/providers/ec2/assets/grub.d/40_custom b/providers/ec2/assets/grub.d/40_custom index 799e887..0946bbd 100644 --- a/providers/ec2/assets/grub.d/40_custom +++ b/providers/ec2/assets/grub.d/40_custom @@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib export TEXTDOMAIN=grub export TEXTDOMAINDIR=${prefix}/share/locale -GRUB_DEVICE=/dev/xvda1 +GRUB_DEVICE=/dev/xvda cat << EOF diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 4cb4577..11e3fd8 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -48,7 +48,7 @@ class BundleImage(Task): bundle_name = 'bundle-{id:x}'.format(id=info.run_id) info.bundle_path = os.path.join(info.workspace, bundle_name) log_check_call(['/usr/bin/euca-bundle-image', - '--image', info.loopback_file, + '--image', info.volume.image_path, '--user', info.credentials['user-id'], '--privatekey', info.credentials['private-key'], '--cert', info.credentials['certificate'], @@ -94,49 +94,109 @@ class RegisterAMI(Task): phase = phases.image_registration after = [Snapshot, UploadImage] - kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', - 'i386': 'aki-b6aa75df'}, - 'us-west-1': {'amd64': 'aki-f77e26b2', - 'i386': 'aki-f57e26b0'}, - 'us-west-2': {'amd64': 'aki-fc37bacc', - 'i386': 'aki-fa37baca'}, - 'eu-west-1': {'amd64': 'aki-71665e05', - 'i386': 'aki-75665e01'}, - 'ap-southeast-1': {'amd64': 'aki-fe1354ac', - 'i386': 'aki-f81354aa'}, - 'ap-southeast-2': {'amd64': 'aki-31990e0b', - 'i386': 'aki-33990e09'}, - 'ap-northeast-1': {'amd64': 'aki-44992845', - 'i386': 'aki-42992843'}, - 'sa-east-1': {'amd64': 'aki-c48f51d9', - 'i386': 'aki-ca8f51d7'}, - 'us-gov-west-1': {'amd64': 'aki-79a4c05a', - 'i386': 'aki-7ba4c058'}} + # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs + kernel_mapping = {'ap-northeast-1': { # Asia Pacific (Tokyo) Region + 'hd0': {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-176bf516'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-196bf518', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-1f6bf51e'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-1': { # Asia Pacific (Singapore) Region + 'hd0': {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-503e7402'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-563e7404', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5e3e740c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-2': { # Asia Pacific (Sydney) Region + 'hd0': {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-c362fff9'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-c162fffb', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-3b1d8001'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'eu-west-1': { # EU (Ireland) Region + 'hd0': {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-52a34525'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5ea34529', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-58a3452f'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'sa-east-1': { # South America (Sao Paulo) Region + 'hd0': {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-5553f448'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5753f44a', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5153f44c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-east-1': { # US East (Northern Virginia) Region + 'hd0': {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-919dcaf8'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-659ccb0c', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-499ccb20'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-gov-west-1': { # AWS GovCloud (US) + 'hd0': {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-1de98d3e'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-63e98d40', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-61e98d42'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-1': { # US West (Northern California) Region + 'hd0': {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-880531cd'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-960531d3', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-920531d7'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-2': { # US West (Oregon) Region + 'hd0': {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-fc8f11cc'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-e28f11d2', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-e68f11d6'} # pv-grub-hd00_1.04-x86_64.gz + }} def run(self, info): + if info.manifest.volume['backing'] == 'ebs': + self.run_ebs(info) + if info.manifest.volume['backing'] == 's3': + self.run_s3(info) + + def run_ebs(self, info): arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) - kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - from providers.ec2.ebsvolume import EBSVolume - from common.fs.loopbackvolume import LoopbackVolume + from base.fs.partitionmaps.none import NoPartitions + if isinstance(info.volume.partition_map, NoPartitions): + grub_boot_device = 'hd0' + root_device_name = '/dev/sda' + else: + grub_boot_device = 'hd00' + root_idx = info.volume.partition_map.root.get_index() + root_device_name = '/dev/sda{idx}'.format(idx=root_idx) - if isinstance(info.volume, EBSVolume): - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.volume.partition_map.get_total_size()/1024) - block_device_map = BlockDeviceMapping() - block_device_map['/dev/sda1'] = block_device + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get(grub_boot_device) + .get(info.manifest.system['architecture'])) - info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - block_device_map=block_device_map) - if isinstance(info.volume, LoopbackVolume): - image_location = ('{bucket}/{ami_name}.manifest.xml' - .format(bucket=info.manifest.image['bucket'], - ami_name=info.ami_name)) - info.image = info.connection.register_image(description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - image_location=image_location) + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=info.volume.partition_map.get_total_size()/1024) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda'] = block_device + + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name=root_device_name, + block_device_map=block_device_map) + + def run_s3(self, info): + arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get('hd0') + .get(info.manifest.system['architecture'])) + + image_manifest = ('{bucket}/{ami_name}.manifest.xml' + .format(bucket=info.manifest.image['bucket'], + ami_name=info.ami_name)) + info.image = info.connection.register_image(description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='dev/sda1', + image_location=image_manifest) diff --git a/providers/ec2/tasks/boot.py b/providers/ec2/tasks/boot.py index cee03e8..48f8d27 100644 --- a/providers/ec2/tasks/boot.py +++ b/providers/ec2/tasks/boot.py @@ -24,6 +24,19 @@ class ConfigureGrub(Task): copy(script_src, script_dst) os.chmod(script_dst, rwxr_xr_x) + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + from common.tools import sed_i + root_idx = info.volume.partition_map.root.get_index() + grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx) + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device) + grub_root = '\troot (hd0,{idx})'.format(idx=root_idx-1) + sed_i(script_dst, '^\troot \(hd0\)$', grub_root) + + if info.manifest.volume['backing'] == 's3': + from common.tools import sed_i + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1') + from common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' diff --git a/providers/ec2/tasks/filesystem.py b/providers/ec2/tasks/filesystem.py new file mode 100644 index 0000000..c2de2a1 --- /dev/null +++ b/providers/ec2/tasks/filesystem.py @@ -0,0 +1,27 @@ +from base import Task +from common import phases + + +class S3FStab(Task): + description = 'Adding the S3 root partition to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + root = info.volume.partition_map.root + + fstab_lines = [] + mount_opts = ['defaults'] + fstab_lines.append('{device_path}{idx} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}' + .format(device_path='/dev/xvda', + idx=1, + mountpoint='/', + filesystem=root.filesystem, + mount_opts=','.join(mount_opts), + dump='1', + pass_num='1')) + + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) + fstab.write('\n')