AWS provider fully working again (S3 & EBS)

EBS booted AMIs now also support partitions

providers/ec2/__init__.py

@@ -8,7 +8,8 @@ from common.tasks import volume as volume_tasks
 from tasks import ebs
 from common.tasks import partitioning
 from common.tasks import loopback
-from common.tasks import filesystem
+from common.tasks import filesystem as common_filesystem
+from tasks import filesystem
 from common.tasks import bootstrap
 from tasks import boot
 from common.tasks import boot as common_boot
@@ -66,15 +67,16 @@ def tasks(tasklist, manifest):
 
 	backing_specific_tasks = {'ebs': [ebs.Create,
 	                                  ebs.Attach,
+	                                  common_filesystem.FStab,
 	                                  ebs.Snapshot],
 	                          's3': [loopback.Create,
 	                                 volume_tasks.Attach,
+	                                 filesystem.S3FStab,
 	                                 ami.BundleImage,
 	                                 ami.UploadImage,
 	                                 ami.RemoveBundle]}
 	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
-	tasklist.add(filesystem.Format,
+	tasklist.add(common_filesystem.Format,
-	             filesystem.FStab,
 	             volume_tasks.Detach,
 	             volume_tasks.Delete)
 
@@ -103,10 +105,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 	counter_task(volume_tasks.Attach, volume_tasks.Detach)
 
 	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
-	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
+	counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir)
-	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
+	counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials)
 
-	counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
+	counter_task(common_filesystem.MountRoot, common_filesystem.UnmountRoot)
-	counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
+	counter_task(common_filesystem.MountBoot, common_filesystem.UnmountBoot)
 	counter_task(volume_tasks.Attach, volume_tasks.Detach)
 	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)
+	counter_task(ami.BundleImage, ami.RemoveBundle)

providers/ec2/assets/grub.d/40_custom

@@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib
 export TEXTDOMAIN=grub
 export TEXTDOMAINDIR=${prefix}/share/locale
 
-GRUB_DEVICE=/dev/xvda1
+GRUB_DEVICE=/dev/xvda
 
 
 cat << EOF

providers/ec2/tasks/ami.py

@@ -48,7 +48,7 @@ class BundleImage(Task):
 		bundle_name = 'bundle-{id:x}'.format(id=info.run_id)
 		info.bundle_path = os.path.join(info.workspace, bundle_name)
 		log_check_call(['/usr/bin/euca-bundle-image',
-		                '--image', info.loopback_file,
+		                '--image', info.volume.image_path,
 		                '--user', info.credentials['user-id'],
 		                '--privatekey', info.credentials['private-key'],
 		                '--cert', info.credentials['certificate'],
@@ -94,49 +94,109 @@ class RegisterAMI(Task):
 	phase = phases.image_registration
 	after = [Snapshot, UploadImage]
 
-	kernel_mapping = {'us-east-1':      {'amd64': 'aki-88aa75e1',
+	# Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs
-	                                     'i386':  'aki-b6aa75df'},
+	kernel_mapping = {'ap-northeast-1': {  # Asia Pacific (Tokyo) Region
-	                  'us-west-1':      {'amd64': 'aki-f77e26b2',
+	                  'hd0':  {'i386':  'aki-136bf512',   # pv-grub-hd0_1.04-i386.gz
-	                                     'i386':  'aki-f57e26b0'},
+	                           'amd64': 'aki-176bf516'},  # pv-grub-hd0_1.04-x86_64.gz
-	                  'us-west-2':      {'amd64': 'aki-fc37bacc',
+	                  'hd00': {'i386':  'aki-196bf518',   # pv-grub-hd00_1.04-i386.gz
-	                                     'i386':  'aki-fa37baca'},
+	                           'amd64': 'aki-1f6bf51e'}   # pv-grub-hd00_1.04-x86_64.gz
-	                  'eu-west-1':      {'amd64': 'aki-71665e05',
+	                  },
-	                                     'i386':  'aki-75665e01'},
+	                  'ap-southeast-1': {  # Asia Pacific (Singapore) Region
-	                  'ap-southeast-1': {'amd64': 'aki-fe1354ac',
+	                  'hd0':  {'i386':  'aki-ae3973fc',   # pv-grub-hd0_1.04-i386.gz
-	                                     'i386':  'aki-f81354aa'},
+	                           'amd64': 'aki-503e7402'},  # pv-grub-hd0_1.04-x86_64.gz
-	                  'ap-southeast-2': {'amd64': 'aki-31990e0b',
+	                  'hd00': {'i386':  'aki-563e7404',   # pv-grub-hd00_1.04-i386.gz
-	                                     'i386':  'aki-33990e09'},
+	                           'amd64': 'aki-5e3e740c'}   # pv-grub-hd00_1.04-x86_64.gz
-	                  'ap-northeast-1': {'amd64': 'aki-44992845',
+	                  },
-	                                     'i386':  'aki-42992843'},
+	                  'ap-southeast-2': {  # Asia Pacific (Sydney) Region
-	                  'sa-east-1':      {'amd64': 'aki-c48f51d9',
+	                  'hd0':  {'i386':  'aki-cd62fff7',   # pv-grub-hd0_1.04-i386.gz
-	                                     'i386':  'aki-ca8f51d7'},
+	                           'amd64': 'aki-c362fff9'},  # pv-grub-hd0_1.04-x86_64.gz
-	                  'us-gov-west-1':  {'amd64': 'aki-79a4c05a',
+	                  'hd00': {'i386':  'aki-c162fffb',   # pv-grub-hd00_1.04-i386.gz
-	                                     'i386':  'aki-7ba4c058'}}
+	                           'amd64': 'aki-3b1d8001'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'eu-west-1': {  # EU (Ireland) Region
+	                  'hd0':  {'i386':  'aki-68a3451f',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-52a34525'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-5ea34529',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-58a3452f'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'sa-east-1': {  # South America (Sao Paulo) Region
+	                  'hd0':  {'i386':  'aki-5b53f446',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-5553f448'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-5753f44a',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-5153f44c'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-east-1': {  # US East (Northern Virginia) Region
+	                  'hd0':  {'i386':  'aki-8f9dcae6',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-919dcaf8'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-659ccb0c',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-499ccb20'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-gov-west-1': {  # AWS GovCloud (US)
+	                  'hd0':  {'i386':  'aki-1fe98d3c',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-1de98d3e'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-63e98d40',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-61e98d42'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-west-1': {  # US West (Northern California) Region
+	                  'hd0':  {'i386':  'aki-8e0531cb',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-880531cd'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-960531d3',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-920531d7'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-west-2': {  # US West (Oregon) Region
+	                  'hd0':  {'i386':  'aki-f08f11c0',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-fc8f11cc'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-e28f11d2',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-e68f11d6'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  }}
 
 	def run(self, info):
+		if info.manifest.volume['backing'] == 'ebs':
+			self.run_ebs(info)
+		if info.manifest.volume['backing'] == 's3':
+			self.run_s3(info)
+
+	def run_ebs(self, info):
 		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
-		kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
 
-		from providers.ec2.ebsvolume import EBSVolume
+		from base.fs.partitionmaps.none import NoPartitions
-		from common.fs.loopbackvolume import LoopbackVolume
+		if isinstance(info.volume.partition_map, NoPartitions):
+			grub_boot_device = 'hd0'
+			root_device_name = '/dev/sda'
+		else:
+			grub_boot_device = 'hd00'
+			root_idx = info.volume.partition_map.root.get_index()
+			root_device_name = '/dev/sda{idx}'.format(idx=root_idx)
 
-		if isinstance(info.volume, EBSVolume):
+		kernel_id = (self.kernel_mapping
-			from boto.ec2.blockdevicemapping import BlockDeviceType
+		             .get(info.host['region'])
-			from boto.ec2.blockdevicemapping import BlockDeviceMapping
+		             .get(grub_boot_device)
-			block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
+		             .get(info.manifest.system['architecture']))
-			                               size=info.volume.partition_map.get_total_size()/1024)
-			block_device_map = BlockDeviceMapping()
-			block_device_map['/dev/sda1'] = block_device
 
-			info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
+		from boto.ec2.blockdevicemapping import BlockDeviceType
-			                                            architecture=arch, kernel_id=kernel_id,
+		from boto.ec2.blockdevicemapping import BlockDeviceMapping
-			                                            root_device_name='/dev/sda1',
+		block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
-			                                            block_device_map=block_device_map)
+		                               size=info.volume.partition_map.get_total_size()/1024)
-		if isinstance(info.volume, LoopbackVolume):
+		block_device_map = BlockDeviceMapping()
-			image_location = ('{bucket}/{ami_name}.manifest.xml'
+		block_device_map['/dev/sda'] = block_device
-			                  .format(bucket=info.manifest.image['bucket'],
+
-			                          ami_name=info.ami_name))
+		info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
-			info.image = info.connection.register_image(description=info.ami_description,
+		                                            architecture=arch, kernel_id=kernel_id,
-			                                            architecture=arch, kernel_id=kernel_id,
+		                                            root_device_name=root_device_name,
-			                                            root_device_name='/dev/sda1',
+		                                            block_device_map=block_device_map)
-			                                            image_location=image_location)
+
+	def run_s3(self, info):
+		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
+
+		kernel_id = (self.kernel_mapping
+		             .get(info.host['region'])
+		             .get('hd0')
+		             .get(info.manifest.system['architecture']))
+
+		image_manifest = ('{bucket}/{ami_name}.manifest.xml'
+		                  .format(bucket=info.manifest.image['bucket'],
+		                          ami_name=info.ami_name))
+		info.image = info.connection.register_image(description=info.ami_description,
+		                                            architecture=arch, kernel_id=kernel_id,
+		                                            root_device_name='dev/sda1',
+		                                            image_location=image_manifest)

providers/ec2/tasks/boot.py

@@ -24,6 +24,19 @@ class ConfigureGrub(Task):
 		copy(script_src, script_dst)
 		os.chmod(script_dst, rwxr_xr_x)
 
+		from base.fs.partitionmaps.none import NoPartitions
+		if not isinstance(info.volume.partition_map, NoPartitions):
+			from common.tools import sed_i
+			root_idx = info.volume.partition_map.root.get_index()
+			grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx)
+			sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device)
+			grub_root = '\troot (hd0,{idx})'.format(idx=root_idx-1)
+			sed_i(script_dst, '^\troot \(hd0\)$', grub_root)
+
+		if info.manifest.volume['backing'] == 's3':
+			from common.tools import sed_i
+			sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1')
+
 		from common.tools import sed_i
 		grub_def = os.path.join(info.root, 'etc/default/grub')
 		sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n'

providers/ec2/tasks/filesystem.py

@@ -0,0 +1,27 @@
+from base import Task
+from common import phases
+
+
+class S3FStab(Task):
+	description = 'Adding the S3 root partition to the fstab'
+	phase = phases.system_modification
+
+	def run(self, info):
+		import os.path
+		root = info.volume.partition_map.root
+
+		fstab_lines = []
+		mount_opts = ['defaults']
+		fstab_lines.append('{device_path}{idx} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}'
+		                   .format(device_path='/dev/xvda',
+		                   	       idx=1,
+		                           mountpoint='/',
+		                           filesystem=root.filesystem,
+		                           mount_opts=','.join(mount_opts),
+		                           dump='1',
+		                           pass_num='1'))
+
+		fstab_path = os.path.join(info.root, 'etc/fstab')
+		with open(fstab_path, 'w') as fstab:
+			fstab.write('\n'.join(fstab_lines))
+			fstab.write('\n')