EC2 provider can now bootstrap EBS volumes again

Use tasksets in EC2 provider

base/fs/__init__.py

@@ -2,7 +2,7 @@
 
 def load_volume(data):
 	from common.fs.loopbackvolume import LoopbackVolume
-	from providers.ec2.volume import EBSVolume
+	from providers.ec2.ebsvolume import EBSVolume
 	from providers.virtualbox.volume import VirtualBoxVolume
 	from partitionmaps.gpt import GPTPartitionMap
 	from partitionmaps.mbr import MBRPartitionMap

base/fs/partitions/abstract.py

@@ -1,5 +1,4 @@
 from abc import ABCMeta
-from abc import abstractmethod
 from common.tools import log_check_call
 from common.fsm_proxy import FSMProxy
 
@@ -26,13 +25,6 @@ class AbstractPartition(FSMProxy):
 		[uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path])
 		return uuid
 
-	def create(self, volume):
-		self.fsm.create(volume=volume)
-
-	@abstractmethod
-	def _before_create(self, e):
-		pass
-
 	def _before_format(self, e):
 		mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem)
 		log_check_call([mkfs, self.device_path])

base/fs/partitions/base.py

@@ -18,6 +18,9 @@ class BasePartition(AbstractPartition):
 		self.previous = previous
 		super(BasePartition, self).__init__(size, filesystem)
 
+	def create(self, volume):
+		self.fsm.create(volume=volume)
+
 	def get_index(self):
 		if self.previous is None:
 			return 1

base/fs/partitions/single.py

@@ -2,6 +2,4 @@ from abstract import AbstractPartition
 
 
 class SinglePartition(AbstractPartition):
-
-	def _before_create(self, e):
 		pass

common/task_sets.py

@@ -1,6 +1,7 @@
 from common.tasks import workspace
 from common.tasks import packages
 from common.tasks import host
+from common.tasks import bootstrap
 from common.tasks import volume
 from common.tasks import filesystem
 from common.tasks import partitioning
@@ -13,11 +14,14 @@ base_set = [workspace.CreateWorkspace,
             packages.HostPackages,
             packages.ImagePackages,
             host.CheckPackages,
+            bootstrap.Bootstrap,
             workspace.DeleteWorkspace,
             ]
 
 volume_set = [volume.Attach,
               volume.Detach,
+              filesystem.Format,
+              filesystem.FStab,
               ]
 
 partitioning_set = [partitioning.PartitionVolume,

providers/ec2/__init__.py

@@ -1,18 +1,15 @@
 from manifest import Manifest
 import logging
 from tasks import packages
-from common.tasks import packages as common_packages
 from tasks import connection
 from tasks import host
-from common.tasks import host as common_host
 from tasks import ami
-from common.tasks import volume
+from common.tasks import volume as volume_tasks
 from tasks import ebs
+from common.tasks import partitioning
 from common.tasks import loopback
 from common.tasks import filesystem
 from common.tasks import bootstrap
-from common.tasks import locale
-from common.tasks import apt
 from tasks import boot
 from common.tasks import boot as common_boot
 from common.tasks import security
@@ -21,6 +18,7 @@ from common.tasks import network as common_network
 from tasks import initd
 from common.tasks import initd as common_initd
 from common.tasks import cleanup
+from common.tasks import workspace
 
 
 def initialize():
@@ -29,74 +27,66 @@ def initialize():
 
 
 def tasks(tasklist, manifest):
-	tasklist.add(packages.HostPackages(),
+	from common.task_sets import base_set
-	             common_packages.HostPackages(),
+	from common.task_sets import mounting_set
-	             packages.ImagePackages(),
+	from common.task_sets import apt_set
-	             common_packages.ImagePackages(),
+	from common.task_sets import locale_set
-	             common_host.CheckPackages(),
+	from common.task_sets import ssh_set
-	             connection.GetCredentials(),
+	tasklist.add(*base_set)
-	             host.GetInfo(),
+	tasklist.add(*mounting_set)
-	             ami.AMIName(),
+	tasklist.add(*apt_set)
-	             connection.Connect(),
+	tasklist.add(*locale_set)
+	tasklist.add(*ssh_set)
 
-	             filesystem.FormatVolume(),
+	if manifest.volume['partitions']['type'] != 'none':
-	             filesystem.CreateMountDir(),
+		from common.task_sets import partitioning_set
-	             filesystem.MountVolume(),
+		tasklist.add(*partitioning_set)
 
-	             bootstrap.Bootstrap(),
+	tasklist.add(packages.HostPackages,
-	             filesystem.MountSpecials(),
+	             packages.ImagePackages,
-	             locale.GenerateLocale(),
+	             connection.GetCredentials,
-	             locale.SetTimezone(),
+	             host.GetInfo,
-	             apt.DisableDaemonAutostart(),
+	             ami.AMIName,
-	             apt.AptSources(),
+	             connection.Connect,
-	             apt.AptUpgrade(),
-	             boot.ConfigureGrub(),
-	             filesystem.FStab(),
-	             common_boot.BlackListModules(),
-	             common_boot.DisableGetTTYs(),
-	             security.EnableShadowConfig(),
-	             security.DisableSSHPasswordAuthentication(),
-	             security.DisableSSHDNSLookup(),
-	             common_network.RemoveDNSInfo(),
-	             common_network.ConfigureNetworkIF(),
-	             network.EnableDHCPCDDNS(),
-	             common_initd.ResolveInitScripts(),
-	             initd.AddEC2InitScripts(),
-	             common_initd.InstallInitScripts(),
-	             cleanup.ClearMOTD(),
-	             cleanup.ShredHostkeys(),
-	             cleanup.CleanTMP(),
-	             apt.PurgeUnusedPackages(),
-	             apt.AptClean(),
-	             apt.EnableDaemonAutostart(),
-	             filesystem.UnmountSpecials(),
 
-	             filesystem.UnmountVolume(),
+	             boot.ConfigureGrub,
-	             filesystem.DeleteMountDir(),
+	             common_boot.BlackListModules,
-	             ami.RegisterAMI())
+	             common_boot.DisableGetTTYs,
+	             security.EnableShadowConfig,
+	             common_network.RemoveDNSInfo,
+	             common_network.ConfigureNetworkIF,
+	             network.EnableDHCPCDDNS,
+	             common_initd.ResolveInitScripts,
+	             initd.AddEC2InitScripts,
+	             common_initd.InstallInitScripts,
+	             cleanup.ClearMOTD,
+	             cleanup.CleanTMP,
+
+	             ami.RegisterAMI)
+
+	backing_specific_tasks = {'ebs': [ebs.Create,
+	                                  ebs.Attach,
+	                                  ebs.Snapshot],
+	                          's3': [loopback.Create,
+	                                 volume_tasks.Attach,
+	                                 ami.BundleImage,
+	                                 ami.UploadImage,
+	                                 ami.RemoveBundle]}
+	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
+	tasklist.add(filesystem.Format,
+	             filesystem.FStab,
+	             volume_tasks.Detach,
+	             volume_tasks.Delete)
 
 	if manifest.bootstrapper.get('tarball', False):
-		tasklist.add(bootstrap.MakeTarball())
+		tasklist.add(bootstrap.MakeTarball)
 
-	backing_specific_tasks = {'ebs': [ebs.Create(),
+	from common.task_sets import get_fs_specific_set
-	                                  volume.Attach(),
+	tasklist.add(*get_fs_specific_set(manifest.volume['partitions']))
-	                                  volume.Detach(),
-	                                  ebs.Snapshot(),
-	                                  volume.Delete()],
-	                          's3': [loopback.Create(),
-	                                 volume.Attach(),
-	                                 volume.Detach(),
-	                                 ami.BundleImage(),
-	                                 ami.UploadImage(),
-	                                 volume.Delete(),
-	                                 ami.RemoveBundle()]}
-	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
 
-	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
+	if 'boot' in manifest.volume['partitions']:
-	                             'ext2': [filesystem.TuneVolumeFS()],
+		from common.task_sets import boot_partition_set
-	                             'ext3': [filesystem.TuneVolumeFS()],
+		tasklist.add(*boot_partition_set)
-	                             'ext4': [filesystem.TuneVolumeFS()]}
-	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -104,14 +94,19 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 
 	def counter_task(task, counter):
 		if task in completed and counter not in completed:
-			tasklist.add(counter())
+			tasklist.add(counter)
 
-	if manifest.volume['backing'].lower() == 'ebs':
+	counter_task(ebs.Create, volume_tasks.Delete)
-		counter_task(ebs.Create, volume.Delete)
+	counter_task(ebs.Attach, volume_tasks.Detach)
-		counter_task(volume.Attach, volume.Detach)
+
-	if manifest.volume['backing'].lower() == 's3':
+	counter_task(loopback.Create, volume_tasks.Delete)
-		counter_task(loopback.Create, volume.Delete)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)
-		counter_task(volume.Attach, volume.Detach)
+
+	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
 	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
-	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
+
+	counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
+	counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)
+	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)

providers/ec2/ebsvolume.py

@@ -5,18 +5,18 @@ import time
 
 class EBSVolume(Volume):
 
-	volume = None
-
 	def create(self, conn, zone):
-		super(EBSVolume, self).create(self)
+		self.fsm.create(connection=conn, zone=zone)
+
+	def _before_create(self, e):
+		conn = e.connection
+		zone = e.zone
 		import math
-		# TODO: Warn if volume size is not a multiple of 1024
+		size = int(math.ceil(self.partition_map.get_total_size() / 1024))
-		size = int(math.ceil(self.partition_map.get_volume_size() / 1024))
 		self.volume = conn.create_volume(size, zone)
 		while self.volume.volume_state() != 'available':
 			time.sleep(5)
 			self.volume.update()
-		self.created = True
 
 	def attach(self, instance_id):
 		self.fsm.attach(instance_id=instance_id)
@@ -25,7 +25,7 @@ class EBSVolume(Volume):
 		instance_id = e.instance_id
 		import os.path
 		import string
-		for letter in string.ascii_lowercase:
+		for letter in string.ascii_lowercase[5:]:
 			dev_path = os.path.join('/dev', 'xvd' + letter)
 			if not os.path.exists(dev_path):
 				self.device_path = dev_path
@@ -47,7 +47,6 @@ class EBSVolume(Volume):
 			self.volume.update()
 
 	def _before_delete(self, e):
-		super(EBSVolume, self).delete(self)
 		self.volume.delete()
 
 	def snapshot(self):

providers/ec2/manifest-schema.json

@@ -20,13 +20,9 @@
 				"backing": {
 					"type": "string",
 					"enum": ["ebs", "s3"]
-				},
-				"filesystem": {
-					"type": "string",
-					"enum": ["ext2", "ext3", "ext4", "xfs"]
 				}
 			},
-			"required": ["backing", "filesystem"]
+			"required": ["backing"]
 		}
 	},
 	"required": ["volume"]

providers/ec2/manifest.py

@@ -9,7 +9,8 @@ class Manifest(base.Manifest):
 		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
 		self.schema_validate(data, schema_path)
 		if data['volume']['backing'] == 'ebs':
-			if data['volume']['size'] % 1024 != 0:
+			volume_size = self._calculate_volume_size(data['volume']['partitions'])
+			if volume_size % 1024 != 0:
 				msg = 'The volume size must be a multiple of 1024 when using EBS backing'
 				raise ManifestError(msg, self)
 		else:
@@ -21,5 +22,15 @@ class Manifest(base.Manifest):
 		self.credentials    = data['credentials']
 		self.virtualization = data['virtualization']
 		self.image          = data['image']
-		if data['volume']['backing'] == 'ebs':
+
-			self.ebs_volume_size = data['volume']['size'] / 1024
+	def _calculate_volume_size(self, partitions):
+		if partitions['type'] == 'mbr':
+			size = 1
+		else:
+			size = 0
+		if 'boot' in partitions:
+			size += partitions['boot']['size']
+		size += partitions['root']['size']
+		if 'swap' in partitions:
+			size += partitions['swap']['size']
+		return size

providers/ec2/tasks/ami.py

@@ -117,11 +117,14 @@ class RegisterAMI(Task):
 		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
 		kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
 
-		if info.manifest.volume['backing'] == 'ebs':
+		from providers.ec2.ebsvolume import EBSVolume
+		from common.fs.loopbackvolume import LoopbackVolume
+
+		if isinstance(info.volume, EBSVolume):
 			from boto.ec2.blockdevicemapping import BlockDeviceType
 			from boto.ec2.blockdevicemapping import BlockDeviceMapping
 			block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
-			                               size=info.manifest.ebs_volume_size)
+			                               size=info.volume.partition_map.get_total_size()/1024)
 			block_device_map = BlockDeviceMapping()
 			block_device_map['/dev/sda1'] = block_device
 
@@ -129,7 +132,7 @@ class RegisterAMI(Task):
 			                                            architecture=arch, kernel_id=kernel_id,
 			                                            root_device_name='/dev/sda1',
 			                                            block_device_map=block_device_map)
-		if info.manifest.volume['backing'] == 's3':
+		if isinstance(info.volume, LoopbackVolume):
 			image_location = ('{bucket}/{ami_name}.manifest.xml'
 			                  .format(bucket=info.manifest.image['bucket'],
 			                          ami_name=info.ami_name))

providers/ec2/tasks/ebs.py

@@ -1,17 +1,24 @@
 from base import Task
 from common import phases
-from common.tasks import volume
 
 
 class Create(Task):
 	description = 'Creating the EBS volume'
 	phase = phases.volume_creation
-	before = [volume.Attach]
 
 	def run(self, info):
 		info.volume.create(info.connection, info.host['availabilityZone'])
 
 
+class Attach(Task):
+	description = 'Attaching the volume'
+	phase = phases.volume_creation
+	after = [Create]
+
+	def run(self, info):
+		info.volume.attach(info.host['instanceId'])
+
+
 class Snapshot(Task):
 	description = 'Creating a snapshot of the EBS volume'
 	phase = phases.image_registration

providers/ec2/tasks/packages.py

@@ -11,8 +11,10 @@ class HostPackages(Task):
 	after = [packages.HostPackages]
 
 	def run(self, info):
-		if info.manifest.volume['filesystem'] == 'xfs':
+		for partition in info.volume.partition_map.partitions:
-			info.host_packages.add('xfsprogs')
+			if partition.filesystem == 'xfs':
+				info.host_packages.add('xfsprogs')
+				break
 		if info.manifest.volume['backing'] == 's3':
 			info.host_packages.add('euca2ools')
 
@@ -25,6 +27,7 @@ class ImagePackages(Task):
 	def run(self, info):
 		manifest = info.manifest
 		include, exclude = info.img_packages
+		include.add('openssh-server')
 		include.add('file')  # Needed for the init scripts
 		include.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
 		if manifest.virtualization == 'pvm':

providers/virtualbox/__init__.py

@@ -38,11 +38,8 @@ def tasks(tasklist, manifest):
 	             packages.ImagePackages,
 
 	             loopback.Create,
-	             filesystem.Format,
 
-	             bootstrap.Bootstrap,
 	             boot.ConfigureGrub,
-	             filesystem.FStab,
 	             common_boot.BlackListModules,
 	             common_boot.DisableGetTTYs,
 	             security.EnableShadowConfig,