Implement hvm support for ec2

common/tasks/boot.py

@@ -1,6 +1,7 @@
 from base import Task
 from common import phases
-import os
+from common.tasks import apt
+import os.path
 
 
 class BlackListModules(Task):
@@ -27,3 +28,65 @@ class DisableGetTTYs(Task):
 		for i in range(2, 7):
 			i = str(i)
 			sed_i(inittab_path, '^' + i + ttyx + i, '#' + i + ttyx + i)
+
+
+class InstallGrub(Task):
+	description = 'Installing grub'
+	phase = phases.system_modification
+	predecessors = [apt.AptUpgrade]
+
+	def run(self, info):
+		from common.fs.loopbackvolume import LoopbackVolume
+		from common.tools import log_check_call
+
+		boot_dir = os.path.join(info.root, 'boot')
+		grub_dir = os.path.join(boot_dir, 'grub')
+
+		from base.fs.partitionmaps.none import NoPartitions
+		from base.fs.partitionmaps.gpt import GPTPartitionMap
+		from common.fs import remount
+		p_map = info.volume.partition_map
+
+		def link_fn():
+			info.volume.link_dm_node()
+			if isinstance(p_map, NoPartitions):
+				p_map.root.device_path = info.volume.device_path
+
+		def unlink_fn():
+			info.volume.unlink_dm_node()
+			if isinstance(p_map, NoPartitions):
+				p_map.root.device_path = info.volume.device_path
+
+		# GRUB cannot deal with installing to loopback devices
+		# so we fake a real harddisk with dmsetup.
+		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, link_fn)
+		try:
+			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
+			device_map_path = os.path.join(grub_dir, 'device.map')
+			partition_prefix = 'msdos'
+			if isinstance(p_map, GPTPartitionMap):
+				partition_prefix = 'gpt'
+			with open(device_map_path, 'w') as device_map:
+				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
+				if not isinstance(p_map, NoPartitions):
+					for idx, partition in enumerate(info.volume.partition_map.partitions):
+						[partition_path] = log_check_call(['readlink', '-f', partition.device_path])
+						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
+						                 .format(device_path=partition_path, prefix=partition_prefix, idx=idx+1))
+
+			# Install grub
+			log_check_call(['/usr/sbin/chroot', info.root,
+			                '/usr/sbin/grub-install',
+			                # '--root-directory=' + info.root,
+			                # '--boot-directory=' + boot_dir,
+			                device_path])
+			log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+		except Exception as e:
+			if isinstance(info.volume, LoopbackVolume):
+				remount(info.volume, unlink_fn)
+			raise e
+
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, unlink_fn)

manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json

@@ -0,0 +1,50 @@
+{
+	"provider":       "ec2",
+	"virtualization": "hvm",
+	"credentials":    {
+		// "access-key": null,
+		// "secret-key": null
+	},
+
+	"bootstrapper": {
+		"workspace": "/target"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} AMI ({virtualization})"
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing": "ebs",
+		"partitions": {
+			"type": "none",
+			"root": {
+				"size":       8192,
+				"filesystem": "ext4"
+			}
+		}
+	},
+	"plugins": {
+		"packages": {
+			"sources": {
+				"backports": [
+					"deb     {apt_mirror} {release}-backports main",
+					"deb-src {apt_mirror} {release}-backports main"
+				]
+			},
+			"remote": [
+				"sudo",
+				{ "name": "cloud-init", "target": "{release}-backports" }
+			]
+		},
+		"cloud_init": {
+			"username": "admin"
+		}
+	}
+}

providers/ec2/__init__.py

@@ -50,7 +50,6 @@ def tasks(tasklist, manifest):
 	             ami.AMIName,
 	             connection.Connect,
 
-	             boot.ConfigureGrub,
 	             common_boot.BlackListModules,
 	             common_boot.DisableGetTTYs,
 	             security.EnableShadowConfig,
@@ -68,6 +67,11 @@ def tasks(tasklist, manifest):
 
 	             ami.RegisterAMI)
 
+	if manifest.virtualization == 'pvm':
+		tasklist.add(boot.ConfigurePVGrub)
+	else:
+		tasklist.add(common_boot.InstallGrub)
+
 	backing_specific_tasks = {'ebs': [ebs.Create,
 	                                  ebs.Attach,
 	                                  common_filesystem.FStab,

providers/ec2/manifest-schema.json

@@ -3,6 +3,7 @@
 	"title": "EC2 manifest",
 	"type": "object",
 	"properties": {
+		"virtualization": { "enum": ["pvm", "hvm"] },
 		"image": {
 			"type": "object",
 			"properties": {

providers/ec2/tasks/ami.py

@@ -151,52 +151,38 @@ class RegisterAMI(Task):
 	                  }}
 
 	def run(self, info):
-		if info.manifest.volume['backing'] == 'ebs':
+		registration_params = {'name': info.ami_name,
-			self.run_ebs(info)
+		                       'description': info.ami_description}
+		registration_params['architecture'] = {'i386': 'i386',
+		                                       'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
+
 		if info.manifest.volume['backing'] == 's3':
-			self.run_s3(info)
-
-	def run_ebs(self, info):
-		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
-
-		from base.fs.partitionmaps.none import NoPartitions
-		if isinstance(info.volume.partition_map, NoPartitions):
 			grub_boot_device = 'hd0'
-			root_device_name = '/dev/sda'
+			registration_params['root_device_name'] = 'dev/sda1'
 		else:
-			grub_boot_device = 'hd00'
+			from base.fs.partitionmaps.none import NoPartitions
-			root_idx = info.volume.partition_map.root.get_index()
+			if isinstance(info.volume.partition_map, NoPartitions):
-			root_device_name = '/dev/sda{idx}'.format(idx=root_idx)
+				grub_boot_device = 'hd0'
+				registration_params['root_device_name'] = '/dev/sda'
+			else:
+				grub_boot_device = 'hd00'
+				root_idx = info.volume.partition_map.root.get_index()
+				registration_params['root_device_name'] = '/dev/sda{idx}'.format(idx=root_idx)
 
-		kernel_id = (self.kernel_mapping
+			from boto.ec2.blockdevicemapping import BlockDeviceType
-		             .get(info.host['region'])
+			from boto.ec2.blockdevicemapping import BlockDeviceMapping
-		             .get(grub_boot_device)
+			block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
-		             .get(info.manifest.system['architecture']))
+			                               size=info.volume.partition_map.get_total_size()/1024)
+			registration_params['block_device_map'] = BlockDeviceMapping()
+			registration_params['block_device_map']['/dev/sda'] = block_device
 
-		from boto.ec2.blockdevicemapping import BlockDeviceType
+		if info.manifest.virtualization == 'hvm':
-		from boto.ec2.blockdevicemapping import BlockDeviceMapping
+			registration_params['virtualization_type'] = 'hvm'
-		block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
+		else:
-		                               size=info.volume.partition_map.get_total_size()/1024)
+			registration_params['virtualization_type'] = 'paravirtual'
-		block_device_map = BlockDeviceMapping()
+			registration_params['kernel_id'] = (self.kernel_mapping
-		block_device_map['/dev/sda'] = block_device
+			                                    .get(info.host['region'])
+			                                    .get(grub_boot_device)
+			                                    .get(info.manifest.system['architecture']))
 
-		info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
+		info.image = info.connection.register_image(**registration_params)
-		                                            architecture=arch, kernel_id=kernel_id,
-		                                            root_device_name=root_device_name,
-		                                            block_device_map=block_device_map)
-
-	def run_s3(self, info):
-		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
-
-		kernel_id = (self.kernel_mapping
-		             .get(info.host['region'])
-		             .get('hd0')
-		             .get(info.manifest.system['architecture']))
-
-		image_manifest = ('{bucket}/{ami_name}.manifest.xml'
-		                  .format(bucket=info.manifest.image['bucket'],
-		                          ami_name=info.ami_name))
-		info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
-		                                            architecture=arch, kernel_id=kernel_id,
-		                                            root_device_name='dev/sda1',
-		                                            image_location=image_manifest)

providers/ec2/tasks/boot.py

@@ -3,8 +3,8 @@ from common import phases
 import os
 
 
-class ConfigureGrub(Task):
+class ConfigurePVGrub(Task):
-	description = 'Configuring grub'
+	description = 'Creating grub config files for PVGrub'
 	phase = phases.system_modification
 
 	def run(self, info):

providers/ec2/tasks/packages.py

@@ -26,8 +26,7 @@ class ImagePackages(Task):
 		include.add('openssh-server')
 		include.add('file')  # Needed for the init scripts
 		include.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
-		if manifest.virtualization == 'pvm':
+		include.add('grub-pc')
-			include.add('grub-pc')
 
 		exclude.add('isc-dhcp-client')
 		exclude.add('isc-dhcp-common')

providers/virtualbox/__init__.py

@@ -5,8 +5,7 @@ from common.tasks import loopback
 from common.tasks import partitioning
 from common.tasks import filesystem
 from common.tasks import bootstrap
-from tasks import boot
+from common.tasks import boot
-from common.tasks import boot as common_boot
 from common.tasks import security
 from common.tasks import network
 from common.tasks import initd
@@ -38,9 +37,9 @@ def tasks(tasklist, manifest):
 
 	             loopback.Create,
 
-	             boot.ConfigureGrub,
+	             boot.InstallGrub,
-	             common_boot.BlackListModules,
+	             boot.BlackListModules,
-	             common_boot.DisableGetTTYs,
+	             boot.DisableGetTTYs,
 	             security.EnableShadowConfig,
 	             network.RemoveDNSInfo,
 	             network.ConfigureNetworkIF,

providers/virtualbox/tasks/boot.py

@@ -1,65 +0,0 @@
-from base import Task
-from common import phases
-from common.tasks import apt
-from common.fs.loopbackvolume import LoopbackVolume
-
-
-class ConfigureGrub(Task):
-	description = 'Configuring grub'
-	phase = phases.system_modification
-	predecessors = [apt.AptUpgrade]
-
-	def run(self, info):
-		import os
-		from common.tools import log_check_call
-
-		boot_dir = os.path.join(info.root, 'boot')
-		grub_dir = os.path.join(boot_dir, 'grub')
-
-		from base.fs.partitionmaps.none import NoPartitions
-		from base.fs.partitionmaps.gpt import GPTPartitionMap
-		from common.fs import remount
-		p_map = info.volume.partition_map
-
-		def mk_remount_fn(fn):
-			def set_device_path():
-				fn()
-				if isinstance(p_map, NoPartitions):
-					p_map.root.device_path = info.volume.device_path
-			return set_device_path
-		link_fn = mk_remount_fn(info.volume.link_dm_node)
-		unlink_fn = mk_remount_fn(info.volume.unlink_dm_node)
-
-		# GRUB cannot deal with installing to loopback devices
-		# so we fake a real harddisk with dmsetup.
-		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, link_fn)
-		try:
-			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
-			device_map_path = os.path.join(grub_dir, 'device.map')
-			partition_prefix = 'msdos'
-			if isinstance(p_map, GPTPartitionMap):
-				partition_prefix = 'gpt'
-			with open(device_map_path, 'w') as device_map:
-				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
-				if not isinstance(p_map, NoPartitions):
-					for idx, partition in enumerate(info.volume.partition_map.partitions):
-						[partition_path] = log_check_call(['readlink', '-f', partition.device_path])
-						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
-						                 .format(device_path=partition_path, prefix=partition_prefix, idx=idx+1))
-
-			# Install grub
-			log_check_call(['/usr/sbin/chroot', info.root,
-			                '/usr/sbin/grub-install',
-			                # '--root-directory=' + info.root,
-			                # '--boot-directory=' + boot_dir,
-			                device_path])
-			log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
-		except Exception as e:
-			if isinstance(info.volume, LoopbackVolume):
-				remount(info.volume, unlink_fn)
-			raise e
-
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, unlink_fn)