Bootstrapping of instance store AMIs implemented

README.md

@@ -13,11 +13,13 @@ Pull requests are also welcome!
 
 Dependencies
 ------------
-You will need to run debian wheezy with python 2.7 and debootstrap installed.
+You will need to run debian wheezy with **python 2.7** and **debootstrap** installed.
 Also the following python libraries are required:
-* boto
+* **boto**
-* jsomschema ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip)
+* **jsomschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip)
-* termcolor
+* **termcolor**
+
+Bootstrapping instance store AMIs requires **euca2ools** to be installed.
 
 Highlights
 ----------

manifests/ec2-s3-pvm.manifest.json

@@ -3,7 +3,10 @@
 	"virtualization": "pvm",
 	"credentials":    {
 		"access-key":  null,
-		"secret-key": null
+		"secret-key":  null,
+		"certificate": null,
+		"private-key": null,
+		"user-id":     null
 	},
 
 	"bootstrapper": {
@@ -25,5 +28,11 @@
 		"backing":      "s3",
 		"filesystem":   "ext4",
 		"size":         1024
+	},
+	"plugins": {
+		"prebootstrapped": {
+			"enabled":  false,
+			"image": null
+		}
 	}
 }

plugins/prebootstrapped/__init__.py

@@ -1,20 +1,35 @@
 from tasks import Snapshot
+from tasks import CopyImage
 from tasks import CreateFromSnapshot
+from tasks import CreateFromImage
 from providers.ec2.tasks import ebs
+from providers.ec2.tasks import loopback
 
 
 def tasks(tasklist, manifest):
 	from providers.ec2.tasks import bootstrap
 	from providers.ec2.tasks import filesystem
-	if manifest.plugins['prebootstrapped']['snapshot'] == "":
+	settings = manifest.plugins['prebootstrapped']
-		tasklist.add(Snapshot())
+	if manifest.volume['backing'] == 'ebs':
-	else:
+		if 'snapshot' in settings and settings['snapshot'] is not None:
 			tasklist.replace(ebs.Create, CreateFromSnapshot())
 			tasklist.remove(filesystem.FormatVolume,
 			                filesystem.TuneVolumeFS,
 			                filesystem.AddXFSProgs,
 			                bootstrap.MakeTarball,
 			                bootstrap.Bootstrap)
+		else:
+			tasklist.add(Snapshot())
+	else:
+		if 'image' in settings and settings['image'] is not None:
+			tasklist.replace(loopback.Create, CreateFromImage())
+			tasklist.remove(filesystem.FormatVolume,
+			                filesystem.TuneVolumeFS,
+			                filesystem.AddXFSProgs,
+			                bootstrap.MakeTarball,
+			                bootstrap.Bootstrap)
+		else:
+			tasklist.add(CopyImage())
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -24,7 +39,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 		if task in completed and counter not in completed:
 			tasklist.add(counter())
 
+	if manifest.volume['backing'] == 'ebs':
 		counter_task(CreateFromSnapshot, ebs.Delete)
+	else:
+		counter_task(CreateFromImage, loopback.Delete)
 
 
 def validate_manifest(data, schema_validate):

plugins/prebootstrapped/manifest-schema.json

@@ -11,9 +11,11 @@
 					"properties": {
 						"snapshot": {
 							"type": "string"
-						}
 						},
-					"required": ["snapshot"]
+						"image": {
+							"type": "string"
+						}
+					}
 				}
 			},
 			"required": ["prebootstrapped"]

plugins/prebootstrapped/tasks.py

@@ -1,17 +1,27 @@
 from base import Task
 from common import phases
-from providers.ec2.tasks import connection
 from providers.ec2.tasks import ebs
+from providers.ec2.tasks import loopback
 from providers.ec2.tasks import bootstrap
 import time
 import logging
 log = logging.getLogger(__name__)
 
 
+class Snapshot(ebs.Snapshot):
+	description = 'Creating a snapshot of the bootstrapped volume'
+	phase = phases.os_installation
+	after = [bootstrap.Bootstrap]
+
+	def run(self, info):
+		super(Snapshot, self).run(info)
+		msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id)
+		log.info(msg)
+
+
 class CreateFromSnapshot(Task):
 	description = 'Creating EBS volume from a snapshot'
 	phase = phases.volume_creation
-	after = [connection.Connect]
 	before = [ebs.Attach]
 
 	def run(self, info):
@@ -25,12 +35,30 @@ class CreateFromSnapshot(Task):
 			info.volume.update()
 
 
-class Snapshot(ebs.Snapshot):
+class CopyImage(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
 	phase = phases.os_installation
 	after = [bootstrap.Bootstrap]
 
 	def run(self, info):
-		super(Snapshot, self).run(info)
+		import os.path
-		msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id)
+		from shutil import copyfile
+		loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id)
+		image_copy_path = os.path.join('/tmp', loopback_backup_name)
+		copyfile(info.loopback_file, image_copy_path)
+		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path)
 		log.info(msg)
+
+
+class CreateFromImage(Task):
+	description = 'Creating loopback image from a copy'
+	phase = phases.volume_creation
+	before = [loopback.Attach]
+
+	def run(self, info):
+		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
+		import os.path
+		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		loopback_backup_path = info.manifest.plugins['prebootstrapped']['image']
+		from shutil import copyfile
+		copyfile(loopback_backup_path, info.loopback_file)

providers/ec2/__init__.py

@@ -62,7 +62,8 @@ def tasks(tasklist, manifest):
 	             apt.EnableDaemonAutostart(),
 	             filesystem.UnmountSpecials(),
 	             filesystem.UnmountVolume(),
-	             filesystem.DeleteMountDir())
+	             filesystem.DeleteMountDir(),
+	             ami.RegisterAMI())
 
 	if manifest.bootstrapper['tarball']:
 		tasklist.add(bootstrap.MakeTarball())
@@ -71,12 +72,14 @@ def tasks(tasklist, manifest):
 	                                  ebs.Attach(),
 	                                  ebs.Detach(),
 	                                  ebs.Snapshot(),
-	                                  ami.RegisterAMI(),
 	                                  ebs.Delete()],
 	                          's3': [loopback.Create(),
 	                                 loopback.Attach(),
 	                                 loopback.Detach(),
-	                                 loopback.Delete()]}
+	                                 ami.BundleImage(),
+	                                 ami.UploadImage(),
+	                                 loopback.Delete(),
+	                                 ami.RemoveBundle()]}
 	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
 
 	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
@@ -96,6 +99,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 	if manifest.volume['backing'].lower() == 'ebs':
 		counter_task(ebs.Create, ebs.Delete)
 		counter_task(ebs.Attach, ebs.Detach)
+	if manifest.volume['backing'].lower() == 's3':
+		counter_task(loopback.Create, loopback.Delete)
+		counter_task(loopback.Attach, loopback.Detach)
 	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
 	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)

providers/ec2/assets/certs/cert-ec2.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIIDzjCCAzegAwIBAgIJALDnZV+lpZdSMA0GCSqGSIb3DQEBBQUAMIGhMQswCQYD
+VQQGEwJaQTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRv
+d24xJzAlBgNVBAoTHkFtYXpvbiBEZXZlbG9wbWVudCBDZW50cmUgKFNBKTEMMAoG
+A1UECxMDQUVTMREwDwYDVQQDEwhBRVMgVGVzdDEdMBsGCSqGSIb3DQEJARYOYWVz
+QGFtYXpvbi5jb20wHhcNMDUwODA5MTYwMTA5WhcNMDYwODA5MTYwMTA5WjCBoTEL
+MAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2Fw
+ZSBUb3duMScwJQYDVQQKEx5BbWF6b24gRGV2ZWxvcG1lbnQgQ2VudHJlIChTQSkx
+DDAKBgNVBAsTA0FFUzERMA8GA1UEAxMIQUVTIFRlc3QxHTAbBgkqhkiG9w0BCQEW
+DmFlc0BhbWF6b24uY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8v/X5
+zZv8CAVfNmvBM0br/RUcf1wU8xC5d2otFQQsQKB3qiWoj3oHeOWskOlTPFVZ8N+/
+hEaMjyOUkg2+g6XEagCQtFCEBzUVoMjiQIBPiWj5CWkFtlav2zt33LZ0ErTND4xl
+j7FQFqbaytHU9xuQcFO2p12bdITiBs5Kwoi9bQIDAQABo4IBCjCCAQYwHQYDVR0O
+BBYEFPQnsX1kDVzPtX+38ACV8RhoYcw8MIHWBgNVHSMEgc4wgcuAFPQnsX1kDVzP
+tX+38ACV8RhoYcw8oYGnpIGkMIGhMQswCQYDVQQGEwJaQTEVMBMGA1UECBMMV2Vz
+dGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xJzAlBgNVBAoTHkFtYXpvbiBE
+ZXZlbG9wbWVudCBDZW50cmUgKFNBKTEMMAoGA1UECxMDQUVTMREwDwYDVQQDEwhB
+RVMgVGVzdDEdMBsGCSqGSIb3DQEJARYOYWVzQGFtYXpvbi5jb22CCQCw52VfpaWX
+UjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAJJlWll4uGlrqBzeIw7u
+M3RvomlxMESwGKb9gI+ZeORlnHAyZxvd9XngIcjPuU+8uc3wc10LRQUCn45a5hFs
+zaCp9BSewLCCirn6awZn2tP8JlagSbjrN9YShStt8S3S/Jj+eBoRvc7jJnmEeMkx
+O0wHOzp5ZHRDK7tGULD6jCfU
+-----END CERTIFICATE-----

providers/ec2/manifest-schema-s3.json

@@ -3,6 +3,20 @@
 	"title": "EC2 manifest for instance store AMIs",
 	"type": "object",
 	"properties": {
+		"credentials": {
+			"type": "object",
+			"properties": {
+				"certificate": {
+					"type": "string"
+				},
+				"private-key": {
+					"type": "string"
+				},
+				"user-id": {
+					"type": "string"
+				}
+			}
+		},
 		"image": {
 			"type": "object",
 			"properties": {

providers/ec2/manifest-schema.json

@@ -3,6 +3,17 @@
 	"title": "EC2 manifest",
 	"type": "object",
 	"properties": {
+		"credentials": {
+			"type": "object",
+			"properties": {
+				"access-key": {
+					"type": "string"
+				},
+				"secret-key": {
+					"type": "string"
+				}
+			}
+		},
 		"volume": {
 			"type": "object",
 			"properties": {

providers/ec2/tasks/ami.py

@@ -1,8 +1,12 @@
 from base import Task
 from common import phases
+from common.exceptions import TaskError
+from common.tools import log_check_call
 from ebs import Snapshot
 from connection import Connect
-from common.exceptions import TaskError
+import os.path
+
+cert_ec2 = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/certs/cert-ec2.pem'))
 
 
 class AMIName(Task):
@@ -40,14 +44,37 @@ class BundleImage(Task):
 	phase = phases.image_registration
 
 	def run(self, info):
-		import os.path
 		bundle_name = 'bundle-{id:x}'.format(id=info.run_id)
-		info.bundle_dir = os.path.join(info.manifest.image['bundle_dir'], bundle_name)
+		info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name)
-		# from euca2ools.commands.bundle.bundleimage import BundleImage
+		log_check_call(['/usr/bin/euca-bundle-image',
-		# bundler = BundleImage()
+		                '--image', info.loopback_file,
-		# bundler.
+		                '--user', info.credentials['user-id'],
-		# euca-upload-bundle -b "${S3_BUCKET}" -m "${bundledir}/${ami_name}.manifest.xml"
+		                '--privatekey', info.credentials['private-key'],
-		pass
+		                '--cert', info.credentials['certificate'],
+		                '--ec2cert', cert_ec2,
+		                '--destination', info.bundle_path,
+		                '--prefix', info.ami_name])
+
+
+class UploadImage(Task):
+	description = 'Uploading the image bundle'
+	phase = phases.image_registration
+	after = [BundleImage]
+
+	def run(self, info):
+		manifest_file = os.path.join(info.bundle_path, info.ami_name + '.manifest.xml')
+		if info.host['region'] == 'us-east-1':
+			s3_url = 'https://s3.amazonaws.com/'
+		else:
+			s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region'])
+		log_check_call(['/usr/bin/euca-upload-bundle',
+		                '--bucket', info.manifest.image['bucket'],
+		                '--manifest', manifest_file,
+		                '--access-key', info.credentials['access-key'],
+		                '--secret-key', info.credentials['secret-key'],
+		                '--url', s3_url,
+		                '--region', info.host['region'],
+		                '--ec2cert', cert_ec2])
 
 
 class RemoveBundle(Task):
@@ -56,14 +83,14 @@ class RemoveBundle(Task):
 
 	def run(self, info):
 		from shutil import rmtree
-		rmtree(info.bundle_dir)
+		rmtree(info.bundle_path)
-		del info.bundle_dir
+		del info.bundle_path
 
 
 class RegisterAMI(Task):
 	description = 'Registering the image as an AMI'
 	phase = phases.image_registration
-	after = [Snapshot]
+	after = [Snapshot, UploadImage]
 
 	kernel_mapping = {'us-east-1':      {'amd64': 'aki-88aa75e1',
 	                                     'i386':  'aki-b6aa75df'},
@@ -88,6 +115,7 @@ class RegisterAMI(Task):
 		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
 		kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
 
+		if info.manifest.volume['backing'] == 'ebs':
 			from boto.ec2.blockdevicemapping import BlockDeviceType
 			from boto.ec2.blockdevicemapping import BlockDeviceMapping
 			block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
@@ -99,3 +127,11 @@ class RegisterAMI(Task):
 			                                            architecture=arch, kernel_id=kernel_id,
 			                                            root_device_name='/dev/sda1',
 			                                            block_device_map=block_device_map)
+		if info.manifest.volume['backing'] == 's3':
+			image_location = ('{bucket}/{ami_name}.manifest.xml'
+			                  .format(bucket=info.manifest.image['bucket'],
+			                          ami_name=info.ami_name))
+			info.image = info.connection.register_image(description=info.ami_description,
+			                                            architecture=arch, kernel_id=kernel_id,
+			                                            root_device_name='/dev/sda1',
+			                                            image_location=image_location)

providers/ec2/tasks/connection.py

@@ -8,24 +8,24 @@ class GetCredentials(Task):
 	phase = phases.preparation
 
 	def run(self, info):
-		info.credentials = self.get_credentials(info.manifest)
+		keys = ['access-key', 'secret-key']
+		if info.manifest.volume['backing'] == 's3':
+			keys.extend(['certificate', 'private-key', 'user-id'])
+		info.credentials = self.get_credentials(info.manifest, keys)
 
-	def get_credentials(self, manifest):
+	def get_credentials(self, manifest, keys):
 		from os import getenv
-		# manifest overrides environment
+		creds = {}
-		if(manifest.credentials['access-key'] and manifest.credentials['secret-key']):
+		if all(key in manifest.credentials for key in keys):
-			return {'access_key': manifest.credentials['access-key'],
+			for key in keys:
-			        'secret_key': manifest.credentials['secret-key']}
+				creds[key] = manifest.credentials[key]
-		if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')):
+			return creds
-			return {'access_key': getenv('EC2_ACCESS_KEY'),
+		if all(getenv(key) is not None for key in keys):
-			        'secret_key': getenv('EC2_SECRET_KEY')}
+			for key in keys:
-
+				creds[key] = getenv(key)
-		if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])):
+			return creds
-			raise RuntimeError('Both the access key and secret key must be specified in the manifest.')
+		raise RuntimeError(('No ec2 credentials found, they must all be specified '
-		if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))):
+		                    'exclusively via environment variables or through the manifest.'))
-			raise RuntimeError('Both the access key and secret key must be specified as environment variables.')
-
-		raise RuntimeError('No ec2 credentials found.')
 
 
 class Connect(Task):
@@ -36,5 +36,5 @@ class Connect(Task):
 	def run(self, info):
 		from boto.ec2 import connect_to_region
 		info.connection = connect_to_region(info.host['region'],
-		                                    aws_access_key_id=info.credentials['access_key'],
+		                                    aws_access_key_id=info.credentials['access-key'],
-		                                    aws_secret_access_key=info.credentials['secret_key'])
+		                                    aws_secret_access_key=info.credentials['secret-key'])

providers/ec2/tasks/filesystem.py

@@ -42,7 +42,7 @@ class CreateMountDir(Task):
 	def run(self, info):
 		import os
 		mount_dir = info.manifest.bootstrapper['mount_dir']
-		info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id)
+		info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id)
 		# Works recursively, fails if last part exists, which is exaclty what we want.
 		os.makedirs(info.root)
 

providers/ec2/tasks/loopback.py

@@ -1,5 +1,6 @@
 from base import Task
 from common import phases
+from filesystem import UnmountVolume
 from common.tools import log_check_call
 
 
@@ -13,7 +14,7 @@ class Create(Task):
 		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
 		log_check_call(['/bin/dd',
 		                'if=/dev/zero', 'of='+info.loopback_file,
-		                'bs=1M', 'seek='+info.manifest.volume['size'], 'count=0'])
+		                'bs=1M', 'seek='+str(info.manifest.volume['size']), 'count=0'])
 
 
 class Attach(Task):
@@ -23,13 +24,14 @@ class Attach(Task):
 
 	def run(self, info):
 		info.bootstrap_device = {}
-		info.bootstrap_device['path'] = log_check_call(['/sbin/losetup', '--find'])
+		[info.bootstrap_device['path']] = log_check_call(['/sbin/losetup', '--find'])
 		log_check_call(['/sbin/losetup', info.bootstrap_device['path'], info.loopback_file])
 
 
 class Detach(Task):
 	description = 'Detaching the loopback volume'
 	phase = phases.volume_unmounting
+	after = [UnmountVolume]
 
 	def run(self, info):
 		log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['path']])
@@ -42,5 +44,5 @@ class Delete(Task):
 
 	def run(self, info):
 		from os import remove
-		remove(info.bootstrap_device['path'])
+		remove(info.loopback_file)
 		del info.loopback_file

providers/ec2/tasks/packages.py

@@ -10,6 +10,8 @@ class HostPackages(Task):
 		packages = set(['debootstrap'])
 		if info.manifest.volume['filesystem'] == 'xfs':
 			packages.add('xfsprogs')
+		if info.manifest.volume['backing'] == 's3':
+			packages.add('euca2ools')
 
 		info.host_packages = packages