From 901d0845bf6e6b071b04810752587e45c05b4cac Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 14 Jul 2013 23:16:37 +0200 Subject: [PATCH] Bootstrapping of instance store AMIs implemented --- README.md | 10 ++- manifests/ec2-s3-pvm.manifest.json | 13 +++- plugins/prebootstrapped/__init__.py | 36 ++++++--- plugins/prebootstrapped/manifest-schema.json | 6 +- plugins/prebootstrapped/tasks.py | 38 ++++++++-- providers/ec2/__init__.py | 12 ++- providers/ec2/assets/certs/cert-ec2.pem | 23 ++++++ providers/ec2/manifest-schema-s3.json | 14 ++++ providers/ec2/manifest-schema.json | 11 +++ providers/ec2/tasks/ami.py | 78 ++++++++++++++------ providers/ec2/tasks/connection.py | 36 ++++----- providers/ec2/tasks/filesystem.py | 2 +- providers/ec2/tasks/loopback.py | 8 +- providers/ec2/tasks/packages.py | 2 + 14 files changed, 221 insertions(+), 68 deletions(-) create mode 100644 providers/ec2/assets/certs/cert-ec2.pem diff --git a/README.md b/README.md index a1ca9cd..ea4d99d 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,13 @@ Pull requests are also welcome! Dependencies ------------ -You will need to run debian wheezy with python 2.7 and debootstrap installed. +You will need to run debian wheezy with **python 2.7** and **debootstrap** installed. Also the following python libraries are required: -* boto -* jsomschema ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip) -* termcolor +* **boto** +* **jsomschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip) +* **termcolor** + +Bootstrapping instance store AMIs requires **euca2ools** to be installed. Highlights ---------- diff --git a/manifests/ec2-s3-pvm.manifest.json b/manifests/ec2-s3-pvm.manifest.json index 8b70f67..335be1e 100644 --- a/manifests/ec2-s3-pvm.manifest.json +++ b/manifests/ec2-s3-pvm.manifest.json @@ -2,8 +2,11 @@ "provider": "ec2", "virtualization": "pvm", "credentials": { - "access-key": null, - "secret-key": null + "access-key": null, + "secret-key": null, + "certificate": null, + "private-key": null, + "user-id": null }, "bootstrapper": { @@ -25,5 +28,11 @@ "backing": "s3", "filesystem": "ext4", "size": 1024 + }, + "plugins": { + "prebootstrapped": { + "enabled": false, + "image": null + } } } diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index 72f9102..4b9ecbb 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -1,20 +1,35 @@ from tasks import Snapshot +from tasks import CopyImage from tasks import CreateFromSnapshot +from tasks import CreateFromImage from providers.ec2.tasks import ebs +from providers.ec2.tasks import loopback def tasks(tasklist, manifest): from providers.ec2.tasks import bootstrap from providers.ec2.tasks import filesystem - if manifest.plugins['prebootstrapped']['snapshot'] == "": - tasklist.add(Snapshot()) + settings = manifest.plugins['prebootstrapped'] + if manifest.volume['backing'] == 'ebs': + if 'snapshot' in settings and settings['snapshot'] is not None: + tasklist.replace(ebs.Create, CreateFromSnapshot()) + tasklist.remove(filesystem.FormatVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + bootstrap.MakeTarball, + bootstrap.Bootstrap) + else: + tasklist.add(Snapshot()) else: - tasklist.replace(ebs.Create, CreateFromSnapshot()) - tasklist.remove(filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + if 'image' in settings and settings['image'] is not None: + tasklist.replace(loopback.Create, CreateFromImage()) + tasklist.remove(filesystem.FormatVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + bootstrap.MakeTarball, + bootstrap.Bootstrap) + else: + tasklist.add(CopyImage()) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -24,7 +39,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if task in completed and counter not in completed: tasklist.add(counter()) - counter_task(CreateFromSnapshot, ebs.Delete) + if manifest.volume['backing'] == 'ebs': + counter_task(CreateFromSnapshot, ebs.Delete) + else: + counter_task(CreateFromImage, loopback.Delete) def validate_manifest(data, schema_validate): diff --git a/plugins/prebootstrapped/manifest-schema.json b/plugins/prebootstrapped/manifest-schema.json index 8c853e6..7c35493 100644 --- a/plugins/prebootstrapped/manifest-schema.json +++ b/plugins/prebootstrapped/manifest-schema.json @@ -11,9 +11,11 @@ "properties": { "snapshot": { "type": "string" + }, + "image": { + "type": "string" } - }, - "required": ["snapshot"] + } } }, "required": ["prebootstrapped"] diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 6de08a1..2265d98 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,17 +1,27 @@ from base import Task from common import phases -from providers.ec2.tasks import connection from providers.ec2.tasks import ebs +from providers.ec2.tasks import loopback from providers.ec2.tasks import bootstrap import time import logging log = logging.getLogger(__name__) +class Snapshot(ebs.Snapshot): + description = 'Creating a snapshot of the bootstrapped volume' + phase = phases.os_installation + after = [bootstrap.Bootstrap] + + def run(self, info): + super(Snapshot, self).run(info) + msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id) + log.info(msg) + + class CreateFromSnapshot(Task): description = 'Creating EBS volume from a snapshot' phase = phases.volume_creation - after = [connection.Connect] before = [ebs.Attach] def run(self, info): @@ -25,12 +35,30 @@ class CreateFromSnapshot(Task): info.volume.update() -class Snapshot(ebs.Snapshot): +class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.os_installation after = [bootstrap.Bootstrap] def run(self, info): - super(Snapshot, self).run(info) - msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id) + import os.path + from shutil import copyfile + loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) + image_copy_path = os.path.join('/tmp', loopback_backup_name) + copyfile(info.loopback_file, image_copy_path) + msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) log.info(msg) + + +class CreateFromImage(Task): + description = 'Creating loopback image from a copy' + phase = phases.volume_creation + before = [loopback.Attach] + + def run(self, info): + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + import os.path + info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] + from shutil import copyfile + copyfile(loopback_backup_path, info.loopback_file) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index dd09d3c..57e2284 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -62,7 +62,8 @@ def tasks(tasklist, manifest): apt.EnableDaemonAutostart(), filesystem.UnmountSpecials(), filesystem.UnmountVolume(), - filesystem.DeleteMountDir()) + filesystem.DeleteMountDir(), + ami.RegisterAMI()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) @@ -71,12 +72,14 @@ def tasks(tasklist, manifest): ebs.Attach(), ebs.Detach(), ebs.Snapshot(), - ami.RegisterAMI(), ebs.Delete()], 's3': [loopback.Create(), loopback.Attach(), loopback.Detach(), - loopback.Delete()]} + ami.BundleImage(), + ami.UploadImage(), + loopback.Delete(), + ami.RemoveBundle()]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], @@ -96,6 +99,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if manifest.volume['backing'].lower() == 'ebs': counter_task(ebs.Create, ebs.Delete) counter_task(ebs.Attach, ebs.Detach) + if manifest.volume['backing'].lower() == 's3': + counter_task(loopback.Create, loopback.Delete) + counter_task(loopback.Attach, loopback.Detach) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/ec2/assets/certs/cert-ec2.pem b/providers/ec2/assets/certs/cert-ec2.pem new file mode 100644 index 0000000..4804945 --- /dev/null +++ b/providers/ec2/assets/certs/cert-ec2.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDzjCCAzegAwIBAgIJALDnZV+lpZdSMA0GCSqGSIb3DQEBBQUAMIGhMQswCQYD +VQQGEwJaQTEVMBMGA1UECBMMV2VzdGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRv +d24xJzAlBgNVBAoTHkFtYXpvbiBEZXZlbG9wbWVudCBDZW50cmUgKFNBKTEMMAoG +A1UECxMDQUVTMREwDwYDVQQDEwhBRVMgVGVzdDEdMBsGCSqGSIb3DQEJARYOYWVz +QGFtYXpvbi5jb20wHhcNMDUwODA5MTYwMTA5WhcNMDYwODA5MTYwMTA5WjCBoTEL +MAkGA1UEBhMCWkExFTATBgNVBAgTDFdlc3Rlcm4gQ2FwZTESMBAGA1UEBxMJQ2Fw +ZSBUb3duMScwJQYDVQQKEx5BbWF6b24gRGV2ZWxvcG1lbnQgQ2VudHJlIChTQSkx +DDAKBgNVBAsTA0FFUzERMA8GA1UEAxMIQUVTIFRlc3QxHTAbBgkqhkiG9w0BCQEW +DmFlc0BhbWF6b24uY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8v/X5 +zZv8CAVfNmvBM0br/RUcf1wU8xC5d2otFQQsQKB3qiWoj3oHeOWskOlTPFVZ8N+/ +hEaMjyOUkg2+g6XEagCQtFCEBzUVoMjiQIBPiWj5CWkFtlav2zt33LZ0ErTND4xl +j7FQFqbaytHU9xuQcFO2p12bdITiBs5Kwoi9bQIDAQABo4IBCjCCAQYwHQYDVR0O +BBYEFPQnsX1kDVzPtX+38ACV8RhoYcw8MIHWBgNVHSMEgc4wgcuAFPQnsX1kDVzP +tX+38ACV8RhoYcw8oYGnpIGkMIGhMQswCQYDVQQGEwJaQTEVMBMGA1UECBMMV2Vz +dGVybiBDYXBlMRIwEAYDVQQHEwlDYXBlIFRvd24xJzAlBgNVBAoTHkFtYXpvbiBE +ZXZlbG9wbWVudCBDZW50cmUgKFNBKTEMMAoGA1UECxMDQUVTMREwDwYDVQQDEwhB +RVMgVGVzdDEdMBsGCSqGSIb3DQEJARYOYWVzQGFtYXpvbi5jb22CCQCw52VfpaWX +UjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAJJlWll4uGlrqBzeIw7u +M3RvomlxMESwGKb9gI+ZeORlnHAyZxvd9XngIcjPuU+8uc3wc10LRQUCn45a5hFs +zaCp9BSewLCCirn6awZn2tP8JlagSbjrN9YShStt8S3S/Jj+eBoRvc7jJnmEeMkx +O0wHOzp5ZHRDK7tGULD6jCfU +-----END CERTIFICATE----- diff --git a/providers/ec2/manifest-schema-s3.json b/providers/ec2/manifest-schema-s3.json index d807841..654629e 100644 --- a/providers/ec2/manifest-schema-s3.json +++ b/providers/ec2/manifest-schema-s3.json @@ -3,6 +3,20 @@ "title": "EC2 manifest for instance store AMIs", "type": "object", "properties": { + "credentials": { + "type": "object", + "properties": { + "certificate": { + "type": "string" + }, + "private-key": { + "type": "string" + }, + "user-id": { + "type": "string" + } + } + }, "image": { "type": "object", "properties": { diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index 1d3bdd7..ffa360d 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -3,6 +3,17 @@ "title": "EC2 manifest", "type": "object", "properties": { + "credentials": { + "type": "object", + "properties": { + "access-key": { + "type": "string" + }, + "secret-key": { + "type": "string" + } + } + }, "volume": { "type": "object", "properties": { diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 00eac3c..be82f43 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -1,8 +1,12 @@ from base import Task from common import phases +from common.exceptions import TaskError +from common.tools import log_check_call from ebs import Snapshot from connection import Connect -from common.exceptions import TaskError +import os.path + +cert_ec2 = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/certs/cert-ec2.pem')) class AMIName(Task): @@ -40,14 +44,37 @@ class BundleImage(Task): phase = phases.image_registration def run(self, info): - import os.path bundle_name = 'bundle-{id:x}'.format(id=info.run_id) - info.bundle_dir = os.path.join(info.manifest.image['bundle_dir'], bundle_name) - # from euca2ools.commands.bundle.bundleimage import BundleImage - # bundler = BundleImage() - # bundler. - # euca-upload-bundle -b "${S3_BUCKET}" -m "${bundledir}/${ami_name}.manifest.xml" - pass + info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name) + log_check_call(['/usr/bin/euca-bundle-image', + '--image', info.loopback_file, + '--user', info.credentials['user-id'], + '--privatekey', info.credentials['private-key'], + '--cert', info.credentials['certificate'], + '--ec2cert', cert_ec2, + '--destination', info.bundle_path, + '--prefix', info.ami_name]) + + +class UploadImage(Task): + description = 'Uploading the image bundle' + phase = phases.image_registration + after = [BundleImage] + + def run(self, info): + manifest_file = os.path.join(info.bundle_path, info.ami_name + '.manifest.xml') + if info.host['region'] == 'us-east-1': + s3_url = 'https://s3.amazonaws.com/' + else: + s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region']) + log_check_call(['/usr/bin/euca-upload-bundle', + '--bucket', info.manifest.image['bucket'], + '--manifest', manifest_file, + '--access-key', info.credentials['access-key'], + '--secret-key', info.credentials['secret-key'], + '--url', s3_url, + '--region', info.host['region'], + '--ec2cert', cert_ec2]) class RemoveBundle(Task): @@ -56,14 +83,14 @@ class RemoveBundle(Task): def run(self, info): from shutil import rmtree - rmtree(info.bundle_dir) - del info.bundle_dir + rmtree(info.bundle_path) + del info.bundle_path class RegisterAMI(Task): description = 'Registering the image as an AMI' phase = phases.image_registration - after = [Snapshot] + after = [Snapshot, UploadImage] kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', 'i386': 'aki-b6aa75df'}, @@ -88,14 +115,23 @@ class RegisterAMI(Task): arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.manifest.ebs_volume_size) - block_device_map = BlockDeviceMapping() - block_device_map['/dev/sda1'] = block_device + if info.manifest.volume['backing'] == 'ebs': + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=info.manifest.ebs_volume_size) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda1'] = block_device - info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - block_device_map=block_device_map) + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='/dev/sda1', + block_device_map=block_device_map) + if info.manifest.volume['backing'] == 's3': + image_location = ('{bucket}/{ami_name}.manifest.xml' + .format(bucket=info.manifest.image['bucket'], + ami_name=info.ami_name)) + info.image = info.connection.register_image(description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='/dev/sda1', + image_location=image_location) diff --git a/providers/ec2/tasks/connection.py b/providers/ec2/tasks/connection.py index 2c2ce19..c6d47df 100644 --- a/providers/ec2/tasks/connection.py +++ b/providers/ec2/tasks/connection.py @@ -8,24 +8,24 @@ class GetCredentials(Task): phase = phases.preparation def run(self, info): - info.credentials = self.get_credentials(info.manifest) + keys = ['access-key', 'secret-key'] + if info.manifest.volume['backing'] == 's3': + keys.extend(['certificate', 'private-key', 'user-id']) + info.credentials = self.get_credentials(info.manifest, keys) - def get_credentials(self, manifest): + def get_credentials(self, manifest, keys): from os import getenv - # manifest overrides environment - if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): - return {'access_key': manifest.credentials['access-key'], - 'secret_key': manifest.credentials['secret-key']} - if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')): - return {'access_key': getenv('EC2_ACCESS_KEY'), - 'secret_key': getenv('EC2_SECRET_KEY')} - - if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])): - raise RuntimeError('Both the access key and secret key must be specified in the manifest.') - if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))): - raise RuntimeError('Both the access key and secret key must be specified as environment variables.') - - raise RuntimeError('No ec2 credentials found.') + creds = {} + if all(key in manifest.credentials for key in keys): + for key in keys: + creds[key] = manifest.credentials[key] + return creds + if all(getenv(key) is not None for key in keys): + for key in keys: + creds[key] = getenv(key) + return creds + raise RuntimeError(('No ec2 credentials found, they must all be specified ' + 'exclusively via environment variables or through the manifest.')) class Connect(Task): @@ -36,5 +36,5 @@ class Connect(Task): def run(self, info): from boto.ec2 import connect_to_region info.connection = connect_to_region(info.host['region'], - aws_access_key_id=info.credentials['access_key'], - aws_secret_access_key=info.credentials['secret_key']) + aws_access_key_id=info.credentials['access-key'], + aws_secret_access_key=info.credentials['secret-key']) diff --git a/providers/ec2/tasks/filesystem.py b/providers/ec2/tasks/filesystem.py index a16e852..83c940c 100644 --- a/providers/ec2/tasks/filesystem.py +++ b/providers/ec2/tasks/filesystem.py @@ -42,7 +42,7 @@ class CreateMountDir(Task): def run(self, info): import os mount_dir = info.manifest.bootstrapper['mount_dir'] - info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id) + info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) # Works recursively, fails if last part exists, which is exaclty what we want. os.makedirs(info.root) diff --git a/providers/ec2/tasks/loopback.py b/providers/ec2/tasks/loopback.py index e1f15d1..631c995 100644 --- a/providers/ec2/tasks/loopback.py +++ b/providers/ec2/tasks/loopback.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from filesystem import UnmountVolume from common.tools import log_check_call @@ -13,7 +14,7 @@ class Create(Task): info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) log_check_call(['/bin/dd', 'if=/dev/zero', 'of='+info.loopback_file, - 'bs=1M', 'seek='+info.manifest.volume['size'], 'count=0']) + 'bs=1M', 'seek='+str(info.manifest.volume['size']), 'count=0']) class Attach(Task): @@ -23,13 +24,14 @@ class Attach(Task): def run(self, info): info.bootstrap_device = {} - info.bootstrap_device['path'] = log_check_call(['/sbin/losetup', '--find']) + [info.bootstrap_device['path']] = log_check_call(['/sbin/losetup', '--find']) log_check_call(['/sbin/losetup', info.bootstrap_device['path'], info.loopback_file]) class Detach(Task): description = 'Detaching the loopback volume' phase = phases.volume_unmounting + after = [UnmountVolume] def run(self, info): log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['path']]) @@ -42,5 +44,5 @@ class Delete(Task): def run(self, info): from os import remove - remove(info.bootstrap_device['path']) + remove(info.loopback_file) del info.loopback_file diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index b5474b4..9b8c739 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -10,6 +10,8 @@ class HostPackages(Task): packages = set(['debootstrap']) if info.manifest.volume['filesystem'] == 'xfs': packages.add('xfsprogs') + if info.manifest.volume['backing'] == 's3': + packages.add('euca2ools') info.host_packages = packages