diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37410c2..9e0498c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,21 +1,27 @@ Changelog ========= + +2016-06-02 +---------- +Peter Wagner + * Added ec2_publish plugin + 2016-06-02 ---------- Zach Marano: - * Fix expand-root script to work with newer version of growpart (in jessie-backports and beyond). - * Overhaul Google Compute Engine image build. - * Add support for Google Cloud repositories. - * Google Cloud SDK install uses a deb package from a Google Cloud repository. - * Google Compute Engine guest software is installed from a Google Cloud repository. - * Google Compute Engine guest software for Debian 8 is updated to new refactor. - * Google Compute Engine wheezy and wheezy-backports manifests are deprecated. + * Fix expand-root script to work with newer version of growpart (in jessie-backports and beyond). + * Overhaul Google Compute Engine image build. + * Add support for Google Cloud repositories. + * Google Cloud SDK install uses a deb package from a Google Cloud repository. + * Google Compute Engine guest software is installed from a Google Cloud repository. + * Google Compute Engine guest software for Debian 8 is updated to new refactor. + * Google Compute Engine wheezy and wheezy-backports manifests are deprecated. 2016-03-03 ---------- Anders Ingemann: - * Rename integration tests to system tests + * Rename integration tests to system tests 2016-02-23 ---------- diff --git a/bootstrapvz/plugins/ec2_publish/README.rst b/bootstrapvz/plugins/ec2_publish/README.rst new file mode 100644 index 0000000..7e9d76d --- /dev/null +++ b/bootstrapvz/plugins/ec2_publish/README.rst @@ -0,0 +1,20 @@ +EC2 publish +----------- + +This plugin lets you publish an EC2 AMI to multiple regions, make AMIs public, +and output the AMIs generated in each file. + +Settings +~~~~~~~~ + +- ``regions``: EC2 regions to copy the final image to. + ``optional`` +- ``public``: Whether the AMIs should be made public (i.e. available by ALL users). + Valid values: ``true``, ``false`` + Default: ``false``. + ``optional`` +- ``manifest_url``: URL to publish generated AMIs. + Can be a path on the local filesystem, or a URL to S3 (https://bucket.s3-region.amazonaws.com/amis.json) + ``optional`` + + diff --git a/bootstrapvz/plugins/ec2_publish/__init__.py b/bootstrapvz/plugins/ec2_publish/__init__.py new file mode 100644 index 0000000..9e14d49 --- /dev/null +++ b/bootstrapvz/plugins/ec2_publish/__init__.py @@ -0,0 +1,15 @@ +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml')) + validator(data, schema_path) + + +def resolve_tasks(taskset, manifest): + import tasks + taskset.add(tasks.CopyAmiToRegions) + if 'manifest_url' in manifest.plugins['ec2_publish']: + taskset.add(tasks.PublishAmiManifest) + + ami_public = manifest.plugins['ec2_publish'].get('public') + if ami_public: + taskset.add(tasks.PublishAmi) diff --git a/bootstrapvz/plugins/ec2_publish/manifest-schema.yml b/bootstrapvz/plugins/ec2_publish/manifest-schema.yml new file mode 100644 index 0000000..236d21e --- /dev/null +++ b/bootstrapvz/plugins/ec2_publish/manifest-schema.yml @@ -0,0 +1,33 @@ +--- +$schema: http://json-schema.org/draft-04/schema# +title: EC2-publish plugin manifest +type: object +properties: + plugins: + type: object + properties: + ec2_publish: + type: object + properties: + regions: + type: array + items: {$ref: '#/definitions/aws-region'} + uniqueItems: true + manifest_url: {type: string} + public: {type: boolean} + additionalProperties: false +definitions: + aws-region: + enum: + - ap-northeast-1 + - ap-northeast-2 + - ap-southeast-1 + - ap-southeast-2 + - eu-central-1 + - eu-west-1 + - sa-east-1 + - us-east-1 + - us-gov-west-1 + - us-west-1 + - us-west-2 + - cn-north-1 \ No newline at end of file diff --git a/bootstrapvz/plugins/ec2_publish/tasks.py b/bootstrapvz/plugins/ec2_publish/tasks.py new file mode 100644 index 0000000..b4d36c2 --- /dev/null +++ b/bootstrapvz/plugins/ec2_publish/tasks.py @@ -0,0 +1,96 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.providers.ec2.tasks import ami + +import logging + + +class CopyAmiToRegions(Task): + description = 'Copy AWS AMI over other regions' + phase = phases.image_registration + predecessors = [ami.RegisterAMI] + + @classmethod + def run(cls, info): + source_region = info._ec2['region'] + source_ami = info._ec2['image'] + name = info._ec2['ami_name'] + copy_description = "Copied from %s (%s)" % (source_ami, source_region) + + connect_args = { + 'aws_access_key_id': info.credentials['access-key'], + 'aws_secret_access_key': info.credentials['secret-key'] + } + if 'security-token' in info.credentials: + connect_args['security_token'] = info.credentials['security-token'] + + region_amis = {source_region: source_ami} + region_conns = {source_region: info._ec2['connection']} + from boto.ec2 import connect_to_region + regions = info.manifest.plugins['ec2_publish'].get('regions', ()) + for region in regions: + conn = connect_to_region(region, **connect_args) + region_conns[region] = conn + copied_image = conn.copy_image(source_region, source_ami, name=name, description=copy_description) + region_amis[region] = copied_image.image_id + info._ec2['region_amis'] = region_amis + info._ec2['region_conns'] = region_conns + + +class PublishAmiManifest(Task): + description = 'Publish a manifest of generated AMIs' + phase = phases.image_registration + predecessors = [CopyAmiToRegions] + + @classmethod + def run(cls, info): + manifest_url = info.manifest.plugins['ec2_publish']['manifest_url'] + + import json + amis_json = json.dumps(info._ec2['region_amis']) + + from urlparse import urlparse + parsed_url = urlparse(manifest_url) + parsed_host = parsed_url.netloc + if not parsed_url.scheme: + with open(parsed_url.path, 'w') as local_out: + local_out.write(amis_json) + elif parsed_host.endswith('amazonaws.com') and 's3' in parsed_host: + region = 'us-east-1' + path = parsed_url.path[1:] + if 's3-' in parsed_host: + loc = parsed_host.find('s3-') + 3 + region = parsed_host[loc:parsed_host.find('.', loc)] + + if '.s3' in parsed_host: + bucket = parsed_host[:parsed_host.find('.s3')] + else: + bucket, path = path.split('/', 1) + + from boto.s3 import connect_to_region + conn = connect_to_region(region) + key = conn.get_bucket(bucket, validate=False).new_key(path) + headers = {'Content-Type': 'application/json'} + key.set_contents_from_string(amis_json, headers=headers, policy='public-read') + + +class PublishAmi(Task): + description = 'Make generated AMIs public' + phase = phases.image_registration + predecessors = [CopyAmiToRegions] + + @classmethod + def run(cls, info): + region_conns = info._ec2['region_conns'] + region_amis = info._ec2['region_amis'] + logger = logging.getLogger(__name__) + + import time + for region, region_ami in region_amis.items(): + conn = region_conns[region] + current_image = conn.get_image(region_ami) + while current_image.state == 'pending': + logger.debug('Waiting for %s in %s (currently: %s)', region_ami, region, current_image.state) + time.sleep(5) + current_image = conn.get_image(region_ami) + conn.modify_image_attribute(region_ami, attribute='launchPermission', operation='add', groups='all')