ec2_publish plugin

Fixes #271, #91

CHANGELOG.rst

@@ -1,10 +1,16 @@
 Changelog
 =========
 
+
+2016-06-02
+----------
+Peter Wagner
+	* Added ec2_publish plugin
+
 2016-03-03
 ----------
 Anders Ingemann:
-  * Rename integration tests to system tests
+	* Rename integration tests to system tests
 
 2016-02-23
 ----------

bootstrapvz/plugins/ec2_publish/README.rst

@@ -0,0 +1,20 @@
+EC2 publish
+-----------
+
+This plugin lets you publish an EC2 AMI to multiple regions, make AMIs public,
+and output the AMIs generated in each file.
+
+Settings
+~~~~~~~~
+
+-  ``regions``: EC2 regions to copy the final image to.
+   ``optional``
+-  ``public``: Whether the AMIs should be made public (i.e. available by ALL users).
+   Valid values: ``true``, ``false``
+   Default: ``false``.
+   ``optional``
+-  ``manifest_url``: URL to publish generated AMIs.
+   Can be a path on the local filesystem, or a URL to S3 (https://bucket.s3-region.amazonaws.com/amis.json)
+   ``optional``
+
+

bootstrapvz/plugins/ec2_publish/__init__.py

@@ -0,0 +1,15 @@
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+	validator(data, schema_path)
+
+
+def resolve_tasks(taskset, manifest):
+	import tasks
+	taskset.add(tasks.CopyAmiToRegions)
+	if 'manifest_url' in manifest.plugins['ec2_publish']:
+		taskset.add(tasks.PublishAmiManifest)
+
+	ami_public = manifest.plugins['ec2_publish'].get('public')
+	if ami_public:
+		taskset.add(tasks.PublishAmi)

bootstrapvz/plugins/ec2_publish/manifest-schema.yml

@@ -0,0 +1,33 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: EC2-publish plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      ec2_publish:
+        type: object
+        properties:
+          regions:
+            type: array
+            items: {$ref: '#/definitions/aws-region'}
+            uniqueItems: true
+          manifest_url: {type: string}
+          public: {type: boolean}
+        additionalProperties: false
+definitions:
+  aws-region:
+    enum:
+    - ap-northeast-1
+    - ap-northeast-2
+    - ap-southeast-1
+    - ap-southeast-2
+    - eu-central-1
+    - eu-west-1
+    - sa-east-1
+    - us-east-1
+    - us-gov-west-1
+    - us-west-1
+    - us-west-2
+    - cn-north-1

bootstrapvz/plugins/ec2_publish/tasks.py

@@ -0,0 +1,96 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.providers.ec2.tasks import ami
+
+import logging
+
+
+class CopyAmiToRegions(Task):
+	description = 'Copy AWS AMI over other regions'
+	phase = phases.image_registration
+	predecessors = [ami.RegisterAMI]
+
+	@classmethod
+	def run(cls, info):
+		source_region = info._ec2['region']
+		source_ami = info._ec2['image']
+		name = info._ec2['ami_name']
+		copy_description = "Copied from %s (%s)" % (source_ami, source_region)
+
+		connect_args = {
+			'aws_access_key_id': info.credentials['access-key'],
+			'aws_secret_access_key': info.credentials['secret-key']
+		}
+		if 'security-token' in info.credentials:
+			connect_args['security_token'] = info.credentials['security-token']
+
+		region_amis = {source_region: source_ami}
+		region_conns = {source_region: info._ec2['connection']}
+		from boto.ec2 import connect_to_region
+		regions = info.manifest.plugins['ec2_publish'].get('regions', ())
+		for region in regions:
+			conn = connect_to_region(region, **connect_args)
+			region_conns[region] = conn
+			copied_image = conn.copy_image(source_region, source_ami, name=name, description=copy_description)
+			region_amis[region] = copied_image.image_id
+		info._ec2['region_amis'] = region_amis
+		info._ec2['region_conns'] = region_conns
+
+
+class PublishAmiManifest(Task):
+	description = 'Publish a manifest of generated AMIs'
+	phase = phases.image_registration
+	predecessors = [CopyAmiToRegions]
+
+	@classmethod
+	def run(cls, info):
+		manifest_url = info.manifest.plugins['ec2_publish']['manifest_url']
+
+		import json
+		amis_json = json.dumps(info._ec2['region_amis'])
+
+		from urlparse import urlparse
+		parsed_url = urlparse(manifest_url)
+		parsed_host = parsed_url.netloc
+		if not parsed_url.scheme:
+			with open(parsed_url.path, 'w') as local_out:
+				local_out.write(amis_json)
+		elif parsed_host.endswith('amazonaws.com') and 's3' in parsed_host:
+			region = 'us-east-1'
+			path = parsed_url.path[1:]
+			if 's3-' in parsed_host:
+				loc = parsed_host.find('s3-') + 3
+				region = parsed_host[loc:parsed_host.find('.', loc)]
+
+			if '.s3' in parsed_host:
+				bucket = parsed_host[:parsed_host.find('.s3')]
+			else:
+				bucket, path = path.split('/', 1)
+
+			from boto.s3 import connect_to_region
+			conn = connect_to_region(region)
+			key = conn.get_bucket(bucket, validate=False).new_key(path)
+			headers = {'Content-Type': 'application/json'}
+			key.set_contents_from_string(amis_json, headers=headers, policy='public-read')
+
+
+class PublishAmi(Task):
+	description = 'Make generated AMIs public'
+	phase = phases.image_registration
+	predecessors = [CopyAmiToRegions]
+
+	@classmethod
+	def run(cls, info):
+		region_conns = info._ec2['region_conns']
+		region_amis = info._ec2['region_amis']
+		logger = logging.getLogger(__name__)
+
+		import time
+		for region, region_ami in region_amis.items():
+			conn = region_conns[region]
+			current_image = conn.get_image(region_ami)
+			while current_image.state == 'pending':
+				logger.debug('Waiting for %s in %s (currently: %s)', region_ami, region, current_image.state)
+				time.sleep(5)
+				current_image = conn.get_image(region_ami)
+			conn.modify_image_attribute(region_ami, attribute='launchPermission', operation='add', groups='all')