Integration testing for EC2

bootstrapvz/remote/build_servers/build-servers-schema.yml

@@ -31,6 +31,18 @@ definitions:
     type: object
     properties:
       guest_additions: {$ref: '#/definitions/absolute_path'}
+      ec2-credentials:
+        required: [access-key, secret-key]
+        type: object
+        properties:
+          access-key: {type: string}
+          secret-key: {type: string}
+          certificate: {type: string}
+          private-key: {type: string}
+          user-id:
+            type: string
+            pattern: (^arn:aws:iam::\d*:user/\w.*$)|(^\d{4}-\d{4}-\d{4}$)
+        additional_properties: false
       apt_proxy:
         type: object
         properties:

bootstrapvz/remote/build_servers/build_server.py

@@ -14,4 +14,10 @@ class BuildServer(object):
 			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
 		if 'apt_proxy' in self.build_settings:
 			manifest_data.get('plugins', {})['apt_proxy'] = self.build_settings['apt_proxy']
+		if 'ec2-credentials' in self.build_settings:
+			if 'credentials' not in manifest_data['provider']:
+				manifest_data['provider']['credentials'] = {}
+			for key in ['access-key', 'secret-key', 'certificate', 'private-key', 'user-id']:
+				if key in self.build_settings['ec2-credentials']:
+					manifest_data['provider']['credentials'][key] = self.build_settings['ec2-credentials'][key]
 		return manifest_data

tests/integration/ec2.py

@@ -0,0 +1,19 @@
+from manifests import merge_manifest_data
+from tools import boot_manifest
+
+partials = {'ec2': 'provider: {name: ec2}',
+            'ebs': 'volume: {backing: ebs}',
+            's3': 'volume: {backing: s3}',
+            'pvm': '{provider: {virtualization: pvm}, system: {bootloader: pvgrub}}',
+            'hvm-extlinux': '{provider: {virtualization: hvm}, system: {bootloader: extlinux}}',
+            'hvm-grub': '{provider: {virtualization: hvm}, system: {bootloader: grub}}',
+            }
+
+
+def test_unpartitioned_ebs_pvgrub_stable():
+	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)

tests/integration/images/__init__.py

@@ -4,3 +4,8 @@ def initialize_image(manifest, build_server, bootstrap_info):
 	if manifest.provider['name'] == 'virtualbox':
 		import vbox
 		return vbox.initialize_image(manifest, build_server, bootstrap_info)
+	if manifest.provider['name'] == 'ec2':
+		import ami
+		credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
+		               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
+		return ami.initialize_image(manifest, credentials, bootstrap_info)

tests/integration/images/ami.py

@@ -0,0 +1,33 @@
+from image import Image
+import logging
+from contextlib import contextmanager
+log = logging.getLogger(__name__)
+
+
+def initialize_image(manifest, credentials, bootstrap_info):
+	from boto.ec2 import connect_to_region
+	connection = connect_to_region(bootstrap_info._ec2['region'],
+	                               aws_access_key_id=credentials['access-key'],
+	                               aws_secret_access_key=credentials['secret-key'])
+	ami = connection.get_image(bootstrap_info._ec2['image'])
+	image = AmazonMachineImage(manifest, ami)
+	return image
+
+
+class AmazonMachineImage(Image):
+
+	def __init__(self, manifest, ami):
+		super(AmazonMachineImage, self).__init__(manifest)
+		self.ami = ami
+
+	def destroy(self):
+		log.debug('Deleting AMI')
+		self.ami.deregister(delete_snapshot=True)
+		del self.ami
+
+	@contextmanager
+	def get_instance(self, instance_type='t1.micro'):
+		from ..instances.ec2 import boot_image
+		name = 'bootstrap-vz test instance'
+		with boot_image(name, self.ami, instance_type) as instance:
+			yield instance

tests/integration/instances/ec2.py

@@ -0,0 +1,32 @@
+from contextlib import contextmanager
+from ..tools import waituntil
+import logging
+log = logging.getLogger(__name__)
+
+
+@contextmanager
+def boot_image(name, image, instance_type):
+	instance = None
+	try:
+		log.debug('Booting ec2 instance')
+		reservation = image.run(instance_type=instance_type)
+		[instance] = reservation.instances
+		instance.add_tag('Name', name)
+
+		def instance_running():
+			instance.update()
+			return instance.state == 'running'
+		if not waituntil(instance_running, timeout=120, interval=3):
+			raise EC2InstanceStartupException('Timeout while booting instance')
+
+		if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
+			raise EC2InstanceStartupException('Timeout while fetching console output')
+
+		yield instance
+	finally:
+		if instance is not None:
+			instance.terminate()
+
+
+class EC2InstanceStartupException(Exception):
+	pass

tests/integration/tools/__init__.py

@@ -7,7 +7,7 @@ register_deserialization_handlers()
 
 
 @contextmanager
-def boot_manifest(manifest_data):
+def boot_manifest(manifest_data, boot_vars={}):
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data('build-servers.yml')
 	from bootstrapvz.remote.build_servers import pick_build_server
@@ -24,7 +24,7 @@ def boot_manifest(manifest_data):
 	from ..images import initialize_image
 	image = initialize_image(manifest, build_server, bootstrap_info)
 	try:
-		with image.get_instance() as instance:
+		with image.get_instance(**boot_vars) as instance:
 			yield instance
 	finally:
 		image.destroy()