Simplify test harness architecture

by reducing the amount of interfacing between generic and provider specific code

tests/integration/images/__init__.py

@@ -1,11 +0,0 @@
-
-
-def initialize_image(manifest, build_server, bootstrap_info):
-	if manifest.provider['name'] == 'virtualbox':
-		import vbox
-		return vbox.initialize_image(manifest, build_server, bootstrap_info)
-	if manifest.provider['name'] == 'ec2':
-		import ami
-		credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
-		               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
-		return ami.initialize_image(manifest, credentials, bootstrap_info)

tests/integration/images/ami.py

@@ -1,38 +0,0 @@
-from image import Image
-import logging
-from contextlib import contextmanager
-log = logging.getLogger(__name__)
-
-
-def initialize_image(manifest, credentials, bootstrap_info):
-	image = AmazonMachineImage(manifest, bootstrap_info._ec2['image'],
-	                           bootstrap_info._ec2['region'], credentials)
-	return image
-
-
-class AmazonMachineImage(Image):
-
-	def __init__(self, manifest, image_id, region, credentials):
-		super(AmazonMachineImage, self).__init__(manifest)
-
-		from boto.ec2 import connect_to_region as ec2_connect
-		self.ec2_connection = ec2_connect(region, aws_access_key_id=credentials['access-key'],
-		                                  aws_secret_access_key=credentials['secret-key'])
-		from boto.vpc import connect_to_region as vpc_connect
-		self.vpc_connection = vpc_connect(region, aws_access_key_id=credentials['access-key'],
-		                                  aws_secret_access_key=credentials['secret-key'])
-
-		self.ami = self.ec2_connection.get_image(image_id)
-
-	def destroy(self):
-		log.debug('Deleting AMI')
-		self.ami.deregister()
-		for device, block_device_type in self.ami.block_device_mapping.items():
-			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
-		del self.ami
-
-	@contextmanager
-	def get_instance(self, instance_type):
-		from ..instances.ec2 import boot_image
-		with boot_image(self.ami, instance_type, self.ec2_connection, self.vpc_connection) as instance:
-			yield instance

tests/integration/images/image.py

@@ -1,6 +0,0 @@
-
-
-class Image(object):
-
-	def __init__(self, manifest):
-		self.manifest = manifest

tests/integration/images/vbox.py

@@ -1,64 +0,0 @@
-from image import Image
-import virtualbox
-import logging
-from contextlib import contextmanager
-log = logging.getLogger(__name__)
-
-
-def initialize_image(manifest, build_server, bootstrap_info):
-	from bootstrapvz.remote.build_servers.local import LocalBuildServer
-	if isinstance(build_server, LocalBuildServer):
-		image_path = bootstrap_info.volume.image_path
-	else:
-		import tempfile
-		handle, image_path = tempfile.mkstemp()
-		import os
-		os.close(handle)
-		try:
-			build_server.download(bootstrap_info.volume.image_path, image_path)
-		except (Exception, KeyboardInterrupt):
-			os.remove(image_path)
-			raise
-		finally:
-			build_server.delete(bootstrap_info.volume.image_path)
-	image = VirtualBoxImage(manifest, image_path)
-	return image
-
-
-class VirtualBoxImage(Image):
-
-	def __init__(self, manifest, image_path):
-		super(VirtualBoxImage, self).__init__(manifest)
-		self.image_path = image_path
-		self.vbox = virtualbox.VirtualBox()
-
-	def open(self):
-		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
-		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
-		                                    virtualbox.library.AccessMode.read_only,  # access_mode
-		                                    False)  # force_new_uuid
-
-	def close(self):
-		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
-		self.medium.close()
-
-	def destroy(self):
-		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
-		import os
-		os.remove(self.image_path)
-		del self.image_path
-
-	@contextmanager
-	def get_instance(self):
-		import hashlib
-		image_hash = hashlib.sha1(self.image_path).hexdigest()
-		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
-
-		self.open()
-		try:
-			from ..instances.vbox import boot_image
-			with boot_image(name, self) as instance:
-				yield instance
-		finally:
-			self.close()

tests/integration/instances/instance.py

@@ -1,16 +0,0 @@
-
-
-class Instance(object):
-
-	def __init__(self, name, image):
-		self.name = name
-		self.image = image
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass

tests/integration/providers/ec2/__init__.py

@@ -1,11 +1,36 @@
 from contextlib import contextmanager
-from ..tools import waituntil
+from tests.integration.tools import waituntil
 import logging
 log = logging.getLogger(__name__)
 
 
 @contextmanager
-def boot_image(image, instance_type, ec2_connection, vpc_connection):
+def boot_image(manifest, build_server, bootstrap_info, instance_type=None):
+
+	credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
+	               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
+	from boto.ec2 import connect_to_region as ec2_connect
+	ec2_connection = ec2_connect(bootstrap_info._ec2['region'],
+	                             aws_access_key_id=credentials['access-key'],
+	                             aws_secret_access_key=credentials['secret-key'])
+	from boto.vpc import connect_to_region as vpc_connect
+	vpc_connection = vpc_connect(bootstrap_info._ec2['region'],
+	                             aws_access_key_id=credentials['access-key'],
+	                             aws_secret_access_key=credentials['secret-key'])
+
+	if manifest.volume['backing'] == 'ebs':
+		from images import EBSImage
+		image = EBSImage(bootstrap_info._ec2['image'], ec2_connection)
+
+	try:
+		with run_instance(image, instance_type, ec2_connection, vpc_connection) as instance:
+			yield instance
+	finally:
+		image.destroy()
+
+
+@contextmanager
+def run_instance(image, instance_type, ec2_connection, vpc_connection):
 
 	with create_env(ec2_connection, vpc_connection) as boot_env:
 
@@ -18,7 +43,7 @@ def boot_image(image, instance_type, ec2_connection, vpc_connection):
 		instance = None
 		try:
 			log.debug('Booting ec2 instance')
-			reservation = image.run(instance_type=instance_type,
+			reservation = image.ami.run(instance_type=instance_type,
 			                            subnet_id=boot_env['subnet_id'])
 			[instance] = reservation.instances
 			instance.add_tag('Name', 'bootstrap-vz test instance')

tests/integration/providers/ec2/images.py

@@ -0,0 +1,19 @@
+import logging
+log = logging.getLogger(__name__)
+
+
+class AmazonMachineImage(object):
+
+	def __init__(self, image_id, ec2_connection):
+		self.ec2_connection = ec2_connection
+		self.ami = self.ec2_connection.get_image(image_id)
+
+
+class EBSImage(AmazonMachineImage):
+
+	def destroy(self):
+		log.debug('Deleting AMI')
+		self.ami.deregister()
+		for device, block_device_type in self.ami.block_device_mapping.items():
+			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
+		del self.ami

tests/integration/providers/virtualbox/__init__.py

@@ -0,0 +1,55 @@
+from contextlib import contextmanager
+import logging
+log = logging.getLogger(__name__)
+
+
+@contextmanager
+def boot_image(manifest, build_server, bootstrap_info):
+	from bootstrapvz.remote.build_servers.local import LocalBuildServer
+	if isinstance(build_server, LocalBuildServer):
+		image_path = bootstrap_info.volume.image_path
+	else:
+		import tempfile
+		handle, image_path = tempfile.mkstemp()
+		import os
+		os.close(handle)
+		try:
+			build_server.download(bootstrap_info.volume.image_path, image_path)
+		except (Exception, KeyboardInterrupt):
+			os.remove(image_path)
+			raise
+		finally:
+			build_server.delete(bootstrap_info.volume.image_path)
+
+	from image import VirtualBoxImage
+	image = VirtualBoxImage(image_path)
+
+	import hashlib
+	image_hash = hashlib.sha1(image_path).hexdigest()
+	instance_name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
+
+	try:
+		image.open()
+		try:
+			with run_instance(image, instance_name, manifest) as instance:
+				yield instance
+		finally:
+			image.close()
+	finally:
+		image.destroy()
+
+
+@contextmanager
+def run_instance(image, instance_name, manifest):
+	from instance import VirtualBoxInstance
+	instance = VirtualBoxInstance(image, instance_name,
+	                              manifest.system['architecture'], manifest.system['release'])
+	try:
+		instance.create()
+		try:
+			instance.boot()
+			yield instance
+		finally:
+			instance.shutdown()
+	finally:
+		instance.destroy()

tests/integration/providers/virtualbox/image.py

@@ -0,0 +1,27 @@
+import virtualbox
+import logging
+log = logging.getLogger(__name__)
+
+
+class VirtualBoxImage(object):
+
+	def __init__(self, image_path):
+		self.image_path = image_path
+		self.vbox = virtualbox.VirtualBox()
+
+	def open(self):
+		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
+		                                    virtualbox.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
+
+	def close(self):
+		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
+		self.medium.close()
+
+	def destroy(self):
+		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
+		import os
+		os.remove(self.image_path)
+		del self.image_path

tests/integration/providers/virtualbox/instance.py

@@ -1,32 +1,20 @@
-from instance import Instance
 import virtualbox
 from contextlib import contextmanager
-from ..tools import waituntil
+from tests.integration.tools import waituntil
 import logging
 log = logging.getLogger(__name__)
 
 
-@contextmanager
+class VirtualBoxInstance(object):
-def boot_image(name, image):
-	instance = VirtualBoxInstance(name, image)
-	try:
-		instance.create()
-		try:
-			instance.boot()
-			yield instance
-		finally:
-			instance.shutdown()
-	finally:
-		instance.destroy()
-
-
-class VirtualBoxInstance(Instance):
 
 	cpus = 1
 	memory = 256
 
-	def __init__(self, name, image):
+	def __init__(self, image, name, arch, release):
-		super(VirtualBoxInstance, self).__init__(name, image)
+		self.image = image
+		self.name = name
+		self.arch = arch
+		self.release = release
 		self.vbox = virtualbox.VirtualBox()
 		manager = virtualbox.Manager()
 		self.session = manager.get_session()
@@ -35,7 +23,7 @@ class VirtualBoxInstance(Instance):
 		log.debug('Creating vbox machine `{name}\''.format(name=self.name))
 		# create machine
 		os_type = {'x86': 'Debian',
-		           'amd64': 'Debian_64'}.get(self.image.manifest.system['architecture'])
+		           'amd64': 'Debian_64'}.get(self.arch)
 		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
 		                                        groups=[], os_type_id=os_type, flags='')
 		self.machine.cpu_count = self.cpus
@@ -71,12 +59,12 @@ class VirtualBoxInstance(Instance):
 	def boot(self):
 		log.debug('Booting vbox machine `{name}\''.format(name=self.name))
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
-		from ..tools import read_from_socket
+		from tests.integration.tools import read_from_socket
 		# Gotta figure out a more reliable way to check when the system is done booting.
 		# Maybe bootstrapped unit test images should have a startup script that issues
 		# a callback to the host.
 		from bootstrapvz.common.tools import get_codename
-		if get_codename(self.image.manifest.system['release']) in ['squeeze', 'wheezy']:
+		if get_codename(self.release) in ['squeeze', 'wheezy']:
 			termination_string = 'INIT: Entering runlevel: 2'
 		else:
 			termination_string = 'Debian GNU/Linux'

tests/integration/tools/__init__.py

@@ -26,13 +26,10 @@ def boot_manifest(manifest_data, boot_vars={}):
 		bootstrap_info = connection.run(manifest)
 
 	log.info('Creating and booting instance')
-	from ..images import initialize_image
+	import importlib
-	image = initialize_image(manifest, build_server, bootstrap_info)
+	provider_module = importlib.import_module('tests.integration.providers.' + manifest.provider['name'])
-	try:
+	with provider_module.boot_image(manifest, build_server, bootstrap_info, **boot_vars) as instance:
-		with image.get_instance(**boot_vars) as instance:
 		yield instance
-	finally:
-		image.destroy()
 
 
 def waituntil(predicate, timeout=5, interval=0.05):