Prepare integration testing framework to work with providers other than virtualbox

Also, convert with context handlers in classes to functions with generators
This makes the code a lot more readable

bootstrapvz/remote/build_servers.py

@@ -53,7 +53,10 @@ class BuildServer(object):
 
 
 class LocalBuildServer(BuildServer):
-	pass
+
+	def run(self, manifest):
+		from bootstrapvz.base.main import run
+		return run(manifest)
 
 
 class RemoteBuildServer(BuildServer):
@@ -152,6 +155,10 @@ class RemoteBuildServer(BuildServer):
 		                  '--'] + command
 		log_check_call(ssh_cmd)
 
+	def run(self, manifest):
+		from bootstrapvz.remote.main import run
+		return run(manifest, self)
+
 
 def getNPorts(n, port_range=(1024, 65535)):
 	import random

tests/integration/images/__init__.py

@@ -1,6 +1,6 @@
 
 
-class Image(object):
-
-	def __init__(self, manifest):
-		self.manifest = manifest
+def initialize_image(manifest, build_server, bootstrap_info):
+	if manifest.provider['name'] == 'virtualbox':
+		import vbox
+		return vbox.initialize_image(manifest, build_server, bootstrap_info)

tests/integration/images/image.py

@@ -0,0 +1,6 @@
+
+
+class Image(object):
+
+	def __init__(self, manifest):
+		self.manifest = manifest

tests/integration/images/vbox.py

@@ -0,0 +1,64 @@
+from image import Image
+import virtualbox
+import logging
+from contextlib import contextmanager
+log = logging.getLogger(__name__)
+
+
+def initialize_image(manifest, build_server, bootstrap_info):
+	from bootstrapvz.remote.build_servers import LocalBuildServer
+	if isinstance(build_server, LocalBuildServer):
+		image_path = bootstrap_info.volume.image_path
+	else:
+		import tempfile
+		handle, image_path = tempfile.mkstemp()
+		import os
+		os.close(handle)
+		try:
+			build_server.download(bootstrap_info.volume.image_path, image_path)
+		except (Exception, KeyboardInterrupt):
+			os.remove(image_path)
+			raise
+		finally:
+			build_server.delete(bootstrap_info.volume.image_path)
+	image = VirtualBoxImage(manifest, image_path)
+	return image
+
+
+class VirtualBoxImage(Image):
+
+	def __init__(self, manifest, image_path):
+		super(VirtualBoxImage, self).__init__(manifest)
+		self.image_path = image_path
+		self.vbox = virtualbox.VirtualBox()
+
+	def open(self):
+		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
+		                                    virtualbox.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
+
+	def close(self):
+		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
+		self.medium.close()
+
+	def destroy(self):
+		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
+		import os
+		os.remove(self.image_path)
+		del self.image_path
+
+	@contextmanager
+	def get_instance(self):
+		import hashlib
+		image_hash = hashlib.sha1(self.image_path).hexdigest()
+		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
+
+		self.open()
+		try:
+			from ..instances.vbox import boot_image
+			with boot_image(name, self) as instance:
+				yield instance
+		finally:
+			self.close()

tests/integration/images/virtualbox.py

@@ -1,44 +0,0 @@
-from __future__ import absolute_import
-from . import Image
-import virtualbox as vboxapi
-import logging
-log = logging.getLogger(__name__)
-
-
-class VirtualBoxImage(Image):
-
-	def __init__(self, manifest, image_path):
-		super(VirtualBoxImage, self).__init__(manifest)
-		self.image_path = image_path
-		self.vbox = vboxapi.VirtualBox()
-
-	def open(self):
-		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
-		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    vboxapi.library.DeviceType.hard_disk,  # device_type
-		                                    vboxapi.library.AccessMode.read_only,  # access_mode
-		                                    False)  # force_new_uuid
-
-	def close(self):
-		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
-		self.medium.close()
-
-	def destroy(self):
-		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
-		import os
-		os.remove(self.image_path)
-		del self.image_path
-
-	def get_instance(self):
-		import hashlib
-		from ..instances.virtualbox import VirtualBoxInstance
-		image_hash = hashlib.sha1(self.image_path).hexdigest()
-		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
-		return VirtualBoxInstance(name, self)
-
-	def __enter__(self):
-		self.open()
-		return self.get_instance()
-
-	def __exit__(self, type, value, traceback):
-		self.close()

tests/integration/instances/__init__.py

@@ -1,16 +0,0 @@
-
-
-class Instance(object):
-
-	def __init__(self, name, image):
-		self.name = name
-		self.image = image
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass

tests/integration/instances/instance.py

@@ -0,0 +1,16 @@
+
+
+class Instance(object):
+
+	def __init__(self, name, image):
+		self.name = name
+		self.image = image
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass

tests/integration/instances/vbox.py

@@ -1,10 +1,24 @@
-from __future__ import absolute_import
-from . import Instance
-import virtualbox as vboxapi
+from instance import Instance
+import virtualbox
+from contextlib import contextmanager
 import logging
 log = logging.getLogger(__name__)
 
 
+@contextmanager
+def boot_image(name, image):
+	instance = VirtualBoxInstance(name, image)
+	try:
+		instance.create()
+		try:
+			instance.boot()
+			yield instance
+		finally:
+			instance.shutdown()
+	finally:
+		instance.destroy()
+
+
 class VirtualBoxInstance(Instance):
 
 	cpus = 1
@@ -12,8 +26,8 @@ class VirtualBoxInstance(Instance):
 
 	def __init__(self, name, image):
 		super(VirtualBoxInstance, self).__init__(name, image)
-		self.vbox = vboxapi.VirtualBox()
-		manager = vboxapi.Manager()
+		self.vbox = virtualbox.VirtualBox()
+		manager = virtualbox.Manager()
 		self.session = manager.get_session()
 
 	def create(self):
@@ -30,18 +44,18 @@ class VirtualBoxInstance(Instance):
 
 		# attach image
 		log.debug('Attaching SATA storage controller to vbox machine `{name}\''.format(name=self.name))
-		with self.Lock(self.machine, self.session) as machine:
+		with lock(self.machine, self.session) as machine:
 			strg_ctrl = machine.add_storage_controller('SATA Controller',
-			                                           vboxapi.library.StorageBus.sata)
+			                                           virtualbox.library.StorageBus.sata)
 			strg_ctrl.port_count = 1
 			machine.attach_device(name='SATA Controller', controller_port=0, device=0,
-			                      type_p=vboxapi.library.DeviceType.hard_disk,
+			                      type_p=virtualbox.library.DeviceType.hard_disk,
 			                      medium=self.image.medium)
 			machine.save_settings()
 
 		# redirect serial port
 		log.debug('Enabling serial port on vbox machine `{name}\''.format(name=self.name))
-		with self.Lock(self.machine, self.session) as machine:
+		with lock(self.machine, self.session) as machine:
 			serial_port = machine.get_serial_port(0)
 			serial_port.enabled = True
 			import tempfile
@@ -49,7 +63,7 @@ class VirtualBoxInstance(Instance):
 			import os
 			os.close(handle)
 			serial_port.path = self.serial_port_path
-			serial_port.host_mode = vboxapi.library.PortMode.host_pipe
+			serial_port.host_mode = virtualbox.library.PortMode.host_pipe
 			serial_port.server = True  # Create the socket on startup
 			machine.save_settings()
 
@@ -70,70 +84,36 @@ class VirtualBoxInstance(Instance):
 	def shutdown(self):
 		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))
 		self.session.console.power_down().wait_for_completion(-1)
-		self.Lock(self.machine, self.session).unlock()
+		lock(self.machine, self.session).unlock()
 
 	def destroy(self):
 		log.debug('Destroying vbox machine `{name}\''.format(name=self.name))
 		if hasattr(self, 'machine'):
 			try:
 				log.debug('Detaching SATA storage controller from vbox machine `{name}\''.format(name=self.name))
-				with self.Lock(self.machine, self.session) as machine:
+				with lock(self.machine, self.session) as machine:
 					machine.detach_device(name='SATA Controller', controller_port=0, device=0)
 					machine.save_settings()
-			except vboxapi.library.VBoxErrorObjectNotFound:
+			except virtualbox.library.VBoxErrorObjectNotFound:
 				pass
 			log.debug('Unregistering and removing vbox machine `{name}\''.format(name=self.name))
-			self.machine.unregister(vboxapi.library.CleanupMode.unregister_only)
+			self.machine.unregister(virtualbox.library.CleanupMode.unregister_only)
 			self.machine.remove(delete=True)
 		else:
 			log.debug('vbox machine `{name}\' was not created, skipping destruction'.format(name=self.name))
 
-	def up(self):
-		try:
-			self.create()
-			try:
-				self.boot()
-			except (Exception, KeyboardInterrupt):
-				self.shutdown()
-				raise
-		except (Exception, KeyboardInterrupt):
-			self.destroy()
-			raise
 
-	def down(self):
-		self.shutdown()
-		self.destroy()
-
-	def __enter__(self):
-		self.up()
-		return self
-
-	def __exit__(self, type, value, traceback):
-		self.down()
-
-	class Lock(object):
-		def __init__(self, machine, session):
-			self.machine = machine
-			self.session = session
-
-		def __enter__(self):
-			return self.lock()
-
-		def __exit__(self, type, value, traceback):
-			return self.unlock()
-
-		def lock(self):
-			self.machine.lock_machine(self.session, vboxapi.library.LockType.write)
-			return self.session.machine
-
-		def unlock(self):
-			from ..tools import waituntil
-			if self.machine.session_state == vboxapi.library.SessionState.unlocked:
-				return
-			if self.machine.session_state == vboxapi.library.SessionState.unlocking:
-				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
-				return
-			if self.machine.session_state == vboxapi.library.SessionState.spawning:
-				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.locked)
-			self.session.unlock_machine()
-			waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
+@contextmanager
+def lock(machine, session):
+	machine.lock_machine(session, virtualbox.library.LockType.write)
+	yield session.machine
+	from ..tools import waituntil
+	if machine.session_state == virtualbox.library.SessionState.unlocked:
+		return
+	if machine.session_state == virtualbox.library.SessionState.unlocking:
+		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)
+		return
+	if machine.session_state == virtualbox.library.SessionState.spawning:
+		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.locked)
+	session.unlock_machine()
+	waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)

tests/integration/tools/__init__.py

@@ -1,9 +1,33 @@
+from contextlib import contextmanager
+from bootstrapvz.remote import register_deserialization_handlers
+
 # Register deserialization handlers for objects
 # that will pass between server and client
-from bootstrapvz.remote import register_deserialization_handlers
 register_deserialization_handlers()
 
 
+@contextmanager
+def boot_manifest(manifest_data):
+	from bootstrapvz.common.tools import load_data
+	build_servers = load_data('build-servers.yml')
+	from bootstrapvz.remote.build_servers import pick_build_server
+	build_server = pick_build_server(build_servers, manifest_data)
+
+	manifest_data = build_server.apply_build_settings(manifest_data)
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(data=manifest_data)
+
+	bootstrap_info = build_server.run(manifest)
+
+	from ..images import initialize_image
+	image = initialize_image(manifest, build_server, bootstrap_info)
+	try:
+		with image.get_instance() as instance:
+			yield instance
+	finally:
+		image.destroy()
+
+
 def waituntil(predicate, timeout=5, interval=0.05):
 	import time
 	threshhold = time.time() + timeout

tests/integration/tools/bootable_manifest.py

@@ -1,69 +0,0 @@
-from bootstrapvz.remote.build_servers import LocalBuildServer
-from ..images.virtualbox import VirtualBoxImage
-
-
-class BootableManifest(object):
-
-	def __init__(self, manifest_data):
-		self.manifest_data = manifest_data
-
-	def pick_build_server(self, path='build-servers.yml'):
-		from bootstrapvz.common.tools import load_data
-		build_servers = load_data(path)
-		from bootstrapvz.remote.build_servers import pick_build_server
-		return pick_build_server(build_servers, self.manifest_data)
-
-	def get_manifest(self, build_server):
-		manifest_data = build_server.apply_build_settings(self.manifest_data)
-		from bootstrapvz.base.manifest import Manifest
-		return Manifest(data=manifest_data)
-
-	def bootstrap(self, manifest, build_server):
-		if isinstance(build_server, LocalBuildServer):
-			from bootstrapvz.base.main import run
-			bootstrap_info = run(manifest)
-		else:
-			from bootstrapvz.remote.main import run
-			bootstrap_info = run(manifest, build_server)
-		return bootstrap_info
-
-	def get_image(self, build_server, bootstrap_info, manifest):
-		if isinstance(build_server, LocalBuildServer):
-			image_path = bootstrap_info.volume.image_path
-		else:
-			import tempfile
-			handle, image_path = tempfile.mkstemp()
-			import os
-			os.close(handle)
-			try:
-				build_server.download(bootstrap_info.volume.image_path, image_path)
-			except (Exception, KeyboardInterrupt) as e:
-				os.remove(image_path)
-				raise e
-			finally:
-				build_server.delete(bootstrap_info.volume.image_path)
-		image_type = {'virtualbox': VirtualBoxImage}
-		return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path)
-
-	def __enter__(self):
-		try:
-			self.build_server = self.pick_build_server()
-			self.manifest = self.get_manifest(self.build_server)
-			self.bootstrap_info = self.bootstrap(self.manifest, self.build_server)
-			self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest)
-			self.image.open()
-			self.instance = self.image.get_instance()
-			self.instance.up()
-		except (Exception, KeyboardInterrupt):
-			if hasattr(self, 'image'):
-				self.image.close()
-				self.image.destroy()
-			raise
-		return self.instance
-
-	def __exit__(self, type, value, traceback):
-		if hasattr(self, 'instance'):
-			self.instance.down()
-		if hasattr(self, 'image'):
-			self.image.close()
-			self.image.destroy()

tests/integration/virtualbox_tests.py

@@ -1,5 +1,5 @@
 from manifests import merge_manifest_data
-from tools.bootable_manifest import BootableManifest
+from tools import boot_manifest
 from unittest.case import SkipTest
 
 partials = {'vbox': 'provider: {name: virtualbox}',
@@ -12,7 +12,7 @@ def test_unpartitioned_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -20,7 +20,7 @@ def test_msdos_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -28,7 +28,7 @@ def test_gpt_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -37,7 +37,7 @@ def test_msdos_grub_oldstable():
 	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -46,7 +46,7 @@ def test_gpt_grub_oldstable():
 	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -54,7 +54,7 @@ def test_unpartitioned_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -62,7 +62,7 @@ def test_msdos_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -70,7 +70,7 @@ def test_gpt_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -78,7 +78,7 @@ def test_msdos_grub():
 	std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -86,7 +86,7 @@ def test_gpt_grub():
 	std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -94,7 +94,7 @@ def test_unpartitioned_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -102,7 +102,7 @@ def test_msdos_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -110,7 +110,7 @@ def test_gpt_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -118,7 +118,7 @@ def test_msdos_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -126,5 +126,5 @@ def test_gpt_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)