diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py index e1981e6..46b3758 100644 --- a/bootstrapvz/remote/build_servers.py +++ b/bootstrapvz/remote/build_servers.py @@ -53,7 +53,10 @@ class BuildServer(object): class LocalBuildServer(BuildServer): - pass + + def run(self, manifest): + from bootstrapvz.base.main import run + return run(manifest) class RemoteBuildServer(BuildServer): @@ -152,6 +155,10 @@ class RemoteBuildServer(BuildServer): '--'] + command log_check_call(ssh_cmd) + def run(self, manifest): + from bootstrapvz.remote.main import run + return run(manifest, self) + def getNPorts(n, port_range=(1024, 65535)): import random diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py index 5260afa..041717f 100644 --- a/tests/integration/images/__init__.py +++ b/tests/integration/images/__init__.py @@ -1,6 +1,6 @@ -class Image(object): - - def __init__(self, manifest): - self.manifest = manifest +def initialize_image(manifest, build_server, bootstrap_info): + if manifest.provider['name'] == 'virtualbox': + import vbox + return vbox.initialize_image(manifest, build_server, bootstrap_info) diff --git a/tests/integration/images/image.py b/tests/integration/images/image.py new file mode 100644 index 0000000..5260afa --- /dev/null +++ b/tests/integration/images/image.py @@ -0,0 +1,6 @@ + + +class Image(object): + + def __init__(self, manifest): + self.manifest = manifest diff --git a/tests/integration/images/vbox.py b/tests/integration/images/vbox.py new file mode 100644 index 0000000..300ae45 --- /dev/null +++ b/tests/integration/images/vbox.py @@ -0,0 +1,64 @@ +from image import Image +import virtualbox +import logging +from contextlib import contextmanager +log = logging.getLogger(__name__) + + +def initialize_image(manifest, build_server, bootstrap_info): + from bootstrapvz.remote.build_servers import LocalBuildServer + if isinstance(build_server, LocalBuildServer): + image_path = bootstrap_info.volume.image_path + else: + import tempfile + handle, image_path = tempfile.mkstemp() + import os + os.close(handle) + try: + build_server.download(bootstrap_info.volume.image_path, image_path) + except (Exception, KeyboardInterrupt): + os.remove(image_path) + raise + finally: + build_server.delete(bootstrap_info.volume.image_path) + image = VirtualBoxImage(manifest, image_path) + return image + + +class VirtualBoxImage(Image): + + def __init__(self, manifest, image_path): + super(VirtualBoxImage, self).__init__(manifest) + self.image_path = image_path + self.vbox = virtualbox.VirtualBox() + + def open(self): + log.debug('Opening vbox medium `{path}\''.format(path=self.image_path)) + self.medium = self.vbox.open_medium(self.image_path, # location + virtualbox.library.DeviceType.hard_disk, # device_type + virtualbox.library.AccessMode.read_only, # access_mode + False) # force_new_uuid + + def close(self): + log.debug('Closing vbox medium `{path}\''.format(path=self.image_path)) + self.medium.close() + + def destroy(self): + log.debug('Deleting vbox image `{path}\''.format(path=self.image_path)) + import os + os.remove(self.image_path) + del self.image_path + + @contextmanager + def get_instance(self): + import hashlib + image_hash = hashlib.sha1(self.image_path).hexdigest() + name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8]) + + self.open() + try: + from ..instances.vbox import boot_image + with boot_image(name, self) as instance: + yield instance + finally: + self.close() diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py deleted file mode 100644 index a09fde3..0000000 --- a/tests/integration/images/virtualbox.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import absolute_import -from . import Image -import virtualbox as vboxapi -import logging -log = logging.getLogger(__name__) - - -class VirtualBoxImage(Image): - - def __init__(self, manifest, image_path): - super(VirtualBoxImage, self).__init__(manifest) - self.image_path = image_path - self.vbox = vboxapi.VirtualBox() - - def open(self): - log.debug('Opening vbox medium `{path}\''.format(path=self.image_path)) - self.medium = self.vbox.open_medium(self.image_path, # location - vboxapi.library.DeviceType.hard_disk, # device_type - vboxapi.library.AccessMode.read_only, # access_mode - False) # force_new_uuid - - def close(self): - log.debug('Closing vbox medium `{path}\''.format(path=self.image_path)) - self.medium.close() - - def destroy(self): - log.debug('Deleting vbox image `{path}\''.format(path=self.image_path)) - import os - os.remove(self.image_path) - del self.image_path - - def get_instance(self): - import hashlib - from ..instances.virtualbox import VirtualBoxInstance - image_hash = hashlib.sha1(self.image_path).hexdigest() - name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8]) - return VirtualBoxInstance(name, self) - - def __enter__(self): - self.open() - return self.get_instance() - - def __exit__(self, type, value, traceback): - self.close() diff --git a/tests/integration/instances/__init__.py b/tests/integration/instances/__init__.py index 049fc0f..e69de29 100644 --- a/tests/integration/instances/__init__.py +++ b/tests/integration/instances/__init__.py @@ -1,16 +0,0 @@ - - -class Instance(object): - - def __init__(self, name, image): - self.name = name - self.image = image - - def boot(self): - pass - - def shutdown(self): - pass - - def destroy(self): - pass diff --git a/tests/integration/instances/instance.py b/tests/integration/instances/instance.py new file mode 100644 index 0000000..049fc0f --- /dev/null +++ b/tests/integration/instances/instance.py @@ -0,0 +1,16 @@ + + +class Instance(object): + + def __init__(self, name, image): + self.name = name + self.image = image + + def boot(self): + pass + + def shutdown(self): + pass + + def destroy(self): + pass diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/vbox.py similarity index 60% rename from tests/integration/instances/virtualbox.py rename to tests/integration/instances/vbox.py index ad908ef..1adc8d4 100644 --- a/tests/integration/instances/virtualbox.py +++ b/tests/integration/instances/vbox.py @@ -1,10 +1,24 @@ -from __future__ import absolute_import -from . import Instance -import virtualbox as vboxapi +from instance import Instance +import virtualbox +from contextlib import contextmanager import logging log = logging.getLogger(__name__) +@contextmanager +def boot_image(name, image): + instance = VirtualBoxInstance(name, image) + try: + instance.create() + try: + instance.boot() + yield instance + finally: + instance.shutdown() + finally: + instance.destroy() + + class VirtualBoxInstance(Instance): cpus = 1 @@ -12,8 +26,8 @@ class VirtualBoxInstance(Instance): def __init__(self, name, image): super(VirtualBoxInstance, self).__init__(name, image) - self.vbox = vboxapi.VirtualBox() - manager = vboxapi.Manager() + self.vbox = virtualbox.VirtualBox() + manager = virtualbox.Manager() self.session = manager.get_session() def create(self): @@ -30,18 +44,18 @@ class VirtualBoxInstance(Instance): # attach image log.debug('Attaching SATA storage controller to vbox machine `{name}\''.format(name=self.name)) - with self.Lock(self.machine, self.session) as machine: + with lock(self.machine, self.session) as machine: strg_ctrl = machine.add_storage_controller('SATA Controller', - vboxapi.library.StorageBus.sata) + virtualbox.library.StorageBus.sata) strg_ctrl.port_count = 1 machine.attach_device(name='SATA Controller', controller_port=0, device=0, - type_p=vboxapi.library.DeviceType.hard_disk, + type_p=virtualbox.library.DeviceType.hard_disk, medium=self.image.medium) machine.save_settings() # redirect serial port log.debug('Enabling serial port on vbox machine `{name}\''.format(name=self.name)) - with self.Lock(self.machine, self.session) as machine: + with lock(self.machine, self.session) as machine: serial_port = machine.get_serial_port(0) serial_port.enabled = True import tempfile @@ -49,7 +63,7 @@ class VirtualBoxInstance(Instance): import os os.close(handle) serial_port.path = self.serial_port_path - serial_port.host_mode = vboxapi.library.PortMode.host_pipe + serial_port.host_mode = virtualbox.library.PortMode.host_pipe serial_port.server = True # Create the socket on startup machine.save_settings() @@ -70,70 +84,36 @@ class VirtualBoxInstance(Instance): def shutdown(self): log.debug('Shutting down vbox machine `{name}\''.format(name=self.name)) self.session.console.power_down().wait_for_completion(-1) - self.Lock(self.machine, self.session).unlock() + lock(self.machine, self.session).unlock() def destroy(self): log.debug('Destroying vbox machine `{name}\''.format(name=self.name)) if hasattr(self, 'machine'): try: log.debug('Detaching SATA storage controller from vbox machine `{name}\''.format(name=self.name)) - with self.Lock(self.machine, self.session) as machine: + with lock(self.machine, self.session) as machine: machine.detach_device(name='SATA Controller', controller_port=0, device=0) machine.save_settings() - except vboxapi.library.VBoxErrorObjectNotFound: + except virtualbox.library.VBoxErrorObjectNotFound: pass log.debug('Unregistering and removing vbox machine `{name}\''.format(name=self.name)) - self.machine.unregister(vboxapi.library.CleanupMode.unregister_only) + self.machine.unregister(virtualbox.library.CleanupMode.unregister_only) self.machine.remove(delete=True) else: log.debug('vbox machine `{name}\' was not created, skipping destruction'.format(name=self.name)) - def up(self): - try: - self.create() - try: - self.boot() - except (Exception, KeyboardInterrupt): - self.shutdown() - raise - except (Exception, KeyboardInterrupt): - self.destroy() - raise - def down(self): - self.shutdown() - self.destroy() - - def __enter__(self): - self.up() - return self - - def __exit__(self, type, value, traceback): - self.down() - - class Lock(object): - def __init__(self, machine, session): - self.machine = machine - self.session = session - - def __enter__(self): - return self.lock() - - def __exit__(self, type, value, traceback): - return self.unlock() - - def lock(self): - self.machine.lock_machine(self.session, vboxapi.library.LockType.write) - return self.session.machine - - def unlock(self): - from ..tools import waituntil - if self.machine.session_state == vboxapi.library.SessionState.unlocked: - return - if self.machine.session_state == vboxapi.library.SessionState.unlocking: - waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked) - return - if self.machine.session_state == vboxapi.library.SessionState.spawning: - waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.locked) - self.session.unlock_machine() - waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked) +@contextmanager +def lock(machine, session): + machine.lock_machine(session, virtualbox.library.LockType.write) + yield session.machine + from ..tools import waituntil + if machine.session_state == virtualbox.library.SessionState.unlocked: + return + if machine.session_state == virtualbox.library.SessionState.unlocking: + waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked) + return + if machine.session_state == virtualbox.library.SessionState.spawning: + waituntil(lambda: machine.session_state == virtualbox.library.SessionState.locked) + session.unlock_machine() + waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked) diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py index 625070b..42bc46a 100644 --- a/tests/integration/tools/__init__.py +++ b/tests/integration/tools/__init__.py @@ -1,9 +1,33 @@ +from contextlib import contextmanager +from bootstrapvz.remote import register_deserialization_handlers + # Register deserialization handlers for objects # that will pass between server and client -from bootstrapvz.remote import register_deserialization_handlers register_deserialization_handlers() +@contextmanager +def boot_manifest(manifest_data): + from bootstrapvz.common.tools import load_data + build_servers = load_data('build-servers.yml') + from bootstrapvz.remote.build_servers import pick_build_server + build_server = pick_build_server(build_servers, manifest_data) + + manifest_data = build_server.apply_build_settings(manifest_data) + from bootstrapvz.base.manifest import Manifest + manifest = Manifest(data=manifest_data) + + bootstrap_info = build_server.run(manifest) + + from ..images import initialize_image + image = initialize_image(manifest, build_server, bootstrap_info) + try: + with image.get_instance() as instance: + yield instance + finally: + image.destroy() + + def waituntil(predicate, timeout=5, interval=0.05): import time threshhold = time.time() + timeout diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py deleted file mode 100644 index d537996..0000000 --- a/tests/integration/tools/bootable_manifest.py +++ /dev/null @@ -1,69 +0,0 @@ -from bootstrapvz.remote.build_servers import LocalBuildServer -from ..images.virtualbox import VirtualBoxImage - - -class BootableManifest(object): - - def __init__(self, manifest_data): - self.manifest_data = manifest_data - - def pick_build_server(self, path='build-servers.yml'): - from bootstrapvz.common.tools import load_data - build_servers = load_data(path) - from bootstrapvz.remote.build_servers import pick_build_server - return pick_build_server(build_servers, self.manifest_data) - - def get_manifest(self, build_server): - manifest_data = build_server.apply_build_settings(self.manifest_data) - from bootstrapvz.base.manifest import Manifest - return Manifest(data=manifest_data) - - def bootstrap(self, manifest, build_server): - if isinstance(build_server, LocalBuildServer): - from bootstrapvz.base.main import run - bootstrap_info = run(manifest) - else: - from bootstrapvz.remote.main import run - bootstrap_info = run(manifest, build_server) - return bootstrap_info - - def get_image(self, build_server, bootstrap_info, manifest): - if isinstance(build_server, LocalBuildServer): - image_path = bootstrap_info.volume.image_path - else: - import tempfile - handle, image_path = tempfile.mkstemp() - import os - os.close(handle) - try: - build_server.download(bootstrap_info.volume.image_path, image_path) - except (Exception, KeyboardInterrupt) as e: - os.remove(image_path) - raise e - finally: - build_server.delete(bootstrap_info.volume.image_path) - image_type = {'virtualbox': VirtualBoxImage} - return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path) - - def __enter__(self): - try: - self.build_server = self.pick_build_server() - self.manifest = self.get_manifest(self.build_server) - self.bootstrap_info = self.bootstrap(self.manifest, self.build_server) - self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest) - self.image.open() - self.instance = self.image.get_instance() - self.instance.up() - except (Exception, KeyboardInterrupt): - if hasattr(self, 'image'): - self.image.close() - self.image.destroy() - raise - return self.instance - - def __exit__(self, type, value, traceback): - if hasattr(self, 'instance'): - self.instance.down() - if hasattr(self, 'image'): - self.image.close() - self.image.destroy() diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py index ef98abc..a5930aa 100644 --- a/tests/integration/virtualbox_tests.py +++ b/tests/integration/virtualbox_tests.py @@ -1,5 +1,5 @@ from manifests import merge_manifest_data -from tools.bootable_manifest import BootableManifest +from tools import boot_manifest from unittest.case import SkipTest partials = {'vbox': 'provider: {name: virtualbox}', @@ -12,7 +12,7 @@ def test_unpartitioned_extlinux_oldstable(): std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -20,7 +20,7 @@ def test_msdos_extlinux_oldstable(): std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -28,7 +28,7 @@ def test_gpt_extlinux_oldstable(): std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -37,7 +37,7 @@ def test_msdos_grub_oldstable(): std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -46,7 +46,7 @@ def test_gpt_grub_oldstable(): std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -54,7 +54,7 @@ def test_unpartitioned_extlinux(): std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -62,7 +62,7 @@ def test_msdos_extlinux(): std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -70,7 +70,7 @@ def test_gpt_extlinux(): std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -78,7 +78,7 @@ def test_msdos_grub(): std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -86,7 +86,7 @@ def test_gpt_grub(): std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -94,7 +94,7 @@ def test_unpartitioned_extlinux_unstable(): std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -102,7 +102,7 @@ def test_msdos_extlinux_unstable(): std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -110,7 +110,7 @@ def test_gpt_extlinux_unstable(): std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -118,7 +118,7 @@ def test_msdos_grub_unstable(): std_partials = ['base', 'unstable64', 'grub', 'msdos', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output) @@ -126,5 +126,5 @@ def test_gpt_grub_unstable(): std_partials = ['base', 'unstable64', 'grub', 'gpt', 'partitioned', 'root_password'] custom_partials = [partials['vbox'], partials['vmdk']] manifest_data = merge_manifest_data(std_partials, custom_partials) - with BootableManifest(manifest_data) as instance: + with boot_manifest(manifest_data) as instance: print(instance.console_output)