From 9c6af89e7837761f7442d4bb187b042d096cc238 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 20 Dec 2014 16:52:31 +0100 Subject: [PATCH] Abstract bootstrapping, instance creation, booting etc.. This all now runs using a simple with: statement --- tests/integration/images/__init__.py | 4 - tests/integration/images/virtualbox.py | 18 ++++- tests/integration/instances/virtualbox.py | 23 ++++++ tests/integration/manifests/__init__.py | 26 +++++++ tests/integration/manifests/extlinux.yml | 3 + tests/integration/manifests/gpt.yml | 4 + tests/integration/manifests/grub.yml | 3 + tests/integration/manifests/msdos.yml | 4 + .../manifests/single_partition.yml | 6 ++ tests/integration/manifests/unpartitioned.yml | 2 +- tests/integration/tools/__init__.py | 34 --------- tests/integration/tools/bootable_manifest.py | 57 ++++++++++++++ tests/integration/virtualbox_tests.py | 74 ++++--------------- 13 files changed, 158 insertions(+), 100 deletions(-) create mode 100644 tests/integration/manifests/extlinux.yml create mode 100644 tests/integration/manifests/gpt.yml create mode 100644 tests/integration/manifests/grub.yml create mode 100644 tests/integration/manifests/msdos.yml create mode 100644 tests/integration/manifests/single_partition.yml create mode 100644 tests/integration/tools/bootable_manifest.py diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py index 9945ad8..5260afa 100644 --- a/tests/integration/images/__init__.py +++ b/tests/integration/images/__init__.py @@ -4,7 +4,3 @@ class Image(object): def __init__(self, manifest): self.manifest = manifest - - def destroy(self): - pass - diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py index 1584140..acb9274 100644 --- a/tests/integration/images/virtualbox.py +++ b/tests/integration/images/virtualbox.py @@ -9,10 +9,26 @@ class VirtualBoxImage(Image): super(VirtualBoxImage, self).__init__(manifest) self.image_path = image_path self.vbox = vboxapi.VirtualBox() + + def open(self): self.medium = self.vbox.open_medium(self.image_path, # location vboxapi.library.DeviceType.hard_disk, # decive_type vboxapi.library.AccessMode.read_only, # access_mode False) # force_new_uuid - def destroy(self): + def close(self): self.medium.close() + + def get_instance(self): + import hashlib + from ..instances.virtualbox import VirtualBoxInstance + image_hash = hashlib.sha1(self.image_path).hexdigest() + name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8]) + return VirtualBoxInstance(name, self) + + def __enter__(self): + self.open() + return self.get_instance() + + def __exit__(self, type, value, traceback): + self.close() diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py index 953e72b..11e6788 100644 --- a/tests/integration/instances/virtualbox.py +++ b/tests/integration/instances/virtualbox.py @@ -68,6 +68,29 @@ class VirtualBoxInstance(Instance): self.machine.unregister(vboxapi.library.CleanupMode.unregister_only) self.machine.remove(delete=True) + def up(self): + try: + self.create() + try: + self.boot() + except Exception as e: + self.shutdown() + raise e + except Exception as e: + self.destroy() + raise e + + def down(self): + self.shutdown() + self.destroy() + + def __enter__(self): + self.up() + return self + + def __exit__(self, type, value, traceback): + self.down() + class Lock(object): def __init__(self, machine, session): self.machine = machine diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py index a535f30..4eb6dfa 100644 --- a/tests/integration/manifests/__init__.py +++ b/tests/integration/manifests/__init__.py @@ -18,3 +18,29 @@ import string pool = string.ascii_uppercase + string.ascii_lowercase + string.digits random_password = ''.join(random.choice(pool) for _ in range(16)) partials['root_password']['plugins']['root_password']['password'] = random_password + + +def merge_manifest_data(standard_partials=[], custom=[]): + import yaml + manifest_data = [partials[name] for name in standard_partials] + manifest_data.extend(yaml.load(data) for data in custom) + return merge_dicts(*manifest_data) + + +# Snatched from here: http://stackoverflow.com/a/7205107 +def merge_dicts(*args): + def merge(a, b, path=None): + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass + else: + raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) + else: + a[key] = b[key] + return a + return reduce(merge, args, {}) diff --git a/tests/integration/manifests/extlinux.yml b/tests/integration/manifests/extlinux.yml new file mode 100644 index 0000000..13a8a43 --- /dev/null +++ b/tests/integration/manifests/extlinux.yml @@ -0,0 +1,3 @@ +--- +system: + bootloader: extlinux diff --git a/tests/integration/manifests/gpt.yml b/tests/integration/manifests/gpt.yml new file mode 100644 index 0000000..1ece4c2 --- /dev/null +++ b/tests/integration/manifests/gpt.yml @@ -0,0 +1,4 @@ +--- +volume: + partitions: + type: gpt diff --git a/tests/integration/manifests/grub.yml b/tests/integration/manifests/grub.yml new file mode 100644 index 0000000..4dfa67f --- /dev/null +++ b/tests/integration/manifests/grub.yml @@ -0,0 +1,3 @@ +--- +system: + bootloader: grub diff --git a/tests/integration/manifests/msdos.yml b/tests/integration/manifests/msdos.yml new file mode 100644 index 0000000..5795f95 --- /dev/null +++ b/tests/integration/manifests/msdos.yml @@ -0,0 +1,4 @@ +--- +volume: + partitions: + type: msdos diff --git a/tests/integration/manifests/single_partition.yml b/tests/integration/manifests/single_partition.yml new file mode 100644 index 0000000..ce4c247 --- /dev/null +++ b/tests/integration/manifests/single_partition.yml @@ -0,0 +1,6 @@ +--- +volume: + partitions: + root: + filesystem: ext4 + size: 1GiB diff --git a/tests/integration/manifests/unpartitioned.yml b/tests/integration/manifests/unpartitioned.yml index ef7f97d..4f881ba 100644 --- a/tests/integration/manifests/unpartitioned.yml +++ b/tests/integration/manifests/unpartitioned.yml @@ -1,7 +1,7 @@ --- volume: - type: none partitions: + type: none root: filesystem: ext4 size: 1GiB diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py index 7760a1d..66910b6 100644 --- a/tests/integration/tools/__init__.py +++ b/tests/integration/tools/__init__.py @@ -4,25 +4,6 @@ from bootstrapvz.remote import register_deserialization_handlers register_deserialization_handlers() -# Snatched from here: http://stackoverflow.com/a/7205107 -def merge_dicts(*args): - def merge(a, b, path=None): - if path is None: - path = [] - for key in b: - if key in a: - if isinstance(a[key], dict) and isinstance(b[key], dict): - merge(a[key], b[key], path + [str(key)]) - elif a[key] == b[key]: - pass - else: - raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) - else: - a[key] = b[key] - return a - return reduce(merge, args, {}) - - def waituntil(predicate, timeout=5, interval=0.05): import time threshhold = time.time() + timeout @@ -71,18 +52,3 @@ def read_from_socket(socket_path, termination_string, timeout): raise SocketReadTimeout(msg) console.close() return output - - -def bootstrap(manifest, build_server): - from bootstrapvz.remote.build_servers import LocalBuildServer - if isinstance(build_server, LocalBuildServer): - from bootstrapvz.base.main import run - bootstrap_info = run(manifest) - else: - from bootstrapvz.remote.main import run - bootstrap_info = run(manifest, build_server) - return bootstrap_info - - -def test(instance): - pass diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py new file mode 100644 index 0000000..5d86245 --- /dev/null +++ b/tests/integration/tools/bootable_manifest.py @@ -0,0 +1,57 @@ +from bootstrapvz.remote.build_servers import LocalBuildServer +from ..images.virtualbox import VirtualBoxImage + + +class BootableManifest(object): + + def __init__(self, manifest_data): + self.manifest_data = manifest_data + + def pick_build_server(self, path='build-servers.yml'): + from bootstrapvz.common.tools import load_data + build_servers = load_data(path) + from bootstrapvz.remote.build_servers import pick_build_server + return pick_build_server(build_servers, self.manifest_data) + + def get_manifest(self, build_server): + manifest_data = build_server.apply_build_settings(self.manifest_data) + from bootstrapvz.base.manifest import Manifest + return Manifest(data=manifest_data) + + def bootstrap(self, manifest, build_server): + if isinstance(build_server, LocalBuildServer): + from bootstrapvz.base.main import run + bootstrap_info = run(manifest) + else: + from bootstrapvz.remote.main import run + bootstrap_info = run(manifest, build_server) + return bootstrap_info + + def get_image(self, build_server, bootstrap_info, manifest): + if isinstance(build_server, LocalBuildServer): + image_path = bootstrap_info.volume.image_path + else: + import tempfile + handle, image_path = tempfile.mkstemp() + import os + os.close(handle) + build_server.download(bootstrap_info.volume.image_path, image_path) + build_server.delete(bootstrap_info.volume.image_path) + image_type = {'virtualbox': VirtualBoxImage} + return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path) + + def __enter__(self): + self.build_server = self.pick_build_server() + self.manifest = self.get_manifest(self.build_server) + self.bootstrap_info = self.bootstrap(self.manifest, self.build_server) + self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest) + self.image.open() + self.instance = self.image.get_instance() + self.instance.up() + return self.instance + + def __exit__(self, type, value, traceback): + if hasattr(self, 'instance'): + self.instance.down() + if hasattr(self, 'image'): + self.image.close() diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py index 36fb8ad..6e2e122 100644 --- a/tests/integration/virtualbox_tests.py +++ b/tests/integration/virtualbox_tests.py @@ -1,62 +1,16 @@ -import tools -from manifests import partials -from bootstrapvz.base.manifest import Manifest -from bootstrapvz.remote.build_servers import pick_build_server -from . import build_servers -from images.virtualbox import VirtualBoxImage -from instances.virtualbox import VirtualBoxInstance +from manifests import merge_manifest_data +from tools.bootable_manifest import BootableManifest + +partials = {'vbox': 'provider: {name: virtualbox}', + 'vdi': 'volume: {backing: vdi}', + 'vmdk': 'volume: {backing: vmdk}', + } -def test_virtualbox_unpartitioned_extlinux(): - import yaml - manifest_data = yaml.load(""" -provider: - name: virtualbox -system: - bootloader: extlinux -volume: - backing: vdi - partitions: - type: msdos -""") - manifest_data = tools.merge_dicts(partials['base'], partials['stable64'], partials['unpartitioned'], - partials['root_password'], partials['apt_proxy'], - manifest_data) - - build_server = pick_build_server(build_servers, manifest_data) - manifest_data = build_server.apply_build_settings(manifest_data) - manifest = Manifest(data=manifest_data) - - # bootstrap_info = tools.bootstrap(manifest, build_server) - from bootstrapvz.base.bootstrapinfo import BootstrapInformation - bootstrap_info = BootstrapInformation(manifest) - bootstrap_info.volume.image_path = '/target/debian-wheezy-amd64-141218.vdi' - - from bootstrapvz.remote.build_servers import LocalBuildServer - if isinstance(build_server, LocalBuildServer): - image_path = bootstrap_info.volume.image_path - else: - # import tempfile - # handle, image_path = tempfile.mkstemp() - # import os - # os.close(handle) - # build_server.download(bootstrap_info.volume.image_path, image_path) - # build_server.delete(bootstrap_info.volume.image_path) - image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk' - - image = VirtualBoxImage(manifest, image_path) - try: - instance = VirtualBoxInstance('unpartitioned_extlinux', image) - try: - instance.create() - try: - instance.boot() - # tools.reachable_with_ssh(instance) - finally: - instance.shutdown() - finally: - instance.destroy() - finally: - image.destroy() - import os - os.remove(image_path) +def test_virtualbox_partitioned_extlinux(): + std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition', + 'root_password', 'apt_proxy'] + custom_partials = [partials['vbox'], partials['vdi']] + manifest_data = merge_manifest_data(std_partials, custom_partials) + with BootableManifest(manifest_data) as instance: + print(instance.console_output)