From 072969065c01d9d8a4e3ac2f7dca4318614759df Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 13 Dec 2015 19:26:11 +0100 Subject: [PATCH] Implement docker integration test provider and docker tests Also make build_server.remote_command() public. It's quite useful. --- CHANGELOG.rst | 2 +- bootstrapvz/remote/README.rst | 4 ++ .../build_servers/build-servers-schema.yml | 6 ++ bootstrapvz/remote/build_servers/remote.py | 6 +- tests/integration/docker_tests.py | 29 ++++++++++ tests/integration/providers/docker/README.rst | 7 +++ .../integration/providers/docker/__init__.py | 57 +++++++++++++++++++ tests/integration/providers/docker/image.py | 55 ++++++++++++++++++ 8 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 tests/integration/docker_tests.py create mode 100644 tests/integration/providers/docker/README.rst create mode 100644 tests/integration/providers/docker/__init__.py create mode 100644 tests/integration/providers/docker/image.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9abeb78..eaabc2c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ Changelog 2015-12-13 ---------- Anders Ingemann: - * Docker provider implemented + * Docker provider implemented (including integration testing harness & tests) * minimize_size: Added various size reduction options for dpkg and apt * Removed image section in manifest. Provider specific options have been moved to the provider section. diff --git a/bootstrapvz/remote/README.rst b/bootstrapvz/remote/README.rst index e592eaa..977af12 100644 --- a/bootstrapvz/remote/README.rst +++ b/bootstrapvz/remote/README.rst @@ -185,6 +185,10 @@ This is useful only when running integration tests. ec2-credentials: access-key: AFAKEACCESSKEYFORAWS secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva + docker: + machine: default * ``ec2-credentials`` contains the access key and secret key used to boot an EC2 AMI. +* ``docker.machine`` The docker machine on which an image built for docker + should run. diff --git a/bootstrapvz/remote/build_servers/build-servers-schema.yml b/bootstrapvz/remote/build_servers/build-servers-schema.yml index 588687b..41c49f0 100644 --- a/bootstrapvz/remote/build_servers/build-servers-schema.yml +++ b/bootstrapvz/remote/build_servers/build-servers-schema.yml @@ -26,6 +26,7 @@ definitions: - virtualbox - ec2-ebs - ec2-s3 + - docker build_settings: type: object @@ -60,6 +61,11 @@ definitions: access-key: {type: string} secret-key: {type: string} additional_properties: false + docker: + type: object + properties: + machine: {type: string} + additional_properties: false ssh: type: object diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py index 2177550..c117c64 100644 --- a/bootstrapvz/remote/build_servers/remote.py +++ b/bootstrapvz/remote/build_servers/remote.py @@ -80,16 +80,16 @@ class RemoteBuildServer(BuildServer): 'build server `{name}\' to `{dst}\'' .format(src=src, dst=dst, name=self.name)) # Make sure we can read the file as {user} - self._remote_command(['sudo', 'chown', self.username, src]) + self.remote_command(['sudo', 'chown', self.username, src]) src_arg = '{user}@{host}:{path}'.format(user=self.username, host=self.address, path=src) log_check_call(['scp', '-i', self.keyfile, '-P', str(self.port), src_arg, dst]) def delete(self, path): log.debug('Deleting file `{path}\' on build server `{name}\''.format(path=path, name=self.name)) - self._remote_command(['sudo', 'rm', path]) + self.remote_command(['sudo', 'rm', path]) - def _remote_command(self, command): + def remote_command(self, command): ssh_cmd = ['ssh', '-i', self.keyfile, '-p', str(self.port), self.username + '@' + self.address, diff --git a/tests/integration/docker_tests.py b/tests/integration/docker_tests.py new file mode 100644 index 0000000..cb0771f --- /dev/null +++ b/tests/integration/docker_tests.py @@ -0,0 +1,29 @@ +from manifests import merge_manifest_data +from tools import boot_manifest + +partials = {'docker': ''' +provider: + name: docker + virtualization: hvm + dockerfile: CMD /bin/bash +bootstrapper: + variant: minbase +system: + bootloader: none +volume: + backing: folder + partitions: + type: none + root: + filesystem: ext4 + size: 1GiB +''', + } + + +def test_stable(): + std_partials = ['base', 'stable64'] + custom_partials = [partials['docker']] + manifest_data = merge_manifest_data(std_partials, custom_partials) + with boot_manifest(manifest_data) as instance: + print('\n'.join(instance.run(['echo', 'test']))) diff --git a/tests/integration/providers/docker/README.rst b/tests/integration/providers/docker/README.rst new file mode 100644 index 0000000..555c568 --- /dev/null +++ b/tests/integration/providers/docker/README.rst @@ -0,0 +1,7 @@ +Docker +------ + + +Dependencies +~~~~~~~~~~~~ +The host machine running the integration tests must have docker installed. diff --git a/tests/integration/providers/docker/__init__.py b/tests/integration/providers/docker/__init__.py new file mode 100644 index 0000000..85ad41b --- /dev/null +++ b/tests/integration/providers/docker/__init__.py @@ -0,0 +1,57 @@ +from contextlib import contextmanager +import logging +log = logging.getLogger(__name__) + + +@contextmanager +def boot_image(manifest, build_server, bootstrap_info): + image_id = None + try: + import os + from bootstrapvz.common.tools import log_check_call + docker_machine = build_server.run_settings.get('docker', {}).get('machine', None) + docker_env = os.environ.copy() + if docker_machine is not None: + cmd = ('eval "$(docker-machine env {machine})" && ' + 'echo $DOCKER_HOST && echo $DOCKER_CERT_PATH && echo $DOCKER_TLS_VERIFY' + .format(machine=docker_machine)) + [docker_host, docker_cert_path, docker_tls] = log_check_call([cmd], shell=True) + docker_env['DOCKER_TLS_VERIFY'] = docker_tls + docker_env['DOCKER_HOST'] = docker_host + docker_env['DOCKER_CERT_PATH'] = docker_cert_path + docker_env['DOCKER_MACHINE_NAME'] = docker_machine + from bootstrapvz.remote.build_servers.local import LocalBuildServer + image_id = bootstrap_info._docker['image_id'] + if not isinstance(build_server, LocalBuildServer): + import tempfile + handle, image_path = tempfile.mkstemp() + os.close(handle) + remote_image_path = os.path.join('/tmp', image_id) + try: + log.debug('Saving remote image to file') + build_server.remote_command([ + 'sudo', 'docker', 'save', + '--output=' + remote_image_path, + image_id, + ]) + log.debug('Downloading remote image') + build_server.download(remote_image_path, image_path) + log.debug('Importing image') + log_check_call(['docker', 'load', '--input=' + image_path], env=docker_env) + except (Exception, KeyboardInterrupt): + raise + finally: + log.debug('Deleting exported image from build server and locally') + build_server.delete(remote_image_path) + os.remove(image_path) + log.debug('Deleting image from build server') + build_server.remote_command(['sudo', 'docker', 'rmi', + bootstrap_info._docker['image_id']]) + + from image import Image + with Image(image_id, docker_env) as container: + yield container + finally: + if image_id is not None: + log.debug('Deleting image') + log_check_call(['docker', 'rmi', image_id], env=docker_env) diff --git a/tests/integration/providers/docker/image.py b/tests/integration/providers/docker/image.py new file mode 100644 index 0000000..12794f2 --- /dev/null +++ b/tests/integration/providers/docker/image.py @@ -0,0 +1,55 @@ +from bootstrapvz.common.tools import log_check_call +import logging +log = logging.getLogger(__name__) + + +class Image(object): + + def __init__(self, image_id, docker_env): + self.image_id = image_id + self.docker_env = docker_env + + def __enter__(self): + self.container = Container(self.image_id, self.docker_env) + self.container.create() + try: + self.container.start() + except: + self.container.destroy() + raise + return self.container + + def __exit__(self, exc_type, exc_value, traceback): + try: + self.container.stop() + self.container.destroy() + except Exception as e: + log.exception(e) + + +class Container(object): + + def __init__(self, image_id, docker_env): + self.image_id = image_id + self.docker_env = docker_env + + def create(self): + log.debug('Creating container') + [self.container_id] = log_check_call(['docker', 'create', '--tty=true', self.image_id], env=self.docker_env) + + def start(self): + log.debug('Starting container') + log_check_call(['docker', 'start', self.container_id], env=self.docker_env) + + def run(self, command): + log.debug('Running command in container') + return log_check_call(['docker', 'exec', self.container_id] + command, env=self.docker_env) + + def stop(self): + log.debug('Stopping container') + log_check_call(['docker', 'stop', self.container_id], env=self.docker_env) + + def destroy(self): + log.debug('Deleting container') + log_check_call(['docker', 'rm', self.container_id], env=self.docker_env) + del self.container_id