From cee05e3fd06713e057e9e2a62df6148506db61b0 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 30 Nov 2014 15:42:38 +0100 Subject: [PATCH] Refactor... --- .gitignore | 2 +- bootstrapvz/remote/build-servers-schema.yml | 49 ++++++++++ bootstrapvz/remote/build_servers.py | 103 +++++++++++++++++++- bootstrapvz/remote/main.py | 58 ++++++----- tests/integration/tools/__init__.py | 19 +--- tests/integration/virtualbox_tests.py | 3 +- 6 files changed, 187 insertions(+), 47 deletions(-) create mode 100644 bootstrapvz/remote/build-servers-schema.yml diff --git a/.gitignore b/.gitignore index d2b3547..3422ad6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ # Testing /.coverage /.tox/ -/build_servers.yml +/build-servers.yml diff --git a/bootstrapvz/remote/build-servers-schema.yml b/bootstrapvz/remote/build-servers-schema.yml new file mode 100644 index 0000000..3990704 --- /dev/null +++ b/bootstrapvz/remote/build-servers-schema.yml @@ -0,0 +1,49 @@ +--- +$schema: http://json-schema.org/draft-04/schema# +title: Build server settings list +type: object +properties: + local: + type: object + properties: + type: {enum: [local]} + can_bootstrap: {$ref: '#/definitions/can_bootstrap'} + release: {type: string} + build_settings: {$ref: '#/definitions/build_settings'} + required: [type, can_bootstrap, release] +patternProperties: + ^(?!local).*$: {$ref: '#/definitions/ssh'} + +definitions: + absolute_path: + type: string + pattern: ^/[^\0]+$ + + can_bootstrap: + type: array + items: + enum: + - virtualbox + - ec2-ebs + - ec2-s3 + + build_settings: + type: object + properties: + guest_additions: {$ref: '#/definitions/absolute_path'} + + ssh: + type: object + properties: + type: {enum: [ssh]} + can_bootstrap: {$ref: '#/definitions/can_bootstrap'} + build_settings: {$ref: '#/definitions/build_settings'} + release: {type: string} + address: {type: string} + port: {type: integer} + username: {type: string} + password: {type: string} + root_password: {type: string} + keyfile: {$ref: '#/definitions/absolute_path'} + server_bin: {$ref: '#/definitions/absolute_path'} + required: [type, can_bootstrap, release] diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py index 95daf41..64ce968 100644 --- a/bootstrapvz/remote/build_servers.py +++ b/bootstrapvz/remote/build_servers.py @@ -1,4 +1,38 @@ from bootstrapvz.common.tools import log_check_call +import logging +log = logging.getLogger(__name__) + + +def pick_build_server(build_servers, preferences, manifest): + # Validate the build servers list + from bootstrapvz.common.tools import load_data + import os.path + schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml'))) + import jsonschema + jsonschema.validate(build_servers, schema) + + if manifest.provider['name'] == 'ec2': + must_bootstrap = 'ec2-' + manifest.volume['backing'] + else: + must_bootstrap = manifest.provider['name'] + + def matches(name, settings): + if preferences.get('name', name) != name: + return False + if preferences.get('release', settings['release']) != settings['release']: + return False + if must_bootstrap not in settings['can_bootstrap']: + return False + return True + + for name, settings in build_servers.iteritems(): + if not matches(name, settings): + continue + if settings['type'] == 'local': + return LocalBuildServer(settings) + else: + return RemoteBuildServer(settings) + raise Exception('Unable to find a build server that matches your preferences.') class BuildServer(object): @@ -17,6 +51,7 @@ class LocalBuildServer(BuildServer): class RemoteBuildServer(BuildServer): def __init__(self, settings): + super(RemoteBuildServer, self).__init__(settings) self.address = settings['address'] self.port = settings['port'] self.username = settings['username'] @@ -24,7 +59,61 @@ class RemoteBuildServer(BuildServer): self.root_password = settings['root_password'] self.keyfile = settings['keyfile'] self.server_bin = settings['server_bin'] - super(RemoteBuildServer, self).__init__(settings) + + # We can't use :0 for the forwarding ports because + # A: It's quite hard to retrieve the port on the remote after the daemon has started + # B: SSH doesn't accept 0:localhost:0 as a port forwarding option + [self.local_server_port, self.local_callback_port] = getNPorts(2) + [self.remote_server_port, self.remote_callback_port] = getNPorts(2) + + def connect(self): + log.debug('Opening SSH connection') + import subprocess + + server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)] + + addr_arg = '{user}@{host}'.format(user=self.username, host=self.address) + ssh_cmd = ['ssh', '-i', self.settings['keyfile'], + '-p', str(self.settings['port']), + '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port), + '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port), + addr_arg] + full_cmd = ssh_cmd + ['--'] + server_cmd + import sys + self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr) + + # Check that we can connect to the server + try: + import Pyro4 + server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port) + self.connection = Pyro4.Proxy(server_uri) + + log.debug('Connecting to the RPC daemon') + remaining_retries = 5 + while True: + try: + self.connection.ping() + break + except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e: + if remaining_retries > 0: + remaining_retries -= 1 + from time import sleep + sleep(2) + else: + raise e + except (Exception, KeyboardInterrupt) as e: + self.ssh_process.terminate() + raise e + return self.connection + + def disconnect(self): + if hasattr(self, 'connection'): + log.debug('Stopping the RPC daemon') + self.connection.stop() + self.connection._pyroRelease() + if hasattr(self, 'ssh_process'): + log.debug('Waiting for the SSH connection to terminate') + self.ssh_process.wait() def download(self, src, dst): src_arg = '{user}@{host}:{path}'.format(self.username, self.address, src) @@ -38,3 +127,15 @@ class RemoteBuildServer(BuildServer): '--', 'sudo', 'rm', path] log_check_call(ssh_cmd) + + +def getNPorts(n, port_range=(1024, 65535)): + import random + ports = [] + for i in range(0, n): + while True: + port = random.randrange(*port_range) + if port not in ports: + ports.append(port) + break + return ports diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py index aa0d719..88019cd 100644 --- a/bootstrapvz/remote/main.py +++ b/bootstrapvz/remote/main.py @@ -12,9 +12,17 @@ def main(): from bootstrapvz.base.manifest import Manifest manifest = Manifest(path=opts['MANIFEST']) - # Load the build servers + # load the build servers file from bootstrapvz.common.tools import load_data build_servers = load_data(opts['--servers']) + # Pick a build server + from build_servers import pick_build_server + preferences = {} + if opts['--name'] is not None: + preferences['name'] = opts['--name'] + if opts['--release'] is not None: + preferences['release'] = opts['--release'] + build_server = pick_build_server(build_servers, preferences, manifest) # Set up logging from bootstrapvz.base.main import setup_loggers @@ -27,7 +35,7 @@ def main(): # Everything has been set up, connect to the server and begin the bootstrapping process run(manifest, - build_servers[opts['SERVER']], + build_server, debug=opts['--debug'], dry_run=opts['--dry-run']) @@ -38,56 +46,54 @@ def get_opts(): from docopt import docopt usage = """bootstrap-vz-remote -Usage: bootstrap-vz-remote [options] --servers= SERVER MANIFEST +Usage: bootstrap-vz-remote [options] --servers= MANIFEST Options: - --servers Path to list of build servers - --log Log to given directory [default: /var/log/bootstrap-vz] - If is `-' file logging will be disabled. - --pause-on-error Pause on error, before rollback - --dry-run Don't actually run the tasks + --servers Path to list of build servers + --name Selects specific server from the build servers list + --release Require the build server OS to be a specific release + --log Log to given directory [default: /var/log/bootstrap-vz] + If is `-' file logging will be disabled. + --pause-on-error Pause on error, before rollback + --dry-run Don't actually run the tasks --color=auto|always|never Colorize the console output [default: auto] - --debug Print debugging information - -h, --help show this help + --debug Print debugging information + -h, --help show this help """ return docopt(usage) -def run(manifest, server, debug=False, dry_run=False): +def run(manifest, build_server, debug=False, dry_run=False): """Connects to the remote build server, starts an RPC daemin on the other side and initiates a remote bootstrapping procedure """ bootstrap_info = None - - from ssh_rpc_manager import SSHRPCManager - manager = SSHRPCManager(server) try: - # Connect to the build server and start the RPC daemon - manager.start() - server = manager.rpc_server + # Connect to the build server + connection = build_server.connect() # Start a callback server on this side, so that we may receive log entries from callback import CallbackServer - callback_server = CallbackServer(listen_port=manager.local_callback_port, - remote_port=manager.remote_callback_port) + callback_server = CallbackServer(listen_port=build_server.local_callback_port, + remote_port=build_server.remote_callback_port) from bootstrapvz.base.log import LogServer log_server = LogServer() try: # Start the callback server (in a background thread) callback_server.start(log_server) # Tell the RPC daemon about the callback server - server.set_log_server(log_server) + connection.set_log_server(log_server) # Everything has been set up, begin the bootstrapping process - bootstrap_info = server.run(manifest, - debug=debug, - # We can't pause the bootstrapping process remotely, yet... - pause_on_error=False, - dry_run=dry_run) + bootstrap_info = connection.run(manifest, + debug=debug, + # We can't pause the bootstrapping process remotely, yet... + pause_on_error=False, + dry_run=dry_run) finally: # Stop the callback server callback_server.stop() finally: # Stop the RPC daemon and close the SSH connection - manager.stop() + build_server.disconnect() return bootstrap_info diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py index 7ccf1cc..e2009c0 100644 --- a/tests/integration/tools/__init__.py +++ b/tests/integration/tools/__init__.py @@ -1,6 +1,4 @@ -from bootstrapvz.common.tools import load_data from bootstrapvz.remote.build_servers import LocalBuildServer -from bootstrapvz.remote.build_servers import RemoteBuildServer # Register deserialization handlers for objects # that will pass between server and client @@ -27,28 +25,13 @@ def merge_dicts(*args): return reduce(merge, args, {}) -def pick_build_server(manifest): - if manifest['provider']['name'] == 'ec2': - img_type = 'ec2-' + manifest['volume']['backing'] - else: - img_type = manifest['provider']['name'] - - # tox makes sure that the cwd is the project root - build_servers = load_data('build_servers.yml') - settings = next((server for name, server in build_servers.iteritems() if img_type in server['can_bootstrap']), None) - if settings['type'] == 'local': - return LocalBuildServer(settings) - else: - return RemoteBuildServer(settings) - - def bootstrap(manifest, build_server): if isinstance(build_server, LocalBuildServer): from bootstrapvz.base.main import run bootstrap_info = run(manifest) else: from bootstrapvz.remote.main import run - bootstrap_info = run(manifest, build_server.settings) + bootstrap_info = run(manifest, build_server) return bootstrap_info diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py index ea7eaed..cd357da 100644 --- a/tests/integration/virtualbox_tests.py +++ b/tests/integration/virtualbox_tests.py @@ -18,7 +18,8 @@ volume: manifest_data = tools.merge_dicts(partials['base'], partials['stable64'], partials['unpartitioned'], manifest_data) - build_server = tools.pick_build_server(manifest_data) + manifest = Manifest(data=manifest_data) + build_server = tools.pick_build_server(manifest) manifest_data['provider']['guest_additions'] = build_server.build_settings['guest_additions'] manifest = Manifest(data=manifest_data)