diff --git a/bootstrap-vz b/bootstrap-vz index 46f8fbf..de62f32 100755 --- a/bootstrap-vz +++ b/bootstrap-vz @@ -1,5 +1,5 @@ #!/usr/bin/env python if __name__ == '__main__': - from bootstrapvz.base import main + from bootstrapvz.base.main import main main() diff --git a/bootstrap-vz-remote b/bootstrap-vz-remote new file mode 100755 index 0000000..81fa633 --- /dev/null +++ b/bootstrap-vz-remote @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +if __name__ == '__main__': + from bootstrapvz.base.remote import main + main() diff --git a/bootstrap-vz-server b/bootstrap-vz-server new file mode 100755 index 0000000..a79f005 --- /dev/null +++ b/bootstrap-vz-server @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +if __name__ == '__main__': + from bootstrapvz.base.remote.server import main + main() diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py index be224ce..305d2e3 100644 --- a/bootstrapvz/base/__init__.py +++ b/bootstrapvz/base/__init__.py @@ -1,6 +1,5 @@ from phase import Phase from task import Task -from main import main __all__ = ['Phase', 'Task', 'main'] diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py index 7e1c054..acfacd3 100644 --- a/bootstrapvz/base/main.py +++ b/bootstrapvz/base/main.py @@ -86,6 +86,8 @@ def setup_loggers(opts): def run(manifest, debug=False, pause_on_error=False, dry_run=False): + log.info('test') + return 'derp' """Runs the bootstrapping process :params Manifest manifest: The manifest to run the bootstrapping process for diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py new file mode 100644 index 0000000..27a7e05 --- /dev/null +++ b/bootstrapvz/base/remote/__init__.py @@ -0,0 +1,100 @@ +import Pyro4 +from threading import Thread +"""Remote module containing methods to bootstrap remotely +""" + + +import logging +log = logging.getLogger(__name__) + +stop = False + + +def main(): + """Main function for invoking the bootstrap process remotely + """ + # Get the commandline arguments + opts = get_opts() + + # Load the manifest + from bootstrapvz.base.manifest import Manifest + manifest = Manifest(path=opts['MANIFEST']) + + from bootstrapvz.common.tools import load_data + build_servers = load_data(opts['--servers']) + + # Set up logging + from bootstrapvz.base.main import setup_loggers + setup_loggers(opts) + + from ssh_rpc_manager import SSHRPCManager + manager = SSHRPCManager(build_servers[opts['SERVER']]) + try: + manager.start() + server = manager.rpc_server + + # Everything has been set up, begin the bootstrapping process + print('run') + server.run(None, + debug=opts['--debug'], + pause_on_error=False, + dry_run=opts['--dry-run']) + print('hasrun') + finally: + manager.stop() + + +def get_opts(): + """Creates an argument parser and returns the arguments it has parsed + """ + from docopt import docopt + usage = """bootstrap-vz-remote + +Usage: bootstrap-vz-remote [options] --servers= SERVER MANIFEST + +Options: + --servers Path to list of build servers + --log Log to given directory [default: /var/log/bootstrap-vz] + If is `-' file logging will be disabled. + --pause-on-error Pause on error, before rollback + --dry-run Don't actually run the tasks + --debug Print debugging information + -h, --help show this help + """ + return docopt(usage) + + +def setup_interrupt_server(manager): + + def on_error(e): + raw_input('Press Enter to commence rollback') + return True + + daemon = Pyro4.Daemon() + daemon.register(on_error) + + def serve(): + daemon.requestLoop(loopCondition=lambda: manager.current == 'rpc_started') + + thread = Thread(target=serve) + thread.start() + return (thread, on_error) + + +def setup_log_server(manager): + from log import LogServer + log_server = LogServer() + daemon = Pyro4.Daemon() + daemon.register(log_server) + + def serve(): + def check(): + import logging + log = logging.getLogger(__name__) + log.info(stop) + return not stop + daemon.requestLoop(loopCondition=check) + + thread = Thread(target=serve) + thread.start() + return (thread, log_server) diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py new file mode 100644 index 0000000..80fe22c --- /dev/null +++ b/bootstrapvz/base/remote/callback.py @@ -0,0 +1,31 @@ + + +class CallbackServer(object): + + def __init__(self, listen_port): + self.listen_port = listen_port + self.stop_serving = False + + from log import LogServer + self.log_server = LogServer() + + def start(self, rpc_server): + import Pyro4 + Pyro4.config.COMMTIMEOUT = 0.5 + daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None) + daemon.register(self.log_server) + + def serve(): + daemon.requestLoop(loopCondition=lambda: not self.stop_serving) + from threading import Thread + self.thread = Thread(target=serve) + self.thread.start() + + rpc_server.set_log_server(self.log_server) + + def stop(self): + self.stop_serving = True + if hasattr(self, 'thread'): + print('joining') + self.thread.join() + print('joined') diff --git a/bootstrapvz/base/remote/log.py b/bootstrapvz/base/remote/log.py new file mode 100644 index 0000000..169ce52 --- /dev/null +++ b/bootstrapvz/base/remote/log.py @@ -0,0 +1,24 @@ +import logging +import pickle + +class LogForwarder(logging.Handler): + + def __init__(self, level=logging.NOTSET): + self.server = None + super(LogForwarder, self).__init__(level) + + def set_server(self, server): + self.server = server + + def emit(self, record): + if self.server is not None: + self.server.handle(pickle.dumps(record)) + + +class LogServer(object): + + def handle(self, pickled_record): + import logging + log = logging.getLogger() + record = pickle.loads(pickled_record) + log.handle(record) diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py new file mode 100644 index 0000000..837349b --- /dev/null +++ b/bootstrapvz/base/remote/server.py @@ -0,0 +1,50 @@ + + +def main(): + opts = getopts() + log_forwarder = setup_logging() + serve(opts, log_forwarder) + + +def setup_logging(): + import logging + from log import LogForwarder + log_forwarder = LogForwarder() + root = logging.getLogger() + root.addHandler(log_forwarder) + root.setLevel(logging.NOTSET) + return log_forwarder + + +def serve(opts, log_forwarder): + class Server(object): + + def run(self, *args, **kwargs): + from bootstrapvz.base.main import run + return run(*args, **kwargs) + + def set_log_server(self, server): + return log_forwarder.set_server(server) + + def ping(self): + return 'pong' + + server = Server() + + import Pyro4 + daemon = Pyro4.Daemon('localhost', port=int(opts['--listen']), unixsocket=None) + daemon.register(server, 'server') + daemon.requestLoop() + + +def getopts(): + from docopt import docopt + usage = """bootstrap-vz-server + +Usage: bootstrap-vz-server [options] + +Options: + --listen Serve on specified port [default: 46675] + -h, --help show this help +""" + return docopt(usage) diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py new file mode 100644 index 0000000..ef14317 --- /dev/null +++ b/bootstrapvz/base/remote/ssh_rpc_manager.py @@ -0,0 +1,67 @@ +from fysom import Fysom + +import logging +log = logging.getLogger(__name__) + + +class SSHRPCManager(object): + + def __init__(self, settings): + self.settings = settings + + import random + self.local_server_port = random.randrange(1024, 65535) + self.local_callback_port = random.randrange(1024, 65535) + # self.remote_server_port = random.randrange(1024, 65535) + # self.remote_callback_port = random.randrange(1024, 65535) + self.remote_server_port = self.local_server_port + self.remote_callback_port = self.local_callback_port + + def start(self): + log.debug('Opening SSH connection') + import subprocess + + ssh_cmd = ['ssh', '-i', self.settings['keyfile'], + '-p', str(self.settings['port']), + '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port), + '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port), + self.settings['username'] + '@' + self.settings['address'], + '--', + 'sudo', self.settings['server-bin'], + '--listen', str(self.remote_server_port)] + import sys + self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr) + + # Check that we can connect to the server + try: + import Pyro4 + server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port) + self.rpc_server = Pyro4.Proxy(server_uri) + + log.debug('Connecting to PYRO') + remaining_retries = 5 + while True: + try: + self.rpc_server.ping() + break + except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e: + if remaining_retries > 0: + remaining_retries -= 1 + from time import sleep + sleep(2) + else: + raise e + except (Exception, KeyboardInterrupt) as e: + print('terminateE') + self.process.terminate() + raise e + + from callback import CallbackServer + self.callback_server = CallbackServer(self.local_callback_port) + self.callback_server.start(self.rpc_server) + + def stop(self): + print('terminate') + self.process.terminate() + if hasattr(self, 'callback_server'): + self.callback_server.stop() diff --git a/build_servers.yml b/build_servers.yml new file mode 100644 index 0000000..dcfabd1 --- /dev/null +++ b/build_servers.yml @@ -0,0 +1,23 @@ +virtualbox: + can_bootstrap: + - virtualbox + - ec2-s3 + address: 127.0.0.1 + port: 2222 + username: vagrant + password: vagrant + root_password: vagrant + keyfile: /Users/anders/.vagrant.d/insecure_private_key + server-bin: /root/bootstrap/bootstrap-vz-server +ec2: + can_bootstrap: + - virtualbox + - ec2-ebs + - ec2-s3 + address: 127.0.0.1 + port: 2222 + username: vagrant + password: vagrant + root_password: vagrant + keyfile: /Users/anders/.vagrant.d/insecure_private_key + server-bin: /root/bootstrap/bootstrap-vz-server diff --git a/remote/client.py b/remote/client.py new file mode 100755 index 0000000..da3cc35 --- /dev/null +++ b/remote/client.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import random +import Pyro4 + +# We need to set either a socket communication timeout, +# or use the select based server. Otherwise the daemon requestLoop +# will block indefinitely and is never able to evaluate the loopCondition. +Pyro4.config.COMMTIMEOUT = 0.5 + +NUM_WORKERS = 5 + +from ssh_wrapper import RemoteServer +srv = RemoteServer() +srv.start() + + +class CallbackHandler(object): + workdone = 0 + + def done(self, number): + print("callback: worker %d reports work is done!" % number) + CallbackHandler.workdone += 1 + + +class LogServer(object): + + def handle(self, record): + print('logging' + record) + # import logging + # log = logging.getLogger() + # (handler.handle(record) for handler in log.handlers) + + +with Pyro4.Daemon('localhost', port=srv.client_port, unixsocket=None) as daemon: + # register our callback handler + callback = CallbackHandler() + daemon.register(callback) + logger = LogServer() + daemon.register(logger) + + # contact the server and put it to work + + def serve(): + daemon.requestLoop(loopCondition=lambda: CallbackHandler.workdone < NUM_WORKERS) + from threading import Thread + thread = Thread(target=serve) + thread.start() + + print("creating a bunch of workers") + with Pyro4.core.Proxy("PYRO:srv@localhost:" + str(srv.server_port)) as server: + server.set_log_server(logger) + for _ in range(NUM_WORKERS): + worker = server.addworker(callback) # provide our callback handler! + # worker._pyroOneway.add("work") # to be able to run in the background + worker.work(0.5) + server.stop() + + print("waiting for all work complete...") + thread.join() + print("done!") + +srv.stop() diff --git a/remote/server.py b/remote/server.py new file mode 100755 index 0000000..081d66d --- /dev/null +++ b/remote/server.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +import time +import Pyro4 +import logging +Pyro4.config.COMMTIMEOUT = 5 + +log = logging.getLogger(__name__) + +class Worker(object): + def __init__(self, number, callback): + self.number = number + self.callback = callback + log.info("Worker %d created" % self.number) + + def work(self, amount): + print("Worker %d busy..." % self.number) + time.sleep(amount) + print("Worker %d done. Informing callback client." % self.number) + self._pyroDaemon.unregister(self) + self.callback.done(self.number) # invoke the callback object + + +class LogForwarder(logging.Handler): + + def __init__(self, level=logging.NOTSET): + self.server = None + super(LogForwarder, self).__init__(level) + + def set_server(self, server): + self.server = server + + def emit(self, record): + if self.server is not None: + self.server.handle('hans') + + +class CallbackServer(object): + + def __init__(self): + self.number = 0 + self.serve = True + + def addworker(self, callback): + self.number += 1 + print("server: adding worker %d" % self.number) + worker = Worker(self.number, callback) + self._pyroDaemon.register(worker) # make it a Pyro object + return worker + + def stop(self): + print('called stop()') + self.serve = False + + def still_serve(self): + print('called still_serve()') + return self.serve + + def set_log_server(self, server): + import logging + log_forwarder = LogForwarder() + root = logging.getLogger() + root.addHandler(log_forwarder) + root.setLevel(logging.NOTSET) + log_forwarder.set_server(server) + + def test(self, msg): + import logging + root = logging.getLogger() + root.info(msg) + + +def main(): + opts = getopts() + with Pyro4.Daemon('localhost', port=int(opts['--listen-port']), unixsocket=None) as daemon: + obj = CallbackServer() + uri = daemon.register(obj, 'srv') + print uri + print("Server ready.") + daemon.requestLoop(loopCondition=lambda: obj.still_serve()) + + +def getopts(): + from docopt import docopt + usage = """bootstrap-vz-server + +Usage: bootstrap-vz-server [options] + +Options: + --listen-port Serve on specified port [default: 46675] + --callback-port Connect callback to specified port [default: 46674] + -h, --help show this help +""" + return docopt(usage) + +if __name__ == '__main__': + main() diff --git a/remote/ssh_wrapper.py b/remote/ssh_wrapper.py new file mode 100644 index 0000000..f1f9d5d --- /dev/null +++ b/remote/ssh_wrapper.py @@ -0,0 +1,28 @@ + + +class RemoteServer(object): + + def __init__(self): + import random + self.server_port = random.randrange(1024, 65535) + self.client_port = random.randrange(1024, 65535) + + def start(self): + import subprocess + + command = ['ssh', '-i', '/Users/anders/.vagrant.d/insecure_private_key', + '-t', # Force pseudo-tty allocation so that server.py quits when we close the connection + '-p', '2222', + '-L' + str(self.server_port) + ':localhost:' + str(self.server_port), + '-R' + str(self.client_port) + ':localhost:' + str(self.client_port), + 'vagrant@localhost', + '--', + 'sudo', '/root/bootstrap/remote/server.py', + '--listen-port', str(self.server_port), + '--callback-port', str(self.client_port)] + self.process = subprocess.Popen(args=command) + import time + time.sleep(2) + + def stop(self): + self.process.terminate() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..ccce1cc --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,9 @@ +import os.path +from bootstrapvz.common.tools import load_data + +combine_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests') +manifests = {'base': load_data(os.path.join(combine_manifests_path, 'base.yml')), + 'partitioned': load_data(os.path.join(combine_manifests_path, 'partitioned.yml')), + 'unpartitioned': load_data(os.path.join(combine_manifests_path, 'unpartitioned.yml')), + } +build_settings = load_data(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'build_settings.yml')) diff --git a/tests/integration/build/__init__.py b/tests/integration/build/__init__.py new file mode 100644 index 0000000..9416fa0 --- /dev/null +++ b/tests/integration/build/__init__.py @@ -0,0 +1,3 @@ + +def get_client(build_settings): + pass diff --git a/tests/integration/build/local/__init__.py b/tests/integration/build/local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/build/local/client.py b/tests/integration/build/local/client.py new file mode 100644 index 0000000..85473cc --- /dev/null +++ b/tests/integration/build/local/client.py @@ -0,0 +1,5 @@ +from .. import BaseClient + + +class Client(BaseClient): + pass diff --git a/tests/integration/build/remote/__init__.py b/tests/integration/build/remote/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/build/remote/client.py b/tests/integration/build/remote/client.py new file mode 100644 index 0000000..85473cc --- /dev/null +++ b/tests/integration/build/remote/client.py @@ -0,0 +1,5 @@ +from .. import BaseClient + + +class Client(BaseClient): + pass diff --git a/tests/integration/build/remote/forward.py b/tests/integration/build/remote/forward.py new file mode 100644 index 0000000..96e1700 --- /dev/null +++ b/tests/integration/build/remote/forward.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python + +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Sample script showing how to do local port forwarding over paramiko. + +This script connects to the requested SSH server and sets up local port +forwarding (the openssh -L option) from a local port through a tunneled +connection to a destination reachable from the SSH server machine. +""" + +import getpass +import os +import socket +import select +try: + import SocketServer +except ImportError: + import socketserver as SocketServer + +import sys +from optparse import OptionParser + +import paramiko + +SSH_PORT = 22 +DEFAULT_PORT = 4000 + +g_verbose = True + + +class ForwardServer (SocketServer.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + +class Handler (SocketServer.BaseRequestHandler): + + def handle(self): + try: + chan = self.ssh_transport.open_channel('direct-tcpip', + (self.chain_host, self.chain_port), + self.request.getpeername()) + except Exception as e: + verbose('Incoming request to %s:%d failed: %s' % (self.chain_host, + self.chain_port, + repr(e))) + return + if chan is None: + verbose('Incoming request to %s:%d was rejected by the SSH server.' % + (self.chain_host, self.chain_port)) + return + + verbose('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(), + chan.getpeername(), (self.chain_host, self.chain_port))) + while True: + r, w, x = select.select([self.request, chan], [], []) + if self.request in r: + data = self.request.recv(1024) + if len(data) == 0: + break + chan.send(data) + if chan in r: + data = chan.recv(1024) + if len(data) == 0: + break + self.request.send(data) + + peername = self.request.getpeername() + chan.close() + self.request.close() + verbose('Tunnel closed from %r' % (peername,)) + + +def forward_tunnel(local_port, remote_host, remote_port, transport): + # this is a little convoluted, but lets me configure things for the Handler + # object. (SocketServer doesn't give Handlers any way to access the outer + # server normally.) + class SubHander (Handler): + chain_host = remote_host + chain_port = remote_port + ssh_transport = transport + ForwardServer(('', local_port), SubHander).serve_forever() + + +def verbose(s): + if g_verbose: + print(s) + + +HELP = """\ +Set up a forward tunnel across an SSH server, using paramiko. A local port +(given with -p) is forwarded across an SSH session to an address:port from +the SSH server. This is similar to the openssh -L option. +""" + + +def get_host_port(spec, default_port): + "parse 'hostname:22' into a host and port, with the port optional" + args = (spec.split(':', 1) + [default_port])[:2] + args[1] = int(args[1]) + return args[0], args[1] + + +def parse_options(): + global g_verbose + + parser = OptionParser(usage='usage: %prog [options] [:]', + version='%prog 1.0', description=HELP) + parser.add_option('-q', '--quiet', action='store_false', dest='verbose', default=True, + help='squelch all informational output') + parser.add_option('-p', '--local-port', action='store', type='int', dest='port', + default=DEFAULT_PORT, + help='local port to forward (default: %d)' % DEFAULT_PORT) + parser.add_option('-u', '--user', action='store', type='string', dest='user', + default=getpass.getuser(), + help='username for SSH authentication (default: %s)' % getpass.getuser()) + parser.add_option('-K', '--key', action='store', type='string', dest='keyfile', + default=None, + help='private key file to use for SSH authentication') + parser.add_option('', '--no-key', action='store_false', dest='look_for_keys', default=True, + help='don\'t look for or use a private key file') + parser.add_option('-P', '--password', action='store_true', dest='readpass', default=False, + help='read password (for key or password auth) from stdin') + parser.add_option('-r', '--remote', action='store', type='string', dest='remote', default=None, metavar='host:port', + help='remote host and port to forward to') + options, args = parser.parse_args() + + if len(args) != 1: + parser.error('Incorrect number of arguments.') + if options.remote is None: + parser.error('Remote address required (-r).') + + g_verbose = options.verbose + server_host, server_port = get_host_port(args[0], SSH_PORT) + remote_host, remote_port = get_host_port(options.remote, SSH_PORT) + return options, (server_host, server_port), (remote_host, remote_port) + + +def main(): + options, server, remote = parse_options() + + password = None + if options.readpass: + password = getpass.getpass('Enter SSH password: ') + + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + + verbose('Connecting to ssh host %s:%d ...' % (server[0], server[1])) + try: + client.connect(server[0], server[1], username=options.user, key_filename=options.keyfile, + look_for_keys=options.look_for_keys, password=password) + except Exception as e: + print('*** Failed to connect to %s:%d: %r' % (server[0], server[1], e)) + sys.exit(1) + + verbose('Now forwarding port %d to %s:%d ...' % (options.port, remote[0], remote[1])) + + try: + forward_tunnel(options.port, remote[0], remote[1], client.get_transport()) + except KeyboardInterrupt: + print('C-c: Port forwarding stopped.') + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/build/remote/test.py b/tests/integration/build/remote/test.py new file mode 100755 index 0000000..abe74a4 --- /dev/null +++ b/tests/integration/build/remote/test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +from remote.ssh_rpc_client import SSHRPCClient + +import logging +log = logging.getLogger(__name__) + + +def main(): + import os.path + + settings_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'settings.yml')) + with open(settings_path, 'r') as stream: + import yaml + settings = yaml.safe_load(stream) + + bootstrapvz_root = os.path.normpath(os.path.join(os.path.dirname(__file__), '../../../')) + import sys + sys.path.append(bootstrapvz_root) + + from bootstrapvz.base.log import get_console_handler + console_handler = get_console_handler(debug=True) + + root_logger = logging.getLogger() + root_logger.setLevel(logging.NOTSET) + root_logger.addHandler(console_handler) + + rpc_server = SSHRPCClient(settings) + try: + rpc_server.start_server() + log.info('connecting to Pyro on remote') + import Pyro4 + main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port) + main = Pyro4.Proxy(main_uri) + log.info('running command') + remaining_retries = 5 + while True: + try: + main.run('eogubhswg') + break + except Pyro4.errors.ConnectionClosedError as e: + if remaining_retries > 0: + remaining_retries -= 1 + from time import sleep + sleep(2) + else: + raise e + log.info('stopping server') + rpc_server.stop_server() + except (Exception, KeyboardInterrupt) as e: + log.error(e.__class__.__name__ + ': ' + str(e)) + finally: + print 'cleaning up' + rpc_server.cleanup() + +if __name__ == '__main__': + main() diff --git a/tests/integration/build/settings.yml b/tests/integration/build/settings.yml new file mode 100644 index 0000000..2788774 --- /dev/null +++ b/tests/integration/build/settings.yml @@ -0,0 +1,8 @@ +build_host: + address: 127.0.0.1 + port: 2222 + username: vagrant + password: vagrant + root_password: vagrant + keyfile: /Users/anders/.vagrant.d/insecure_private_key + server-bin: /root/bootstrap/tests/integration/build/server.py diff --git a/tests/integration/image/__init__.py b/tests/integration/image/__init__.py new file mode 100644 index 0000000..67bc2b1 --- /dev/null +++ b/tests/integration/image/__init__.py @@ -0,0 +1,21 @@ + + +class Image(object): + + def create_instance(self): + return Instance() + + def destroy(self): + pass + + +class Instance(object): + + def boot(self): + pass + + def shutdown(self): + pass + + def destroy(self): + pass diff --git a/tests/integration/manifests/base.yml b/tests/integration/manifests/base.yml new file mode 100644 index 0000000..849a31a --- /dev/null +++ b/tests/integration/manifests/base.yml @@ -0,0 +1,14 @@ +--- +provider: {} +bootstrapper: + workspace: /target +image: + name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d} + description: Debian {system.release} {system.architecture} +system: + charmap: UTF-8 + locale: en_US + timezone: UTC +volume: + partitions: {} +packages: {} diff --git a/tests/integration/manifests/partitioned.yml b/tests/integration/manifests/partitioned.yml new file mode 100644 index 0000000..e6686c4 --- /dev/null +++ b/tests/integration/manifests/partitioned.yml @@ -0,0 +1,11 @@ +--- +volume: + partitions: + boot: + filesystem: ext2 + size: 32MiB + root: + filesystem: ext4 + size: 864MiB + swap: + size: 128MiB diff --git a/tests/integration/manifests/unpartitioned.yml b/tests/integration/manifests/unpartitioned.yml new file mode 100644 index 0000000..ef7f97d --- /dev/null +++ b/tests/integration/manifests/unpartitioned.yml @@ -0,0 +1,7 @@ +--- +volume: + type: none + partitions: + root: + filesystem: ext4 + size: 1GiB diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py new file mode 100644 index 0000000..3bdad63 --- /dev/null +++ b/tests/integration/tools/__init__.py @@ -0,0 +1,56 @@ + + +# Snatched from here: http://stackoverflow.com/a/7205107 +def merge_dicts(*args): + def merge(a, b, path=None): + if path is None: + path = [] + for key in b: + if key in a: + if isinstance(a[key], dict) and isinstance(b[key], dict): + merge(a[key], b[key], path + [str(key)]) + elif a[key] == b[key]: + pass + else: + raise Exception('Conflict at %s' % '.'.join(path + [str(key)])) + else: + a[key] = b[key] + return a + return reduce(merge, args, {}) + + +def bootstrap(manifest, build_settings): + # if 'build_host' in build_settings: + # run = get_remote_run(build_settings) + # else: + # run = __import__('bootstrapvz.base.run') + # run(manifest) + from ..image import Image + return Image() + + +def test_instance(instance, build_settings): + pass + + +def get_remote_run(build_settings): + from ..build.client import SSHRPCServer + + rpc_server = SSHRPCServer(settings) + try: + rpc_server.start() + from time import sleep + sleep(2) + log.info('connection to Pyro on remote') + import Pyro4 + main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port) + main = Pyro4.Proxy(main_uri) + log.info('running command') + main.run('eogubhswg') + log.info('stopping server') + rpc_server.stop() + except (Exception, KeyboardInterrupt) as e: + log.error(e.__class__.__name__ + ': ' + str(e)) + finally: + print 'cleaning up' + rpc_server.cleanup() diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py new file mode 100644 index 0000000..d3b317b --- /dev/null +++ b/tests/integration/virtualbox_tests.py @@ -0,0 +1,28 @@ +import tools +from . import manifests +from . import build_settings + + +def test_virtualbox_unpartitioned_extlinux(): + specific_settings = {} + specific_settings['provider'] = {'name': 'virtualbox', + 'guest_additions': build_settings['virtualbox']['guest_additions']} + specific_settings['system'] = {'release': 'wheezy', + 'architecture': 'amd64', + 'bootloader': 'extlinux'} + specific_settings['volume'] = {'backing': 'vdi', + 'partitions': {'type': 'msdos'}} + manifest = tools.merge_dicts(manifests['base'], manifests['unpartitioned'], specific_settings) + + client = tools.get_client(build_settings['virtualbox']) + + image = client.bootstrap(manifest, build_settings['virtualbox']) + instance = image.create_instance() + instance.boot() + + tools.test_instance(instance, build_settings['virtualbox']) + + instance.destroy() + image.destroy() + + client.shutdown() diff --git a/tox.ini b/tox.ini index 999071f..122ee83 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,10 @@ deps = nose-cov commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive +[testenv:integration] +deps = nose +commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive + [testenv:docs] changedir = docs deps =