diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py index 7e1c054..fd3064d 100644 --- a/bootstrapvz/base/main.py +++ b/bootstrapvz/base/main.py @@ -1,9 +1,6 @@ """Main module containing all the setup necessary for running the bootstrapping process """ -import logging -log = logging.getLogger(__name__) - def main(): """Main function for invoking the bootstrap process @@ -12,6 +9,7 @@ def main(): """ # Get the commandline arguments opts = get_opts() + # Require root privileges, except when doing a dry-run where they aren't needed import os if os.geteuid() != 0 and not opts['--dry-run']: @@ -104,6 +102,8 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False): from bootstrapinfo import BootstrapInformation bootstrap_info = BootstrapInformation(manifest=manifest, debug=debug) + import logging + log = logging.getLogger(__name__) try: # Run all the tasks the tasklist has gathered tasklist.run(info=bootstrap_info, dry_run=dry_run) diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py index 8e604e5..3c61646 100644 --- a/bootstrapvz/base/remote/__init__.py +++ b/bootstrapvz/base/remote/__init__.py @@ -8,6 +8,12 @@ def register_deserialization_handlers(): SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo) +def unregister_deserialization_handlers(): + from Pyro4.util import SerializerBase + SerializerBase.unregister_dict_to_class('bootstrapvz.base.manifest.Manifest') + SerializerBase.unregister_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation') + + def deserialize_manifest(classname, state): from bootstrapvz.base.manifest import Manifest return Manifest(path=state['path'], data=state['data']) diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py index a4fc6d8..25b197b 100644 --- a/bootstrapvz/base/remote/callback.py +++ b/bootstrapvz/base/remote/callback.py @@ -1,3 +1,5 @@ +import logging +log = logging.getLogger(__name__) class CallbackServer(object): @@ -17,6 +19,7 @@ class CallbackServer(object): self.daemon.requestLoop() from threading import Thread self.thread = Thread(target=serve) + log.debug('Starting the callback server') self.thread.start() def stop(self): diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/base/remote/remote.py index 1f1e96c..aeffa31 100644 --- a/bootstrapvz/base/remote/remote.py +++ b/bootstrapvz/base/remote/remote.py @@ -1,6 +1,11 @@ +"""Main module containing all the setup necessary for running the remote bootstrapping process +""" + def main(): """Main function for invoking the bootstrap process remotely + + """ # Get the commandline arguments opts = get_opts() @@ -9,6 +14,7 @@ def main(): from bootstrapvz.base.manifest import Manifest manifest = Manifest(path=opts['MANIFEST']) + # Load the build servers from bootstrapvz.common.tools import load_data build_servers = load_data(opts['--servers']) @@ -16,35 +22,16 @@ def main(): from bootstrapvz.base.main import setup_loggers setup_loggers(opts) + # Register deserialization handlers for objects + # that will pass between server and client from . import register_deserialization_handlers register_deserialization_handlers() - bootstrap_info = None - - from ssh_rpc_manager import SSHRPCManager - manager = SSHRPCManager(build_servers[opts['SERVER']]) - try: - manager.start() - server = manager.rpc_server - from callback import CallbackServer - callback_server = CallbackServer(listen_port=manager.local_callback_port, - remote_port=manager.remote_callback_port) - from bootstrapvz.base.log import LogServer - log_server = LogServer() - try: - callback_server.start(log_server) - server.set_log_server(log_server) - - # Everything has been set up, begin the bootstrapping process - bootstrap_info = server.run(manifest, - debug=opts['--debug'], - pause_on_error=False, - dry_run=opts['--dry-run']) - finally: - callback_server.stop() - finally: - manager.stop() - return bootstrap_info + # Everything has been set up, connect to the server and begin the bootstrapping process + run(manifest, + build_servers[opts['SERVER']], + debug=opts['--debug'], + dry_run=opts['--dry-run']) def get_opts(): @@ -65,3 +52,42 @@ Options: -h, --help show this help """ return docopt(usage) + + +def run(manifest, server, debug=False, dry_run=False): + """Connects to the remote build server, starts an RPC daemin + on the other side and initiates a remote bootstrapping procedure + """ + bootstrap_info = None + + from ssh_rpc_manager import SSHRPCManager + manager = SSHRPCManager(server) + try: + # Connect to the build server and start the RPC daemon + manager.start() + server = manager.rpc_server + # Start a callback server on this side, so that we may receive log entries + from callback import CallbackServer + callback_server = CallbackServer(listen_port=manager.local_callback_port, + remote_port=manager.remote_callback_port) + from bootstrapvz.base.log import LogServer + log_server = LogServer() + try: + # Start the callback server (in a background thread) + callback_server.start(log_server) + # Tell the RPC daemon about the callback server + server.set_log_server(log_server) + + # Everything has been set up, begin the bootstrapping process + bootstrap_info = server.run(manifest, + debug=debug, + # We can't pause the bootstrapping process remotely, yet... + pause_on_error=False, + dry_run=dry_run) + finally: + # Stop the callback server + callback_server.stop() + finally: + # Stop the RPC daemon and close the SSH connection + manager.stop() + return bootstrap_info diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py index 483bbb4..d41d7ea 100644 --- a/bootstrapvz/base/remote/server.py +++ b/bootstrapvz/base/remote/server.py @@ -1,3 +1,5 @@ +import logging +log = logging.getLogger(__name__) def main(): @@ -10,7 +12,6 @@ def main(): def setup_logging(): - import logging from bootstrapvz.base.log import LogForwarder log_forwarder = LogForwarder() root = logging.getLogger() @@ -52,7 +53,8 @@ class Server(object): return run(*args, **kwargs) def set_log_server(self, server): - return self.log_forwarder.set_server(server) + self.log_forwarder.set_server(server) + log.debug('Successfully set the log forwarding server') def ping(self): return 'pong' diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py index 558dcfb..8d280f6 100644 --- a/bootstrapvz/base/remote/ssh_rpc_manager.py +++ b/bootstrapvz/base/remote/ssh_rpc_manager.py @@ -37,7 +37,7 @@ class SSHRPCManager(object): 'sudo', self.settings['server-bin'], '--listen', str(self.remote_server_port)] import sys - self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr) + self.ssh_process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr) # Check that we can connect to the server try: @@ -45,7 +45,7 @@ class SSHRPCManager(object): server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port) self.rpc_server = Pyro4.Proxy(server_uri) - log.debug('Connecting to PYRO') + log.debug('Connecting to the RPC daemon') remaining_retries = 5 while True: try: @@ -59,10 +59,14 @@ class SSHRPCManager(object): else: raise e except (Exception, KeyboardInterrupt) as e: - self.process.terminate() + self.ssh_process.terminate() raise e def stop(self): - self.rpc_server.stop() - self.rpc_server._pyroRelease() - self.process.wait() + if hasattr(self, 'rpc_server'): + log.debug('Stopping the RPC daemon') + self.rpc_server.stop() + self.rpc_server._pyroRelease() + if hasattr(self, 'ssh_process'): + log.debug('Waiting for the SSH connection to terminate') + self.ssh_process.wait()