Refactor, logging, comments

bootstrapvz/base/main.py

@@ -1,9 +1,6 @@
 """Main module containing all the setup necessary for running the bootstrapping process
 """
 
-import logging
-log = logging.getLogger(__name__)
-
 
 def main():
 	"""Main function for invoking the bootstrap process
@@ -12,6 +9,7 @@ def main():
 	"""
 	# Get the commandline arguments
 	opts = get_opts()
+
 	# Require root privileges, except when doing a dry-run where they aren't needed
 	import os
 	if os.geteuid() != 0 and not opts['--dry-run']:
@@ -104,6 +102,8 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	from bootstrapinfo import BootstrapInformation
 	bootstrap_info = BootstrapInformation(manifest=manifest, debug=debug)
 
+	import logging
+	log = logging.getLogger(__name__)
 	try:
 		# Run all the tasks the tasklist has gathered
 		tasklist.run(info=bootstrap_info, dry_run=dry_run)

bootstrapvz/base/remote/__init__.py

@@ -8,6 +8,12 @@ def register_deserialization_handlers():
 	SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo)
 
 
+def unregister_deserialization_handlers():
+	from Pyro4.util import SerializerBase
+	SerializerBase.unregister_dict_to_class('bootstrapvz.base.manifest.Manifest')
+	SerializerBase.unregister_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation')
+
+
 def deserialize_manifest(classname, state):
 	from bootstrapvz.base.manifest import Manifest
 	return Manifest(path=state['path'], data=state['data'])

bootstrapvz/base/remote/callback.py

@@ -1,3 +1,5 @@
+import logging
+log = logging.getLogger(__name__)
 
 
 class CallbackServer(object):
@@ -17,6 +19,7 @@ class CallbackServer(object):
 			self.daemon.requestLoop()
 		from threading import Thread
 		self.thread = Thread(target=serve)
+		log.debug('Starting the callback server')
 		self.thread.start()
 
 	def stop(self):

bootstrapvz/base/remote/remote.py

@@ -1,6 +1,11 @@
+"""Main module containing all the setup necessary for running the remote bootstrapping process
+"""
+
 
 def main():
 	"""Main function for invoking the bootstrap process remotely
+
+
 	"""
 	# Get the commandline arguments
 	opts = get_opts()
@@ -9,6 +14,7 @@ def main():
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(path=opts['MANIFEST'])
 
+	# Load the build servers
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data(opts['--servers'])
 
@@ -16,35 +22,16 @@ def main():
 	from bootstrapvz.base.main import setup_loggers
 	setup_loggers(opts)
 
+	# Register deserialization handlers for objects
+	# that will pass between server and client
 	from . import register_deserialization_handlers
 	register_deserialization_handlers()
 
-	bootstrap_info = None
+	# Everything has been set up, connect to the server and begin the bootstrapping process
-
+	run(manifest,
-	from ssh_rpc_manager import SSHRPCManager
+	    build_servers[opts['SERVER']],
-	manager = SSHRPCManager(build_servers[opts['SERVER']])
+	    debug=opts['--debug'],
-	try:
+	    dry_run=opts['--dry-run'])
-		manager.start()
-		server = manager.rpc_server
-		from callback import CallbackServer
-		callback_server = CallbackServer(listen_port=manager.local_callback_port,
-		                                 remote_port=manager.remote_callback_port)
-		from bootstrapvz.base.log import LogServer
-		log_server = LogServer()
-		try:
-			callback_server.start(log_server)
-			server.set_log_server(log_server)
-
-			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = server.run(manifest,
-			                            debug=opts['--debug'],
-			                            pause_on_error=False,
-			                            dry_run=opts['--dry-run'])
-		finally:
-			callback_server.stop()
-	finally:
-		manager.stop()
-	return bootstrap_info
 
 
 def get_opts():
@@ -65,3 +52,42 @@ Options:
   -h, --help         show this help
 	"""
 	return docopt(usage)
+
+
+def run(manifest, server, debug=False, dry_run=False):
+	"""Connects to the remote build server, starts an RPC daemin
+	on the other side and initiates a remote bootstrapping procedure
+	"""
+	bootstrap_info = None
+
+	from ssh_rpc_manager import SSHRPCManager
+	manager = SSHRPCManager(server)
+	try:
+		# Connect to the build server and start the RPC daemon
+		manager.start()
+		server = manager.rpc_server
+		# Start a callback server on this side, so that we may receive log entries
+		from callback import CallbackServer
+		callback_server = CallbackServer(listen_port=manager.local_callback_port,
+		                                 remote_port=manager.remote_callback_port)
+		from bootstrapvz.base.log import LogServer
+		log_server = LogServer()
+		try:
+			# Start the callback server (in a background thread)
+			callback_server.start(log_server)
+			# Tell the RPC daemon about the callback server
+			server.set_log_server(log_server)
+
+			# Everything has been set up, begin the bootstrapping process
+			bootstrap_info = server.run(manifest,
+			                            debug=debug,
+			                            # We can't pause the bootstrapping process remotely, yet...
+			                            pause_on_error=False,
+			                            dry_run=dry_run)
+		finally:
+			# Stop the callback server
+			callback_server.stop()
+	finally:
+		# Stop the RPC daemon and close the SSH connection
+		manager.stop()
+	return bootstrap_info

bootstrapvz/base/remote/server.py

@@ -1,3 +1,5 @@
+import logging
+log = logging.getLogger(__name__)
 
 
 def main():
@@ -10,7 +12,6 @@ def main():
 
 
 def setup_logging():
-	import logging
 	from bootstrapvz.base.log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
@@ -52,7 +53,8 @@ class Server(object):
 		return run(*args, **kwargs)
 
 	def set_log_server(self, server):
-		return self.log_forwarder.set_server(server)
+		self.log_forwarder.set_server(server)
+		log.debug('Successfully set the log forwarding server')
 
 	def ping(self):
 		return 'pong'

bootstrapvz/base/remote/ssh_rpc_manager.py

@@ -37,7 +37,7 @@ class SSHRPCManager(object):
 		                  'sudo', self.settings['server-bin'],
 		                  '--listen', str(self.remote_server_port)]
 		import sys
-		self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
+		self.ssh_process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
 
 		# Check that we can connect to the server
 		try:
@@ -45,7 +45,7 @@ class SSHRPCManager(object):
 			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
 			self.rpc_server = Pyro4.Proxy(server_uri)
 
-			log.debug('Connecting to PYRO')
+			log.debug('Connecting to the RPC daemon')
 			remaining_retries = 5
 			while True:
 				try:
@@ -59,10 +59,14 @@ class SSHRPCManager(object):
 					else:
 						raise e
 		except (Exception, KeyboardInterrupt) as e:
-			self.process.terminate()
+			self.ssh_process.terminate()
 			raise e
 
 	def stop(self):
-		self.rpc_server.stop()
+		if hasattr(self, 'rpc_server'):
-		self.rpc_server._pyroRelease()
+			log.debug('Stopping the RPC daemon')
-		self.process.wait()
+			self.rpc_server.stop()
+			self.rpc_server._pyroRelease()
+		if hasattr(self, 'ssh_process'):
+			log.debug('Waiting for the SSH connection to terminate')
+			self.ssh_process.wait()