Support SIGINT over the network

bootstrapvz/base/main.py

@@ -83,7 +83,7 @@ def setup_loggers(opts):
 	root.addHandler(console_handler)
 
 
-def run(manifest, debug=False, pause_on_error=False, dry_run=False):
+def run(manifest, debug=False, pause_on_error=False, dry_run=False, check_continue=None):
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
@@ -106,7 +106,7 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	log = logging.getLogger(__name__)
 	try:
 		# Run all the tasks the tasklist has gathered
-		tasklist.run(info=bootstrap_info, dry_run=dry_run)
+		tasklist.run(info=bootstrap_info, dry_run=dry_run, check_continue=check_continue)
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
 	except (Exception, KeyboardInterrupt) as e:

bootstrapvz/base/tasklist.py

@@ -15,7 +15,7 @@ class TaskList(object):
 		self.tasks = tasks
 		self.tasks_completed = []
 
-	def run(self, info, dry_run=False):
+	def run(self, info, dry_run=False, check_continue=None):
 		"""Converts the taskgraph into a list and runs all tasks in that list
 
 		:param dict info: The bootstrap information object
@@ -27,6 +27,9 @@ class TaskList(object):
 		log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list))))
 
 		for task in task_list:
+			# Check if we should abort the run (used for asynchronous run abortion through remote building)
+			if callable(check_continue) and not check_continue():
+				raise TaskListError('Run was aborted.')
 			# Tasks are not required to have a description
 			if hasattr(task, 'description'):
 				log.info(task.description)

bootstrapvz/remote/build_servers.py

@@ -77,6 +77,13 @@ class RemoteBuildServer(BuildServer):
 
 		server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)]
 
+		def set_process_group():
+			# Changes the process group of a command so that any SIGINT
+			# for the main thread will not be propagated to it.
+			# We'd like to handle SIGINT ourselves (i.e. propagate the shutdown to the serverside)
+			import os
+			os.setpgrp()
+
 		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
 		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
 		                  '-p', str(self.settings['port']),
@@ -85,7 +92,8 @@ class RemoteBuildServer(BuildServer):
 		                  addr_arg]
 		full_cmd = ssh_cmd + ['--'] + server_cmd
 		import sys
-		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr)
+		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr,
+		                                    preexec_fn=set_process_group)
 
 		# Check that we can connect to the server
 		try:

bootstrapvz/remote/callback.py

@@ -12,6 +12,7 @@ class CallbackServer(object):
 		                           nathost='localhost', natport=remote_port,
 		                           unixsocket=None)
 		self.daemon.register(self)
+		self.abort = False
 
 	def start(self):
 		def serve():
@@ -36,3 +37,9 @@ class CallbackServer(object):
 		record.extra['source'] = 'remote'
 		log.handle(record)
 
+	@Pyro4.expose
+	def get_abort_run(self):
+		return self.abort
+
+	def abort_run(self):
+		self.abort = True

bootstrapvz/remote/main.py

@@ -88,12 +88,23 @@ def run(manifest, build_server, debug=False, dry_run=False):
 			# Tell the RPC daemon about the callback server
 			connection.set_callback_server(callback_server)
 
+			# Replace the standard SIGINT handler with a remote call to the server
+			# so that it may abort the run.
+			def abort(signum, frame):
+				import logging
+				logging.getLogger(__name__).warn('SIGINT received, asking remote to abort.')
+				callback_server.abort_run()
+			import signal
+			orig_sigint = signal.signal(signal.SIGINT, abort)
+
 			# Everything has been set up, begin the bootstrapping process
 			bootstrap_info = connection.run(manifest,
 			                                debug=debug,
 			                                # We can't pause the bootstrapping process remotely, yet...
 			                                pause_on_error=False,
 			                                dry_run=dry_run)
+			# Restore the old SIGINT handler
+			signal.signal(signal.SIGINT, orig_sigint)
 		finally:
 			# Stop the callback server
 			callback_server.stop()

bootstrapvz/remote/server.py

@@ -52,7 +52,11 @@ class Server(object):
 
 	@Pyro4.expose
 	def run(self, *args, **kwargs):
+
+		def abort_run():
+			return not self.callback_server.get_abort_run()
 		from bootstrapvz.base.main import run
+		kwargs['check_continue'] = abort_run
 		return run(*args, **kwargs)
 
 	@Pyro4.expose