Refactor...

.gitignore

@@ -12,4 +12,4 @@
 # Testing
 /.coverage
 /.tox/
-/build_servers.yml
+/build-servers.yml

bootstrapvz/remote/build-servers-schema.yml

@@ -0,0 +1,49 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Build server settings list
+type: object
+properties:
+  local:
+    type: object
+    properties:
+      type: {enum: [local]}
+      can_bootstrap: {$ref: '#/definitions/can_bootstrap'}
+      release: {type: string}
+      build_settings: {$ref: '#/definitions/build_settings'}
+    required: [type, can_bootstrap, release]
+patternProperties:
+  ^(?!local).*$: {$ref: '#/definitions/ssh'}
+
+definitions:
+  absolute_path:
+    type: string
+    pattern: ^/[^\0]+$
+
+  can_bootstrap:
+    type: array
+    items:
+      enum:
+      - virtualbox
+      - ec2-ebs
+      - ec2-s3
+
+  build_settings:
+    type: object
+    properties:
+      guest_additions: {$ref: '#/definitions/absolute_path'}
+
+  ssh:
+    type: object
+    properties:
+      type: {enum: [ssh]}
+      can_bootstrap: {$ref: '#/definitions/can_bootstrap'}
+      build_settings: {$ref: '#/definitions/build_settings'}
+      release: {type: string}
+      address: {type: string}
+      port: {type: integer}
+      username: {type: string}
+      password: {type: string}
+      root_password: {type: string}
+      keyfile: {$ref: '#/definitions/absolute_path'}
+      server_bin: {$ref: '#/definitions/absolute_path'}
+    required: [type, can_bootstrap, release]

bootstrapvz/remote/build_servers.py

@@ -1,4 +1,38 @@
 from bootstrapvz.common.tools import log_check_call
+import logging
+log = logging.getLogger(__name__)
+
+
+def pick_build_server(build_servers, preferences, manifest):
+	# Validate the build servers list
+	from bootstrapvz.common.tools import load_data
+	import os.path
+	schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml')))
+	import jsonschema
+	jsonschema.validate(build_servers, schema)
+
+	if manifest.provider['name'] == 'ec2':
+		must_bootstrap = 'ec2-' + manifest.volume['backing']
+	else:
+		must_bootstrap = manifest.provider['name']
+
+	def matches(name, settings):
+		if preferences.get('name', name) != name:
+			return False
+		if preferences.get('release', settings['release']) != settings['release']:
+			return False
+		if must_bootstrap not in settings['can_bootstrap']:
+			return False
+		return True
+
+	for name, settings in build_servers.iteritems():
+		if not matches(name, settings):
+			continue
+		if settings['type'] == 'local':
+			return LocalBuildServer(settings)
+		else:
+			return RemoteBuildServer(settings)
+	raise Exception('Unable to find a build server that matches your preferences.')
 
 
 class BuildServer(object):
@@ -17,6 +51,7 @@ class LocalBuildServer(BuildServer):
 class RemoteBuildServer(BuildServer):
 
 	def __init__(self, settings):
+		super(RemoteBuildServer, self).__init__(settings)
 		self.address = settings['address']
 		self.port = settings['port']
 		self.username = settings['username']
@@ -24,7 +59,61 @@ class RemoteBuildServer(BuildServer):
 		self.root_password = settings['root_password']
 		self.keyfile = settings['keyfile']
 		self.server_bin = settings['server_bin']
-		super(RemoteBuildServer, self).__init__(settings)
+
+		# We can't use :0 for the forwarding ports because
+		# A: It's quite hard to retrieve the port on the remote after the daemon has started
+		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
+		[self.local_server_port, self.local_callback_port] = getNPorts(2)
+		[self.remote_server_port, self.remote_callback_port] = getNPorts(2)
+
+	def connect(self):
+		log.debug('Opening SSH connection')
+		import subprocess
+
+		server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)]
+
+		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
+		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
+		                  '-p', str(self.settings['port']),
+		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
+		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
+		                  addr_arg]
+		full_cmd = ssh_cmd + ['--'] + server_cmd
+		import sys
+		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr)
+
+		# Check that we can connect to the server
+		try:
+			import Pyro4
+			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
+			self.connection = Pyro4.Proxy(server_uri)
+
+			log.debug('Connecting to the RPC daemon')
+			remaining_retries = 5
+			while True:
+				try:
+					self.connection.ping()
+					break
+				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
+					if remaining_retries > 0:
+						remaining_retries -= 1
+						from time import sleep
+						sleep(2)
+					else:
+						raise e
+		except (Exception, KeyboardInterrupt) as e:
+			self.ssh_process.terminate()
+			raise e
+		return self.connection
+
+	def disconnect(self):
+		if hasattr(self, 'connection'):
+			log.debug('Stopping the RPC daemon')
+			self.connection.stop()
+			self.connection._pyroRelease()
+		if hasattr(self, 'ssh_process'):
+			log.debug('Waiting for the SSH connection to terminate')
+			self.ssh_process.wait()
 
 	def download(self, src, dst):
 		src_arg = '{user}@{host}:{path}'.format(self.username, self.address, src)
@@ -38,3 +127,15 @@ class RemoteBuildServer(BuildServer):
 		                  '--',
 		                  'sudo', 'rm', path]
 		log_check_call(ssh_cmd)
+
+
+def getNPorts(n, port_range=(1024, 65535)):
+	import random
+	ports = []
+	for i in range(0, n):
+		while True:
+			port = random.randrange(*port_range)
+			if port not in ports:
+				ports.append(port)
+				break
+	return ports

bootstrapvz/remote/main.py

@@ -12,9 +12,17 @@ def main():
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(path=opts['MANIFEST'])
 
-	# Load the build servers
+	# load the build servers file
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data(opts['--servers'])
+	# Pick a build server
+	from build_servers import pick_build_server
+	preferences = {}
+	if opts['--name'] is not None:
+		preferences['name'] = opts['--name']
+	if opts['--release'] is not None:
+		preferences['release'] = opts['--release']
+	build_server = pick_build_server(build_servers, preferences, manifest)
 
 	# Set up logging
 	from bootstrapvz.base.main import setup_loggers
@@ -27,7 +35,7 @@ def main():
 
 	# Everything has been set up, connect to the server and begin the bootstrapping process
 	run(manifest,
-	    build_servers[opts['SERVER']],
+	    build_server,
 	    debug=opts['--debug'],
 	    dry_run=opts['--dry-run'])
 
@@ -38,10 +46,12 @@ def get_opts():
 	from docopt import docopt
 	usage = """bootstrap-vz-remote
 
-Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
+Usage: bootstrap-vz-remote [options] --servers=<path> MANIFEST
 
 Options:
   --servers <path>    Path to list of build servers
+  --name <name>       Selects specific server from the build servers list
+  --release <release> Require the build server OS to be a specific release
   --log <path>        Log to given directory [default: /var/log/bootstrap-vz]
                       If <path> is `-' file logging will be disabled.
   --pause-on-error    Pause on error, before rollback
@@ -54,32 +64,28 @@ Options:
 	return docopt(usage)
 
 
-def run(manifest, server, debug=False, dry_run=False):
+def run(manifest, build_server, debug=False, dry_run=False):
 	"""Connects to the remote build server, starts an RPC daemin
 	on the other side and initiates a remote bootstrapping procedure
 	"""
 	bootstrap_info = None
-
-	from ssh_rpc_manager import SSHRPCManager
-	manager = SSHRPCManager(server)
 	try:
-		# Connect to the build server and start the RPC daemon
+		# Connect to the build server
-		manager.start()
+		connection = build_server.connect()
-		server = manager.rpc_server
 		# Start a callback server on this side, so that we may receive log entries
 		from callback import CallbackServer
-		callback_server = CallbackServer(listen_port=manager.local_callback_port,
+		callback_server = CallbackServer(listen_port=build_server.local_callback_port,
-		                                 remote_port=manager.remote_callback_port)
+		                                 remote_port=build_server.remote_callback_port)
 		from bootstrapvz.base.log import LogServer
 		log_server = LogServer()
 		try:
 			# Start the callback server (in a background thread)
 			callback_server.start(log_server)
 			# Tell the RPC daemon about the callback server
-			server.set_log_server(log_server)
+			connection.set_log_server(log_server)
 
 			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = server.run(manifest,
+			bootstrap_info = connection.run(manifest,
 			                                debug=debug,
 			                                # We can't pause the bootstrapping process remotely, yet...
 			                                pause_on_error=False,
@@ -89,5 +95,5 @@ def run(manifest, server, debug=False, dry_run=False):
 			callback_server.stop()
 	finally:
 		# Stop the RPC daemon and close the SSH connection
-		manager.stop()
+		build_server.disconnect()
 	return bootstrap_info

tests/integration/tools/__init__.py

@@ -1,6 +1,4 @@
-from bootstrapvz.common.tools import load_data
 from bootstrapvz.remote.build_servers import LocalBuildServer
-from bootstrapvz.remote.build_servers import RemoteBuildServer
 
 # Register deserialization handlers for objects
 # that will pass between server and client
@@ -27,28 +25,13 @@ def merge_dicts(*args):
 	return reduce(merge, args, {})
 
 
-def pick_build_server(manifest):
-	if manifest['provider']['name'] == 'ec2':
-		img_type = 'ec2-' + manifest['volume']['backing']
-	else:
-		img_type = manifest['provider']['name']
-
-	# tox makes sure that the cwd is the project root
-	build_servers = load_data('build_servers.yml')
-	settings = next((server for name, server in build_servers.iteritems() if img_type in server['can_bootstrap']), None)
-	if settings['type'] == 'local':
-		return LocalBuildServer(settings)
-	else:
-		return RemoteBuildServer(settings)
-
-
 def bootstrap(manifest, build_server):
 	if isinstance(build_server, LocalBuildServer):
 		from bootstrapvz.base.main import run
 		bootstrap_info = run(manifest)
 	else:
 		from bootstrapvz.remote.main import run
-		bootstrap_info = run(manifest, build_server.settings)
+		bootstrap_info = run(manifest, build_server)
 	return bootstrap_info
 
 

tests/integration/virtualbox_tests.py

@@ -18,7 +18,8 @@ volume:
 	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
 	                                  partials['unpartitioned'], manifest_data)
 
-	build_server = tools.pick_build_server(manifest_data)
+	manifest = Manifest(data=manifest_data)
+	build_server = tools.pick_build_server(manifest)
 	manifest_data['provider']['guest_additions'] = build_server.build_settings['guest_additions']
 	manifest = Manifest(data=manifest_data)