Initial work on integration testing started.

The work consists of three parts:
* Allow for bootstrapping remotely,
  this makes it possible to run the tests
  on e.g. OSX with VirtualBox installed
* Make bootstrapping a fully automated process
  where the manifests can be generated by the tests
  and the tests can call the bootstrapper directly in python
* Create a framework wherein instances can be booted up
  using the bootstrapped images and subsequently tested

bootstrap-vz

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 
 if __name__ == '__main__':
-	from bootstrapvz.base import main
+	from bootstrapvz.base.main import main
 	main()

bootstrap-vz-remote

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+if __name__ == '__main__':
+	from bootstrapvz.base.remote import main
+	main()

bootstrap-vz-server

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+if __name__ == '__main__':
+	from bootstrapvz.base.remote.server import main
+	main()

bootstrapvz/base/__init__.py

@@ -1,6 +1,5 @@
 from phase import Phase
 from task import Task
-from main import main
 
 __all__ = ['Phase', 'Task', 'main']
 

bootstrapvz/base/main.py

@@ -86,6 +86,8 @@ def setup_loggers(opts):
 
 
 def run(manifest, debug=False, pause_on_error=False, dry_run=False):
+	log.info('test')
+	return 'derp'
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for

bootstrapvz/base/remote/__init__.py

@@ -0,0 +1,100 @@
+import Pyro4
+from threading import Thread
+"""Remote module containing methods to bootstrap remotely
+"""
+
+
+import logging
+log = logging.getLogger(__name__)
+
+stop = False
+
+
+def main():
+	"""Main function for invoking the bootstrap process remotely
+	"""
+	# Get the commandline arguments
+	opts = get_opts()
+
+	# Load the manifest
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(path=opts['MANIFEST'])
+
+	from bootstrapvz.common.tools import load_data
+	build_servers = load_data(opts['--servers'])
+
+	# Set up logging
+	from bootstrapvz.base.main import setup_loggers
+	setup_loggers(opts)
+
+	from ssh_rpc_manager import SSHRPCManager
+	manager = SSHRPCManager(build_servers[opts['SERVER']])
+	try:
+		manager.start()
+		server = manager.rpc_server
+
+		# Everything has been set up, begin the bootstrapping process
+		print('run')
+		server.run(None,
+		           debug=opts['--debug'],
+		           pause_on_error=False,
+		           dry_run=opts['--dry-run'])
+		print('hasrun')
+	finally:
+		manager.stop()
+
+
+def get_opts():
+	"""Creates an argument parser and returns the arguments it has parsed
+	"""
+	from docopt import docopt
+	usage = """bootstrap-vz-remote
+
+Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
+
+Options:
+  --servers <path>   Path to list of build servers
+  --log <path>       Log to given directory [default: /var/log/bootstrap-vz]
+                     If <path> is `-' file logging will be disabled.
+  --pause-on-error   Pause on error, before rollback
+  --dry-run          Don't actually run the tasks
+  --debug            Print debugging information
+  -h, --help         show this help
+	"""
+	return docopt(usage)
+
+
+def setup_interrupt_server(manager):
+
+	def on_error(e):
+		raw_input('Press Enter to commence rollback')
+		return True
+
+	daemon = Pyro4.Daemon()
+	daemon.register(on_error)
+
+	def serve():
+		daemon.requestLoop(loopCondition=lambda: manager.current == 'rpc_started')
+
+	thread = Thread(target=serve)
+	thread.start()
+	return (thread, on_error)
+
+
+def setup_log_server(manager):
+	from log import LogServer
+	log_server = LogServer()
+	daemon = Pyro4.Daemon()
+	daemon.register(log_server)
+
+	def serve():
+		def check():
+			import logging
+			log = logging.getLogger(__name__)
+			log.info(stop)
+			return not stop
+		daemon.requestLoop(loopCondition=check)
+
+	thread = Thread(target=serve)
+	thread.start()
+	return (thread, log_server)

bootstrapvz/base/remote/callback.py

@@ -0,0 +1,31 @@
+
+
+class CallbackServer(object):
+
+	def __init__(self, listen_port):
+		self.listen_port = listen_port
+		self.stop_serving = False
+
+		from log import LogServer
+		self.log_server = LogServer()
+
+	def start(self, rpc_server):
+		import Pyro4
+		Pyro4.config.COMMTIMEOUT = 0.5
+		daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
+		daemon.register(self.log_server)
+
+		def serve():
+			daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
+		from threading import Thread
+		self.thread = Thread(target=serve)
+		self.thread.start()
+
+		rpc_server.set_log_server(self.log_server)
+
+	def stop(self):
+		self.stop_serving = True
+		if hasattr(self, 'thread'):
+			print('joining')
+			self.thread.join()
+			print('joined')

bootstrapvz/base/remote/log.py

@@ -0,0 +1,24 @@
+import logging
+import pickle
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			self.server.handle(pickle.dumps(record))
+
+
+class LogServer(object):
+
+	def handle(self, pickled_record):
+		import logging
+		log = logging.getLogger()
+		record = pickle.loads(pickled_record)
+		log.handle(record)

bootstrapvz/base/remote/server.py

@@ -0,0 +1,50 @@
+
+
+def main():
+	opts = getopts()
+	log_forwarder = setup_logging()
+	serve(opts, log_forwarder)
+
+
+def setup_logging():
+	import logging
+	from log import LogForwarder
+	log_forwarder = LogForwarder()
+	root = logging.getLogger()
+	root.addHandler(log_forwarder)
+	root.setLevel(logging.NOTSET)
+	return log_forwarder
+
+
+def serve(opts, log_forwarder):
+	class Server(object):
+
+		def run(self, *args, **kwargs):
+			from bootstrapvz.base.main import run
+			return run(*args, **kwargs)
+
+		def set_log_server(self, server):
+			return log_forwarder.set_server(server)
+
+		def ping(self):
+			return 'pong'
+
+	server = Server()
+
+	import Pyro4
+	daemon = Pyro4.Daemon('localhost', port=int(opts['--listen']), unixsocket=None)
+	daemon.register(server, 'server')
+	daemon.requestLoop()
+
+
+def getopts():
+	from docopt import docopt
+	usage = """bootstrap-vz-server
+
+Usage: bootstrap-vz-server [options]
+
+Options:
+  --listen <port>    Serve on specified port [default: 46675]
+  -h, --help         show this help
+"""
+	return docopt(usage)

bootstrapvz/base/remote/ssh_rpc_manager.py

@@ -0,0 +1,67 @@
+from fysom import Fysom
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class SSHRPCManager(object):
+
+	def __init__(self, settings):
+		self.settings = settings
+
+		import random
+		self.local_server_port = random.randrange(1024, 65535)
+		self.local_callback_port = random.randrange(1024, 65535)
+		# self.remote_server_port = random.randrange(1024, 65535)
+		# self.remote_callback_port = random.randrange(1024, 65535)
+		self.remote_server_port = self.local_server_port
+		self.remote_callback_port = self.local_callback_port
+
+	def start(self):
+		log.debug('Opening SSH connection')
+		import subprocess
+
+		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
+		                  '-p', str(self.settings['port']),
+		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
+		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
+		                  self.settings['username'] + '@' + self.settings['address'],
+		                  '--',
+		                  'sudo', self.settings['server-bin'],
+		                  '--listen', str(self.remote_server_port)]
+		import sys
+		self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
+
+		# Check that we can connect to the server
+		try:
+			import Pyro4
+			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
+			self.rpc_server = Pyro4.Proxy(server_uri)
+
+			log.debug('Connecting to PYRO')
+			remaining_retries = 5
+			while True:
+				try:
+					self.rpc_server.ping()
+					break
+				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
+					if remaining_retries > 0:
+						remaining_retries -= 1
+						from time import sleep
+						sleep(2)
+					else:
+						raise e
+		except (Exception, KeyboardInterrupt) as e:
+			print('terminateE')
+			self.process.terminate()
+			raise e
+
+		from callback import CallbackServer
+		self.callback_server = CallbackServer(self.local_callback_port)
+		self.callback_server.start(self.rpc_server)
+
+	def stop(self):
+		print('terminate')
+		self.process.terminate()
+		if hasattr(self, 'callback_server'):
+			self.callback_server.stop()

build_servers.yml

@@ -0,0 +1,23 @@
+virtualbox:
+  can_bootstrap:
+    - virtualbox
+    - ec2-s3
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/bootstrap-vz-server
+ec2:
+  can_bootstrap:
+    - virtualbox
+    - ec2-ebs
+    - ec2-s3
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/bootstrap-vz-server

remote/client.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import random
+import Pyro4
+
+# We need to set either a socket communication timeout,
+# or use the select based server. Otherwise the daemon requestLoop
+# will block indefinitely and is never able to evaluate the loopCondition.
+Pyro4.config.COMMTIMEOUT = 0.5
+
+NUM_WORKERS = 5
+
+from ssh_wrapper import RemoteServer
+srv = RemoteServer()
+srv.start()
+
+
+class CallbackHandler(object):
+	workdone = 0
+
+	def done(self, number):
+		print("callback: worker %d reports work is done!" % number)
+		CallbackHandler.workdone += 1
+
+
+class LogServer(object):
+
+	def handle(self, record):
+		print('logging' + record)
+		# import logging
+		# log = logging.getLogger()
+		# (handler.handle(record) for handler in log.handlers)
+
+
+with Pyro4.Daemon('localhost', port=srv.client_port, unixsocket=None) as daemon:
+	# register our callback handler
+	callback = CallbackHandler()
+	daemon.register(callback)
+	logger = LogServer()
+	daemon.register(logger)
+
+	# contact the server and put it to work
+
+	def serve():
+		daemon.requestLoop(loopCondition=lambda: CallbackHandler.workdone < NUM_WORKERS)
+	from threading import Thread
+	thread = Thread(target=serve)
+	thread.start()
+
+	print("creating a bunch of workers")
+	with Pyro4.core.Proxy("PYRO:srv@localhost:" + str(srv.server_port)) as server:
+		server.set_log_server(logger)
+		for _ in range(NUM_WORKERS):
+			worker = server.addworker(callback)   # provide our callback handler!
+			# worker._pyroOneway.add("work")      # to be able to run in the background
+			worker.work(0.5)
+		server.stop()
+
+	print("waiting for all work complete...")
+	thread.join()
+	print("done!")
+
+srv.stop()

remote/server.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+import time
+import Pyro4
+import logging
+Pyro4.config.COMMTIMEOUT = 5
+
+log = logging.getLogger(__name__)
+
+class Worker(object):
+	def __init__(self, number, callback):
+		self.number = number
+		self.callback = callback
+		log.info("Worker %d created" % self.number)
+
+	def work(self, amount):
+		print("Worker %d busy..." % self.number)
+		time.sleep(amount)
+		print("Worker %d done. Informing callback client." % self.number)
+		self._pyroDaemon.unregister(self)
+		self.callback.done(self.number)  # invoke the callback object
+
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			self.server.handle('hans')
+
+
+class CallbackServer(object):
+
+	def __init__(self):
+		self.number = 0
+		self.serve = True
+
+	def addworker(self, callback):
+		self.number += 1
+		print("server: adding worker %d" % self.number)
+		worker = Worker(self.number, callback)
+		self._pyroDaemon.register(worker)   # make it a Pyro object
+		return worker
+
+	def stop(self):
+		print('called stop()')
+		self.serve = False
+
+	def still_serve(self):
+		print('called still_serve()')
+		return self.serve
+
+	def set_log_server(self, server):
+		import logging
+		log_forwarder = LogForwarder()
+		root = logging.getLogger()
+		root.addHandler(log_forwarder)
+		root.setLevel(logging.NOTSET)
+		log_forwarder.set_server(server)
+
+	def test(self, msg):
+		import logging
+		root = logging.getLogger()
+		root.info(msg)
+
+
+def main():
+	opts = getopts()
+	with Pyro4.Daemon('localhost', port=int(opts['--listen-port']), unixsocket=None) as daemon:
+		obj = CallbackServer()
+		uri = daemon.register(obj, 'srv')
+		print uri
+		print("Server ready.")
+		daemon.requestLoop(loopCondition=lambda: obj.still_serve())
+
+
+def getopts():
+	from docopt import docopt
+	usage = """bootstrap-vz-server
+
+Usage: bootstrap-vz-server [options]
+
+Options:
+  --listen-port <port>   Serve on specified port [default: 46675]
+  --callback-port <port> Connect callback to specified port [default: 46674]
+  -h, --help          show this help
+"""
+	return docopt(usage)
+
+if __name__ == '__main__':
+	main()

remote/ssh_wrapper.py

@@ -0,0 +1,28 @@
+
+
+class RemoteServer(object):
+
+	def __init__(self):
+		import random
+		self.server_port = random.randrange(1024, 65535)
+		self.client_port = random.randrange(1024, 65535)
+
+	def start(self):
+		import subprocess
+
+		command = ['ssh', '-i', '/Users/anders/.vagrant.d/insecure_private_key',
+		                  '-t',  # Force pseudo-tty allocation so that server.py quits when we close the connection
+		                  '-p', '2222',
+		                  '-L' + str(self.server_port) + ':localhost:' + str(self.server_port),
+		                  '-R' + str(self.client_port) + ':localhost:' + str(self.client_port),
+		                  'vagrant@localhost',
+		                  '--',
+		                  'sudo', '/root/bootstrap/remote/server.py',
+		                  '--listen-port', str(self.server_port),
+		                  '--callback-port', str(self.client_port)]
+		self.process = subprocess.Popen(args=command)
+		import time
+		time.sleep(2)
+
+	def stop(self):
+		self.process.terminate()

tests/integration/__init__.py

@@ -0,0 +1,9 @@
+import os.path
+from bootstrapvz.common.tools import load_data
+
+combine_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests')
+manifests = {'base': load_data(os.path.join(combine_manifests_path, 'base.yml')),
+             'partitioned': load_data(os.path.join(combine_manifests_path, 'partitioned.yml')),
+             'unpartitioned': load_data(os.path.join(combine_manifests_path, 'unpartitioned.yml')),
+             }
+build_settings = load_data(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'build_settings.yml'))

tests/integration/build/__init__.py

@@ -0,0 +1,3 @@
+
+def get_client(build_settings):
+	pass

tests/integration/build/local/client.py

@@ -0,0 +1,5 @@
+from .. import BaseClient
+
+
+class Client(BaseClient):
+	pass

tests/integration/build/remote/client.py

@@ -0,0 +1,5 @@
+from .. import BaseClient
+
+
+class Client(BaseClient):
+	pass

tests/integration/build/remote/forward.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2003-2007  Robey Pointer <robeypointer@gmail.com>
+#
+# This file is part of paramiko.
+#
+# Paramiko is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+
+"""
+Sample script showing how to do local port forwarding over paramiko.
+
+This script connects to the requested SSH server and sets up local port
+forwarding (the openssh -L option) from a local port through a tunneled
+connection to a destination reachable from the SSH server machine.
+"""
+
+import getpass
+import os
+import socket
+import select
+try:
+    import SocketServer
+except ImportError:
+    import socketserver as SocketServer
+
+import sys
+from optparse import OptionParser
+
+import paramiko
+
+SSH_PORT = 22
+DEFAULT_PORT = 4000
+
+g_verbose = True
+
+
+class ForwardServer (SocketServer.ThreadingTCPServer):
+    daemon_threads = True
+    allow_reuse_address = True
+    
+
+class Handler (SocketServer.BaseRequestHandler):
+
+    def handle(self):
+        try:
+            chan = self.ssh_transport.open_channel('direct-tcpip',
+                                                   (self.chain_host, self.chain_port),
+                                                   self.request.getpeername())
+        except Exception as e:
+            verbose('Incoming request to %s:%d failed: %s' % (self.chain_host,
+                                                              self.chain_port,
+                                                              repr(e)))
+            return
+        if chan is None:
+            verbose('Incoming request to %s:%d was rejected by the SSH server.' %
+                    (self.chain_host, self.chain_port))
+            return
+
+        verbose('Connected!  Tunnel open %r -> %r -> %r' % (self.request.getpeername(),
+                                                            chan.getpeername(), (self.chain_host, self.chain_port)))
+        while True:
+            r, w, x = select.select([self.request, chan], [], [])
+            if self.request in r:
+                data = self.request.recv(1024)
+                if len(data) == 0:
+                    break
+                chan.send(data)
+            if chan in r:
+                data = chan.recv(1024)
+                if len(data) == 0:
+                    break
+                self.request.send(data)
+                
+        peername = self.request.getpeername()
+        chan.close()
+        self.request.close()
+        verbose('Tunnel closed from %r' % (peername,))
+
+
+def forward_tunnel(local_port, remote_host, remote_port, transport):
+    # this is a little convoluted, but lets me configure things for the Handler
+    # object.  (SocketServer doesn't give Handlers any way to access the outer
+    # server normally.)
+    class SubHander (Handler):
+        chain_host = remote_host
+        chain_port = remote_port
+        ssh_transport = transport
+    ForwardServer(('', local_port), SubHander).serve_forever()
+
+
+def verbose(s):
+    if g_verbose:
+        print(s)
+
+
+HELP = """\
+Set up a forward tunnel across an SSH server, using paramiko. A local port
+(given with -p) is forwarded across an SSH session to an address:port from
+the SSH server. This is similar to the openssh -L option.
+"""
+
+
+def get_host_port(spec, default_port):
+    "parse 'hostname:22' into a host and port, with the port optional"
+    args = (spec.split(':', 1) + [default_port])[:2]
+    args[1] = int(args[1])
+    return args[0], args[1]
+
+
+def parse_options():
+    global g_verbose
+    
+    parser = OptionParser(usage='usage: %prog [options] <ssh-server>[:<server-port>]',
+                          version='%prog 1.0', description=HELP)
+    parser.add_option('-q', '--quiet', action='store_false', dest='verbose', default=True,
+                      help='squelch all informational output')
+    parser.add_option('-p', '--local-port', action='store', type='int', dest='port',
+                      default=DEFAULT_PORT,
+                      help='local port to forward (default: %d)' % DEFAULT_PORT)
+    parser.add_option('-u', '--user', action='store', type='string', dest='user',
+                      default=getpass.getuser(),
+                      help='username for SSH authentication (default: %s)' % getpass.getuser())
+    parser.add_option('-K', '--key', action='store', type='string', dest='keyfile',
+                      default=None,
+                      help='private key file to use for SSH authentication')
+    parser.add_option('', '--no-key', action='store_false', dest='look_for_keys', default=True,
+                      help='don\'t look for or use a private key file')
+    parser.add_option('-P', '--password', action='store_true', dest='readpass', default=False,
+                      help='read password (for key or password auth) from stdin')
+    parser.add_option('-r', '--remote', action='store', type='string', dest='remote', default=None, metavar='host:port',
+                      help='remote host and port to forward to')
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error('Incorrect number of arguments.')
+    if options.remote is None:
+        parser.error('Remote address required (-r).')
+    
+    g_verbose = options.verbose
+    server_host, server_port = get_host_port(args[0], SSH_PORT)
+    remote_host, remote_port = get_host_port(options.remote, SSH_PORT)
+    return options, (server_host, server_port), (remote_host, remote_port)
+
+
+def main():
+    options, server, remote = parse_options()
+    
+    password = None
+    if options.readpass:
+        password = getpass.getpass('Enter SSH password: ')
+    
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    client.set_missing_host_key_policy(paramiko.WarningPolicy())
+
+    verbose('Connecting to ssh host %s:%d ...' % (server[0], server[1]))
+    try:
+        client.connect(server[0], server[1], username=options.user, key_filename=options.keyfile,
+                       look_for_keys=options.look_for_keys, password=password)
+    except Exception as e:
+        print('*** Failed to connect to %s:%d: %r' % (server[0], server[1], e))
+        sys.exit(1)
+
+    verbose('Now forwarding port %d to %s:%d ...' % (options.port, remote[0], remote[1]))
+
+    try:
+        forward_tunnel(options.port, remote[0], remote[1], client.get_transport())
+    except KeyboardInterrupt:
+        print('C-c: Port forwarding stopped.')
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/integration/build/remote/test.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+from remote.ssh_rpc_client import SSHRPCClient
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def main():
+	import os.path
+
+	settings_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'settings.yml'))
+	with open(settings_path, 'r') as stream:
+		import yaml
+		settings = yaml.safe_load(stream)
+
+	bootstrapvz_root = os.path.normpath(os.path.join(os.path.dirname(__file__), '../../../'))
+	import sys
+	sys.path.append(bootstrapvz_root)
+
+	from bootstrapvz.base.log import get_console_handler
+	console_handler = get_console_handler(debug=True)
+
+	root_logger = logging.getLogger()
+	root_logger.setLevel(logging.NOTSET)
+	root_logger.addHandler(console_handler)
+
+	rpc_server = SSHRPCClient(settings)
+	try:
+		rpc_server.start_server()
+		log.info('connecting to Pyro on remote')
+		import Pyro4
+		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
+		main = Pyro4.Proxy(main_uri)
+		log.info('running command')
+		remaining_retries = 5
+		while True:
+			try:
+				main.run('eogubhswg')
+				break
+			except Pyro4.errors.ConnectionClosedError as e:
+				if remaining_retries > 0:
+					remaining_retries -= 1
+					from time import sleep
+					sleep(2)
+				else:
+					raise e
+		log.info('stopping server')
+		rpc_server.stop_server()
+	except (Exception, KeyboardInterrupt) as e:
+		log.error(e.__class__.__name__ + ': ' + str(e))
+	finally:
+		print 'cleaning up'
+		rpc_server.cleanup()
+
+if __name__ == '__main__':
+	main()

tests/integration/build/settings.yml

@@ -0,0 +1,8 @@
+build_host:
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/tests/integration/build/server.py

tests/integration/image/__init__.py

@@ -0,0 +1,21 @@
+
+
+class Image(object):
+
+	def create_instance(self):
+		return Instance()
+
+	def destroy(self):
+		pass
+
+
+class Instance(object):
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass

tests/integration/manifests/base.yml

@@ -0,0 +1,14 @@
+---
+provider: {}
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  partitions: {}
+packages: {}

tests/integration/manifests/partitioned.yml

@@ -0,0 +1,11 @@
+---
+volume:
+  partitions:
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 864MiB
+    swap:
+      size: 128MiB

tests/integration/manifests/unpartitioned.yml

@@ -0,0 +1,7 @@
+---
+volume:
+  type: none
+  partitions:
+    root:
+      filesystem: ext4
+      size: 1GiB

tests/integration/tools/__init__.py

@@ -0,0 +1,56 @@
+
+
+# Snatched from here: http://stackoverflow.com/a/7205107
+def merge_dicts(*args):
+	def merge(a, b, path=None):
+		if path is None:
+			path = []
+		for key in b:
+			if key in a:
+				if isinstance(a[key], dict) and isinstance(b[key], dict):
+					merge(a[key], b[key], path + [str(key)])
+				elif a[key] == b[key]:
+					pass
+				else:
+					raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
+			else:
+						a[key] = b[key]
+		return a
+	return reduce(merge, args, {})
+
+
+def bootstrap(manifest, build_settings):
+	# if 'build_host' in build_settings:
+	# 	run = get_remote_run(build_settings)
+	# else:
+	# 	run = __import__('bootstrapvz.base.run')
+	# run(manifest)
+	from ..image import Image
+	return Image()
+
+
+def test_instance(instance, build_settings):
+	pass
+
+
+def get_remote_run(build_settings):
+	from ..build.client import SSHRPCServer
+
+	rpc_server = SSHRPCServer(settings)
+	try:
+		rpc_server.start()
+		from time import sleep
+		sleep(2)
+		log.info('connection to Pyro on remote')
+		import Pyro4
+		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
+		main = Pyro4.Proxy(main_uri)
+		log.info('running command')
+		main.run('eogubhswg')
+		log.info('stopping server')
+		rpc_server.stop()
+	except (Exception, KeyboardInterrupt) as e:
+		log.error(e.__class__.__name__ + ': ' + str(e))
+	finally:
+		print 'cleaning up'
+		rpc_server.cleanup()

tests/integration/virtualbox_tests.py

@@ -0,0 +1,28 @@
+import tools
+from . import manifests
+from . import build_settings
+
+
+def test_virtualbox_unpartitioned_extlinux():
+	specific_settings = {}
+	specific_settings['provider'] = {'name': 'virtualbox',
+	                                 'guest_additions': build_settings['virtualbox']['guest_additions']}
+	specific_settings['system'] = {'release': 'wheezy',
+	                               'architecture': 'amd64',
+	                               'bootloader': 'extlinux'}
+	specific_settings['volume'] = {'backing': 'vdi',
+	                               'partitions': {'type': 'msdos'}}
+	manifest = tools.merge_dicts(manifests['base'], manifests['unpartitioned'], specific_settings)
+
+	client = tools.get_client(build_settings['virtualbox'])
+
+	image = client.bootstrap(manifest, build_settings['virtualbox'])
+	instance = image.create_instance()
+	instance.boot()
+
+	tools.test_instance(instance, build_settings['virtualbox'])
+
+	instance.destroy()
+	image.destroy()
+
+	client.shutdown()

tox.ini

@@ -15,6 +15,10 @@ deps =
     nose-cov
 commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
+[testenv:integration]
+deps = nose
+commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
+
 [testenv:docs]
 changedir = docs
 deps =