From fdf11fcc1aa01f686936ad2672b0e72c2d728a67 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Fri, 9 May 2014 00:31:36 -0700
Subject: [PATCH 001/345] providers/gce: add InstallInitScript task

---
 bootstrapvz/providers/gce/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index b705c20..291d201 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -6,6 +6,7 @@ import tasks.image
 import tasks.host
 import tasks.packages
 from bootstrapvz.common.tasks import loopback
+from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import ssh
 import bootstrapvz.plugins.cloud_init.tasks
 
@@ -35,6 +36,7 @@ def resolve_tasks(taskset, manifest):
 
 	                tasks.host.DisableIPv6,
 	                tasks.boot.ConfigureGrub,
+	                initd.InstallInitScripts,
 	                ssh.AddSSHKeyGeneration,
 	                tasks.apt.CleanGoogleRepositoriesAndKeys,
 

From 49475292d76283fc85f0e2fa4da7b9f4e6e43c0d Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Sat, 10 May 2014 21:10:02 -0700
Subject: [PATCH 002/345] Split GCE manifest into regular & backports

Some people want the security team-supported 3.2 kernel, while others
want the better performance of the backports kernel. Both flavors are
currently provided in the Google Compute Engine debian-cloud project.

Also add OpenSSH to the backports flavor, for parity with the current
GCE backports image based on build-debian-cloud. This backport is
included for performance reasons, in line with the purpose of that image
flavor, and as discussed on the debian-cloud list.
---
 manifests/gce-backports.manifest.json | 53 +++++++++++++++++++++++++++
 manifests/gce.manifest.json           | 11 +-----
 2 files changed, 54 insertions(+), 10 deletions(-)
 create mode 100644 manifests/gce-backports.manifest.json

diff --git a/manifests/gce-backports.manifest.json b/manifests/gce-backports.manifest.json
new file mode 100644
index 0000000..1334ab2
--- /dev/null
+++ b/manifests/gce-backports.manifest.json
@@ -0,0 +1,53 @@
+{
+	"provider": "gce",
+	"bootstrapper": {
+		"workspace":      "/target"
+	},
+	"image": {
+		"name":        "disk",
+		"description": "Debian {system.release} {system.architecture}"
+	},
+	"system": {
+		"release":      "wheezy",
+		"sections":     ["main", "contrib", "non-free"],
+		"architecture": "amd64",
+		"bootloader":   "grub",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"packages": {
+		"mirror": "http://gce_debian_mirror.storage.googleapis.com/",
+		"preferences": {
+			"backport-kernel": [
+			{
+				"package": "linux-image-* initramfs-tools",
+				"pin": "release n=wheezy-backports",
+				"pin-priority": 500
+			}
+			],
+			"backport-ssh": [
+			{
+				"package": "init-system-helpers openssh-sftp-server openssh-client openssh-server",
+				"pin": "release n=wheezy-backports",
+				"pin-priority": 500
+			}
+			]
+		}
+	},
+	"plugins": {
+		"ntp": {
+			"servers": ["metadata.google.internal"]
+		}
+	},
+	"volume": {
+		"backing": "raw",
+		"partitions": {
+			"type": "msdos",
+			"root": {
+				"size":       "10GiB",
+				"filesystem": "ext4"
+			}
+		}
+	}
+}
diff --git a/manifests/gce.manifest.json b/manifests/gce.manifest.json
index 2c6fdad..2f2bdb6 100644
--- a/manifests/gce.manifest.json
+++ b/manifests/gce.manifest.json
@@ -17,16 +17,7 @@
 		"charmap":      "UTF-8"
 	},
 	"packages": {
-		"mirror": "http://gce_debian_mirror.storage.googleapis.com/",
-		"preferences": {
-			"backport-kernel": [
-			{
-				"package": "linux-image-* initramfs-tools",
-				"pin": "release n=wheezy-backports",
-				"pin-priority": 500
-			}
-			]
-		}
+		"mirror": "http://gce_debian_mirror.storage.googleapis.com/"
 	},
 	"plugins": {
 		"ntp": {

From 81659321af8e6831f03f71a38e8c1b6fb8ac0d4c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 May 2014 18:29:42 +0200
Subject: [PATCH 003/345] Fix name of chef plugin manifest schema name

---
 bootstrapvz/plugins/chef/manifest-schema.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/chef/manifest-schema.json b/bootstrapvz/plugins/chef/manifest-schema.json
index 9bc9d47..c7b1299 100644
--- a/bootstrapvz/plugins/chef/manifest-schema.json
+++ b/bootstrapvz/plugins/chef/manifest-schema.json
@@ -1,6 +1,6 @@
 {
 	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Puppet plugin manifest",
+	"title": "Chef plugin manifest",
 	"type": "object",
 	"properties": {
 		"plugins": {

From b03ec12c415df68f3c38d3eaf9e186032e4e3e28 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 May 2014 18:30:01 +0200
Subject: [PATCH 004/345] Make "stable" the salt install_source default. Fix a
 few indentation things

---
 bootstrapvz/plugins/salt/manifest-schema.json | 34 +++++++------------
 bootstrapvz/plugins/salt/tasks.py             |  9 ++---
 2 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/bootstrapvz/plugins/salt/manifest-schema.json b/bootstrapvz/plugins/salt/manifest-schema.json
index 82a7952..c6f12ca 100644
--- a/bootstrapvz/plugins/salt/manifest-schema.json
+++ b/bootstrapvz/plugins/salt/manifest-schema.json
@@ -9,30 +9,20 @@
 				"salt": {
 					"type": "object",
 					"properties": {
-						"master": {
-							"type": "string"
-						},
-                        "install_source": {
-                            "type": "string"
-                        },
-						"version": {
-							"type": "string"
-						},
-                        "grains": {
-                            "type": "object",
-                            "patternProperties": {
-						        "^[^\/\\0]+$": {
-							        "type": "string"
-						        }
-					        },
-                            "minItems": 1
-                        }
+						"master": { "type": "string" },
+						"install_source": { "enum": ["stable", "daily", "git"] },
+						"version": { "type": "string" },
+						"grains": {
+							"type": "object",
+							"patternProperties": {
+								"^[^\/\\0]+$": { "type": "string" }
+							},
+							"minItems": 1
+						}
 					},
 					"required": ["install_source"]
 				}
-			},
-			"required": ["salt"]
+			}
 		}
-	},
-	"required": ["plugins"]
+	}
 }
diff --git a/bootstrapvz/plugins/salt/tasks.py b/bootstrapvz/plugins/salt/tasks.py
index 5a1b294..d44cbc3 100644
--- a/bootstrapvz/plugins/salt/tasks.py
+++ b/bootstrapvz/plugins/salt/tasks.py
@@ -34,17 +34,14 @@ class BootstrapSaltMinion(Task):
 
 		# This is needed since bootstrap doesn't handle -X for debian distros properly.
 		# We disable checking for running services at end since we do not start them.
-		sed_i(
-			bootstrap_script, 'install_debian_check_services',
-			"disabled_debian_check_services")
+		sed_i(bootstrap_script, 'install_debian_check_services', 'disabled_debian_check_services')
 
-		bootstrap_command = [
-			'chroot', info.root, 'bash', 'install_salt.sh', '-X']
+		bootstrap_command = ['chroot', info.root, 'bash', 'install_salt.sh', '-X']
 
 		if 'master' in info.manifest.plugins['salt']:
 			bootstrap_command.extend(['-A', info.manifest.plugins['salt']['master']])
 
-		install_source = info.manifest.plugins['salt']['install_source']
+		install_source = info.manifest.plugins['salt'].get('install_source', 'stable')
 
 		bootstrap_command.append(install_source)
 		if install_source == 'git' and ('version' in info.manifest.plugins['salt']):

From 4caf5d1813c0aa915b4fc9c15508bae1f917a2ea Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 May 2014 18:33:21 +0200
Subject: [PATCH 005/345] Make "assets" a required property in chef schema

---
 bootstrapvz/plugins/chef/manifest-schema.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bootstrapvz/plugins/chef/manifest-schema.json b/bootstrapvz/plugins/chef/manifest-schema.json
index c7b1299..9d70622 100644
--- a/bootstrapvz/plugins/chef/manifest-schema.json
+++ b/bootstrapvz/plugins/chef/manifest-schema.json
@@ -11,8 +11,7 @@
 					"properties": {
 						"assets": { "$ref": "#/definitions/absolute_path" }
 					},
-					"minProperties": 1,
-					"additionalProperties": false
+					"required": ["assets"]
 				}
 			}
 		}

From 6fda70a2370aeb1e2d0917b02660de4d99aa4d5d Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 7 May 2014 17:03:16 -0700
Subject: [PATCH 006/345] plugins/docker_daemon: initial import

A plugin that install the docker daemon w/ an init script.
---
 bootstrapvz/plugins/docker_daemon/__init__.py |  23 ++++
 .../docker_daemon/assets/default/docker       |  13 ++
 .../docker_daemon/assets/init.d/docker        | 129 ++++++++++++++++++
 .../docker_daemon/manifest-schema.json        |  24 ++++
 bootstrapvz/plugins/docker_daemon/tasks.py    |  47 +++++++
 5 files changed, 236 insertions(+)
 create mode 100644 bootstrapvz/plugins/docker_daemon/__init__.py
 create mode 100644 bootstrapvz/plugins/docker_daemon/assets/default/docker
 create mode 100644 bootstrapvz/plugins/docker_daemon/assets/init.d/docker
 create mode 100644 bootstrapvz/plugins/docker_daemon/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/docker_daemon/tasks.py

diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
new file mode 100644
index 0000000..c952878
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -0,0 +1,23 @@
+import tasks
+import os.path
+from bootstrapvz.common.exceptions import ManifestError
+
+
+def validate_manifest(data, validator, error):
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	try:
+		validator(data, schema_path)
+	except ManifestError, e:
+		error('docker_daemon manifest validation failed: "%s"' % e.message, e.json_path)
+	if data.get('system', {}).get('release', None) in ['wheezy', 'stable']:
+		# prefs is a generator of apt preferences across files in the manifest
+		prefs = (item for vals in data.get('packages', {}).get('preferences', {}).values() for item in vals)
+		if not any('linux-image' in item['package'] and 'wheezy-backports' in item['pin'] for item in prefs):
+			msg = 'The backports kernel is required for the docker daemon to function properly'
+			error(msg, ['packages', 'preferences'])
+
+
+def resolve_tasks(taskset, manifest):
+	taskset.add(tasks.AddDockerDeps)
+	taskset.add(tasks.AddDockerBinary)
+	taskset.add(tasks.AddDockerInit)
diff --git a/bootstrapvz/plugins/docker_daemon/assets/default/docker b/bootstrapvz/plugins/docker_daemon/assets/default/docker
new file mode 100644
index 0000000..14e6601
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/assets/default/docker
@@ -0,0 +1,13 @@
+# Docker Upstart and SysVinit configuration file
+
+# Customize location of Docker binary (especially for development testing).
+#DOCKER="/usr/local/bin/docker"
+
+# Use DOCKER_OPTS to modify the daemon startup options.
+#DOCKER_OPTS="--dns 8.8.8.8 --dns 8.8.4.4"
+
+# If you need Docker to use an HTTP proxy, it can also be specified here.
+#export http_proxy="http://127.0.0.1:3128/"
+
+# This is also a handy place to tweak where Docker's temporary files go.
+#export TMPDIR="/mnt/bigdrive/docker-tmp"
diff --git a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
new file mode 100644
index 0000000..67f0d28
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
@@ -0,0 +1,129 @@
+#!/bin/sh
+
+### BEGIN INIT INFO
+# Provides:           docker
+# Required-Start:     $syslog $remote_fs
+# Required-Stop:      $syslog $remote_fs
+# Default-Start:      2 3 4 5
+# Default-Stop:       0 1 6
+# Short-Description:  Create lightweight, portable, self-sufficient containers.
+# Description:
+#  Docker is an open-source project to easily create lightweight, portable,
+#  self-sufficient containers from any application. The same container that a
+#  developer builds and tests on a laptop can run at scale, in production, on
+#  VMs, bare metal, OpenStack clusters, public clouds and more.
+### END INIT INFO
+
+export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
+
+BASE=$(basename $0)
+
+# modify these in /etc/default/$BASE (/etc/default/docker)
+DOCKER=/usr/bin/$BASE
+DOCKER_PIDFILE=/var/run/$BASE.pid
+DOCKER_LOGFILE=/var/log/$BASE.log
+DOCKER_OPTS=
+DOCKER_DESC="Docker"
+
+# Get lsb functions
+. /lib/lsb/init-functions
+
+if [ -f /etc/default/$BASE ]; then
+	. /etc/default/$BASE
+fi
+
+# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it)
+if [ -x /sbin/initctl ] && /sbin/initctl version 2>/dev/null | grep -q upstart; then
+	log_failure_msg "$DOCKER_DESC is managed via upstart, try using service $BASE $1"
+	exit 1
+fi
+
+# Check docker is present
+if [ ! -x $DOCKER ]; then
+	log_failure_msg "$DOCKER not present or not executable"
+	exit 1
+fi
+
+fail_unless_root() {
+	if [ "$(id -u)" != '0' ]; then
+		log_failure_msg "$DOCKER_DESC must be run as root"
+		exit 1
+	fi
+}
+
+cgroupfs_mount() {
+	# see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
+	if grep -v '^#' /etc/fstab | grep -q cgroup \
+		|| [ ! -e /proc/cgroups ] \
+		|| [ ! -d /sys/fs/cgroup ]; then
+		return
+	fi
+	if ! mountpoint -q /sys/fs/cgroup; then
+		mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
+	fi
+	(
+		cd /sys/fs/cgroup
+		for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
+			mkdir -p $sys
+			if ! mountpoint -q $sys; then
+				if ! mount -n -t cgroup -o $sys cgroup $sys; then
+					rmdir $sys || true
+				fi
+			fi
+		done
+	)
+}
+
+case "$1" in
+	start)
+		fail_unless_root
+
+		cgroupfs_mount
+
+		touch "$DOCKER_LOGFILE"
+		chgrp docker "$DOCKER_LOGFILE"
+
+		log_begin_msg "Starting $DOCKER_DESC: $BASE"
+		start-stop-daemon --start --background \
+			--no-close \
+			--exec "$DOCKER" \
+			--pidfile "$DOCKER_PIDFILE" \
+			-- \
+				-d -p "$DOCKER_PIDFILE" \
+				$DOCKER_OPTS \
+					>> "$DOCKER_LOGFILE" 2>&1
+		log_end_msg $?
+		;;
+
+	stop)
+		fail_unless_root
+		log_begin_msg "Stopping $DOCKER_DESC: $BASE"
+		start-stop-daemon --stop --pidfile "$DOCKER_PIDFILE"
+		log_end_msg $?
+		;;
+
+	restart)
+		fail_unless_root
+		docker_pid=`cat "$DOCKER_PIDFILE" 2>/dev/null`
+		[ -n "$docker_pid" ] \
+			&& ps -p $docker_pid > /dev/null 2>&1 \
+			&& $0 stop
+		$0 start
+		;;
+
+	force-reload)
+		fail_unless_root
+		$0 restart
+		;;
+
+	status)
+		status_of_proc -p "$DOCKER_PIDFILE" "$DOCKER" docker
+		;;
+
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 1
+		;;
+esac
+
+exit 0
diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.json b/bootstrapvz/plugins/docker_daemon/manifest-schema.json
new file mode 100644
index 0000000..365ccfa
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/manifest-schema.json
@@ -0,0 +1,24 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Install Docker plugin manifest",
+	"type": "object",
+	"properties": {
+		"system": {
+			"type": "object",
+			"properties": {
+				"architecture": {
+					"type": "string",
+					"enum": ["amd64"]
+					// Docker runs on x86_64 only
+				},
+				"release": {
+					"not": {
+						"type": "string",
+						"enum": ["squeeze", "oldstable"]
+						// Docker needs at least wheezy + backports.
+					}
+				}
+			}
+		}
+	}
+}
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
new file mode 100644
index 0000000..b2b1bbe
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -0,0 +1,47 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import initd
+import os
+import os.path
+import shutil
+
+ASSETS_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets'))
+
+
+class AddDockerDeps(Task):
+	description = 'Add packages for docker deps'
+	phase = phases.package_installation
+	DOCKER_DEPS = ['aufs-tools', 'btrfs-tools', 'git', 'iptables',
+	               'procps', 'xz-utils', 'ca-certificates']
+
+	@classmethod
+	def run(cls, info):
+		for pkg in cls.DOCKER_DEPS:
+			info.packages.add(pkg)
+
+
+class AddDockerBinary(Task):
+	description = 'Add docker binary'
+	phase = phases.system_modification
+	DOCKER_URL = 'https://get.docker.io/builds/Linux/x86_64/docker-latest'
+
+	@classmethod
+	def run(cls, info):
+		import urllib
+		bin_docker = os.path.join(info.root, 'usr/bin/docker')
+		urllib.urlretrieve(cls.DOCKER_URL, bin_docker)
+		os.chmod(bin_docker, 0755)
+
+
+class AddDockerInit(Task):
+	description = 'Add docker init script'
+	phase = phases.system_modification
+	successors = [initd.InstallInitScripts]
+
+	@classmethod
+	def run(cls, info):
+		init_src = os.path.join(ASSETS_DIR, 'init.d/docker')
+		info.initd['install']['docker'] = init_src
+		default_src = os.path.join(ASSETS_DIR, 'default/docker')
+		default_dest = os.path.join(info.root, 'etc/default/docker')
+		shutil.copy(default_src, default_dest)

From 7fe72feb2cbf2eab355fcd6f5a5b3b1f9ab46dee Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 18 May 2014 22:26:46 +0200
Subject: [PATCH 007/345] Add --assume-yes to apt-get autoremove

---
 bootstrapvz/common/tasks/apt.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 0d3faef..87732f2 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -154,7 +154,8 @@ class PurgeUnusedPackages(Task):
 	def run(cls, info):
 		log_check_call(['chroot', info.root,
 		                'apt-get', 'autoremove',
-		                           '--purge'])
+		                           '--purge',
+		                           '--assume-yes'])
 
 
 class AptClean(Task):

From 4f45749e13800e673e2230f27b416e348882de85 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Thu, 22 May 2014 17:28:17 +0200
Subject: [PATCH 008/345] Check if qemu-img is available

---
 bootstrapvz/providers/azure/__init__.py      | 1 +
 bootstrapvz/providers/gce/__init__.py        | 1 +
 bootstrapvz/providers/kvm/__init__.py        | 1 +
 bootstrapvz/providers/virtualbox/__init__.py | 1 +
 4 files changed, 4 insertions(+)

diff --git a/bootstrapvz/providers/azure/__init__.py b/bootstrapvz/providers/azure/__init__.py
index c86c549..8b40123 100644
--- a/bootstrapvz/providers/azure/__init__.py
+++ b/bootstrapvz/providers/azure/__init__.py
@@ -21,6 +21,7 @@ def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
 
 	taskset.update([tasks.packages.DefaultPackages,
+	                loopback.AddRequiredCommands,
 	                loopback.Create,
 	                initd.InstallInitScripts,
 	                ssh.AddOpenSSHPackage,
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 291d201..7c0cab6 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -25,6 +25,7 @@ def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
 
 	taskset.update([bootstrapvz.plugins.cloud_init.tasks.AddBackports,
+	                loopback.AddRequiredCommands,
 	                loopback.Create,
 	                tasks.apt.SetPackageRepositories,
 	                tasks.apt.ImportGoogleKey,
diff --git a/bootstrapvz/providers/kvm/__init__.py b/bootstrapvz/providers/kvm/__init__.py
index f09b464..ecded2b 100644
--- a/bootstrapvz/providers/kvm/__init__.py
+++ b/bootstrapvz/providers/kvm/__init__.py
@@ -19,6 +19,7 @@ def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
 
 	taskset.update([tasks.packages.DefaultPackages,
+	                loopback.AddRequiredCommands,
 	                loopback.Create,
 	                initd.InstallInitScripts,
 	                ssh.AddOpenSSHPackage,
diff --git a/bootstrapvz/providers/virtualbox/__init__.py b/bootstrapvz/providers/virtualbox/__init__.py
index 0ba1c1e..ea4ce88 100644
--- a/bootstrapvz/providers/virtualbox/__init__.py
+++ b/bootstrapvz/providers/virtualbox/__init__.py
@@ -17,6 +17,7 @@ def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
 
 	taskset.update([tasks.packages.DefaultPackages,
+	                loopback.AddRequiredCommands,
 	                loopback.Create,
 	                loopback.MoveImage,
 	                ])

From 4ba701cfada88e4b99e41a0525b1ff31aba10f86 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Fri, 23 May 2014 09:51:17 -0300
Subject: [PATCH 009/345] Add CheckAptProxy task

Check if the specified APT proxy server can be reached, informing the
user if this can't be done. That will help them to debug the errors that
will be raised by `apt-get` because of the misleading proxy
configuration.

This closes #95.
---
 bootstrapvz/plugins/apt_proxy/__init__.py |  1 +
 bootstrapvz/plugins/apt_proxy/tasks.py    | 24 ++++++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/apt_proxy/__init__.py b/bootstrapvz/plugins/apt_proxy/__init__.py
index a5b086c..6ca8f71 100644
--- a/bootstrapvz/plugins/apt_proxy/__init__.py
+++ b/bootstrapvz/plugins/apt_proxy/__init__.py
@@ -6,6 +6,7 @@ def validate_manifest(data, validator, error):
 
 def resolve_tasks(taskset, manifest):
 	import tasks
+	taskset.add(tasks.CheckAptProxy)
 	taskset.add(tasks.SetAptProxy)
 	if not manifest.plugins['apt_proxy'].get('persistent', False):
 		taskset.add(tasks.RemoveAptProxy)
diff --git a/bootstrapvz/plugins/apt_proxy/tasks.py b/bootstrapvz/plugins/apt_proxy/tasks.py
index 9d8dbc3..22ac4d9 100644
--- a/bootstrapvz/plugins/apt_proxy/tasks.py
+++ b/bootstrapvz/plugins/apt_proxy/tasks.py
@@ -2,12 +2,34 @@ from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
 import os
+import urllib2
+
+
+class CheckAptProxy(Task):
+	description = 'Checking APT proxy server'
+	phase = phases.package_installation
+
+	@classmethod
+	def run(cls, info):
+		proxy_address = info.manifest.plugins['apt_proxy']['address']
+		proxy_port = info.manifest.plugins['apt_proxy']['port']
+		proxy_url = 'http://{address}:{port}'.format(address=proxy_address, port=proxy_port)
+		try:
+			urllib2.urlopen(proxy_url, timeout=5)
+		except Exception as e:
+			# Default response from `apt-cacher-ng`
+			if isinstance(e, urllib2.HTTPError) and e.code == 404 and e.msg == 'Usage Information':
+				pass
+			else:
+				import logging
+				log = logging.getLogger(__name__)
+				log.error('The APT proxy server couldn\'t be reached. `apt-get` commands may fail from now on.')
 
 
 class SetAptProxy(Task):
 	description = 'Setting proxy for APT'
 	phase = phases.package_installation
-	successors = [apt.AptUpdate]
+	successors = [apt.AptUpdate, CheckAptProxy]
 
 	@classmethod
 	def run(cls, info):

From 08c0a88459854e943160cb4567f516c34a2cf2e6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 23 May 2014 20:19:09 +0200
Subject: [PATCH 010/345] Might as well check apt proxy early Moved
 CheckAptProxy to preparation phase Changed the logging to output a warning
 instead of an error Changed the error message a little

---
 bootstrapvz/plugins/apt_proxy/tasks.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/plugins/apt_proxy/tasks.py b/bootstrapvz/plugins/apt_proxy/tasks.py
index 22ac4d9..6447b70 100644
--- a/bootstrapvz/plugins/apt_proxy/tasks.py
+++ b/bootstrapvz/plugins/apt_proxy/tasks.py
@@ -6,8 +6,8 @@ import urllib2
 
 
 class CheckAptProxy(Task):
-	description = 'Checking APT proxy server'
-	phase = phases.package_installation
+	description = 'Checking reachability of APT proxy server'
+	phase = phases.preparation
 
 	@classmethod
 	def run(cls, info):
@@ -23,13 +23,13 @@ class CheckAptProxy(Task):
 			else:
 				import logging
 				log = logging.getLogger(__name__)
-				log.error('The APT proxy server couldn\'t be reached. `apt-get` commands may fail from now on.')
+				log.warning('The APT proxy server couldn\'t be reached. `apt-get\' commands may fail.')
 
 
 class SetAptProxy(Task):
 	description = 'Setting proxy for APT'
 	phase = phases.package_installation
-	successors = [apt.AptUpdate, CheckAptProxy]
+	successors = [apt.AptUpdate]
 
 	@classmethod
 	def run(cls, info):

From 669ccc3a8ba4b6c0a3fabfc1e29c2edd3e1d2a88 Mon Sep 17 00:00:00 2001
From: Ilya Margolin <ilya@jimdo.com>
Date: Wed, 28 May 2014 19:00:35 +0200
Subject: [PATCH 011/345] Added pip_install plugin

configuration is like

"pip_install": {"packages": ["a_package", "another_package==1.0.1"]}

Installs build-essential and python-dev
---
 CHANGELOG                                     |  3 ++
 bootstrapvz/plugins/pip_install/__init__.py   | 12 +++++++
 .../plugins/pip_install/manifest-schema.json  | 30 ++++++++++++++++++
 bootstrapvz/plugins/pip_install/tasks.py      | 31 +++++++++++++++++++
 4 files changed, 76 insertions(+)
 create mode 100644 bootstrapvz/plugins/pip_install/__init__.py
 create mode 100644 bootstrapvz/plugins/pip_install/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/pip_install/tasks.py

diff --git a/CHANGELOG b/CHANGELOG
index f030ecc..31efe54 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,6 @@
+2014-06-06:
+	Ilya Margolin:
+		* pip_install plugin
 2014-05-04:
 	Dhananjay Balan:
 		* Salt minion installation & configuration plugin
diff --git a/bootstrapvz/plugins/pip_install/__init__.py b/bootstrapvz/plugins/pip_install/__init__.py
new file mode 100644
index 0000000..fe75aae
--- /dev/null
+++ b/bootstrapvz/plugins/pip_install/__init__.py
@@ -0,0 +1,12 @@
+import tasks
+
+
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	validator(data, schema_path)
+
+
+def resolve_tasks(taskset, manifest):
+	taskset.add(tasks.AddPipPackage)
+	taskset.add(tasks.PipInstallCommand)
diff --git a/bootstrapvz/plugins/pip_install/manifest-schema.json b/bootstrapvz/plugins/pip_install/manifest-schema.json
new file mode 100644
index 0000000..6d3f27e
--- /dev/null
+++ b/bootstrapvz/plugins/pip_install/manifest-schema.json
@@ -0,0 +1,30 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Pip install plugin manifest",
+	"type": "object",
+	"properties": {
+		"plugins": {
+			"type": "object",
+			"properties": {
+				"pip_install": {
+					"type": "object",
+					"properties": {
+						"packages": { "$ref": "#/definitions/packages" }
+					},
+					"minProperties": 1,
+					"additionalProperties": false
+				}
+			}
+		}
+	},
+	"definitions": {
+		"packages": {
+			"type":    "array",
+			"items": {
+				"type": "string"
+			},
+			"minItems": 1,
+			"uniqueItems": true
+		}
+	}
+}
diff --git a/bootstrapvz/plugins/pip_install/tasks.py b/bootstrapvz/plugins/pip_install/tasks.py
new file mode 100644
index 0000000..a4e6eb5
--- /dev/null
+++ b/bootstrapvz/plugins/pip_install/tasks.py
@@ -0,0 +1,31 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import network
+from bootstrapvz.common.tasks import packages
+from bootstrapvz.common.tasks import apt
+
+
+class AddPipPackage(Task):
+	description = 'Adding `pip\' and Co. to the image packages'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources]
+	successors = [packages.InstallPackages]
+
+	@classmethod
+	def run(cls, info):
+		for package_name in ('python-pip', 'build-essential', 'python-dev'):
+			info.packages.add(package_name)
+
+
+class PipInstallCommand(Task):
+	description = 'Install python packages from pypi with pip'
+	phase = phases.system_modification
+	successors = [network.RemoveDNSInfo]
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import log_check_call
+		packages = info.manifest.plugins['pip_install']['packages']
+		pip_install_command = ['chroot', info.root, 'pip', 'install']
+		pip_install_command.extend(packages)
+		log_check_call(pip_install_command)

From aa1f8cf1893bc23f1e8162a2bbfa1d55ef33c951 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 7 Jun 2014 12:36:50 -0300
Subject: [PATCH 012/345] Update volume image path after move it

The `loopback.MoveImage` task wasn't updating `info.volume.image_path`
after moving the volume image file. That made `volume.Delete` to fail
when called, because it would try to remove a file that didn't exist
anymore.
---
 bootstrapvz/common/tasks/loopback.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/common/tasks/loopback.py b/bootstrapvz/common/tasks/loopback.py
index 8283320..77f562a 100644
--- a/bootstrapvz/common/tasks/loopback.py
+++ b/bootstrapvz/common/tasks/loopback.py
@@ -45,6 +45,7 @@ class MoveImage(Task):
 		destination = os.path.join(info.manifest.bootstrapper['workspace'], filename)
 		import shutil
 		shutil.move(info.volume.image_path, destination)
+		info.volume.image_path = destination
 		import logging
 		log = logging.getLogger(__name__)
 		log.info('The volume image has been moved to ' + destination)

From 80d0dfb9394a4234cc191eea0eb0d8df41f58ca2 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 7 Jun 2014 12:45:58 -0300
Subject: [PATCH 013/345] Add `volume.Delete` to GCE taskset

This closes #97.
---
 bootstrapvz/providers/gce/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 7c0cab6..f3e3454 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -8,6 +8,7 @@ import tasks.packages
 from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import ssh
+from bootstrapvz.common.tasks import volume
 import bootstrapvz.plugins.cloud_init.tasks
 
 
@@ -43,6 +44,7 @@ def resolve_tasks(taskset, manifest):
 
 	                loopback.MoveImage,
 	                tasks.image.CreateTarball,
+	                volume.Delete,
 	                ])
 
 	if 'gcs_destination' in manifest.image:

From 65bdb34d77549cbdec66eb3b179044e60ff5dbd8 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 7 Jun 2014 13:31:10 -0300
Subject: [PATCH 014/345] Save downloaded `gsutil` tarball to workspace

`gsutil` tarball was being downloaded to the current working directory
and wasn't removed after its extraction.

This will be useful until #87 is merged.
---
 bootstrapvz/providers/gce/tasks/packages.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index 1667d35..4d8f3b5 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -3,7 +3,6 @@ from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tools import log_check_call
 import os
-import os.path
 
 
 class DefaultPackages(Task):
@@ -48,9 +47,12 @@ class InstallGSUtil(Task):
 
 	@classmethod
 	def run(cls, info):
-		log_check_call(['wget', 'http://storage.googleapis.com/pub/gsutil.tar.gz'])
+		gsutil_tarball = os.path.join(info.manifest.bootstrapper['workspace'], 'gsutil.tar.gz')
+		log_check_call(['wget', '--output-document', gsutil_tarball,
+		                'http://storage.googleapis.com/pub/gsutil.tar.gz'])
 		gsutil_directory = os.path.join(info.root, 'usr/local/share/google')
 		gsutil_binary = os.path.join(os.path.join(info.root, 'usr/local/bin'), 'gsutil')
 		os.makedirs(gsutil_directory)
-		log_check_call(['tar', 'xaf', 'gsutil.tar.gz', '-C', gsutil_directory])
+		log_check_call(['tar', 'xaf', gsutil_tarball, '-C', gsutil_directory])
+		os.remove(gsutil_tarball)
 		log_check_call(['ln', '-s', '../share/google/gsutil/gsutil', gsutil_binary])

From 68d9ee1096413be71fe85a038c783c99bc89781b Mon Sep 17 00:00:00 2001
From: Vladimir Vitkov <vvitkov@linux-bg.org>
Date: Thu, 19 Jun 2014 17:55:05 +0300
Subject: [PATCH 015/345] Improve ami listing performance

 * no need to list all available ami's, just self owned should be enough
---
 bootstrapvz/providers/ec2/tasks/ami.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index cb8d459..61ef26e 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -21,7 +21,7 @@ class AMIName(Task):
 		ami_name = info.manifest.image['name'].format(**info.manifest_vars)
 		ami_description = info.manifest.image['description'].format(**info.manifest_vars)
 
-		images = info._ec2['connection'].get_all_images()
+		images = info._ec2['connection'].get_all_images(owner=['self'])
 		for image in images:
 			if ami_name == image.name:
 				msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name)

From a9a5be7717c696aace53048d6d47602041709ce0 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Thu, 19 Jun 2014 14:50:59 -0300
Subject: [PATCH 016/345] Allow stable/oldstable on manifest

This closes #94.
---
 bootstrapvz/base/manifest-schema.json | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/manifest-schema.json b/bootstrapvz/base/manifest-schema.json
index 066c175..5a836f3 100644
--- a/bootstrapvz/base/manifest-schema.json
+++ b/bootstrapvz/base/manifest-schema.json
@@ -41,7 +41,12 @@
 		"system": {
 			"type": "object",
 			"properties": {
-				"release":      { "enum": ["squeeze", "wheezy", "jessie", "testing", "unstable"] },
+				"release": {
+					"enum": [
+						"squeeze", "wheezy", "jessie", "sid",
+						"oldstable", "stable", "testing", "unstable"
+					]
+				},
 				"architecture": { "enum": ["i386", "amd64"] },
 				"bootloader":   { "enum": ["pvgrub", "grub", "extlinux"] },
 				"timezone":     { "type": "string" },

From 19240dc2018047e81a76059e756c984358c43847 Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Thu, 19 Jun 2014 17:14:43 -0700
Subject: [PATCH 017/345] Support executing commands in a specific cwd

Simply plumbed through to Popen(), which already supports this.

Change-Id: If1fdf0a33c96f3aad42407fdc7c9c9f7d4b95c00
---
 bootstrapvz/common/tools.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 4b75d3f..a2704da 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -1,12 +1,12 @@
-def log_check_call(command, stdin=None, env=None, shell=False):
-	status, stdout, stderr = log_call(command, stdin, env, shell)
+def log_check_call(command, stdin=None, env=None, shell=False, cwd=None):
+	status, stdout, stderr = log_call(command, stdin, env, shell, cwd)
 	if status != 0:
 		from subprocess import CalledProcessError
 		raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr))
 	return stdout
 
 
-def log_call(command, stdin=None, env=None, shell=False):
+def log_call(command, stdin=None, env=None, shell=False, cwd=None):
 	import subprocess
 	import logging
 	from multiprocessing.dummy import Pool as ThreadPool
@@ -16,7 +16,7 @@ def log_call(command, stdin=None, env=None, shell=False):
 	log = logging.getLogger(__name__ + command_log)
 	log.debug('Executing: {command}'.format(command=' '.join(command)))
 
-	process = subprocess.Popen(args=command, env=env, shell=shell,
+	process = subprocess.Popen(args=command, env=env, shell=shell, cwd=cwd,
 	                           stdin=subprocess.PIPE,
 	                           stdout=subprocess.PIPE,
 	                           stderr=subprocess.PIPE)

From fb8507c0f4c8b80ddd75b318406612da76bca402 Mon Sep 17 00:00:00 2001
From: Victor Marmol <vmarmol@google.com>
Date: Sun, 22 Jun 2014 05:54:44 +0000
Subject: [PATCH 018/345] Enable the memory cgroup for the Docker plugin.

This will allow for the enforcement and tracking of memory limits and usage.
---
 bootstrapvz/plugins/docker_daemon/__init__.py |  1 +
 bootstrapvz/plugins/docker_daemon/tasks.py    | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index c952878..f32e6b1 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -21,3 +21,4 @@ def resolve_tasks(taskset, manifest):
 	taskset.add(tasks.AddDockerDeps)
 	taskset.add(tasks.AddDockerBinary)
 	taskset.add(tasks.AddDockerInit)
+	taskset.add(tasks.EnableMemoryCgroup)
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index b2b1bbe..d44a661 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -1,6 +1,8 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
+from bootstrapvz.providers.gce.tasks import boot as gceboot
 import os
 import os.path
 import shutil
@@ -45,3 +47,16 @@ class AddDockerInit(Task):
 		default_src = os.path.join(ASSETS_DIR, 'default/docker')
 		default_dest = os.path.join(info.root, 'etc/default/docker')
 		shutil.copy(default_src, default_dest)
+
+
+class EnableMemoryCgroup(Task):
+	description = 'Change grub configuration to enable the memory cgroup'
+	phase = phases.system_modification
+	successors = [boot.InstallGrub]
+	predecessors = [boot.ConfigureGrub, gceboot.ConfigureGrub]
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import sed_i
+		grub_config = os.path.join(info.root, 'etc/default/grub')
+		sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1 cgroup_enable=memory"')

From 091030affd8752cb734b2206a259119833d53b24 Mon Sep 17 00:00:00 2001
From: "Razvan Musaloiu-E." <razvan@musaloiu.com>
Date: Thu, 26 Jun 2014 07:05:46 -0700
Subject: [PATCH 019/345] Fix a typo: s/topoligcally/topologically/

---
 docs/howitworks.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/howitworks.rst b/docs/howitworks.rst
index 7d600cd..cca75c9 100644
--- a/docs/howitworks.rst
+++ b/docs/howitworks.rst
@@ -36,7 +36,7 @@ successors.
 
 The final task list that will be executed is computed by enumerating
 all tasks in the package, placing them in the graph and
-`sorting them topoligcally <http://en.wikipedia.org/wiki/Topological_sort>`_.
+`sorting them topologically <http://en.wikipedia.org/wiki/Topological_sort>`_.
 Subsequently the list returned is filtered to contain only the tasks the
 provider and the plugins added to the taskset.
 

From 869d7d770c1e4b7cfd6edb202ffccd1ed4f5cbbb Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Mon, 30 Jun 2014 20:02:50 +0200
Subject: [PATCH 020/345] Return information about created image.

---
 bootstrapvz/base/main.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index ec14ea6..ad64db6 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -78,6 +78,7 @@ def run(opts):
 		tasklist.run(info=bootstrap_info, dry_run=opts['--dry-run'])
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
+		return bootstrap_info
 	except (Exception, KeyboardInterrupt) as e:
 		# When an error occurs, log it and begin rollback
 		log.exception(e)

From 7412ca26d0cde5176dec99e9bdd627b49f7f243d Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Mon, 30 Jun 2014 20:23:44 +0200
Subject: [PATCH 021/345] Do not require AWS credentials in manifest file

Boto allows for storing credentials in ~/.boto file; user those
if user has not provider one in manifest file.
---
 bootstrapvz/providers/ec2/tasks/connection.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/bootstrapvz/providers/ec2/tasks/connection.py b/bootstrapvz/providers/ec2/tasks/connection.py
index cf203e4..e05cf6c 100644
--- a/bootstrapvz/providers/ec2/tasks/connection.py
+++ b/bootstrapvz/providers/ec2/tasks/connection.py
@@ -29,6 +29,15 @@ class GetCredentials(Task):
 			for key in keys:
 				creds[key] = getenv(env_key(key))
 			return creds
+
+		def provider_key(key):
+			return key.replace('-', '_')
+		import boto.provider
+		provider = boto.provider.Provider('aws')
+		if all(getattr(provider, provider_key(key)) is not None for key in keys):
+			for key in keys:
+				creds[key] = getattr(provider, provider_key(key))
+			return creds
 		raise RuntimeError(('No ec2 credentials found, they must all be specified '
 		                    'exclusively via environment variables or through the manifest.'))
 

From 94b3e4605a11f0af926c0e14afd421fb97febd5c Mon Sep 17 00:00:00 2001
From: Vladimir Vitkov <vvitkov@linux-bg.org>
Date: Wed, 2 Jul 2014 15:04:54 +0300
Subject: [PATCH 022/345] Improvement in grub options

 * Be more robust when setting GRUB_CMDLINE_LINUX_DEFAULT
 * Stop console from blanking
 * Switch elevator to noop as disks are not real disks and this yields
   better performance.
---
 bootstrapvz/providers/ec2/tasks/boot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/tasks/boot.py b/bootstrapvz/providers/ec2/tasks/boot.py
index c325513..11b322f 100644
--- a/bootstrapvz/providers/ec2/tasks/boot.py
+++ b/bootstrapvz/providers/ec2/tasks/boot.py
@@ -44,7 +44,7 @@ class ConfigurePVGrub(Task):
 		sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n'
 		                                        'GRUB_HIDDEN_TIMEOUT=true')
 		sed_i(grub_def, '^#GRUB_TERMINAL=console', 'GRUB_TERMINAL=console')
-		sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT="quiet"', 'GRUB_CMDLINE_LINUX_DEFAULT="console=hvc0"')
+		sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT=.*', 'GRUB_CMDLINE_LINUX_DEFAULT="consoleblank=0 console=hvc0 elevator=noop"')
 
 		from bootstrapvz.common.tools import log_check_call
 		log_check_call(['chroot', info.root, 'update-grub'])

From 55c853c348ca0d85c64c61b8e5d6db142b32cc5c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 5 Jul 2014 11:08:21 +0200
Subject: [PATCH 023/345] Fix call to boto get_all_images()

---
 bootstrapvz/providers/ec2/tasks/ami.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 61ef26e..dcf90ae 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -21,7 +21,7 @@ class AMIName(Task):
 		ami_name = info.manifest.image['name'].format(**info.manifest_vars)
 		ami_description = info.manifest.image['description'].format(**info.manifest_vars)
 
-		images = info._ec2['connection'].get_all_images(owner=['self'])
+		images = info._ec2['connection'].get_all_images(owners=['self'])
 		for image in images:
 			if ami_name == image.name:
 				msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name)

From 5433963f4ec656cd001e00e55f9f0792c5f12f75 Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Sun, 6 Jul 2014 19:05:00 +0200
Subject: [PATCH 024/345] Jessie contains dhcpcd5 instaed of dhcpcd; fixing
 configuration.

---
 bootstrapvz/providers/ec2/tasks/network.py  | 9 +++++----
 bootstrapvz/providers/ec2/tasks/packages.py | 6 +++++-
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/bootstrapvz/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py
index 8d3f695..65aff26 100644
--- a/bootstrapvz/providers/ec2/tasks/network.py
+++ b/bootstrapvz/providers/ec2/tasks/network.py
@@ -11,10 +11,11 @@ class EnableDHCPCDDNS(Task):
 	@classmethod
 	def run(cls, info):
 		# The dhcp client that ships with debian sets the DNS servers per default.
-		# For dhcpcd we need to configure it to do that.
-		from bootstrapvz.common.tools import sed_i
-		dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
-		sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')
+		# For dhcpcd in Wheezy and earlier we need to configure it to do that.
+		if info.release_codename in {'jessie', 'sid'}:
+			from bootstrapvz.common.tools import sed_i
+			dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
+			sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')
 
 
 class AddBuildEssentialPackage(Task):
diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py
index 596d3fb..62b21b5 100644
--- a/bootstrapvz/providers/ec2/tasks/packages.py
+++ b/bootstrapvz/providers/ec2/tasks/packages.py
@@ -11,7 +11,11 @@ class DefaultPackages(Task):
 	@classmethod
 	def run(cls, info):
 		info.packages.add('file')  # Needed for the init scripts
-		info.packages.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
+		# isc-dhcp-client doesn't work properly with ec2
+		if info.release_codename in {'jessie', 'sid'}:
+			info.packages.add('dhcpcd5')
+		else:
+			info.packages.add('dhcpcd')
 
 		info.exclude_packages.add('isc-dhcp-client')
 		info.exclude_packages.add('isc-dhcp-common')

From ebee46b57f2018296c0eca4a6038db43d96ac96f Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Sun, 6 Jul 2014 19:13:46 +0200
Subject: [PATCH 025/345] Fix wrong condition in dhcpcd configuration in
 Jessie.

---
 bootstrapvz/providers/ec2/tasks/network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py
index 65aff26..100f9c8 100644
--- a/bootstrapvz/providers/ec2/tasks/network.py
+++ b/bootstrapvz/providers/ec2/tasks/network.py
@@ -12,7 +12,7 @@ class EnableDHCPCDDNS(Task):
 	def run(cls, info):
 		# The dhcp client that ships with debian sets the DNS servers per default.
 		# For dhcpcd in Wheezy and earlier we need to configure it to do that.
-		if info.release_codename in {'jessie', 'sid'}:
+		if info.release_codename not in {'jessie', 'sid'}:
 			from bootstrapvz.common.tools import sed_i
 			dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
 			sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')

From 62b3e9c2553298ae8de673d11cee73b4c7622e26 Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Tue, 8 Jul 2014 20:59:16 +0200
Subject: [PATCH 026/345] Jessie has different kernel for i386, fixing name for
 EC2.

---
 bootstrapvz/providers/ec2/tasks/packages-kernels.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/providers/ec2/tasks/packages-kernels.json b/bootstrapvz/providers/ec2/tasks/packages-kernels.json
index adbbf97..9fcf825 100644
--- a/bootstrapvz/providers/ec2/tasks/packages-kernels.json
+++ b/bootstrapvz/providers/ec2/tasks/packages-kernels.json
@@ -7,9 +7,9 @@
 	{"i386":  "linux-image-686",
 	 "amd64": "linux-image-amd64"},
 "jessie":
-	{"i386":  "linux-image-686",
+	{"i386":  "linux-image-686-pae",
 	 "amd64": "linux-image-amd64"},
 "sid":
-	{"i386":  "linux-image-686",
+	{"i386":  "linux-image-686-pae",
 	 "amd64": "linux-image-amd64"}
 }

From ebba39f59b3fd9ef14ea8132986f5d351853821b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 5 Jul 2014 18:14:29 +0200
Subject: [PATCH 027/345] Convert "provider" into provider specific section
 This is where all provider specific settings belong like waagent on azure,
 guest additions iso path on vbox and virtualization type on ec2

---
 bootstrapvz/base/manifest-schema.json         |  6 +-
 bootstrapvz/base/manifest.py                  |  2 +-
 bootstrapvz/plugins/cloud_init/tasks.py       |  4 +-
 .../providers/azure/manifest-schema.json      | 17 ++++--
 bootstrapvz/providers/azure/tasks/packages.py |  2 +-
 bootstrapvz/providers/ec2/__init__.py         |  2 +-
 .../providers/ec2/manifest-schema-s3.json     | 25 +++++----
 .../providers/ec2/manifest-schema.json        | 30 ++++++----
 bootstrapvz/providers/ec2/tasks/ami.py        |  4 +-
 bootstrapvz/providers/ec2/tasks/connection.py |  4 +-
 bootstrapvz/providers/kvm/__init__.py         |  2 +-
 .../providers/kvm/manifest-schema.json        |  9 ++-
 bootstrapvz/providers/kvm/tasks/virtio.py     |  2 +-
 bootstrapvz/providers/virtualbox/__init__.py  |  2 +-
 .../virtualbox/tasks/guest_additions.py       |  4 +-
 manifests/azure.manifest.json                 | 12 ++--
 ...fficial-amd64-hvm-cn-north-1.manifest.json | 13 +++--
 ...bs-debian-official-amd64-hvm.manifest.json | 13 +++--
 ...fficial-amd64-pvm-cn-north-1.manifest.json | 13 +++--
 ...bs-debian-official-amd64-pvm.manifest.json | 13 +++--
 ...ebs-debian-official-i386-pvm.manifest.json | 13 +++--
 ...ebs-debian-testing-amd64-pvm.manifest.json | 13 +++--
 ...-ebs-debian-testing-amd64-pvm.manifest.yml | 55 ++++++++++---------
 ...bs-debian-unstable-amd64-pvm.manifest.json | 13 +++--
 ...n-unstable-contrib-amd64-pvm.manifest.json | 13 +++--
 manifests/ec2-ebs-partitioned.manifest.json   | 13 +++--
 manifests/ec2-ebs-single.manifest.json        | 13 +++--
 ...fficial-amd64-pvm-cn-north-1.manifest.json | 16 +++---
 manifests/ec2-s3.manifest.json                | 19 ++++---
 manifests/gce-backports.manifest.json         |  6 +-
 manifests/gce.manifest.json                   |  6 +-
 manifests/kvm-virtio.manifest.json            | 11 ++--
 manifests/kvm.manifest.json                   |  7 ++-
 ...bs-debian-official-amd64-pvm.manifest.json | 12 ++--
 ...ebs-debian-official-i386-pvm.manifest.json | 12 ++--
 manifests/virtualbox-vagrant.manifest.json    |  8 ++-
 manifests/virtualbox.manifest.json            |  8 ++-
 37 files changed, 234 insertions(+), 183 deletions(-)

diff --git a/bootstrapvz/base/manifest-schema.json b/bootstrapvz/base/manifest-schema.json
index 5a836f3..7a5040c 100644
--- a/bootstrapvz/base/manifest-schema.json
+++ b/bootstrapvz/base/manifest-schema.json
@@ -4,7 +4,11 @@
 	"type": "object",
 	"properties": {
 		"provider": {
-			"type": "string"
+			"type": "object",
+			"properties": {
+				"name": { "type": "string" }
+			},
+			"required": ["name"]
 		},
 		"bootstrapper": {
 			"type": "object",
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index e1d6d95..f982b44 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -39,7 +39,7 @@ class Manifest(object):
 			self.data = load_yaml(self.path)
 
 		# Get the provider name from the manifest and load the corresponding module
-		provider_modname = 'bootstrapvz.providers.' + self.data['provider']
+		provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name']
 		log.debug('Loading provider ' + provider_modname)
 		# Create a modules dict that contains the loaded provider and plugins
 		import importlib
diff --git a/bootstrapvz/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py
index 007ad86..44e2dff 100644
--- a/bootstrapvz/plugins/cloud_init/tasks.py
+++ b/bootstrapvz/plugins/cloud_init/tasks.py
@@ -63,10 +63,10 @@ class SetMetadataSource(Task):
 			sources = info.manifest.plugins['cloud_init']['metadata_sources']
 		else:
 			source_mapping = {'ec2': 'Ec2'}
-			sources = source_mapping.get(info.manifest.provider, None)
+			sources = source_mapping.get(info.manifest.provider['name'], None)
 			if sources is None:
 				msg = ('No cloud-init metadata source mapping found for provider `{provider}\', '
-				       'skipping selections setting.').format(provider=info.manifest.provider)
+				       'skipping selections setting.').format(provider=info.manifest.provider['name'])
 				logging.getLogger(__name__).warn(msg)
 				return
 		sources = "cloud-init	cloud-init/datasources	multiselect	" + sources
diff --git a/bootstrapvz/providers/azure/manifest-schema.json b/bootstrapvz/providers/azure/manifest-schema.json
index 97b4330..52516e6 100644
--- a/bootstrapvz/providers/azure/manifest-schema.json
+++ b/bootstrapvz/providers/azure/manifest-schema.json
@@ -3,13 +3,9 @@
 	"title": "Azure manifest",
 	"type": "object",
 	"properties": {
-		"system": {
+		"provider": {
 			"type": "object",
 			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["grub", "extlinux"]
-				},
 				"waagent": {
 					"type": "object",
 					"properties": {
@@ -22,9 +18,18 @@
 					},
 					"required": ["version"]
 				}
-			},
+			}
 			"required": ["waagent"]
 		},
+		"system": {
+			"type": "object",
+			"properties": {
+				"bootloader": {
+					"type": "string",
+					"enum": ["grub", "extlinux"]
+				}
+			},
+		},
 		"volume": {
 			"type": "object",
 			"properties": {
diff --git a/bootstrapvz/providers/azure/tasks/packages.py b/bootstrapvz/providers/azure/tasks/packages.py
index 6cb4fa0..40a51e2 100644
--- a/bootstrapvz/providers/azure/tasks/packages.py
+++ b/bootstrapvz/providers/azure/tasks/packages.py
@@ -29,7 +29,7 @@ class Waagent(Task):
 	def run(cls, info):
 		from bootstrapvz.common.tools import log_check_call
 		import os
-		waagent_version = info.manifest.system['waagent']['version']
+		waagent_version = info.manifest.provider['waagent']['version']
 		waagent_file = 'WALinuxAgent-' + waagent_version + '.tar.gz'
 		waagent_url = 'https://github.com/Azure/WALinuxAgent/archive/' + waagent_file
 		log_check_call(['wget', '-P', info.root, waagent_url])
diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index fd5c804..4a06d39 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -38,7 +38,7 @@ def validate_manifest(data, validator, error):
 		validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json'))
 
 	bootloader = data['system']['bootloader']
-	virtualization = data['virtualization']
+	virtualization = data['provider']['virtualization']
 	backing = data['volume']['backing']
 	partition_type = data['volume']['partitions']['type']
 
diff --git a/bootstrapvz/providers/ec2/manifest-schema-s3.json b/bootstrapvz/providers/ec2/manifest-schema-s3.json
index 79cd334..6c92f33 100644
--- a/bootstrapvz/providers/ec2/manifest-schema-s3.json
+++ b/bootstrapvz/providers/ec2/manifest-schema-s3.json
@@ -3,18 +3,23 @@
 	"title": "EC2 manifest for instance store AMIs",
 	"type": "object",
 	"properties": {
-		"credentials": {
+		"provider": {
 			"type": "object",
 			"properties": {
-				"certificate": {
-					"type": "string"
-				},
-				"private-key": {
-					"type": "string"
-				},
-				"user-id": {
-					"type": "string",
-					"pattern": "(^arn:aws:iam::\\d*:user/\\w.*$)|(^\\d{4}-\\d{4}-\\d{4}$)"
+				"credentials": {
+					"type": "object",
+					"properties": {
+						"certificate": {
+							"type": "string"
+						},
+						"private-key": {
+							"type": "string"
+						},
+						"user-id": {
+							"type": "string",
+							"pattern": "(^arn:aws:iam::\\d*:user/\\w.*$)|(^\\d{4}-\\d{4}-\\d{4}$)"
+						}
+					}
 				}
 			}
 		},
diff --git a/bootstrapvz/providers/ec2/manifest-schema.json b/bootstrapvz/providers/ec2/manifest-schema.json
index bb4bd97..2baed68 100644
--- a/bootstrapvz/providers/ec2/manifest-schema.json
+++ b/bootstrapvz/providers/ec2/manifest-schema.json
@@ -3,7 +3,24 @@
 	"title": "EC2 manifest",
 	"type": "object",
 	"properties": {
-		"virtualization": { "enum": ["pvm", "hvm"] },
+		"provider": {
+			"type": "object",
+			"properties": {
+				"virtualization": { "enum": ["pvm", "hvm"] },
+				"credentials": {
+					"type": "object",
+					"properties": {
+						"access-key": {
+							"type": "string"
+						},
+						"secret-key": {
+							"type": "string"
+						}
+					}
+				}
+			},
+			"required": ["virtualization"]
+		},
 		"image": {
 			"type": "object",
 			"properties": {
@@ -12,17 +29,6 @@
 				}
 			}
 		},
-		"credentials": {
-			"type": "object",
-			"properties": {
-				"access-key": {
-					"type": "string"
-				},
-				"secret-key": {
-					"type": "string"
-				}
-			}
-		},
 		"system": {
 			"type": "object",
 			"properties": {
diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index dcf90ae..2561769 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -103,7 +103,7 @@ class RegisterAMI(Task):
 			registration_params['image_location'] = info._ec2['manifest_location']
 		else:
 			root_dev_name = {'pvm': '/dev/sda',
-			                 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization'])
+			                 'hvm': '/dev/xvda'}.get(info.manifest.provider['virtualization'])
 			registration_params['root_device_name'] = root_dev_name
 
 			from boto.ec2.blockdevicemapping import BlockDeviceType
@@ -113,7 +113,7 @@ class RegisterAMI(Task):
 			registration_params['block_device_map'] = BlockDeviceMapping()
 			registration_params['block_device_map'][root_dev_name] = block_device
 
-		if info.manifest.data['virtualization'] == 'hvm':
+		if info.manifest.provider['virtualization'] == 'hvm':
 			registration_params['virtualization_type'] = 'hvm'
 		else:
 			registration_params['virtualization_type'] = 'paravirtual'
diff --git a/bootstrapvz/providers/ec2/tasks/connection.py b/bootstrapvz/providers/ec2/tasks/connection.py
index e05cf6c..e68cd6d 100644
--- a/bootstrapvz/providers/ec2/tasks/connection.py
+++ b/bootstrapvz/providers/ec2/tasks/connection.py
@@ -18,9 +18,9 @@ class GetCredentials(Task):
 	def get_credentials(cls, manifest, keys):
 		from os import getenv
 		creds = {}
-		if all(key in manifest.data['credentials'] for key in keys):
+		if all(key in manifest.provider['credentials'] for key in keys):
 			for key in keys:
-				creds[key] = manifest.data['credentials'][key]
+				creds[key] = manifest.provider['credentials'][key]
 			return creds
 
 		def env_key(key):
diff --git a/bootstrapvz/providers/kvm/__init__.py b/bootstrapvz/providers/kvm/__init__.py
index ecded2b..f678bb0 100644
--- a/bootstrapvz/providers/kvm/__init__.py
+++ b/bootstrapvz/providers/kvm/__init__.py
@@ -28,7 +28,7 @@ def resolve_tasks(taskset, manifest):
 	                loopback.MoveImage,
 	                ])
 
-	if manifest.bootstrapper.get('virtio', []):
+	if manifest.provider.get('virtio', []):
 		from tasks import virtio
 		taskset.update([virtio.VirtIO])
 
diff --git a/bootstrapvz/providers/kvm/manifest-schema.json b/bootstrapvz/providers/kvm/manifest-schema.json
index 3586e4e..a800409 100644
--- a/bootstrapvz/providers/kvm/manifest-schema.json
+++ b/bootstrapvz/providers/kvm/manifest-schema.json
@@ -3,7 +3,7 @@
 	"title": "KVM manifest",
 	"type": "object",
 	"properties": {
-		"system": {
+		"provider": {
 			"type": "object",
 			"properties": {
 				"virtio": {
@@ -18,7 +18,12 @@
 						         "virtio_ring"]
 					},
 					"minItems": 1
-				},
+				}
+			}
+		},
+		"system": {
+			"type": "object",
+			"properties": {
 				"bootloader": {
 					"type": "string",
 					"enum": ["grub", "extlinux"]
diff --git a/bootstrapvz/providers/kvm/tasks/virtio.py b/bootstrapvz/providers/kvm/tasks/virtio.py
index cb3ae68..402ba1a 100644
--- a/bootstrapvz/providers/kvm/tasks/virtio.py
+++ b/bootstrapvz/providers/kvm/tasks/virtio.py
@@ -12,5 +12,5 @@ class VirtIO(Task):
 		modules = os.path.join(info.root, '/etc/initramfs-tools/modules')
 		with open(modules, "a") as modules_file:
 			modules_file.write("\n")
-			for module in info.manifest.system.get('virtio', []):
+			for module in info.manifest.provider.get('virtio', []):
 				modules_file.write(module + "\n")
diff --git a/bootstrapvz/providers/virtualbox/__init__.py b/bootstrapvz/providers/virtualbox/__init__.py
index ea4ce88..4645ff5 100644
--- a/bootstrapvz/providers/virtualbox/__init__.py
+++ b/bootstrapvz/providers/virtualbox/__init__.py
@@ -22,7 +22,7 @@ def resolve_tasks(taskset, manifest):
 	                loopback.MoveImage,
 	                ])
 
-	if manifest.bootstrapper.get('guest_additions', False):
+	if manifest.provider.get('guest_additions', False):
 		from tasks import guest_additions
 		taskset.update([guest_additions.CheckGuestAdditionsPath,
 		                guest_additions.AddGuestAdditionsPackages,
diff --git a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
index e9a6257..726ef29 100644
--- a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
+++ b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
@@ -11,7 +11,7 @@ class CheckGuestAdditionsPath(Task):
 	@classmethod
 	def run(cls, info):
 		import os.path
-		guest_additions_path = info.manifest.bootstrapper['guest_additions']
+		guest_additions_path = info.manifest.provider['guest_additions']
 		if not os.path.exists(guest_additions_path):
 			msg = 'The file {file} does not exist.'.format(file=guest_additions_path)
 			raise TaskError(msg)
@@ -43,7 +43,7 @@ class InstallGuestAdditions(Task):
 	@classmethod
 	def run(cls, info):
 		import os
-		guest_additions_path = info.manifest.bootstrapper['guest_additions']
+		guest_additions_path = info.manifest.provider['guest_additions']
 		mount_dir = 'mnt/guest_additions'
 		mount_path = os.path.join(info.root, mount_dir)
 		os.mkdir(mount_path)
diff --git a/manifests/azure.manifest.json b/manifests/azure.manifest.json
index 4007691..a8eebc9 100644
--- a/manifests/azure.manifest.json
+++ b/manifests/azure.manifest.json
@@ -1,5 +1,10 @@
 {
-	"provider": "azure",
+	"provider": {
+		"name": "azure",
+		"waagent": {
+			"version":  "2.0.4"
+		}
+	},
 	"bootstrapper": {
 		"workspace":      "/target",
 		"mirror": "http://ftp.fr.debian.org/debian/"
@@ -14,10 +19,7 @@
 		"bootloader":   "grub",
 		"timezone":     "UTC",
 		"locale":       "en_US",
-		"charmap":      "UTF-8",
-		"waagent": {
-			"version":  "2.0.4"
-		}
+		"charmap":      "UTF-8"
 	},
 	"packages": {
 	},
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
index 622879e..0c3d844 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "hvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "hvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
index 651d211..32d2e5f 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "hvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "hvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
index 4c88efb..01e9c63 100644
--- a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
index 34ced43..4f37a23 100644
--- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
index b0a7db3..b7244a8 100644
--- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
+++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
index 0456892..03035d6 100644
--- a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
+++ b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
index cfdbd73..8fb5a53 100644
--- a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
@@ -1,36 +1,37 @@
 ---
-provider: "ec2"
-virtualization: "pvm"
-credentials:
+provider:
+  name: "ec2"
+  virtualization: "pvm"
+  credentials:
     access-key: ""
     secret-key: ""
 bootstrapper:
-    workspace: "/target"
+  workspace: "/target"
 image:
-    name: "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs"
-    description: "Debian {system.release} {system.architecture}"
+  name: "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs"
+  description: "Debian {system.release} {system.architecture}"
 system:
-    release: "testing"
-    architecture: "amd64"
-    bootloader: "pvgrub"
-    timezone: "UTC"
-    locale: "en_US"
-    charmap: "UTF-8"
+  release: "testing"
+  architecture: "amd64"
+  bootloader: "pvgrub"
+  timezone: "UTC"
+  locale: "en_US"
+  charmap: "UTF-8"
 packages:
-    #mirror: "http://cloudfront.debian.net/debian"
-    install_standard: true
+  #mirror: "http://cloudfront.debian.net/debian"
+  install_standard: true
 volume:
-    backing: "ebs"
-    partitions:
-        type: "none"
-        root:
-            size: "8GiB"
-            filesystem: "ext4"
+  backing: "ebs"
+  partitions:
+    type: "none"
+    root:
+      size: "8GiB"
+      filesystem: "ext4"
 plugins:
-    cloud_init:
-        username: "admin"
-        #metadata_sources: "Ec2"
-        disable_modules:
-            - "landscape"
-            - "byobu"
-            - "ssh-import-id"
+  cloud_init:
+    username: "admin"
+    #metadata_sources: "Ec2"
+    disable_modules:
+      - "landscape"
+      - "byobu"
+      - "ssh-import-id"
diff --git a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
index 98e0f0a..04cf637 100644
--- a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
+++ b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
index eaf9584..3af6564 100644
--- a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
+++ b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json
index 441c076..4f2606f 100644
--- a/manifests/ec2-ebs-partitioned.manifest.json
+++ b/manifests/ec2-ebs-partitioned.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-ebs-single.manifest.json b/manifests/ec2-ebs-single.manifest.json
index 1e7f883..72fa0ca 100644
--- a/manifests/ec2-ebs-single.manifest.json
+++ b/manifests/ec2-ebs-single.manifest.json
@@ -1,11 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
index 6039091..d5f989e 100644
--- a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
+++ b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
@@ -1,14 +1,12 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key":  null,
-		// "secret-key":  null,
-		// "certificate": null,
-		// "private-key": null,
-		// "user-id":     null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json
index fcf7ca2..36bcfad 100644
--- a/manifests/ec2-s3.manifest.json
+++ b/manifests/ec2-s3.manifest.json
@@ -1,14 +1,15 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key":  null,
-		// "secret-key":  null,
-		// "certificate": null,
-		// "private-key": null,
-		// "user-id":     null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key":  null,
+			// "secret-key":  null,
+			// "certificate": null,
+			// "private-key": null,
+			// "user-id":     null
+		}
 	},
-
 	"bootstrapper": {
 		"workspace": "/target"
 	},
diff --git a/manifests/gce-backports.manifest.json b/manifests/gce-backports.manifest.json
index 1334ab2..44f087d 100644
--- a/manifests/gce-backports.manifest.json
+++ b/manifests/gce-backports.manifest.json
@@ -1,7 +1,9 @@
 {
-	"provider": "gce",
+	"provider": {
+		"name": "gce"
+	},
 	"bootstrapper": {
-		"workspace":      "/target"
+		"workspace": "/target"
 	},
 	"image": {
 		"name":        "disk",
diff --git a/manifests/gce.manifest.json b/manifests/gce.manifest.json
index 2f2bdb6..9d2e079 100644
--- a/manifests/gce.manifest.json
+++ b/manifests/gce.manifest.json
@@ -1,7 +1,9 @@
 {
-	"provider": "gce",
+	"provider": {
+		"name": "gce"
+	},
 	"bootstrapper": {
-		"workspace":      "/target"
+		"workspace": "/target"
 	},
 	"image": {
 		"name":        "disk",
diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json
index a7a92b0..33c8b23 100644
--- a/manifests/kvm-virtio.manifest.json
+++ b/manifests/kvm-virtio.manifest.json
@@ -1,8 +1,10 @@
 {
-	"provider": "kvm",
+	"provider": {
+		"name": "kvm",
+		"virtio_modules": [ "virtio_pci", "virtio_blk" ]
+	},
 	"bootstrapper": {
-		"workspace":      "/target",
-		"mirror": "http://ftp.fr.debian.org/debian/"
+		"workspace": "/target"
 	},
 	"image": {
 		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
@@ -14,8 +16,7 @@
 		"bootloader":     "grub",
 		"timezone":       "UTC",
 		"locale":         "en_US",
-		"charmap":        "UTF-8",
-		"virtio_modules": [ "virtio_pci", "virtio_blk" ]
+		"charmap":        "UTF-8"
 	},
 	"packages": {},
 	"volume": {
diff --git a/manifests/kvm.manifest.json b/manifests/kvm.manifest.json
index ea4c24a..bd26b50 100644
--- a/manifests/kvm.manifest.json
+++ b/manifests/kvm.manifest.json
@@ -1,8 +1,9 @@
 {
-	"provider": "kvm",
+	"provider": {
+		"name": "kvm"
+	},
 	"bootstrapper": {
-		"workspace":      "/target",
-		"mirror": "http://ftp.fr.debian.org/debian/"
+		"workspace": "/target"
 	},
 	"image": {
 		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
diff --git a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
index 3be23f7..d9e8e29 100644
--- a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
+++ b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
@@ -1,9 +1,11 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
 
 	"bootstrapper": {
diff --git a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
index aaa5669..703eea8 100644
--- a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
+++ b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
@@ -1,9 +1,11 @@
 {
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		// "access-key": null,
-		// "secret-key": null
+	"provider": {
+		"name": "ec2",
+		"virtualization": "pvm",
+		"credentials":    {
+			// "access-key": null,
+			// "secret-key": null
+		}
 	},
 
 	"bootstrapper": {
diff --git a/manifests/virtualbox-vagrant.manifest.json b/manifests/virtualbox-vagrant.manifest.json
index 6641ac2..632c0b4 100644
--- a/manifests/virtualbox-vagrant.manifest.json
+++ b/manifests/virtualbox-vagrant.manifest.json
@@ -1,9 +1,11 @@
 {
-	"provider": "virtualbox",
-	"bootstrapper": {
-		"workspace":       "/target",
+	"provider": {
+		"name":            "virtualbox",
 		"guest_additions": "/root/images/VBoxGuestAdditions.iso"
 	},
+	"bootstrapper": {
+		"workspace": "/target"
+	},
 	"image": {
 		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
 		"description": "Debian {system.release} {system.architecture}"
diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json
index 062cc9c..9e69292 100644
--- a/manifests/virtualbox.manifest.json
+++ b/manifests/virtualbox.manifest.json
@@ -1,8 +1,10 @@
 {
-	"provider": "virtualbox",
+	"provider": {
+		"name":            "virtualbox",
+		"guest_additions": "/root/images/VBoxGuestAdditions.iso"
+	},
 	"bootstrapper": {
-		"workspace":      "/target"
-		// "guest_additions": "/root/images/VBoxGuestAdditions.iso"
+		"workspace": "/target"
 	},
 	"image": {
 		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",

From 03a0746299f8ae73abdb4c03315b7908a1b2b0cd Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 5 Jul 2014 20:01:20 +0200
Subject: [PATCH 028/345] Convert every JSON file to YAML Lines removed: over
 500. Readiblity gained: A shitload Now you can actually get an overview of a
 manifest on a single screen height. I am sure that it will also save a lot of
 hassle when modifying schema in the future. No more "expected property name"
 etc. because of an extraneous comma Comments are of course natively support,
 so there's no need for this minify_json hokey pokey

---
 bootstrapvz/base/__init__.py                  |   2 +-
 bootstrapvz/base/bootstrapinfo.py             |   2 +-
 bootstrapvz/base/manifest-schema.json         | 214 ------------------
 bootstrapvz/base/manifest-schema.yml          | 177 +++++++++++++++
 bootstrapvz/base/manifest.py                  |  19 +-
 bootstrapvz/base/release-codenames.json       |  22 --
 bootstrapvz/base/release-codenames.yml        |  23 ++
 bootstrapvz/common/exceptions.py              |  12 +-
 .../common/tasks/network-configuration.json   |  14 --
 .../common/tasks/network-configuration.yml    |  17 ++
 bootstrapvz/common/tasks/network.py           |   2 +-
 bootstrapvz/common/tools.py                   |  17 +-
 bootstrapvz/plugins/admin_user/__init__.py    |   2 +-
 .../plugins/admin_user/manifest-schema.json   |  21 --
 .../plugins/admin_user/manifest-schema.yml    |  13 ++
 bootstrapvz/plugins/apt_proxy/__init__.py     |   2 +-
 .../plugins/apt_proxy/manifest-schema.json    |  27 ---
 .../plugins/apt_proxy/manifest-schema.yml     |  17 ++
 bootstrapvz/plugins/chef/__init__.py          |   2 +-
 bootstrapvz/plugins/chef/manifest-schema.json |  25 --
 bootstrapvz/plugins/chef/manifest-schema.yml  |  19 ++
 bootstrapvz/plugins/cloud_init/__init__.py    |   2 +-
 .../plugins/cloud_init/manifest-schema.json   |  44 ----
 .../plugins/cloud_init/manifest-schema.yml    |  30 +++
 bootstrapvz/plugins/docker_daemon/__init__.py |   8 +-
 .../docker_daemon/manifest-schema.json        |  24 --
 .../plugins/docker_daemon/manifest-schema.yml |  17 ++
 .../plugins/image_commands/__init__.py        |   2 +-
 .../image_commands/manifest-schema.json       |  29 ---
 .../image_commands/manifest-schema.yml        |  25 ++
 bootstrapvz/plugins/minimize_size/__init__.py |   2 +-
 .../minimize_size/manifest-schema.json        |  19 --
 .../plugins/minimize_size/manifest-schema.yml |  15 ++
 bootstrapvz/plugins/ntp/__init__.py           |   2 +-
 bootstrapvz/plugins/ntp/manifest-schema.json  |  22 --
 bootstrapvz/plugins/ntp/manifest-schema.yml   |  15 ++
 bootstrapvz/plugins/pip_install/__init__.py   |   2 +-
 .../plugins/pip_install/manifest-schema.json  |  30 ---
 .../plugins/pip_install/manifest-schema.yml   |  17 ++
 .../plugins/prebootstrapped/__init__.py       |   2 +-
 .../prebootstrapped/manifest-schema.json      |  34 ---
 .../prebootstrapped/manifest-schema.yml       |  25 ++
 bootstrapvz/plugins/puppet/__init__.py        |   2 +-
 .../plugins/puppet/manifest-schema.json       |  28 ---
 .../plugins/puppet/manifest-schema.yml        |  20 ++
 bootstrapvz/plugins/root_password/__init__.py |   2 +-
 .../root_password/manifest-schema.json        |  21 --
 .../plugins/root_password/manifest-schema.yml |  13 ++
 bootstrapvz/plugins/salt/__init__.py          |   2 +-
 bootstrapvz/plugins/salt/manifest-schema.json |  28 ---
 bootstrapvz/plugins/salt/manifest-schema.yml  |  24 ++
 .../plugins/unattended_upgrades/__init__.py   |   2 +-
 .../unattended_upgrades/manifest-schema.json  |  29 ---
 .../unattended_upgrades/manifest-schema.yml   |  18 ++
 bootstrapvz/plugins/vagrant/__init__.py       |   2 +-
 .../plugins/vagrant/manifest-schema.json      |  34 ---
 .../plugins/vagrant/manifest-schema.yml       |  24 ++
 bootstrapvz/providers/azure/__init__.py       |   2 +-
 .../providers/azure/manifest-schema.json      |  50 ----
 .../providers/azure/manifest-schema.yml       |  38 ++++
 bootstrapvz/providers/ec2/__init__.py         |   4 +-
 .../providers/ec2/manifest-schema-s3.json     |  49 ----
 .../providers/ec2/manifest-schema-s3.yml      |  38 ++++
 .../providers/ec2/manifest-schema.json        |  56 -----
 bootstrapvz/providers/ec2/manifest-schema.yml |  47 ++++
 bootstrapvz/providers/ec2/tasks/ami-akis.json |  34 ---
 bootstrapvz/providers/ec2/tasks/ami-akis.yml  |  33 +++
 bootstrapvz/providers/ec2/tasks/ami.py        |   2 +-
 .../providers/ec2/tasks/packages-kernels.json |  15 --
 .../providers/ec2/tasks/packages-kernels.yml  |  14 ++
 bootstrapvz/providers/ec2/tasks/packages.py   |   2 +-
 bootstrapvz/providers/gce/__init__.py         |   2 +-
 .../providers/gce/manifest-schema.json        |  43 ----
 bootstrapvz/providers/gce/manifest-schema.yml |  29 +++
 bootstrapvz/providers/gce/tasks/packages.py   |   2 +-
 bootstrapvz/providers/kvm/__init__.py         |   2 +-
 .../providers/kvm/manifest-schema.json        |  50 ----
 bootstrapvz/providers/kvm/manifest-schema.yml |  44 ++++
 bootstrapvz/providers/virtualbox/__init__.py  |   2 +-
 .../providers/virtualbox/manifest-schema.json |  40 ----
 .../providers/virtualbox/manifest-schema.yml  |  36 +++
 manifests/azure.manifest.json                 |  45 ----
 manifests/azure.manifest.yml                  |  33 +++
 ...fficial-amd64-hvm-cn-north-1.manifest.json |  44 ----
 ...official-amd64-hvm-cn-north-1.manifest.yml |  32 +++
 ...bs-debian-official-amd64-hvm.manifest.json |  44 ----
 ...ebs-debian-official-amd64-hvm.manifest.yml |  32 +++
 ...fficial-amd64-pvm-cn-north-1.manifest.json |  44 ----
 ...official-amd64-pvm-cn-north-1.manifest.yml |  32 +++
 ...bs-debian-official-amd64-pvm.manifest.json |  44 ----
 ...ebs-debian-official-amd64-pvm.manifest.yml |  32 +++
 ...ebs-debian-official-i386-pvm.manifest.json |  44 ----
 ...-ebs-debian-official-i386-pvm.manifest.yml |  32 +++
 ...ebs-debian-testing-amd64-pvm.manifest.json |  44 ----
 ...-ebs-debian-testing-amd64-pvm.manifest.yml |  49 ++--
 ...bs-debian-unstable-amd64-pvm.manifest.json |  44 ----
 ...ebs-debian-unstable-amd64-pvm.manifest.yml |  32 +++
 ...n-unstable-contrib-amd64-pvm.manifest.json |  45 ----
 ...an-unstable-contrib-amd64-pvm.manifest.yml |  36 +++
 manifests/ec2-ebs-partitioned.manifest.json   |  38 ----
 manifests/ec2-ebs-partitioned.manifest.yml    |  28 +++
 manifests/ec2-ebs-single.manifest.json        |  38 ----
 manifests/ec2-ebs-single.manifest.yml         |  28 +++
 ...fficial-amd64-pvm-cn-north-1.manifest.json |  46 ----
 ...official-amd64-pvm-cn-north-1.manifest.yml |  40 ++++
 manifests/ec2-s3.manifest.json                |  43 ----
 manifests/ec2-s3.manifest.yml                 |  33 +++
 manifests/gce-backports.manifest.json         |  55 -----
 manifests/gce-backports.manifest.yml          |  41 ++++
 manifests/gce.manifest.json                   |  39 ----
 manifests/gce.manifest.yml                    |  32 +++
 manifests/kvm-virtio.manifest.json            |  42 ----
 manifests/kvm-virtio.manifest.yml             |  34 +++
 manifests/kvm.manifest.json                   |  41 ----
 manifests/kvm.manifest.yml                    |  31 +++
 ...bs-debian-official-amd64-pvm.manifest.json |  44 ----
 ...ebs-debian-official-amd64-pvm.manifest.yml |  31 +++
 ...ebs-debian-official-i386-pvm.manifest.json |  44 ----
 ...-ebs-debian-official-i386-pvm.manifest.yml |  31 +++
 manifests/virtualbox-vagrant.manifest.json    |  42 ----
 manifests/virtualbox-vagrant.manifest.yml     |  32 +++
 manifests/virtualbox.manifest.json            |  37 ---
 manifests/virtualbox.manifest.yml             |  29 +++
 123 files changed, 1518 insertions(+), 1987 deletions(-)
 delete mode 100644 bootstrapvz/base/manifest-schema.json
 create mode 100644 bootstrapvz/base/manifest-schema.yml
 delete mode 100644 bootstrapvz/base/release-codenames.json
 create mode 100644 bootstrapvz/base/release-codenames.yml
 delete mode 100644 bootstrapvz/common/tasks/network-configuration.json
 create mode 100644 bootstrapvz/common/tasks/network-configuration.yml
 delete mode 100644 bootstrapvz/plugins/admin_user/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/admin_user/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/apt_proxy/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/apt_proxy/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/chef/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/chef/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/cloud_init/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/cloud_init/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/docker_daemon/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/docker_daemon/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/image_commands/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/image_commands/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/minimize_size/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/minimize_size/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/ntp/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/ntp/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/pip_install/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/pip_install/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/prebootstrapped/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/puppet/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/puppet/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/root_password/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/root_password/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/salt/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/salt/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/unattended_upgrades/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
 delete mode 100644 bootstrapvz/plugins/vagrant/manifest-schema.json
 create mode 100644 bootstrapvz/plugins/vagrant/manifest-schema.yml
 delete mode 100644 bootstrapvz/providers/azure/manifest-schema.json
 create mode 100644 bootstrapvz/providers/azure/manifest-schema.yml
 delete mode 100644 bootstrapvz/providers/ec2/manifest-schema-s3.json
 create mode 100644 bootstrapvz/providers/ec2/manifest-schema-s3.yml
 delete mode 100644 bootstrapvz/providers/ec2/manifest-schema.json
 create mode 100644 bootstrapvz/providers/ec2/manifest-schema.yml
 delete mode 100644 bootstrapvz/providers/ec2/tasks/ami-akis.json
 create mode 100644 bootstrapvz/providers/ec2/tasks/ami-akis.yml
 delete mode 100644 bootstrapvz/providers/ec2/tasks/packages-kernels.json
 create mode 100644 bootstrapvz/providers/ec2/tasks/packages-kernels.yml
 delete mode 100644 bootstrapvz/providers/gce/manifest-schema.json
 create mode 100644 bootstrapvz/providers/gce/manifest-schema.yml
 delete mode 100644 bootstrapvz/providers/kvm/manifest-schema.json
 create mode 100644 bootstrapvz/providers/kvm/manifest-schema.yml
 delete mode 100644 bootstrapvz/providers/virtualbox/manifest-schema.json
 create mode 100644 bootstrapvz/providers/virtualbox/manifest-schema.yml
 delete mode 100644 manifests/azure.manifest.json
 create mode 100644 manifests/azure.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
 delete mode 100644 manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
 delete mode 100644 manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
 create mode 100644 manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
 delete mode 100644 manifests/ec2-ebs-partitioned.manifest.json
 create mode 100644 manifests/ec2-ebs-partitioned.manifest.yml
 delete mode 100644 manifests/ec2-ebs-single.manifest.json
 create mode 100644 manifests/ec2-ebs-single.manifest.yml
 delete mode 100644 manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
 create mode 100644 manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
 delete mode 100644 manifests/ec2-s3.manifest.json
 create mode 100644 manifests/ec2-s3.manifest.yml
 delete mode 100644 manifests/gce-backports.manifest.json
 create mode 100644 manifests/gce-backports.manifest.yml
 delete mode 100644 manifests/gce.manifest.json
 create mode 100644 manifests/gce.manifest.yml
 delete mode 100644 manifests/kvm-virtio.manifest.json
 create mode 100644 manifests/kvm-virtio.manifest.yml
 delete mode 100644 manifests/kvm.manifest.json
 create mode 100644 manifests/kvm.manifest.yml
 delete mode 100644 manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
 create mode 100644 manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
 delete mode 100644 manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
 create mode 100644 manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
 delete mode 100644 manifests/virtualbox-vagrant.manifest.json
 create mode 100644 manifests/virtualbox-vagrant.manifest.yml
 delete mode 100644 manifests/virtualbox.manifest.json
 create mode 100644 manifests/virtualbox.manifest.yml

diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index 1cafa81..b76d520 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -12,7 +12,7 @@ def validate_manifest(data, validator, error):
 	:param function error: The function tha raises an error when the validation fails
 	"""
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 	# Check the bootloader/partitioning configuration.
diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 7aba5b0..8edb5e8 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -33,7 +33,7 @@ class BootstrapInformation(object):
 
 		# Normalize the release codenames so that tasks may query for release codenames rather than
 		# 'stable', 'unstable' etc. This is useful when handling cases that are specific to a release.
-		release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.json')
+		release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.yml')
 		from bootstrapvz.common.tools import config_get
 		self.release_codename = config_get(release_codenames_path, [self.manifest.system['release']])
 
diff --git a/bootstrapvz/base/manifest-schema.json b/bootstrapvz/base/manifest-schema.json
deleted file mode 100644
index 7a5040c..0000000
--- a/bootstrapvz/base/manifest-schema.json
+++ /dev/null
@@ -1,214 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Generic manifest",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "object",
-			"properties": {
-				"name": { "type": "string" }
-			},
-			"required": ["name"]
-		},
-		"bootstrapper": {
-			"type": "object",
-			"properties": {
-				"workspace": { "$ref": "#/definitions/path" },
-				"mirror":    { "type": "string", "format": "uri" },
-				"tarball":   { "type": "boolean" },
-				"include_packages": {
-					"type": "array",
-					"items": {
-						"type":    "string",
-						"pattern": "^[^/]+$"
-					},
-					"minItems": 1
-				},
-				"exclude_packages": {
-					"type": "array",
-					"items": {
-						"type":    "string",
-						"pattern": "^[^/]+$"
-					},
-					"minItems": 1
-				}
-			},
-			"required": ["workspace"]
-		},
-		"image": {
-			"type": "object",
-			"properties": {
-				"name": { "type": "string" }
-			},
-			"required": ["name"]
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"release": {
-					"enum": [
-						"squeeze", "wheezy", "jessie", "sid",
-						"oldstable", "stable", "testing", "unstable"
-					]
-				},
-				"architecture": { "enum": ["i386", "amd64"] },
-				"bootloader":   { "enum": ["pvgrub", "grub", "extlinux"] },
-				"timezone":     { "type": "string" },
-				"locale":       { "type": "string" },
-				"charmap":      { "type": "string" },
-				"hostname": {
-					"type": "string",
-					"pattern": "^\\S+$"
-				}
-			},
-			"required": ["release", "architecture", "bootloader", "timezone", "locale", "charmap"]
-		},
-		"packages": {
-			"type": "object",
-			"properties": {
-				"mirror": { "type": "string", "format": "uri" },
-				"sources": {
-					"type": "object",
-					"patternProperties": {
-						"^[^\/\\0]+$": {
-							"type": "array",
-							"items": {
-								"type": "string",
-								"pattern": "^(deb|deb-src)\\s+(\\[\\s*(.+\\S)?\\s*\\]\\s+)?\\S+\\s+\\S+(\\s+(.+\\S))?\\s*$"
-							},
-							"minItems": 1
-						}
-					},
-					"additionalProperties": false,
-					"minItems": 1
-				},
-				"components": {
-					"type": "array",
-					"items": {"type": "string"},
-					"minItems": 1
-				},
-				"preferences": {
-					"type": "object",
-					"patternProperties": {
-						"^[^\/\\0]+$": {
-							"type": "array",
-							"items": {
-								"type": "object",
-								"properties": {
-									"pin": {
-										"type": "string"
-									},
-									"package": {
-										"type": "string"
-									},
-									"pin-priority": {
-										"type": "integer"
-									}
-								},
-								"required": ["pin", "package", "pin-priority"],
-								"additionalProperties": false
-							},
-							"minItems": 1
-						}
-					},
-					"additionalProperties": false,
-					"minItems": 1
-				},
-				"trusted-keys": {
-					"type": "array",
-					"items": { "$ref": "#/definitions/absolute_path" },
-					"minItems": 1
-				},
-				"install": {
-					"type": "array",
-					"items": {
-						"anyOf": [
-							{ "pattern": "^[^/]+(/[^/]+)?$" },
-							{ "$ref": "#/definitions/absolute_path" }
-						]
-					},
-					"minItems": 1
-				},
-				"install_standard": {
-					"type": "boolean"
-				}
-			},
-			"additionalProperties": false
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": { "type": "string" },
-				"partitions": {
-					"type": "object",
-					"oneOf": [
-						{ "$ref": "#/definitions/no_partitions" },
-						{ "$ref": "#/definitions/partition_table" }
-					]
-				}
-			},
-			"required": ["partitions"]
-		},
-		"plugins": {
-			"type": "object",
-			"patternProperties": {
-				"^\\w+$": {
-					"type": "object"
-				}
-			},
-			"additionalProperties": false
-		}
-	},
-	"required": ["provider", "bootstrapper", "system", "volume"],
-	"definitions": {
-		"path": {
-			"type":    "string",
-			"pattern": "^[^\\0]+$"
-		},
-		"absolute_path": {
-			"type":    "string",
-			"pattern": "^/[^\\0]+$"
-		},
-		"bytes": {
-			"type":    "string",
-			"pattern": "^\\d+([KMGT]i?B|B)$"
-		},
-		"no_partitions": {
-			"type": "object",
-			"properties": {
-				"type": { "enum": ["none"] },
-				"root": { "$ref": "#/definitions/partition" }
-			},
-			"required": ["root"],
-			"additionalProperties": false
-		},
-		"partition_table": {
-			"type": "object",
-			"properties": {
-				"type": { "enum": ["msdos", "gpt"] },
-				"boot": { "$ref": "#/definitions/partition" },
-				"root": { "$ref": "#/definitions/partition" },
-				"swap": {
-					"type": "object",
-					"properties": { "size": { "$ref": "#/definitions/bytes" } },
-					"required": ["size"]
-				}
-			},
-			"required": ["root"],
-			"additionalProperties": false
-		},
-		"partition": {
-			"type": "object",
-			"properties": {
-				"size": { "$ref": "#/definitions/bytes" },
-				"filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] },
-				"format_command": {
-					"type": "array",
-					"items": {"type": "string"},
-					"minItems": 1
-				}
-			},
-			"required": ["size", "filesystem"]
-		}
-	}
-}
diff --git a/bootstrapvz/base/manifest-schema.yml b/bootstrapvz/base/manifest-schema.yml
new file mode 100644
index 0000000..7110215
--- /dev/null
+++ b/bootstrapvz/base/manifest-schema.yml
@@ -0,0 +1,177 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Generic manifest
+type: object
+required: [provider, bootstrapper, system, volume]
+properties:
+  provider:
+    type: object
+    properties:
+      name: {type: string}
+    required: [name]
+  bootstrapper:
+    type: object
+    properties:
+      exclude_packages:
+        type: array
+        items:
+          type: string
+          pattern: '^[^/]+$'
+        minItems: 1
+      include_packages:
+        type: array
+        items:
+          type: string
+          pattern: '^[^/]+$'
+        minItems: 1
+      mirror:
+        type: string
+        format: uri
+      tarball: {type: boolean}
+      workspace:
+        $ref: '#/definitions/path'
+    required: [workspace]
+  image:
+    type: object
+    properties:
+      name: {type: string}
+    required: [name]
+  system:
+    properties:
+      architecture:
+        enum: [i386, amd64]
+      bootloader:
+        enum:
+        - pvgrub
+        - grub
+        - extlinux
+      charmap: {type: string}
+      hostname:
+        type: string
+        pattern: ^\S+$
+      locale: {type: string}
+      release:
+        enum:
+        - squeeze
+        - wheezy
+        - jessie
+        - sid
+        - oldstable
+        - stable
+        - testing
+        - unstable
+      timezone: {type: string}
+    required:
+    - release
+    - architecture
+    - bootloader
+    - timezone
+    - locale
+    - charmap
+    type: object
+  packages:
+    type: object
+    properties:
+      components:
+        type: array
+        items: {type: string}
+        minItems: 1
+      install:
+        type: array
+        items:
+          anyOf:
+          - pattern: ^[^/]+(/[^/]+)?$
+          - $ref: '#/definitions/absolute_path'
+        minItems: 1
+      install_standard: {type: boolean}
+      mirror:
+        type: string
+        format: uri
+      preferences:
+        type: object
+        patternProperties:
+          ^[^/\0]+$:
+            type: array
+            items:
+              type: object
+              properties:
+                package: {type: string}
+                pin: {type: string}
+                pin-priority: {type: integer}
+              required: [pin, package, pin-priority]
+              additionalProperties: false
+            minItems: 1
+        minItems: 1
+        additionalProperties: false
+      sources:
+        type: object
+        patternProperties:
+          ^[^/\0]+$:
+            items:
+              type: string
+              pattern: ^(deb|deb-src)\s+(\[\s*(.+\S)?\s*\]\s+)?\S+\s+\S+(\s+(.+\S))?\s*$
+            minItems: 1
+            type: array
+        minItems: 1
+        additionalProperties: false
+      trusted-keys:
+        type: array
+        items:
+          $ref: '#/definitions/absolute_path'
+        minItems: 1
+    additionalProperties: false
+  plugins:
+    type: object
+    patternProperties:
+      ^\w+$: {type: object}
+  volume:
+    type: object
+    properties:
+      backing: {type: string}
+      partitions:
+        type: object
+        oneOf:
+        - $ref: '#/definitions/no_partitions'
+        - $ref: '#/definitions/partition_table'
+    required: [partitions]
+definitions:
+  absolute_path:
+    type: string
+    pattern: ^/[^\0]+$
+  bytes:
+    pattern: ^\d+([KMGT]i?B|B)$
+    type: string
+  no_partitions:
+    type: object
+    properties:
+      root: {$ref: '#/definitions/partition'}
+      type: {enum: [none]}
+    required: [root]
+    additionalProperties: false
+  partition:
+    type: object
+    properties:
+      filesystem:
+        enum: [ext2, ext3, ext4, xfs]
+      format_command:
+        items: {type: string}
+        minItems: 1
+        type: array
+      size: {$ref: '#/definitions/bytes'}
+    required: [size, filesystem]
+  partition_table:
+    type: object
+    additionalProperties: false
+    properties:
+      boot: {$ref: '#/definitions/partition'}
+      root: {$ref: '#/definitions/partition'}
+      swap:
+        type: object
+        properties:
+          size: {$ref: '#/definitions/bytes'}
+        required: [size]
+      type: {enum: [msdos, gpt]}
+    required: [root]
+  path:
+    type: string
+    pattern: ^[^\0]+$
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index f982b44..44f56f1 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -2,8 +2,7 @@
 to determine which tasks should be added to the tasklist, what arguments various
 invocations should have etc..
 """
-from bootstrapvz.common.tools import load_json
-from bootstrapvz.common.tools import load_yaml
+from bootstrapvz.common.tools import load_data
 import logging
 log = logging.getLogger(__name__)
 
@@ -31,12 +30,7 @@ class Manifest(object):
 		Once they are loaded, the initialize() function is called on each of them (if it exists).
 		The provider must have an initialize function.
 		"""
-		# Load the manifest JSON using the loader in common.tools
-		# It strips comments (which are invalid in strict json) before loading the data.
-		if self.path.endswith('.json'):
-			self.data = load_json(self.path)
-		elif self.path.endswith('.yml') or self.path.endswith('.yaml'):
-			self.data = load_yaml(self.path)
+		self.data = load_data(self.path)
 
 		# Get the provider name from the manifest and load the corresponding module
 		provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name']
@@ -102,19 +96,20 @@ class Manifest(object):
 		:param str schema_path: Path to the json-schema to use for validation
 		"""
 		import jsonschema
-		schema = load_json(schema_path)
+
+		schema = load_data(schema_path)
 		try:
 			jsonschema.validate(data, schema)
 		except jsonschema.ValidationError as e:
 			self.validation_error(e.message, e.path)
 
-	def validation_error(self, message, json_path=None):
+	def validation_error(self, message, data_path=None):
 		"""This function is passed to all validation functions so that they may
 		raise a validation error because a custom validation of the manifest failed.
 
 		:param str message: Message to user about the error
-		:param list json_path: A path to the location in the manifest where the error occurred
+		:param list data_path: A path to the location in the manifest where the error occurred
 		:raises ManifestError: With absolute certainty
 		"""
 		from bootstrapvz.common.exceptions import ManifestError
-		raise ManifestError(message, self.path, json_path)
+		raise ManifestError(message, self.path, data_path)
diff --git a/bootstrapvz/base/release-codenames.json b/bootstrapvz/base/release-codenames.json
deleted file mode 100644
index cac8692..0000000
--- a/bootstrapvz/base/release-codenames.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{ // This is a mapping of Debian release names to their respective codenames
-	"unstable":  "sid",
-	"testing":   "jessie",
-	"stable":    "wheezy",
-	"oldstable": "squeeze",
-
-	"jessie":  "jessie",
-	"wheezy":  "wheezy",
-	"squeeze": "squeeze",
-
-	// The following release names are not supported, but included of completeness sake
-	"lenny":   "lenny",
-	"etch":    "etch",
-	"sarge":   "sarge",
-	"woody":   "woody",
-	"potato":  "potato",
-	"slink":   "slink",
-	"hamm":    "hamm",
-	"bo":      "bo",
-	"rex":     "rex",
-	"buzz":    "buzz"
-}
diff --git a/bootstrapvz/base/release-codenames.yml b/bootstrapvz/base/release-codenames.yml
new file mode 100644
index 0000000..218488d
--- /dev/null
+++ b/bootstrapvz/base/release-codenames.yml
@@ -0,0 +1,23 @@
+---
+# This is a mapping of Debian release names to their respective codenames
+
+unstable:  sid
+testing:   jessie
+stable:    wheezy
+oldstable: squeeze
+
+jessie:  jessie
+wheezy:  wheezy
+squeeze: squeeze
+
+# The following release names are not supported, but included fir completeness sake
+lenny:  lenny
+etch:   etch
+sarge:  sarge
+woody:  woody
+potato: potato
+slink:  slink
+hamm:   hamm
+bo:     bo
+rex:    rex
+buzz:   buz
diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index 940d2b0..a349c07 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -1,16 +1,16 @@
 
 
 class ManifestError(Exception):
-	def __init__(self, message, manifest_path, json_path=None):
+	def __init__(self, message, manifest_path, data_path=None):
 		self.message = message
 		self.manifest_path = manifest_path
-		self.json_path = json_path
+		self.data_path = data_path
 
 	def __str__(self):
-		if self.json_path is not None:
-			path = '.'.join(map(str, self.json_path))
-			return ('{msg}\n  File path: {file}\n  JSON path: {jsonpath}'
-			        .format(msg=self.message, file=self.manifest_path, jsonpath=path))
+		if self.data_path is not None:
+			path = '.'.join(map(str, self.data_path))
+			return ('{msg}\n  File path: {file}\n  Data path: {datapath}'
+			        .format(msg=self.message, file=self.manifest_path, datapath=path))
 		return '{file}: {msg}'.format(msg=self.message, file=self.manifest_path)
 
 
diff --git a/bootstrapvz/common/tasks/network-configuration.json b/bootstrapvz/common/tasks/network-configuration.json
deleted file mode 100644
index 2937384..0000000
--- a/bootstrapvz/common/tasks/network-configuration.json
+++ /dev/null
@@ -1,14 +0,0 @@
-// This is a mapping of Debian release codenames to NIC configurations
-// Every item in an array is a line
-{
-"squeeze": ["auto lo",
-            "iface lo inet loopback",
-            "auto eth0",
-            "iface eth0 inet dhcp"],
-"wheezy": ["auto eth0",
-           "iface eth0 inet dhcp"],
-"jessie": ["auto eth0",
-           "iface eth0 inet dhcp"],
-"sid": ["auto eth0",
-           "iface eth0 inet dhcp"]
-}
diff --git a/bootstrapvz/common/tasks/network-configuration.yml b/bootstrapvz/common/tasks/network-configuration.yml
new file mode 100644
index 0000000..c044069
--- /dev/null
+++ b/bootstrapvz/common/tasks/network-configuration.yml
@@ -0,0 +1,17 @@
+---
+# This is a mapping of Debian release codenames to NIC configurations
+# Every item in an array is a line
+squeeze:
+- auto lo
+- iface lo inet loopback
+- auto eth0
+- iface eth0 inet dhcp
+wheezy:
+- auto eth0
+- iface eth0 inet dhcp
+jessie:
+- auto eth0
+- iface eth0 inet dhcp
+sid:
+- auto eth0
+- iface eth0 inet dhcp
diff --git a/bootstrapvz/common/tasks/network.py b/bootstrapvz/common/tasks/network.py
index a85a34d..72da5c9 100644
--- a/bootstrapvz/common/tasks/network.py
+++ b/bootstrapvz/common/tasks/network.py
@@ -45,7 +45,7 @@ class ConfigureNetworkIF(Task):
 
 	@classmethod
 	def run(cls, info):
-		network_config_path = os.path.join(os.path.dirname(__file__), 'network-configuration.json')
+		network_config_path = os.path.join(os.path.dirname(__file__), 'network-configuration.yml')
 		from ..tools import config_get
 		if_config = config_get(network_config_path, [info.release_codename])
 
diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index a2704da..315291e 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -1,3 +1,5 @@
+
+
 def log_check_call(command, stdin=None, env=None, shell=False, cwd=None):
 	status, stdout, stderr = log_call(command, stdin, env, shell, cwd)
 	if status != 0:
@@ -73,8 +75,21 @@ def load_yaml(path):
 		return yaml.safe_load(fobj)
 
 
+def load_data(path):
+	import os.path
+	filename, extension = os.path.splitext(path)
+	if not os.path.isfile(path):
+		raise Exception('The path {path} does not point to a file.'.format(path=path))
+	if extension == '.json':
+		return load_json(path)
+	elif extension == '.yml' or extension == '.yaml':
+		return load_yaml(path)
+	else:
+		raise Exception('Unrecognized extension: {ext}'.format(ext=extension))
+
+
 def config_get(path, config_path):
-	config = load_json(path)
+	config = load_data(path)
 	for key in config_path:
 		config = config.get(key)
 	return config
diff --git a/bootstrapvz/plugins/admin_user/__init__.py b/bootstrapvz/plugins/admin_user/__init__.py
index 67f3488..b506b0d 100644
--- a/bootstrapvz/plugins/admin_user/__init__.py
+++ b/bootstrapvz/plugins/admin_user/__init__.py
@@ -2,7 +2,7 @@
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/admin_user/manifest-schema.json b/bootstrapvz/plugins/admin_user/manifest-schema.json
deleted file mode 100644
index fc3c421..0000000
--- a/bootstrapvz/plugins/admin_user/manifest-schema.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Admin user plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"admin_user": {
-					"type": "object",
-					"properties": {
-						"username": {
-							"type": "string"
-						}
-					},
-					"required": ["username"]
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/admin_user/manifest-schema.yml b/bootstrapvz/plugins/admin_user/manifest-schema.yml
new file mode 100644
index 0000000..31e8d82
--- /dev/null
+++ b/bootstrapvz/plugins/admin_user/manifest-schema.yml
@@ -0,0 +1,13 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Admin user plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      admin_user:
+        type: object
+        properties:
+          username: {type: string}
+        required: [username]
diff --git a/bootstrapvz/plugins/apt_proxy/__init__.py b/bootstrapvz/plugins/apt_proxy/__init__.py
index 6ca8f71..132c679 100644
--- a/bootstrapvz/plugins/apt_proxy/__init__.py
+++ b/bootstrapvz/plugins/apt_proxy/__init__.py
@@ -1,6 +1,6 @@
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/apt_proxy/manifest-schema.json b/bootstrapvz/plugins/apt_proxy/manifest-schema.json
deleted file mode 100644
index 8c8e932..0000000
--- a/bootstrapvz/plugins/apt_proxy/manifest-schema.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "APT proxy plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"apt_proxy": {
-					"type": "object",
-					"properties": {
-						"address": {
-							"type": "string"
-						},
-						"persistent": {
-							"type": "boolean"
-						},
-						"port": {
-							"type": "integer"
-						}
-					},
-					"required": ["address", "port"]
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/apt_proxy/manifest-schema.yml b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
new file mode 100644
index 0000000..5f3f051
--- /dev/null
+++ b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
@@ -0,0 +1,17 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: APT proxy plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      apt_proxy:
+        type: object
+        properties:
+          address: {type: string}
+          port: {type: integer}
+          persistent: {type: boolean}
+        required:
+        - address
+        - port
diff --git a/bootstrapvz/plugins/chef/__init__.py b/bootstrapvz/plugins/chef/__init__.py
index 7ba2396..5716b20 100644
--- a/bootstrapvz/plugins/chef/__init__.py
+++ b/bootstrapvz/plugins/chef/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/chef/manifest-schema.json b/bootstrapvz/plugins/chef/manifest-schema.json
deleted file mode 100644
index 9d70622..0000000
--- a/bootstrapvz/plugins/chef/manifest-schema.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Chef plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"chef": {
-					"type": "object",
-					"properties": {
-						"assets": { "$ref": "#/definitions/absolute_path" }
-					},
-					"required": ["assets"]
-				}
-			}
-		}
-	},
-	"definitions": {
-		"absolute_path": {
-			"type":    "string",
-			"pattern": "^/[^\\0]+$"
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/chef/manifest-schema.yml b/bootstrapvz/plugins/chef/manifest-schema.yml
new file mode 100644
index 0000000..58fd02f
--- /dev/null
+++ b/bootstrapvz/plugins/chef/manifest-schema.yml
@@ -0,0 +1,19 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Chef plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      chef:
+        type: object
+        properties:
+          assets:
+            $ref: '#/definitions/absolute_path'
+        required:
+        - assets
+definitions:
+  absolute_path:
+    pattern: ^/[^\0]+$
+    type: string
diff --git a/bootstrapvz/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py
index c08eebe..999572b 100644
--- a/bootstrapvz/plugins/cloud_init/__init__.py
+++ b/bootstrapvz/plugins/cloud_init/__init__.py
@@ -2,7 +2,7 @@
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/cloud_init/manifest-schema.json b/bootstrapvz/plugins/cloud_init/manifest-schema.json
deleted file mode 100644
index 08cd153..0000000
--- a/bootstrapvz/plugins/cloud_init/manifest-schema.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "cloud-init plugin manifest",
-	"type": "object",
-	"properties": {
-		"system": {
-			"type": "object",
-			"properties": {
-				"release": {
-					"type": "string",
-					"enum": ["wheezy", "stable",
-					         "jessie", "testing",
-					         "sid", "unstable"]
-				}
-			}
-		},
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"cloud_init": {
-					"type": "object",
-					"properties": {
-						"username": {
-							"type": "string"
-						},
-						"disable_modules": {
-							"type": "array",
-							"items": {
-								"type": "string"
-							},
-							"uniqueItems": true
-						},
-						"metadata_sources": {
-							"type": "string"
-						}
-					},
-					"required": ["username"]
-				},
-				"packages":   {"type": "object"}
-			},
-			"required": ["cloud_init"]
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/cloud_init/manifest-schema.yml b/bootstrapvz/plugins/cloud_init/manifest-schema.yml
new file mode 100644
index 0000000..ea7f64c
--- /dev/null
+++ b/bootstrapvz/plugins/cloud_init/manifest-schema.yml
@@ -0,0 +1,30 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: cloud-init plugin manifest
+type: object
+properties:
+  system:
+    type: object
+    properties:
+      release:
+        type: string
+        enum:
+        - wheezy
+        - stable
+        - jessie
+        - testing
+        - sid
+        - unstable
+  plugins:
+    type: object
+    properties:
+      cloud_init:
+        type: object
+        properties:
+          username: {type: string}
+          metadata_sources: {type: string}
+          disable_modules:
+            type: array
+            items: {type: string}
+            uniqueItems: true
+        required: [username]
diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index f32e6b1..f2eda61 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -1,14 +1,10 @@
 import tasks
 import os.path
-from bootstrapvz.common.exceptions import ManifestError
 
 
 def validate_manifest(data, validator, error):
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	try:
-		validator(data, schema_path)
-	except ManifestError, e:
-		error('docker_daemon manifest validation failed: "%s"' % e.message, e.json_path)
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+	validator(data, schema_path)
 	if data.get('system', {}).get('release', None) in ['wheezy', 'stable']:
 		# prefs is a generator of apt preferences across files in the manifest
 		prefs = (item for vals in data.get('packages', {}).get('preferences', {}).values() for item in vals)
diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.json b/bootstrapvz/plugins/docker_daemon/manifest-schema.json
deleted file mode 100644
index 365ccfa..0000000
--- a/bootstrapvz/plugins/docker_daemon/manifest-schema.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Install Docker plugin manifest",
-	"type": "object",
-	"properties": {
-		"system": {
-			"type": "object",
-			"properties": {
-				"architecture": {
-					"type": "string",
-					"enum": ["amd64"]
-					// Docker runs on x86_64 only
-				},
-				"release": {
-					"not": {
-						"type": "string",
-						"enum": ["squeeze", "oldstable"]
-						// Docker needs at least wheezy + backports.
-					}
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
new file mode 100644
index 0000000..0746d90
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
@@ -0,0 +1,17 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Install Docker plugin manifest
+type: object
+properties:
+  system:
+    type: object
+    properties:
+      architecture:
+        type: string
+        enum: [amd64]
+      release:
+        not:
+          type: string
+          enum:
+          - squeeze
+          - oldstable
diff --git a/bootstrapvz/plugins/image_commands/__init__.py b/bootstrapvz/plugins/image_commands/__init__.py
index 1642e0d..4d1600f 100644
--- a/bootstrapvz/plugins/image_commands/__init__.py
+++ b/bootstrapvz/plugins/image_commands/__init__.py
@@ -2,7 +2,7 @@
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/image_commands/manifest-schema.json b/bootstrapvz/plugins/image_commands/manifest-schema.json
deleted file mode 100644
index 9785c90..0000000
--- a/bootstrapvz/plugins/image_commands/manifest-schema.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Image commands plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"image_commands": {
-					"type": "object",
-					"properties": {
-						"commands": {
-							"type": "array",
-							"items": {
-								"type": "array",
-								"items": {"type": "string"},
-								"minItems": 1
-							},
-							"minItems": 1
-						}
-					},
-					"required": ["commands"]
-				}
-			},
-			"required": ["image_commands"]
-		}
-	},
-	"required": ["plugins"]
-}
diff --git a/bootstrapvz/plugins/image_commands/manifest-schema.yml b/bootstrapvz/plugins/image_commands/manifest-schema.yml
new file mode 100644
index 0000000..da1ad8d
--- /dev/null
+++ b/bootstrapvz/plugins/image_commands/manifest-schema.yml
@@ -0,0 +1,25 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+properties:
+  plugins:
+    properties:
+      image_commands:
+        properties:
+          commands:
+            items:
+              items:
+                type: string
+              minItems: 1
+              type: array
+            minItems: 1
+            type: array
+        required:
+        - commands
+        type: object
+    required:
+    - image_commands
+    type: object
+required:
+- plugins
+title: Image commands plugin manifest
+type: object
diff --git a/bootstrapvz/plugins/minimize_size/__init__.py b/bootstrapvz/plugins/minimize_size/__init__.py
index 3f8c583..107da6e 100644
--- a/bootstrapvz/plugins/minimize_size/__init__.py
+++ b/bootstrapvz/plugins/minimize_size/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.join(os.path.dirname(__file__), 'manifest-schema.json')
+	schema_path = os.path.join(os.path.dirname(__file__), 'manifest-schema.yml')
 	validator(data, schema_path)
 	if data['plugins']['minimize_size'].get('shrink', False) and data['volume']['backing'] != 'vmdk':
 		error('Can only shrink vmdk images', ['plugins', 'minimize_size', 'shrink'])
diff --git a/bootstrapvz/plugins/minimize_size/manifest-schema.json b/bootstrapvz/plugins/minimize_size/manifest-schema.json
deleted file mode 100644
index 0181fa2..0000000
--- a/bootstrapvz/plugins/minimize_size/manifest-schema.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Minimize size plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"minimize_size": {
-					"type": "object",
-					"properties": {
-						"shrink": { "type": "boolean" },
-						"zerofree": { "type": "boolean" }
-					}
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/minimize_size/manifest-schema.yml b/bootstrapvz/plugins/minimize_size/manifest-schema.yml
new file mode 100644
index 0000000..fc84249
--- /dev/null
+++ b/bootstrapvz/plugins/minimize_size/manifest-schema.yml
@@ -0,0 +1,15 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+properties:
+  plugins:
+    properties:
+      minimize_size:
+        properties:
+          shrink:
+            type: boolean
+          zerofree:
+            type: boolean
+        type: object
+    type: object
+title: Minimize size plugin manifest
+type: object
diff --git a/bootstrapvz/plugins/ntp/__init__.py b/bootstrapvz/plugins/ntp/__init__.py
index 9a68d0f..858af72 100644
--- a/bootstrapvz/plugins/ntp/__init__.py
+++ b/bootstrapvz/plugins/ntp/__init__.py
@@ -1,6 +1,6 @@
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/ntp/manifest-schema.json b/bootstrapvz/plugins/ntp/manifest-schema.json
deleted file mode 100644
index c386045..0000000
--- a/bootstrapvz/plugins/ntp/manifest-schema.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "NTP plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"ntp": {
-					"type": "object",
-					"properties": {
-						"servers": {
-							"type": "array",
-							"items": {"type": "string"},
-							"minItems": 1
-						}
-					}
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/ntp/manifest-schema.yml b/bootstrapvz/plugins/ntp/manifest-schema.yml
new file mode 100644
index 0000000..2303d1c
--- /dev/null
+++ b/bootstrapvz/plugins/ntp/manifest-schema.yml
@@ -0,0 +1,15 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: NTP plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      ntp:
+        type: object
+        properties:
+          servers:
+            type: array
+            items: {type: string}
+            minItems: 1
diff --git a/bootstrapvz/plugins/pip_install/__init__.py b/bootstrapvz/plugins/pip_install/__init__.py
index fe75aae..0f6810b 100644
--- a/bootstrapvz/plugins/pip_install/__init__.py
+++ b/bootstrapvz/plugins/pip_install/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/pip_install/manifest-schema.json b/bootstrapvz/plugins/pip_install/manifest-schema.json
deleted file mode 100644
index 6d3f27e..0000000
--- a/bootstrapvz/plugins/pip_install/manifest-schema.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Pip install plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"pip_install": {
-					"type": "object",
-					"properties": {
-						"packages": { "$ref": "#/definitions/packages" }
-					},
-					"minProperties": 1,
-					"additionalProperties": false
-				}
-			}
-		}
-	},
-	"definitions": {
-		"packages": {
-			"type":    "array",
-			"items": {
-				"type": "string"
-			},
-			"minItems": 1,
-			"uniqueItems": true
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/pip_install/manifest-schema.yml b/bootstrapvz/plugins/pip_install/manifest-schema.yml
new file mode 100644
index 0000000..df3b922
--- /dev/null
+++ b/bootstrapvz/plugins/pip_install/manifest-schema.yml
@@ -0,0 +1,17 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Pip install plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      pip_install:
+        type: object
+        properties:
+          packages:
+            type: array
+            items:
+              type: string
+            minItems: 1
+            uniqueItems: true
diff --git a/bootstrapvz/plugins/prebootstrapped/__init__.py b/bootstrapvz/plugins/prebootstrapped/__init__.py
index dbda042..bcdc4fc 100644
--- a/bootstrapvz/plugins/prebootstrapped/__init__.py
+++ b/bootstrapvz/plugins/prebootstrapped/__init__.py
@@ -15,7 +15,7 @@ from bootstrapvz.common.tasks import partitioning
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/prebootstrapped/manifest-schema.json b/bootstrapvz/plugins/prebootstrapped/manifest-schema.json
deleted file mode 100644
index d3a19d7..0000000
--- a/bootstrapvz/plugins/prebootstrapped/manifest-schema.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Prebootstrapped plugin manifest",
-	"type": "object",
-	"properties": {
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["raw", "ebs", "s3", "vdi", "vmdk"]
-				}
-			},
-			"required": ["backing"]
-		},
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"prebootstrapped": {
-					"type": "object",
-					"properties": {
-						"snapshot": {
-							"type": "string"
-						},
-						"image": {
-							"type": "string"
-						}
-					}
-				}
-			}
-		}
-	},
-	"required": ["volume"]
-}
diff --git a/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml b/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
new file mode 100644
index 0000000..68fbbe4
--- /dev/null
+++ b/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
@@ -0,0 +1,25 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Prebootstrapped plugin manifest
+type: object
+properties:
+  volume:
+    type: object
+    properties:
+      backing:
+        type: string
+        enum:
+        - raw
+        - ebs
+        - s3
+        - vdi
+        - vmdk
+    required: [backing]
+  plugins:
+    type: object
+    properties:
+      prebootstrapped:
+        type: object
+        properties:
+          image: {type: string}
+          snapshot: {type: string}
diff --git a/bootstrapvz/plugins/puppet/__init__.py b/bootstrapvz/plugins/puppet/__init__.py
index 5a4fcbf..529c1d5 100644
--- a/bootstrapvz/plugins/puppet/__init__.py
+++ b/bootstrapvz/plugins/puppet/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/puppet/manifest-schema.json b/bootstrapvz/plugins/puppet/manifest-schema.json
deleted file mode 100644
index cd3891f..0000000
--- a/bootstrapvz/plugins/puppet/manifest-schema.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Puppet plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"puppet": {
-					"type": "object",
-					"properties": {
-						"manifest": { "$ref": "#/definitions/absolute_path" },
-						"assets": { "$ref": "#/definitions/absolute_path" },
-						"enable_agent": { "type": "boolean" }
-					},
-					"minProperties": 1,
-					"additionalProperties": false
-				}
-			}
-		}
-	},
-	"definitions": {
-		"absolute_path": {
-			"type":    "string",
-			"pattern": "^/[^\\0]+$"
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/puppet/manifest-schema.yml b/bootstrapvz/plugins/puppet/manifest-schema.yml
new file mode 100644
index 0000000..e874dac
--- /dev/null
+++ b/bootstrapvz/plugins/puppet/manifest-schema.yml
@@ -0,0 +1,20 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Puppet plugin manifest
+type: object
+properties:
+  plugins:
+    properties:
+    type: object
+      puppet:
+        type: object
+        properties:
+          assets: {$ref: '#/definitions/absolute_path'}
+          enable_agent: {type: boolean}
+          manifest: {$ref: '#/definitions/absolute_path'}
+        minProperties: 1
+        additionalProperties: false
+definitions:
+  absolute_path:
+    pattern: ^/[^\0]+$
+    type: string
diff --git a/bootstrapvz/plugins/root_password/__init__.py b/bootstrapvz/plugins/root_password/__init__.py
index 7ca6870..850c6d2 100644
--- a/bootstrapvz/plugins/root_password/__init__.py
+++ b/bootstrapvz/plugins/root_password/__init__.py
@@ -2,7 +2,7 @@
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/root_password/manifest-schema.json b/bootstrapvz/plugins/root_password/manifest-schema.json
deleted file mode 100644
index fee0b5f..0000000
--- a/bootstrapvz/plugins/root_password/manifest-schema.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Root password plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"root_password": {
-					"type": "object",
-					"properties": {
-						"password": {
-							"type": "string"
-						}
-					},
-					"required": ["password"]
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/root_password/manifest-schema.yml b/bootstrapvz/plugins/root_password/manifest-schema.yml
new file mode 100644
index 0000000..7267807
--- /dev/null
+++ b/bootstrapvz/plugins/root_password/manifest-schema.yml
@@ -0,0 +1,13 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Root password plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      root_password:
+        type: object
+        properties:
+          password: {type: string}
+        required: [password]
diff --git a/bootstrapvz/plugins/salt/__init__.py b/bootstrapvz/plugins/salt/__init__.py
index 0b9cf11..f165f49 100644
--- a/bootstrapvz/plugins/salt/__init__.py
+++ b/bootstrapvz/plugins/salt/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/salt/manifest-schema.json b/bootstrapvz/plugins/salt/manifest-schema.json
deleted file mode 100644
index c6f12ca..0000000
--- a/bootstrapvz/plugins/salt/manifest-schema.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Saltstack plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"salt": {
-					"type": "object",
-					"properties": {
-						"master": { "type": "string" },
-						"install_source": { "enum": ["stable", "daily", "git"] },
-						"version": { "type": "string" },
-						"grains": {
-							"type": "object",
-							"patternProperties": {
-								"^[^\/\\0]+$": { "type": "string" }
-							},
-							"minItems": 1
-						}
-					},
-					"required": ["install_source"]
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/salt/manifest-schema.yml b/bootstrapvz/plugins/salt/manifest-schema.yml
new file mode 100644
index 0000000..43b6b3a
--- /dev/null
+++ b/bootstrapvz/plugins/salt/manifest-schema.yml
@@ -0,0 +1,24 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Saltstack plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      salt:
+        type: object
+        properties:
+          grains:
+            type: object
+            patternProperties:
+              ^[^/\0]+$: {type: string}
+            minItems: 1
+          install_source:
+            enum:
+            - stable
+            - daily
+            - git
+          master: {type: string}
+          version: {type: string}
+        required: [install_source]
diff --git a/bootstrapvz/plugins/unattended_upgrades/__init__.py b/bootstrapvz/plugins/unattended_upgrades/__init__.py
index f67e60e..dbf5ebd 100644
--- a/bootstrapvz/plugins/unattended_upgrades/__init__.py
+++ b/bootstrapvz/plugins/unattended_upgrades/__init__.py
@@ -2,7 +2,7 @@
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/unattended_upgrades/manifest-schema.json b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.json
deleted file mode 100644
index 3e45a7b..0000000
--- a/bootstrapvz/plugins/unattended_upgrades/manifest-schema.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Unattended upgrades plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"unattended_upgrades": {
-					"type": "object",
-					"properties": {
-						"update_interval": {
-							"type": "integer"
-						},
-						"download_interval": {
-							"type": "integer"
-						},
-						"upgrade_interval": {
-							"type": "integer"
-						}
-					},
-					"required": ["update_interval", "download_interval", "upgrade_interval"]
-				}
-			},
-			"required": ["unattended_upgrades"]
-		}
-	},
-	"required": ["plugins"]
-}
diff --git a/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
new file mode 100644
index 0000000..7e5fc0b
--- /dev/null
+++ b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
@@ -0,0 +1,18 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Unattended upgrades plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      unattended_upgrades:
+        type: object
+        properties:
+          download_interval: {type: integer}
+          update_interval: {type: integer}
+          upgrade_interval: {type: integer}
+        required:
+        - update_interval
+        - download_interval
+        - upgrade_interval
diff --git a/bootstrapvz/plugins/vagrant/__init__.py b/bootstrapvz/plugins/vagrant/__init__.py
index af0c6c5..b820a8c 100644
--- a/bootstrapvz/plugins/vagrant/__init__.py
+++ b/bootstrapvz/plugins/vagrant/__init__.py
@@ -3,7 +3,7 @@ import tasks
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/plugins/vagrant/manifest-schema.json b/bootstrapvz/plugins/vagrant/manifest-schema.json
deleted file mode 100644
index bb35ab5..0000000
--- a/bootstrapvz/plugins/vagrant/manifest-schema.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Vagrant plugin manifest",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "string",
-			"enum": ["virtualbox"]
-		},
-		"system": {
-			"required": ["hostname"]
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["vmdk"]
-					// VirtualBox only supports vmdk or raw when importing via OVF:
-					// https://www.virtualbox.org/browser/vbox/trunk/src/VBox/Main/src-server/ApplianceImplImport.cpp?rev=51092#L636
-				}
-			},
-			"required": ["backing"]
-		},
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"vagrant": {
-					"type": "object"
-				}
-			}
-		}
-	}
-}
diff --git a/bootstrapvz/plugins/vagrant/manifest-schema.yml b/bootstrapvz/plugins/vagrant/manifest-schema.yml
new file mode 100644
index 0000000..c27172f
--- /dev/null
+++ b/bootstrapvz/plugins/vagrant/manifest-schema.yml
@@ -0,0 +1,24 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Vagrant plugin manifest
+type: object
+properties:
+  provider:
+    type: object
+    properties:
+      name:
+        type: string
+        enum: [virtualbox]
+  system:
+    required: [hostname]
+  volume:
+    type: object
+    properties:
+      backing:
+        type: string
+        enum: [vmdk]
+    required: [backing]
+  plugins:
+    type: object
+    properties:
+      vagrant: {type: object}
diff --git a/bootstrapvz/providers/azure/__init__.py b/bootstrapvz/providers/azure/__init__.py
index 8b40123..a4c2415 100644
--- a/bootstrapvz/providers/azure/__init__.py
+++ b/bootstrapvz/providers/azure/__init__.py
@@ -13,7 +13,7 @@ def initialize():
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/providers/azure/manifest-schema.json b/bootstrapvz/providers/azure/manifest-schema.json
deleted file mode 100644
index 52516e6..0000000
--- a/bootstrapvz/providers/azure/manifest-schema.json
+++ /dev/null
@@ -1,50 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Azure manifest",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "object",
-			"properties": {
-				"waagent": {
-					"type": "object",
-					"properties": {
-						"conf": {
-							"type": "string"
-						},
-						"version": {
-							"type": "string"
-						}
-					},
-					"required": ["version"]
-				}
-			}
-			"required": ["waagent"]
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["grub", "extlinux"]
-				}
-			},
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["raw"]
-				},
-				"partitions": {
-					"type": "object",
-					"properties": {
-						"type": { "enum": ["none", "msdos", "gpt"] }
-					}
-				}
-			},
-			"required": ["backing"]
-		}
-	}
-}
diff --git a/bootstrapvz/providers/azure/manifest-schema.yml b/bootstrapvz/providers/azure/manifest-schema.yml
new file mode 100644
index 0000000..8d9a34d
--- /dev/null
+++ b/bootstrapvz/providers/azure/manifest-schema.yml
@@ -0,0 +1,38 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Azure manifest
+type: object
+properties:
+  provider:
+    type: object
+    properties:
+      waagent:
+        type: object
+        properties:
+          conf: {type: string}
+          version: {type: string}
+        required: [version]
+    required: [waagent]
+  system:
+    type: object
+    properties:
+      bootloader:
+        type: string
+        enum:
+        - grub
+        - extlinux
+  volume:
+    type: object
+    properties:
+      backing:
+        type: string
+        enum: [raw]
+      partitions:
+        type: object
+        properties:
+          type:
+            enum:
+            - none
+            - msdos
+            - gpt
+    required: [backing]
diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index 4a06d39..ed761d1 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -23,7 +23,7 @@ def initialize():
 
 def validate_manifest(data, validator, error):
 	import os.path
-	validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 
 	from bootstrapvz.common.bytes import Bytes
 	if data['volume']['backing'] == 'ebs':
@@ -35,7 +35,7 @@ def validate_manifest(data, validator, error):
 			msg = ('The volume size must be a multiple of 1GiB when using EBS backing')
 			error(msg, ['volume', 'partitions'])
 	else:
-		validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json'))
+		validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.yml'))
 
 	bootloader = data['system']['bootloader']
 	virtualization = data['provider']['virtualization']
diff --git a/bootstrapvz/providers/ec2/manifest-schema-s3.json b/bootstrapvz/providers/ec2/manifest-schema-s3.json
deleted file mode 100644
index 6c92f33..0000000
--- a/bootstrapvz/providers/ec2/manifest-schema-s3.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "EC2 manifest for instance store AMIs",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "object",
-			"properties": {
-				"credentials": {
-					"type": "object",
-					"properties": {
-						"certificate": {
-							"type": "string"
-						},
-						"private-key": {
-							"type": "string"
-						},
-						"user-id": {
-							"type": "string",
-							"pattern": "(^arn:aws:iam::\\d*:user/\\w.*$)|(^\\d{4}-\\d{4}-\\d{4}$)"
-						}
-					}
-				}
-			}
-		},
-		"image": {
-			"type": "object",
-			"properties": {
-				"bucket": {
-					"type": "string"
-				},
-				"region": {
-					"$ref": "#/definitions/aws-region"
-				}
-			},
-			"required": ["bucket", "region"]
-		}
-	},
-	"required": ["image"],
-	"definitions": {
-		"aws-region": {
-			"enum": ["ap-northeast-1", "ap-southeast-1",
-			         "ap-southeast-2", "eu-west-1",
-			         "sa-east-1", "us-east-1",
-			         "us-gov-west-1", "us-west-1",
-			         "us-west-2", "cn-north-1"]
-		}
-	}
-}
diff --git a/bootstrapvz/providers/ec2/manifest-schema-s3.yml b/bootstrapvz/providers/ec2/manifest-schema-s3.yml
new file mode 100644
index 0000000..6e0fc3e
--- /dev/null
+++ b/bootstrapvz/providers/ec2/manifest-schema-s3.yml
@@ -0,0 +1,38 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: EC2 manifest for instance store AMIs
+type: object
+properties:
+  image:
+    type: object
+    properties:
+      bucket: {type: string}
+      region: {$ref: '#/definitions/aws-region'}
+    required:
+    - bucket
+    - region
+  provider:
+    type: object
+    properties:
+      credentials:
+        type: object
+        properties:
+          certificate: {type: string}
+          private-key: {type: string}
+          user-id:
+            type: string
+            pattern: (^arn:aws:iam::\d*:user/\w.*$)|(^\d{4}-\d{4}-\d{4}$)
+required: [image]
+definitions:
+  aws-region:
+    enum:
+    - ap-northeast-1
+    - ap-southeast-1
+    - ap-southeast-2
+    - eu-west-1
+    - sa-east-1
+    - us-east-1
+    - us-gov-west-1
+    - us-west-1
+    - us-west-2
+    - cn-north-1
diff --git a/bootstrapvz/providers/ec2/manifest-schema.json b/bootstrapvz/providers/ec2/manifest-schema.json
deleted file mode 100644
index 2baed68..0000000
--- a/bootstrapvz/providers/ec2/manifest-schema.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "EC2 manifest",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "object",
-			"properties": {
-				"virtualization": { "enum": ["pvm", "hvm"] },
-				"credentials": {
-					"type": "object",
-					"properties": {
-						"access-key": {
-							"type": "string"
-						},
-						"secret-key": {
-							"type": "string"
-						}
-					}
-				}
-			},
-			"required": ["virtualization"]
-		},
-		"image": {
-			"type": "object",
-			"properties": {
-				"description": {
-					"type": "string"
-				}
-			}
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["pvgrub", "extlinux"]
-				}
-			}
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": { "enum": ["ebs", "s3"] },
-				"partitions": {
-					"type": "object",
-					"properties": {
-						"type": { "enum": ["none", "msdos", "gpt"] }
-					}
-				}
-			},
-			"required": ["backing"]
-		}
-	},
-	"required": ["image"]
-}
diff --git a/bootstrapvz/providers/ec2/manifest-schema.yml b/bootstrapvz/providers/ec2/manifest-schema.yml
new file mode 100644
index 0000000..7bd28df
--- /dev/null
+++ b/bootstrapvz/providers/ec2/manifest-schema.yml
@@ -0,0 +1,47 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: EC2 manifest
+type: object
+properties:
+  image:
+    type: object
+    properties:
+      description: {type: string}
+  provider:
+    type: object
+    properties:
+      credentials:
+        type: object
+        properties:
+          access-key: {type: string}
+          secret-key: {type: string}
+      virtualization:
+        enum:
+        - pvm
+        - hvm
+    required: [virtualization]
+  system:
+    type: object
+    properties:
+      bootloader:
+        type: string
+        enum:
+        - pvgrub
+        - extlinux
+  volume:
+    type: object
+    properties:
+      backing:
+        enum:
+        - ebs
+        - s3
+      partitions:
+        type: object
+        properties:
+          type:
+            enum:
+            - none
+            - msdos
+            - gpt
+    required: [backing]
+required: [image]
diff --git a/bootstrapvz/providers/ec2/tasks/ami-akis.json b/bootstrapvz/providers/ec2/tasks/ami-akis.json
deleted file mode 100644
index 79e1b66..0000000
--- a/bootstrapvz/providers/ec2/tasks/ami-akis.json
+++ /dev/null
@@ -1,34 +0,0 @@
-// This is a mapping of EC2 regions to processor architectures to Amazon Kernel Images
-// Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs
-{
-"ap-northeast-1": // Asia Pacific (Tokyo) Region
-	{"i386":  "aki-136bf512", // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-176bf516"}, // pv-grub-hd0_1.04-x86_64.gz
-"ap-southeast-1": // Asia Pacific (Singapore) Region
-	{"i386":   "aki-ae3973fc", // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-503e7402"}, // pv-grub-hd0_1.04-x86_64.gz
-"ap-southeast-2": // Asia Pacific (Sydney) Region
-	{"i386":  "aki-cd62fff7",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-c362fff9"}, // pv-grub-hd0_1.04-x86_64.gz
-"eu-west-1": // EU (Ireland) Region
-	{"i386":  "aki-68a3451f",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-52a34525"}, // pv-grub-hd0_1.04-x86_64.gz
-"sa-east-1": // South America (Sao Paulo) Region
-	{"i386":  "aki-5b53f446",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-5553f448"}, // pv-grub-hd0_1.04-x86_64.gz
-"us-east-1": // US East (Northern Virginia) Region
-	{"i386":  "aki-8f9dcae6",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-919dcaf8"}, // pv-grub-hd0_1.04-x86_64.gz
-"us-gov-west-1": // AWS GovCloud (US)
-	{"i386":  "aki-1fe98d3c",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-1de98d3e"}, // pv-grub-hd0_1.04-x86_64.gz
-"us-west-1": // US West (Northern California) Region
-	{"i386":  "aki-8e0531cb",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-880531cd"}, // pv-grub-hd0_1.04-x86_64.gz
-"us-west-2": // US West (Oregon) Region
-	{"i386":  "aki-f08f11c0",  // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-fc8f11cc"}, // pv-grub-hd0_1.04-x86_64.gz
-"cn-north-1":// China North (Beijing) Region
-	{"i386":  "aki-908f1da9", // pv-grub-hd0_1.04-i386.gz
-	 "amd64": "aki-9e8f1da7"} // pv-grub-hd0_1.04-x86_64.gz
-}
diff --git a/bootstrapvz/providers/ec2/tasks/ami-akis.yml b/bootstrapvz/providers/ec2/tasks/ami-akis.yml
new file mode 100644
index 0000000..f03a64f
--- /dev/null
+++ b/bootstrapvz/providers/ec2/tasks/ami-akis.yml
@@ -0,0 +1,33 @@
+---
+# This is a mapping of EC2 regions to processor architectures to Amazon Kernel Images
+# Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs
+ap-northeast-1:
+  amd64: aki-176bf516
+  i386: aki-136bf512
+ap-southeast-1:
+  amd64: aki-503e7402
+  i386: aki-ae3973fc
+ap-southeast-2:
+  amd64: aki-c362fff9
+  i386: aki-cd62fff7
+eu-west-1:
+  amd64: aki-52a34525
+  i386: aki-68a3451f
+sa-east-1:
+  amd64: aki-5553f448
+  i386: aki-5b53f446
+us-east-1:
+  amd64: aki-919dcaf8
+  i386: aki-8f9dcae6
+us-gov-west-1:
+  amd64: aki-1de98d3e
+  i386: aki-1fe98d3c
+us-west-1:
+  amd64: aki-880531cd
+  i386: aki-8e0531cb
+us-west-2:
+  amd64: aki-fc8f11cc
+  i386: aki-f08f11c0
+cn-north-1:
+  amd64: aki-9e8f1da7
+  i386: aki-908f1da9
diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 2561769..80feda3 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -117,7 +117,7 @@ class RegisterAMI(Task):
 			registration_params['virtualization_type'] = 'hvm'
 		else:
 			registration_params['virtualization_type'] = 'paravirtual'
-			akis_path = os.path.join(os.path.dirname(__file__), 'ami-akis.json')
+			akis_path = os.path.join(os.path.dirname(__file__), 'ami-akis.yml')
 			from bootstrapvz.common.tools import config_get
 			registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'],
 			                                                          info.manifest.system['architecture']])
diff --git a/bootstrapvz/providers/ec2/tasks/packages-kernels.json b/bootstrapvz/providers/ec2/tasks/packages-kernels.json
deleted file mode 100644
index 9fcf825..0000000
--- a/bootstrapvz/providers/ec2/tasks/packages-kernels.json
+++ /dev/null
@@ -1,15 +0,0 @@
-// This is a mapping of Debian release codenames to processor architectures to kernel packages
-{
-"squeeze": // In squeeze, we need a special kernel flavor for xen
-	{"i386":  "linux-image-xen-686",
-	 "amd64": "linux-image-xen-amd64"},
-"wheezy":
-	{"i386":  "linux-image-686",
-	 "amd64": "linux-image-amd64"},
-"jessie":
-	{"i386":  "linux-image-686-pae",
-	 "amd64": "linux-image-amd64"},
-"sid":
-	{"i386":  "linux-image-686-pae",
-	 "amd64": "linux-image-amd64"}
-}
diff --git a/bootstrapvz/providers/ec2/tasks/packages-kernels.yml b/bootstrapvz/providers/ec2/tasks/packages-kernels.yml
new file mode 100644
index 0000000..1d5a4a0
--- /dev/null
+++ b/bootstrapvz/providers/ec2/tasks/packages-kernels.yml
@@ -0,0 +1,14 @@
+---
+# This is a mapping of Debian release codenames to processor architectures to kernel packages
+squeeze: # In squeeze, we need a special kernel flavor for xen
+  amd64: linux-image-xen-amd64
+  i386: linux-image-xen-686
+wheezy:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+jessie:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
+sid:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py
index 62b21b5..a85f7ed 100644
--- a/bootstrapvz/providers/ec2/tasks/packages.py
+++ b/bootstrapvz/providers/ec2/tasks/packages.py
@@ -21,7 +21,7 @@ class DefaultPackages(Task):
 		info.exclude_packages.add('isc-dhcp-common')
 
 		import os.path
-		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.json')
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
 		kernel_package = config_get(kernel_packages_path, [info.release_codename,
 		                                                   info.manifest.system['architecture']])
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index f3e3454..710e893 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -18,7 +18,7 @@ def initialize():
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/providers/gce/manifest-schema.json b/bootstrapvz/providers/gce/manifest-schema.json
deleted file mode 100644
index 4e1d02d..0000000
--- a/bootstrapvz/providers/gce/manifest-schema.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "GCE manifest",
-	"type": "object",
-	"properties": {
-		"image": {
-			"type": "object",
-				"properties": {
-					"description": {
-						"type": "string"
-					},
-					"gcs_destination": {
-						"type": "string"
-					},
-					"gce_project": {
-						"type": "string"
-					}
-			}
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["grub", "extlinux"]
-				}
-			}
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"partitions": {
-					"type": "object",
-					"properties": {
-						"type": { "enum": ["msdos"] }
-					}
-				}
-			},
-			"required": ["partitions"]
-		}
-	}
-}
-
diff --git a/bootstrapvz/providers/gce/manifest-schema.yml b/bootstrapvz/providers/gce/manifest-schema.yml
new file mode 100644
index 0000000..aa6bb59
--- /dev/null
+++ b/bootstrapvz/providers/gce/manifest-schema.yml
@@ -0,0 +1,29 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: GCE manifest
+type: object
+properties:
+  image:
+    type: object
+    properties:
+      description: {type: string}
+      gce_project: {type: string}
+      gcs_destination: {type: string}
+  system:
+    type: object
+    properties:
+      bootloader:
+        type: string
+        enum:
+        - grub
+        - extlinux
+  volume:
+    type: object
+    properties:
+      partitions:
+        type: object
+        properties:
+          type:
+            enum:
+            - msdos
+    required: [partitions]
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index 4d8f3b5..8bee994 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -21,7 +21,7 @@ class DefaultPackages(Task):
 		info.packages.add('openssh-server')
 		info.packages.add('dhcpd')
 
-		kernel_packages_path = os.path.join(os.path.dirname(__file__), '../../ec2/tasks/packages-kernels.json')
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), '../../ec2/tasks/packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
 		kernel_package = config_get(kernel_packages_path, [info.release_codename,
 		                                                   info.manifest.system['architecture']])
diff --git a/bootstrapvz/providers/kvm/__init__.py b/bootstrapvz/providers/kvm/__init__.py
index f678bb0..55879d4 100644
--- a/bootstrapvz/providers/kvm/__init__.py
+++ b/bootstrapvz/providers/kvm/__init__.py
@@ -11,7 +11,7 @@ def initialize():
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/providers/kvm/manifest-schema.json b/bootstrapvz/providers/kvm/manifest-schema.json
deleted file mode 100644
index a800409..0000000
--- a/bootstrapvz/providers/kvm/manifest-schema.json
+++ /dev/null
@@ -1,50 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "KVM manifest",
-	"type": "object",
-	"properties": {
-		"provider": {
-			"type": "object",
-			"properties": {
-				"virtio": {
-					"type": "array",
-					"items": {
-						"type": "string",
-						"enum": ["virtio",
-						         "virtio_pci",
-						         "virtio_balloon",
-						         "virtio_blk",
-						         "virtio_net",
-						         "virtio_ring"]
-					},
-					"minItems": 1
-				}
-			}
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["grub", "extlinux"]
-				}
-			}
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["raw"]
-				},
-				"partitions": {
-					"type": "object",
-					"properties": {
-						"type": { "enum": ["none", "msdos", "gpt"] }
-					}
-				}
-			},
-			"required": ["backing"]
-		}
-	}
-}
diff --git a/bootstrapvz/providers/kvm/manifest-schema.yml b/bootstrapvz/providers/kvm/manifest-schema.yml
new file mode 100644
index 0000000..0deb7da
--- /dev/null
+++ b/bootstrapvz/providers/kvm/manifest-schema.yml
@@ -0,0 +1,44 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: KVM manifest
+type: object
+properties:
+  provider:
+    type: object
+    properties:
+      virtio:
+        type: array
+        items:
+          type: string
+          enum:
+          - virtio
+          - virtio_pci
+          - virtio_balloon
+          - virtio_blk
+          - virtio_net
+          - virtio_ring
+        minItems: 1
+  system:
+    type: object
+    properties:
+      bootloader:
+        type: string
+        enum:
+        - grub
+        - extlinux
+  volume:
+    type: object
+    properties:
+      backing:
+        type: string
+        enum: [raw]
+      partitions:
+        type: object
+        properties:
+          type:
+            type: string
+            enum:
+            - none
+            - msdos
+            - gpt
+    required: [backing]
diff --git a/bootstrapvz/providers/virtualbox/__init__.py b/bootstrapvz/providers/virtualbox/__init__.py
index 4645ff5..83411f8 100644
--- a/bootstrapvz/providers/virtualbox/__init__.py
+++ b/bootstrapvz/providers/virtualbox/__init__.py
@@ -9,7 +9,7 @@ def initialize():
 
 def validate_manifest(data, validator, error):
 	import os.path
-	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
 
diff --git a/bootstrapvz/providers/virtualbox/manifest-schema.json b/bootstrapvz/providers/virtualbox/manifest-schema.json
deleted file mode 100644
index faefa6f..0000000
--- a/bootstrapvz/providers/virtualbox/manifest-schema.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "VirtualBox manifest",
-	"type": "object",
-	"properties": {
-		"bootstrapper": {
-			"type": "object",
-			"properties": {
-				"guest_additions": {
-					"type": "string"
-				}
-			}
-		},
-		"system": {
-			"type": "object",
-			"properties": {
-				"bootloader": {
-					"type": "string",
-					"enum": ["grub", "extlinux"]
-				}
-			}
-		},
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["raw", "vdi", "vmdk"]
-				},
-				"partitions": {
-					"type": "object",
-					"properties": {
-						"type": { "enum": ["none", "msdos", "gpt"] }
-					}
-				}
-			},
-			"required": ["backing"]
-		}
-	}
-}
diff --git a/bootstrapvz/providers/virtualbox/manifest-schema.yml b/bootstrapvz/providers/virtualbox/manifest-schema.yml
new file mode 100644
index 0000000..1141e66
--- /dev/null
+++ b/bootstrapvz/providers/virtualbox/manifest-schema.yml
@@ -0,0 +1,36 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: VirtualBox manifest
+type: object
+properties:
+  bootstrapper:
+    type: object
+    properties:
+      guest_additions: {type: string}
+  system:
+    type: object
+    properties:
+      bootloader:
+        type: string
+        enum:
+        - grub
+        - extlinux
+  volume:
+    type: object
+    properties:
+      backing:
+        type: string
+        enum:
+        - raw
+        - vdi
+        - vmdk
+      partitions:
+        type: object
+        properties:
+          type:
+            type: string
+            enum:
+            - none
+            - msdos
+            - gpt
+    required: [backing]
diff --git a/manifests/azure.manifest.json b/manifests/azure.manifest.json
deleted file mode 100644
index a8eebc9..0000000
--- a/manifests/azure.manifest.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
-	"provider": {
-		"name": "azure",
-		"waagent": {
-			"version":  "2.0.4"
-		}
-	},
-	"bootstrapper": {
-		"workspace":      "/target",
-		"mirror": "http://ftp.fr.debian.org/debian/"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-	},
-	"volume": {
-		"backing": "raw",
-		"partitions": {
-			"type": "msdos",
-			"boot": {
-				"size":       "32MiB",
-				"filesystem": "ext2"
-			},
-			"root": {
-				"size":       "7GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"ntp": {
-			"servers": ["time.windows.com"]
-		}
-	}
-}
diff --git a/manifests/azure.manifest.yml b/manifests/azure.manifest.yml
new file mode 100644
index 0000000..a6cd751
--- /dev/null
+++ b/manifests/azure.manifest.yml
@@ -0,0 +1,33 @@
+---
+provider:
+  name: azure
+  waagent:
+    version: 2.0.4
+bootstrapper:
+  mirror: http://ftp.fr.debian.org/debian/
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: raw
+  partitions:
+    type: msdos
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 7GiB
+packages: {}
+plugins:
+  ntp:
+    servers:
+    - time.windows.com
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
deleted file mode 100644
index 0c3d844..0000000
--- a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "hvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "extlinux",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://ftp.cn.debian.org/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
new file mode 100644
index 0000000..7ec2d57
--- /dev/null
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: hvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: extlinux
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://ftp.cn.debian.org/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
deleted file mode 100644
index 32d2e5f..0000000
--- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "hvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "extlinux",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
new file mode 100644
index 0000000..ad5a030
--- /dev/null
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: hvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: extlinux
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
deleted file mode 100644
index 01e9c63..0000000
--- a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://ftp.cn.debian.org/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
new file mode 100644
index 0000000..9de6156
--- /dev/null
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://ftp.cn.debian.org/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
deleted file mode 100644
index 4f37a23..0000000
--- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
new file mode 100644
index 0000000..78d414e
--- /dev/null
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
deleted file mode 100644
index b7244a8..0000000
--- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "i386",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-                        "metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
new file mode 100644
index 0000000..d6aabc8
--- /dev/null
+++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: i386
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
deleted file mode 100644
index 03035d6..0000000
--- a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "testing",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
index 8fb5a53..6f66f07 100644
--- a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
@@ -1,37 +1,32 @@
 ---
 provider:
-  name: "ec2"
-  virtualization: "pvm"
-  credentials:
-    access-key: ""
-    secret-key: ""
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
 bootstrapper:
-  workspace: "/target"
+  workspace: /target
 image:
-  name: "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs"
-  description: "Debian {system.release} {system.architecture}"
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
 system:
-  release: "testing"
-  architecture: "amd64"
-  bootloader: "pvgrub"
-  timezone: "UTC"
-  locale: "en_US"
-  charmap: "UTF-8"
-packages:
-  #mirror: "http://cloudfront.debian.net/debian"
-  install_standard: true
+  release: testing
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
 volume:
-  backing: "ebs"
+  backing: ebs
   partitions:
-    type: "none"
+    type: none
     root:
-      size: "8GiB"
-      filesystem: "ext4"
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
 plugins:
   cloud_init:
-    username: "admin"
-    #metadata_sources: "Ec2"
-    disable_modules:
-      - "landscape"
-      - "byobu"
-      - "ssh-import-id"
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
deleted file mode 100644
index 04cf637..0000000
--- a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "unstable",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
new file mode 100644
index 0000000..f7bb3ca
--- /dev/null
+++ b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: unstable
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
deleted file mode 100644
index 3af6564..0000000
--- a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "unstable",
-		"sections":     ["main", "contrib", "non-free"],
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"metadata_sources": "Ec2"
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
new file mode 100644
index 0000000..9394c3c
--- /dev/null
+++ b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
@@ -0,0 +1,36 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: unstable
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+  components:
+  - main
+  - contrib
+  - non-free
+plugins:
+  cloud_init:
+    metadata_sources: Ec2
+    username: admin
diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json
deleted file mode 100644
index 4f2606f..0000000
--- a/manifests/ec2-ebs-partitioned.manifest.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture} AMI ({virtualization})"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "msdos",
-			"root": {
-				"size":       "1GiB",
-				"filesystem": "ext4"
-			}
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-partitioned.manifest.yml b/manifests/ec2-ebs-partitioned.manifest.yml
new file mode 100644
index 0000000..9cfc2fe
--- /dev/null
+++ b/manifests/ec2-ebs-partitioned.manifest.yml
@@ -0,0 +1,28 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture} AMI ({virtualization})
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: msdos
+    root:
+      filesystem: ext4
+      size: 1GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
diff --git a/manifests/ec2-ebs-single.manifest.json b/manifests/ec2-ebs-single.manifest.json
deleted file mode 100644
index 72fa0ca..0000000
--- a/manifests/ec2-ebs-single.manifest.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture} AMI ({virtualization})"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "1GiB",
-				"filesystem": "ext4"
-			}
-		}
-	}
-}
diff --git a/manifests/ec2-ebs-single.manifest.yml b/manifests/ec2-ebs-single.manifest.yml
new file mode 100644
index 0000000..d017eae
--- /dev/null
+++ b/manifests/ec2-ebs-single.manifest.yml
@@ -0,0 +1,28 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture} AMI ({virtualization})
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 1GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
diff --git a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
deleted file mode 100644
index d5f989e..0000000
--- a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}",
-		"description": "Debian {system.release} {system.architecture} AMI",
-		"bucket":      "debian-amis-cn-north-1",
-		"region":      "cn-north-1"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://ftp.cn.debian.org/debian"
-	},
-	"volume": {
-		"backing": "s3",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "4GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"cloud_init": {
-			"username": "admin",
-			"disable_modules": [ "landscape", "byobu", "ssh-import-id" ]
-		}
-	}
-}
diff --git a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
new file mode 100644
index 0000000..708e6eb
--- /dev/null
+++ b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
@@ -0,0 +1,40 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key:  AFAKEACCESSKEYFORAWS
+  #   secret-key:  thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+  #   certificate: /path/to/your/certificate.pem
+  #   private-key: /path/to/your/private.key
+  #   user-id:     arn:aws:iam::123456789012:user/iamuser
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}
+  description: Debian {system.release} {system.architecture} AMI
+  bucket: debian-amis-cn-north-1
+  region: cn-north-1
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: s3
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 4GiB
+packages:
+  mirror: http://ftp.cn.debian.org/debian
+plugins:
+  cloud_init:
+    disable_modules:
+    - landscape
+    - byobu
+    - ssh-import-id
+    username: admin
diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json
deleted file mode 100644
index 36bcfad..0000000
--- a/manifests/ec2-s3.manifest.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key":  null,
-			// "secret-key":  null,
-			// "certificate": null,
-			// "private-key": null,
-			// "user-id":     null
-		}
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture} AMI",
-		"bucket":      "debian-amis",
-		"region":      "us-west-1"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "s3",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "4GiB",
-				"filesystem": "ext4"
-			}
-		}
-	}
-}
diff --git a/manifests/ec2-s3.manifest.yml b/manifests/ec2-s3.manifest.yml
new file mode 100644
index 0000000..464fefe
--- /dev/null
+++ b/manifests/ec2-s3.manifest.yml
@@ -0,0 +1,33 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key:  AFAKEACCESSKEYFORAWS
+  #   secret-key:  thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+  #   certificate: /path/to/your/certificate.pem
+  #   private-key: /path/to/your/private.key
+  #   user-id:     arn:aws:iam::123456789012:user/iamuser
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture} AMI
+  bucket: debian-amis
+  region: us-west-1
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: s3
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 4GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
diff --git a/manifests/gce-backports.manifest.json b/manifests/gce-backports.manifest.json
deleted file mode 100644
index 44f087d..0000000
--- a/manifests/gce-backports.manifest.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-	"provider": {
-		"name": "gce"
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "disk",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"sections":     ["main", "contrib", "non-free"],
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://gce_debian_mirror.storage.googleapis.com/",
-		"preferences": {
-			"backport-kernel": [
-			{
-				"package": "linux-image-* initramfs-tools",
-				"pin": "release n=wheezy-backports",
-				"pin-priority": 500
-			}
-			],
-			"backport-ssh": [
-			{
-				"package": "init-system-helpers openssh-sftp-server openssh-client openssh-server",
-				"pin": "release n=wheezy-backports",
-				"pin-priority": 500
-			}
-			]
-		}
-	},
-	"plugins": {
-		"ntp": {
-			"servers": ["metadata.google.internal"]
-		}
-	},
-	"volume": {
-		"backing": "raw",
-		"partitions": {
-			"type": "msdos",
-			"root": {
-				"size":       "10GiB",
-				"filesystem": "ext4"
-			}
-		}
-	}
-}
diff --git a/manifests/gce-backports.manifest.yml b/manifests/gce-backports.manifest.yml
new file mode 100644
index 0000000..76cf7f9
--- /dev/null
+++ b/manifests/gce-backports.manifest.yml
@@ -0,0 +1,41 @@
+---
+provider:
+  name: gce
+bootstrapper:
+  workspace: /target
+image:
+  name: disk
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: raw
+  partitions:
+    type: msdos
+    root:
+      filesystem: ext4
+      size: 10GiB
+packages:
+  mirror: http://gce_debian_mirror.storage.googleapis.com/
+  components:
+  - main
+  - contrib
+  - non-free
+  preferences:
+    backport-kernel:
+    - package: linux-image-* initramfs-tools
+      pin: release n=wheezy-backports
+      pin-priority: 500
+    backport-ssh:
+    - package: init-system-helpers openssh-sftp-server openssh-client openssh-server
+      pin: release n=wheezy-backports
+      pin-priority: 500
+plugins:
+  ntp:
+    servers:
+    - metadata.google.internal
diff --git a/manifests/gce.manifest.json b/manifests/gce.manifest.json
deleted file mode 100644
index 9d2e079..0000000
--- a/manifests/gce.manifest.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
-	"provider": {
-		"name": "gce"
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "disk",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"sections":     ["main", "contrib", "non-free"],
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://gce_debian_mirror.storage.googleapis.com/"
-	},
-	"plugins": {
-		"ntp": {
-			"servers": ["metadata.google.internal"]
-		}
-	},
-	"volume": {
-		"backing": "raw",
-		"partitions": {
-			"type": "msdos",
-			"root": {
-				"size":       "10GiB",
-				"filesystem": "ext4"
-			}
-		}
-	}
-}
diff --git a/manifests/gce.manifest.yml b/manifests/gce.manifest.yml
new file mode 100644
index 0000000..db26dd1
--- /dev/null
+++ b/manifests/gce.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: gce
+bootstrapper:
+  workspace: /target
+image:
+  name: disk
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: raw
+  partitions:
+    type: msdos
+    root:
+      filesystem: ext4
+      size: 10GiB
+packages:
+  mirror: http://gce_debian_mirror.storage.googleapis.com/
+  components:
+  - main
+  - contrib
+  - non-free
+plugins:
+  ntp:
+    servers:
+    - metadata.google.internal
diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json
deleted file mode 100644
index 33c8b23..0000000
--- a/manifests/kvm-virtio.manifest.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-	"provider": {
-		"name": "kvm",
-		"virtio_modules": [ "virtio_pci", "virtio_blk" ]
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":        "wheezy",
-		"architecture":   "amd64",
-		"bootloader":     "grub",
-		"timezone":       "UTC",
-		"locale":         "en_US",
-		"charmap":        "UTF-8"
-	},
-	"packages": {},
-	"volume": {
-		"backing": "raw",
-		"partitions": {
-			"type": "msdos",
-			"boot": {
-				"size":       "32MiB",
-				"filesystem": "ext2"
-			},
-			"root": {
-				"size":       "864MiB",
-				"filesystem": "ext4"
-			},
-			"swap": {"size": "128MiB"}
-		}
-	},
-	"plugins": {
-		"root_password": {
-			"password": "test"
-		}
-	}
-}
diff --git a/manifests/kvm-virtio.manifest.yml b/manifests/kvm-virtio.manifest.yml
new file mode 100644
index 0000000..81e6c59
--- /dev/null
+++ b/manifests/kvm-virtio.manifest.yml
@@ -0,0 +1,34 @@
+---
+provider:
+  name: kvm
+  virtio_modules:
+  - virtio_pci
+  - virtio_blk
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: raw
+  partitions:
+    type: msdos
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 864MiB
+    swap:
+      size: 128MiB
+packages: {}
+plugins:
+  root_password:
+    password: test
diff --git a/manifests/kvm.manifest.json b/manifests/kvm.manifest.json
deleted file mode 100644
index bd26b50..0000000
--- a/manifests/kvm.manifest.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
-	"provider": {
-		"name": "kvm"
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {},
-	"volume": {
-		"backing": "raw",
-		"partitions": {
-			"type": "msdos",
-			"boot": {
-				"size":       "32MiB",
-				"filesystem": "ext2"
-			},
-			"root": {
-				"size":       "864MiB",
-				"filesystem": "ext4"
-			},
-			"swap": {"size": "128MiB"}
-		}
-	},
-	"plugins": {
-		"root_password": {
-			"password": "test"
-		}
-	}
-}
diff --git a/manifests/kvm.manifest.yml b/manifests/kvm.manifest.yml
new file mode 100644
index 0000000..4b35e63
--- /dev/null
+++ b/manifests/kvm.manifest.yml
@@ -0,0 +1,31 @@
+---
+provider:
+  name: kvm
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: raw
+  partitions:
+    type: msdos
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 864MiB
+    swap:
+      size: 128MiB
+packages: {}
+plugins:
+  root_password:
+    password: test
diff --git a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
deleted file mode 100644
index d9e8e29..0000000
--- a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "squeeze",
-		"architecture": "amd64",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"admin_user": {
-			"username": "admin"
-		}
-	}
-}
diff --git a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
new file mode 100644
index 0000000..c175565
--- /dev/null
+++ b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
@@ -0,0 +1,31 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: squeeze
+  architecture: amd64
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  admin_user:
+    username: admin
diff --git a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
deleted file mode 100644
index 703eea8..0000000
--- a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
-	"provider": {
-		"name": "ec2",
-		"virtualization": "pvm",
-		"credentials":    {
-			// "access-key": null,
-			// "secret-key": null
-		}
-	},
-
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "squeeze",
-		"architecture": "i386",
-		"bootloader":   "pvgrub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {
-		"mirror": "http://cloudfront.debian.net/debian"
-	},
-	"volume": {
-		"backing": "ebs",
-		"partitions": {
-			"type": "none",
-			"root": {
-				"size":       "8GiB",
-				"filesystem": "ext4"
-			}
-		}
-	},
-	"plugins": {
-		"admin_user": {
-			"username": "admin"
-		}
-	}
-}
diff --git a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
new file mode 100644
index 0000000..1fb07b6
--- /dev/null
+++ b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
@@ -0,0 +1,31 @@
+---
+provider:
+  name: ec2
+  virtualization: pvm
+  # credentials:
+  #   access-key: AFAKEACCESSKEYFORAWS
+  #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  description: Debian {system.release} {system.architecture}
+system:
+  release: squeeze
+  architecture: i386
+  bootloader: pvgrub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: ebs
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 8GiB
+packages:
+  mirror: http://cloudfront.debian.net/debian
+plugins:
+  admin_user:
+    username: admin
diff --git a/manifests/virtualbox-vagrant.manifest.json b/manifests/virtualbox-vagrant.manifest.json
deleted file mode 100644
index 632c0b4..0000000
--- a/manifests/virtualbox-vagrant.manifest.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-	"provider": {
-		"name":            "virtualbox",
-		"guest_additions": "/root/images/VBoxGuestAdditions.iso"
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8",
-		"hostname":     "localhost"
-	},
-	"packages": {},
-	"volume": {
-		"backing": "vmdk",
-		"partitions": {
-			"type": "msdos",
-			"boot": {
-				"size":       "64MiB",
-				"filesystem": "ext2"
-			},
-			"root": {
-				"size":       "1856MiB",
-				"filesystem": "ext4"
-			},
-			"swap": {"size": "128MiB"}
-		}
-	},
-	"plugins": {
-		"vagrant": {
-		}
-	}
-}
diff --git a/manifests/virtualbox-vagrant.manifest.yml b/manifests/virtualbox-vagrant.manifest.yml
new file mode 100644
index 0000000..e440ca7
--- /dev/null
+++ b/manifests/virtualbox-vagrant.manifest.yml
@@ -0,0 +1,32 @@
+---
+provider:
+  name: virtualbox
+  guest_additions: /root/images/VBoxGuestAdditions.iso
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  hostname: localhost
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: vmdk
+  partitions:
+    type: msdos
+    boot:
+      filesystem: ext2
+      size: 64MiB
+    root:
+      filesystem: ext4
+      size: 1856MiB
+    swap:
+      size: 128MiB
+packages: {}
+plugins:
+  vagrant: {}
diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json
deleted file mode 100644
index 9e69292..0000000
--- a/manifests/virtualbox.manifest.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-	"provider": {
-		"name":            "virtualbox",
-		"guest_additions": "/root/images/VBoxGuestAdditions.iso"
-	},
-	"bootstrapper": {
-		"workspace": "/target"
-	},
-	"image": {
-		"name":        "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}",
-		"description": "Debian {system.release} {system.architecture}"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"bootloader":   "grub",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"packages": {},
-	"volume": {
-		"backing": "vdi",
-		"partitions": {
-			"type": "msdos",
-			"boot": {
-				"size":       "32MiB",
-				"filesystem": "ext2"
-			},
-			"root": {
-				"size":       "864MiB",
-				"filesystem": "ext4"
-			},
-			"swap": {"size": "128MiB"}
-		}
-	}
-}
diff --git a/manifests/virtualbox.manifest.yml b/manifests/virtualbox.manifest.yml
new file mode 100644
index 0000000..6a1b358
--- /dev/null
+++ b/manifests/virtualbox.manifest.yml
@@ -0,0 +1,29 @@
+---
+provider:
+  name: virtualbox
+  guest_additions: /root/images/VBoxGuestAdditions.iso
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  release: wheezy
+  architecture: amd64
+  bootloader: grub
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: vdi
+  partitions:
+    type: msdos
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 864MiB
+    swap:
+      size: 128MiB
+packages: {}

From 376baae583322ab5d14fd374ff08437e358a1102 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 6 Jul 2014 18:24:35 +0200
Subject: [PATCH 029/345] Take advantage of the YAML multiline notation

---
 .../common/tasks/network-configuration.yml    | 29 +++++++++----------
 bootstrapvz/common/tasks/network.py           |  2 +-
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/bootstrapvz/common/tasks/network-configuration.yml b/bootstrapvz/common/tasks/network-configuration.yml
index c044069..12e5f03 100644
--- a/bootstrapvz/common/tasks/network-configuration.yml
+++ b/bootstrapvz/common/tasks/network-configuration.yml
@@ -1,17 +1,16 @@
 ---
 # This is a mapping of Debian release codenames to NIC configurations
-# Every item in an array is a line
-squeeze:
-- auto lo
-- iface lo inet loopback
-- auto eth0
-- iface eth0 inet dhcp
-wheezy:
-- auto eth0
-- iface eth0 inet dhcp
-jessie:
-- auto eth0
-- iface eth0 inet dhcp
-sid:
-- auto eth0
-- iface eth0 inet dhcp
+squeeze: |
+  auto lo
+  iface lo inet loopback
+  auto eth0
+  iface eth0 inet dhcp
+wheezy: |
+  auto eth0
+  iface eth0 inet dhcp
+jessie: |
+  auto eth0
+  iface eth0 inet dhcp
+sid: |
+  auto eth0
+  iface eth0 inet dhcp
diff --git a/bootstrapvz/common/tasks/network.py b/bootstrapvz/common/tasks/network.py
index 72da5c9..858b158 100644
--- a/bootstrapvz/common/tasks/network.py
+++ b/bootstrapvz/common/tasks/network.py
@@ -51,4 +51,4 @@ class ConfigureNetworkIF(Task):
 
 		interfaces_path = os.path.join(info.root, 'etc/network/interfaces')
 		with open(interfaces_path, 'a') as interfaces:
-			interfaces.write('\n'.join(if_config) + '\n')
+			interfaces.write(if_config + '\n')

From 34bb45bb0058d2a568964295bb3acd1a8733e857 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 6 Jul 2014 20:23:48 +0200
Subject: [PATCH 030/345] Factor release codename fetching out into
 common.tools This allows code that runs before the bootstrapinformation
 object has been created to also figure out the release codename

---
 bootstrapvz/base/bootstrapinfo.py                  |  7 ++-----
 bootstrapvz/{base => common}/release-codenames.yml |  0
 bootstrapvz/common/tasks/initd.py                  |  2 +-
 bootstrapvz/common/tasks/ssh.py                    |  4 ++--
 bootstrapvz/common/tools.py                        | 12 ++++++++++--
 bootstrapvz/plugins/cloud_init/__init__.py         |  3 ++-
 bootstrapvz/plugins/cloud_init/tasks.py            |  2 +-
 bootstrapvz/plugins/docker_daemon/__init__.py      |  3 ++-
 bootstrapvz/plugins/opennebula/__init__.py         |  3 ++-
 bootstrapvz/plugins/opennebula/tasks.py            |  2 +-
 10 files changed, 23 insertions(+), 15 deletions(-)
 rename bootstrapvz/{base => common}/release-codenames.yml (100%)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 8edb5e8..0313e3b 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -31,11 +31,8 @@ class BootstrapInformation(object):
 		# The default apt mirror
 		self.apt_mirror = self.manifest.packages.get('mirror', 'http://http.debian.net/debian')
 
-		# Normalize the release codenames so that tasks may query for release codenames rather than
-		# 'stable', 'unstable' etc. This is useful when handling cases that are specific to a release.
-		release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.yml')
-		from bootstrapvz.common.tools import config_get
-		self.release_codename = config_get(release_codenames_path, [self.manifest.system['release']])
+		from bootstrapvz.common.tools import get_codename
+		self.release_codename = get_codename(self.manifest.system['release'])
 
 		# Create the manifest_vars dictionary
 		self.manifest_vars = self.__create_manifest_vars(self.manifest, {'apt_mirror': self.apt_mirror})
diff --git a/bootstrapvz/base/release-codenames.yml b/bootstrapvz/common/release-codenames.yml
similarity index 100%
rename from bootstrapvz/base/release-codenames.yml
rename to bootstrapvz/common/release-codenames.yml
diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py
index fdc14f5..ccf9119 100644
--- a/bootstrapvz/common/tasks/initd.py
+++ b/bootstrapvz/common/tasks/initd.py
@@ -45,7 +45,7 @@ class RemoveHWClock(Task):
 	@classmethod
 	def run(cls, info):
 		info.initd['disable'].append('hwclock.sh')
-		if info.manifest.system['release'] == 'squeeze':
+		if info.release_codename == 'squeeze':
 			info.initd['disable'].append('hwclockfirst.sh')
 
 
diff --git a/bootstrapvz/common/tasks/ssh.py b/bootstrapvz/common/tasks/ssh.py
index 242330d..354e95b 100644
--- a/bootstrapvz/common/tasks/ssh.py
+++ b/bootstrapvz/common/tasks/ssh.py
@@ -30,7 +30,7 @@ class AddSSHKeyGeneration(Task):
 		try:
 			log_check_call(['chroot', info.root,
 			                'dpkg-query', '-W', 'openssh-server'])
-			if info.manifest.system['release'] == 'squeeze':
+			if info.release_codename == 'squeeze':
 				install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'squeeze/generate-ssh-hostkeys')
 			else:
 				install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'generate-ssh-hostkeys')
@@ -70,7 +70,7 @@ class ShredHostkeys(Task):
 	def run(cls, info):
 		ssh_hostkeys = ['ssh_host_dsa_key',
 		                'ssh_host_rsa_key']
-		if info.manifest.system['release'] != 'squeeze':
+		if info.release_codename != 'squeeze':
 			ssh_hostkeys.append('ssh_host_ecdsa_key')
 
 		private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 315291e..777a111 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -1,3 +1,4 @@
+import os
 
 
 def log_check_call(command, stdin=None, env=None, shell=False, cwd=None):
@@ -76,7 +77,6 @@ def load_yaml(path):
 
 
 def load_data(path):
-	import os.path
 	filename, extension = os.path.splitext(path)
 	if not os.path.isfile(path):
 		raise Exception('The path {path} does not point to a file.'.format(path=path))
@@ -95,9 +95,17 @@ def config_get(path, config_path):
 	return config
 
 
+def get_codename(release):
+	"""Normalizes the release codenames
+	This allows tasks to query for release codenames rather than 'stable', 'unstable' etc.
+	"""
+	release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.yml')
+	from bootstrapvz.common.tools import config_get
+	return config_get(release_codenames_path, [release])
+
+
 def copy_tree(from_path, to_path):
 	from shutil import copy
-	import os
 	for abs_prefix, dirs, files in os.walk(from_path):
 		prefix = os.path.normpath(os.path.relpath(abs_prefix, from_path))
 		for path in dirs:
diff --git a/bootstrapvz/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py
index 999572b..0268d61 100644
--- a/bootstrapvz/plugins/cloud_init/__init__.py
+++ b/bootstrapvz/plugins/cloud_init/__init__.py
@@ -12,7 +12,8 @@ def resolve_tasks(taskset, manifest):
 	from bootstrapvz.common.tasks import initd
 	from bootstrapvz.common.tasks import ssh
 
-	if manifest.system['release'] in ['wheezy', 'stable']:
+	from bootstrapvz.common.tools import get_codename
+	if get_codename(manifest.system['release']) == 'wheezy':
 		taskset.add(tasks.AddBackports)
 
 	taskset.update([tasks.SetMetadataSource,
diff --git a/bootstrapvz/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py
index 44e2dff..6f64c38 100644
--- a/bootstrapvz/plugins/cloud_init/tasks.py
+++ b/bootstrapvz/plugins/cloud_init/tasks.py
@@ -29,7 +29,7 @@ class AddCloudInitPackages(Task):
 	@classmethod
 	def run(cls, info):
 		target = None
-		if info.manifest.system['release'] in ['wheezy', 'stable']:
+		if info.release_codename == 'wheezy':
 			target = '{system.release}-backports'
 		info.packages.add('cloud-init', target)
 		info.packages.add('sudo')
diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index f2eda61..b85e32f 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -5,7 +5,8 @@ import os.path
 def validate_manifest(data, validator, error):
 	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
-	if data.get('system', {}).get('release', None) in ['wheezy', 'stable']:
+	from bootstrapvz.common.tools import get_codename
+	if get_codename(data['system']['release']) == 'wheezy':
 		# prefs is a generator of apt preferences across files in the manifest
 		prefs = (item for vals in data.get('packages', {}).get('preferences', {}).values() for item in vals)
 		if not any('linux-image' in item['package'] and 'wheezy-backports' in item['pin'] for item in prefs):
diff --git a/bootstrapvz/plugins/opennebula/__init__.py b/bootstrapvz/plugins/opennebula/__init__.py
index 4ac6056..bb90773 100644
--- a/bootstrapvz/plugins/opennebula/__init__.py
+++ b/bootstrapvz/plugins/opennebula/__init__.py
@@ -2,6 +2,7 @@ import tasks
 
 
 def resolve_tasks(taskset, manifest):
-	if manifest.system['release'] in ['wheezy', 'stable']:
+	from bootstrapvz.common.tools import get_codename
+	if get_codename(manifest.system['release']) == 'wheezy':
 		taskset.add(tasks.AddBackports)
 	taskset.update([tasks.AddONEContextPackage])
diff --git a/bootstrapvz/plugins/opennebula/tasks.py b/bootstrapvz/plugins/opennebula/tasks.py
index 410bc90..4e5c48b 100644
--- a/bootstrapvz/plugins/opennebula/tasks.py
+++ b/bootstrapvz/plugins/opennebula/tasks.py
@@ -26,6 +26,6 @@ class AddONEContextPackage(Task):
 	@classmethod
 	def run(cls, info):
 		target = None
-		if info.manifest.system['release'] in ['wheezy', 'stable']:
+		if info.release_codename == 'wheezy':
 			target = '{system.release}-backports'
 		info.packages.add('opennebula-context', target)

From 1dc9ae18db10b52d547a2692ae2bfb1244bd4bac Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 6 Jul 2014 22:27:36 +0200
Subject: [PATCH 031/345] Do a basic validation of the manifest before
 accessing it. This prevents cryptic error messages

---
 bootstrapvz/base/manifest.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 44f56f1..8151bdd 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -25,13 +25,21 @@ class Manifest(object):
 		self.parse()
 
 	def load(self):
-		"""Loads the manifest.
-		This function not only reads the manifest but also loads the specified provider and plugins.
-		Once they are loaded, the initialize() function is called on each of them (if it exists).
+		"""Loads the manifest and performs a basic validation.
+		This function reads the manifest, loads the specified provider and plugins, and performs
+		some basic validation of the manifest itself to ensure that the properties
+		required for initalization are accessible
+		(otherwise the user would be presented with some cryptic error messages).
+		Once the provider and plugins are loaded,
+		the initialize() function is called on each of them (if it exists).
 		The provider must have an initialize function.
 		"""
 		self.data = load_data(self.path)
 
+		from . import validate_manifest
+		# Validate the manifest with the base validation function in __init__
+		validate_manifest(self.data, self.schema_validator, self.validation_error)
+
 		# Get the provider name from the manifest and load the corresponding module
 		provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name']
 		log.debug('Loading provider ' + provider_modname)
@@ -57,12 +65,9 @@ class Manifest(object):
 				init()
 
 	def validate(self):
-		"""Validates the manifest using the base, provider and plugin validation functions.
+		"""Validates the manifest using the provider and plugin validation functions.
 		Plugins are not required to have a validate_manifest function
 		"""
-		from . import validate_manifest
-		# Validate the manifest with the base validation function in __init__
-		validate_manifest(self.data, self.schema_validator, self.validation_error)
 
 		# Run the provider validation
 		self.modules['provider'].validate_manifest(self.data, self.schema_validator, self.validation_error)

From e4a9cc837ae54ddb3007de71f658cf9a1a1c756c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 21:20:26 +0200
Subject: [PATCH 032/345] Differentiate installation of grub 1.99 and grub 2

---
 bootstrapvz/common/task_groups.py          | 19 +++++++++++----
 bootstrapvz/common/tasks/boot.py           | 28 +++++++++++++---------
 bootstrapvz/plugins/docker_daemon/tasks.py |  2 +-
 bootstrapvz/providers/azure/tasks/boot.py  |  2 +-
 bootstrapvz/providers/gce/tasks/boot.py    |  2 +-
 5 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index c10c177..96e125c 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -30,7 +30,7 @@ def get_standard_groups(manifest):
 	group.extend(get_apt_group(manifest))
 	group.extend(security_group)
 	group.extend(locale_group)
-	group.extend(bootloader_group.get(manifest.system['bootloader'], []))
+	group.extend(get_bootloader_group(manifest))
 	group.extend(cleanup_group)
 	return group
 
@@ -126,9 +126,20 @@ locale_group = [locale.LocaleBootstrapPackage,
                 ]
 
 
-bootloader_group = {'grub':     [boot.AddGrubPackage, boot.ConfigureGrub, boot.InstallGrub],
-                    'extlinux': [boot.AddExtlinuxPackage, boot.InstallExtLinux],
-                    }
+def get_bootloader_group(manifest):
+	group = []
+	if manifest.system['bootloader'] == 'grub':
+		group.extend([boot.AddGrubPackage,
+		              boot.ConfigureGrub])
+		from bootstrapvz.common.tools import get_codename
+		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+			group.append(boot.InstallGrub_1_99)
+		else:
+			group.append(boot.InstallGrub_2)
+	if manifest.system['bootloader'] == 'extlinux':
+		group.extend([boot.AddExtlinuxPackage,
+		              boot.InstallExtLinux])
+	return group
 
 
 def get_fs_specific_group(manifest):
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 05dab75..c0a4d4d 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -1,5 +1,6 @@
 from bootstrapvz.base import Task
 from .. import phases
+from ..tools import log_check_call
 import apt
 import filesystem
 from bootstrapvz.base.fs import partitionmaps
@@ -58,18 +59,13 @@ class ConfigureGrub(Task):
 		                'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0"')
 
 
-class InstallGrub(Task):
-	description = 'Installing grub'
+class InstallGrub_1_99(Task):
+	description = 'Installing grub 1.99'
 	phase = phases.system_modification
 	predecessors = [filesystem.FStab]
 
 	@classmethod
 	def run(cls, info):
-		from ..fs.loopbackvolume import LoopbackVolume
-		from ..tools import log_check_call
-
-		boot_dir = os.path.join(info.root, 'boot')
-		grub_dir = os.path.join(boot_dir, 'grub')
 
 		from ..fs import remount
 		p_map = info.volume.partition_map
@@ -87,11 +83,12 @@ class InstallGrub(Task):
 		# GRUB cannot deal with installing to loopback devices
 		# so we fake a real harddisk with dmsetup.
 		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
+		from ..fs.loopbackvolume import LoopbackVolume
 		if isinstance(info.volume, LoopbackVolume):
 			remount(info.volume, link_fn)
 		try:
 			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
-			device_map_path = os.path.join(grub_dir, 'device.map')
+			device_map_path = os.path.join(info.root, 'boot/grub/device.map')
 			partition_prefix = 'msdos'
 			if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
 				partition_prefix = 'gpt'
@@ -105,8 +102,7 @@ class InstallGrub(Task):
 						                         idx=idx + 1))
 
 			# Install grub
-			log_check_call(['chroot', info.root,
-			                'grub-install', device_path])
+			log_check_call(['chroot', info.root, 'grub-install', device_path])
 			log_check_call(['chroot', info.root, 'update-grub'])
 		except Exception as e:
 			if isinstance(info.volume, LoopbackVolume):
@@ -117,6 +113,17 @@ class InstallGrub(Task):
 			remount(info.volume, unlink_fn)
 
 
+class InstallGrub_2(Task):
+	description = 'Installing grub 2'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+		log_check_call(['chroot', info.root, 'grub-install', info.volume.device_path])
+		log_check_call(['chroot', info.root, 'update-grub'])
+
+
 class AddExtlinuxPackage(Task):
 	description = 'Adding extlinux package'
 	phase = phases.preparation
@@ -136,7 +143,6 @@ class InstallExtLinux(Task):
 
 	@classmethod
 	def run(cls, info):
-		from ..tools import log_check_call
 		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
 			bootloader = '/usr/lib/syslinux/gptmbr.bin'
 		else:
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index d44a661..e82aad2 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -52,7 +52,7 @@ class AddDockerInit(Task):
 class EnableMemoryCgroup(Task):
 	description = 'Change grub configuration to enable the memory cgroup'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub]
+	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
 	predecessors = [boot.ConfigureGrub, gceboot.ConfigureGrub]
 
 	@classmethod
diff --git a/bootstrapvz/providers/azure/tasks/boot.py b/bootstrapvz/providers/azure/tasks/boot.py
index 5fc4756..7f95276 100644
--- a/bootstrapvz/providers/azure/tasks/boot.py
+++ b/bootstrapvz/providers/azure/tasks/boot.py
@@ -6,7 +6,7 @@ from bootstrapvz.common.tasks import boot
 class ConfigureGrub(Task):
 	description = 'Change grub configuration to allow for ttyS0 output'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub]
+	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/gce/tasks/boot.py b/bootstrapvz/providers/gce/tasks/boot.py
index 1100210..6d03025 100644
--- a/bootstrapvz/providers/gce/tasks/boot.py
+++ b/bootstrapvz/providers/gce/tasks/boot.py
@@ -7,7 +7,7 @@ import os.path
 class ConfigureGrub(Task):
 	description = 'Change grub configuration to allow for ttyS0 output'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub]
+	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
 
 	@classmethod
 	def run(cls, info):

From 9d8821235fce1e8c8ee2fbd78571c10bacc2fdf7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 21:40:34 +0200
Subject: [PATCH 033/345] Clone packages-kernels.yml pattern to other providers
 GCE also gets its own file. For now, this scales - but we might want to
 refactor when there is more that just the kernel package we need to choose
 from

---
 .../providers/azure/tasks/packages-kernels.yml     | 14 ++++++++++++++
 bootstrapvz/providers/azure/tasks/packages.py      | 10 +++++++---
 .../providers/gce/tasks/packages-kernels.yml       | 14 ++++++++++++++
 bootstrapvz/providers/gce/tasks/packages.py        |  2 +-
 .../providers/kvm/tasks/packages-kernels.yml       | 14 ++++++++++++++
 bootstrapvz/providers/kvm/tasks/packages.py        |  9 ++++++---
 .../virtualbox/tasks/packages-kernels.yml          | 14 ++++++++++++++
 bootstrapvz/providers/virtualbox/tasks/packages.py |  9 ++++++---
 8 files changed, 76 insertions(+), 10 deletions(-)
 create mode 100644 bootstrapvz/providers/azure/tasks/packages-kernels.yml
 create mode 100644 bootstrapvz/providers/gce/tasks/packages-kernels.yml
 create mode 100644 bootstrapvz/providers/kvm/tasks/packages-kernels.yml
 create mode 100644 bootstrapvz/providers/virtualbox/tasks/packages-kernels.yml

diff --git a/bootstrapvz/providers/azure/tasks/packages-kernels.yml b/bootstrapvz/providers/azure/tasks/packages-kernels.yml
new file mode 100644
index 0000000..1279faf
--- /dev/null
+++ b/bootstrapvz/providers/azure/tasks/packages-kernels.yml
@@ -0,0 +1,14 @@
+---
+# This is a mapping of Debian release codenames to processor architectures to kernel packages
+squeeze:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+wheezy:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+jessie:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
+sid:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
diff --git a/bootstrapvz/providers/azure/tasks/packages.py b/bootstrapvz/providers/azure/tasks/packages.py
index 40a51e2..b7d2558 100644
--- a/bootstrapvz/providers/azure/tasks/packages.py
+++ b/bootstrapvz/providers/azure/tasks/packages.py
@@ -11,14 +11,18 @@ class DefaultPackages(Task):
 
 	@classmethod
 	def run(cls, info):
-		kernels = {'amd64': 'linux-image-amd64',
-		           'i386':  'linux-image-686', }
-		info.packages.add(kernels.get(info.manifest.system['architecture']))
 		info.packages.add('openssl')
 		info.packages.add('python-openssl')
 		info.packages.add('python-pyasn1')
 		info.packages.add('sudo')
 
+		import os.path
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
+		from bootstrapvz.common.tools import config_get
+		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		                                                   info.manifest.system['architecture']])
+		info.packages.add(kernel_package)
+
 
 class Waagent(Task):
 	description = 'Add waagent'
diff --git a/bootstrapvz/providers/gce/tasks/packages-kernels.yml b/bootstrapvz/providers/gce/tasks/packages-kernels.yml
new file mode 100644
index 0000000..1d5a4a0
--- /dev/null
+++ b/bootstrapvz/providers/gce/tasks/packages-kernels.yml
@@ -0,0 +1,14 @@
+---
+# This is a mapping of Debian release codenames to processor architectures to kernel packages
+squeeze: # In squeeze, we need a special kernel flavor for xen
+  amd64: linux-image-xen-amd64
+  i386: linux-image-xen-686
+wheezy:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+jessie:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
+sid:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index 8bee994..dc964e5 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -21,7 +21,7 @@ class DefaultPackages(Task):
 		info.packages.add('openssh-server')
 		info.packages.add('dhcpd')
 
-		kernel_packages_path = os.path.join(os.path.dirname(__file__), '../../ec2/tasks/packages-kernels.yml')
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
 		kernel_package = config_get(kernel_packages_path, [info.release_codename,
 		                                                   info.manifest.system['architecture']])
diff --git a/bootstrapvz/providers/kvm/tasks/packages-kernels.yml b/bootstrapvz/providers/kvm/tasks/packages-kernels.yml
new file mode 100644
index 0000000..1279faf
--- /dev/null
+++ b/bootstrapvz/providers/kvm/tasks/packages-kernels.yml
@@ -0,0 +1,14 @@
+---
+# This is a mapping of Debian release codenames to processor architectures to kernel packages
+squeeze:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+wheezy:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+jessie:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
+sid:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
diff --git a/bootstrapvz/providers/kvm/tasks/packages.py b/bootstrapvz/providers/kvm/tasks/packages.py
index 85ad028..7498c53 100644
--- a/bootstrapvz/providers/kvm/tasks/packages.py
+++ b/bootstrapvz/providers/kvm/tasks/packages.py
@@ -10,6 +10,9 @@ class DefaultPackages(Task):
 
 	@classmethod
 	def run(cls, info):
-		kernels = {'amd64': 'linux-image-amd64',
-		           'i386':  'linux-image-686', }
-		info.packages.add(kernels.get(info.manifest.system['architecture']))
+		import os.path
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
+		from bootstrapvz.common.tools import config_get
+		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		                                                   info.manifest.system['architecture']])
+		info.packages.add(kernel_package)
diff --git a/bootstrapvz/providers/virtualbox/tasks/packages-kernels.yml b/bootstrapvz/providers/virtualbox/tasks/packages-kernels.yml
new file mode 100644
index 0000000..1279faf
--- /dev/null
+++ b/bootstrapvz/providers/virtualbox/tasks/packages-kernels.yml
@@ -0,0 +1,14 @@
+---
+# This is a mapping of Debian release codenames to processor architectures to kernel packages
+squeeze:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+wheezy:
+  amd64: linux-image-amd64
+  i386: linux-image-686
+jessie:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
+sid:
+  amd64: linux-image-amd64
+  i386: linux-image-686-pae
diff --git a/bootstrapvz/providers/virtualbox/tasks/packages.py b/bootstrapvz/providers/virtualbox/tasks/packages.py
index 8235c32..ccdb419 100644
--- a/bootstrapvz/providers/virtualbox/tasks/packages.py
+++ b/bootstrapvz/providers/virtualbox/tasks/packages.py
@@ -10,6 +10,9 @@ class DefaultPackages(Task):
 
 	@classmethod
 	def run(cls, info):
-		kernels = {'amd64': 'linux-image-amd64',
-		           'i386':  'linux-image-686', }
-		info.packages.add(kernels.get(info.manifest.system['architecture']))
+		import os.path
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
+		from bootstrapvz.common.tools import config_get
+		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		                                                   info.manifest.system['architecture']])
+		info.packages.add(kernel_package)

From 7fe9c1ba362e5cd87a2224f2ab82ba7ba91c2459 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 22:20:20 +0200
Subject: [PATCH 034/345] Refactor logging setup to be more modular

---
 bootstrapvz/base/log.py  | 76 ++++++++++++++++++++++------------------
 bootstrapvz/base/main.py | 55 ++++++++++++++++++-----------
 2 files changed, 77 insertions(+), 54 deletions(-)

diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py
index 83ff97f..5699dcc 100644
--- a/bootstrapvz/base/log.py
+++ b/bootstrapvz/base/log.py
@@ -4,6 +4,48 @@ both to a file and to the console.
 import logging
 
 
+def get_console_handler(debug):
+	"""Returns a log handler for the console
+	The handler color codes the different log levels
+
+	:params bool debug: Whether to set the log level to DEBUG (otherwise INFO)
+	:return: The console logging handler
+	"""
+	# Create a console log handler
+	import sys
+	console_handler = logging.StreamHandler(sys.stderr)
+	# We want to colorize the output to the console, so we add a formatter
+	console_handler.setFormatter(ConsoleFormatter())
+	# Set the log level depending on the debug argument
+	if debug:
+		console_handler.setLevel(logging.DEBUG)
+	else:
+		console_handler.setLevel(logging.INFO)
+	return console_handler
+
+
+def get_file_handler(path, debug):
+	"""Returns a log handler for the given path
+	If the parent directory of the logpath does not exist it will be created
+	The handler outputs relative timestamps (to when it was created)
+
+	:params str path: The full path to the logfile
+	:params bool debug: Whether to set the log level to DEBUG (otherwise INFO)
+	:return: The file logging handler
+	"""
+	import os.path
+	if not os.path.exists(os.path.dirname(path)):
+		os.makedirs(os.path.dirname(path))
+	# Create the log handler
+	file_handler = logging.FileHandler(path)
+	# Absolute timestamps are rather useless when bootstrapping, it's much more interesting
+	# to see how long things take, so we log in a relative format instead
+	file_handler.setFormatter(FileFormatter('[%(relativeCreated)s] %(levelname)s: %(message)s'))
+	# The file log handler always logs everything
+	file_handler.setLevel(logging.DEBUG)
+	return file_handler
+
+
 def get_log_filename(manifest_path):
 	"""Returns the path to a logfile given a manifest
 	The logfile name is constructed from the current timestamp and the basename of the manifest
@@ -22,40 +64,6 @@ def get_log_filename(manifest_path):
 	return filename
 
 
-def setup_logger(logfile=None, debug=False):
-	"""Sets up the python logger to log to both a file and the console
-
-	:param str logfile: Path to a logfile
-	:param bool debug: Whether to log debug output to the console
-	"""
-	root = logging.getLogger()
-	# Make sure all logging statements are processed by our handlers, they decide the log level
-	root.setLevel(logging.NOTSET)
-
-	# Only enable logging to file if a destination was supplied
-	if logfile is not None:
-		# Create a file log handler
-		file_handler = logging.FileHandler(logfile)
-		# Absolute timestamps are rather useless when bootstrapping, it's much more interesting
-		# to see how long things take, so we log in a relative format instead
-		file_handler.setFormatter(FileFormatter('[%(relativeCreated)s] %(levelname)s: %(message)s'))
-		# The file log handler always logs everything
-		file_handler.setLevel(logging.DEBUG)
-		root.addHandler(file_handler)
-
-	# Create a console log handler
-	import sys
-	console_handler = logging.StreamHandler(sys.stderr)
-	# We want to colorize the output to the console, so we add a formatter
-	console_handler.setFormatter(ConsoleFormatter())
-	# Set the log level depending on the debug argument
-	if debug:
-		console_handler.setLevel(logging.DEBUG)
-	else:
-		console_handler.setLevel(logging.INFO)
-	root.addHandler(console_handler)
-
-
 class ConsoleFormatter(logging.Formatter):
 	"""Formats log statements for the console
 	"""
diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index ad64db6..80a4c4d 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -17,20 +17,14 @@ def main():
 	if os.geteuid() != 0 and not opts['--dry-run']:
 		raise Exception('This program requires root privileges.')
 
-	import log
-	# Log to file unless --log is a single dash
-	if opts['--log'] != '-':
-		# Setup logging
-		if not os.path.exists(opts['--log']):
-			os.makedirs(opts['--log'])
-		log_filename = log.get_log_filename(opts['MANIFEST'])
-		logfile = os.path.join(opts['--log'], log_filename)
-	else:
-		logfile = None
-	log.setup_logger(logfile=logfile, debug=opts['--debug'])
+	# Set up logging
+	setup_loggers(opts)
 
 	# Everything has been set up, begin the bootstrapping process
-	run(opts)
+	run(opts['MANIFEST'],
+	    debug=opts['--debug'],
+	    pause_on_error=opts['--pause-on-error'],
+	    dry_run=opts['--dry-run'])
 
 
 def get_opts():
@@ -49,18 +43,39 @@ Options:
   --debug            Print debugging information
   -h, --help         show this help
 	"""
-	opts = docopt(usage)
-	return opts
+	return docopt(usage)
 
 
-def run(opts):
+def setup_loggers(opts):
+	"""Sets up the file and console loggers
+
+	:params dict opts: Dictionary of options from the commandline
+	"""
+	import logging
+	root = logging.getLogger()
+	root.setLevel(logging.NOTSET)
+
+	import log
+	# Log to file unless --log is a single dash
+	if opts['--log'] != '-':
+		import os.path
+		log_filename = log.get_log_filename(opts['MANIFEST'])
+		logpath = os.path.join(opts['--log'], log_filename)
+		file_handler = log.get_file_handler(path=logpath, debug=True)
+		root.addHandler(file_handler)
+
+	console_handler = log.get_console_handler(debug=opts['--debug'])
+	root.addHandler(console_handler)
+
+
+def run(manifest_path, debug=False, pause_on_error=False, dry_run=False):
 	"""Runs the bootstrapping process
 
 	:params dict opts: Dictionary of options from the commandline
 	"""
 	# Load the manifest
 	from manifest import Manifest
-	manifest = Manifest(opts['MANIFEST'])
+	manifest = Manifest(manifest_path)
 
 	# Get the tasklist
 	from tasklist import load_tasks
@@ -71,18 +86,18 @@ def run(opts):
 
 	# Create the bootstrap information object that'll be used throughout the bootstrapping process
 	from bootstrapinfo import BootstrapInformation
-	bootstrap_info = BootstrapInformation(manifest=manifest, debug=opts['--debug'])
+	bootstrap_info = BootstrapInformation(manifest=manifest, debug=debug)
 
 	try:
 		# Run all the tasks the tasklist has gathered
-		tasklist.run(info=bootstrap_info, dry_run=opts['--dry-run'])
+		tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
 		return bootstrap_info
 	except (Exception, KeyboardInterrupt) as e:
 		# When an error occurs, log it and begin rollback
 		log.exception(e)
-		if opts['--pause-on-error']:
+		if pause_on_error:
 			# The --pause-on-error is useful when the user wants to inspect the volume before rollback
 			raw_input('Press Enter to commence rollback')
 		log.error('Rolling back')
@@ -106,6 +121,6 @@ def run(opts):
 		rollback_tasklist = TaskList(rollback_tasks)
 
 		# Run the rollback tasklist
-		rollback_tasklist.run(info=bootstrap_info, dry_run=opts['--dry-run'])
+		rollback_tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		log.info('Successfully completed rollback')
 		raise e

From 0cc26d82d075a1100a0a28e4aefa5cc02a7cb8c1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 22:41:40 +0200
Subject: [PATCH 035/345] Allow passing data into the manifest. This makes it
 possible to create dynamically created manifests

---
 bootstrapvz/base/main.py             | 12 +++++-----
 bootstrapvz/base/manifest.py         | 36 ++++++++++++++++++----------
 tests/integration/manifests_tests.py |  2 +-
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 80a4c4d..882ac5f 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -20,8 +20,12 @@ def main():
 	# Set up logging
 	setup_loggers(opts)
 
+	# Load the manifest
+	from manifest import Manifest
+	manifest = Manifest(path=opts['MANIFEST'])
+
 	# Everything has been set up, begin the bootstrapping process
-	run(opts['MANIFEST'],
+	run(manifest,
 	    debug=opts['--debug'],
 	    pause_on_error=opts['--pause-on-error'],
 	    dry_run=opts['--dry-run'])
@@ -68,15 +72,11 @@ def setup_loggers(opts):
 	root.addHandler(console_handler)
 
 
-def run(manifest_path, debug=False, pause_on_error=False, dry_run=False):
+def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	"""Runs the bootstrapping process
 
 	:params dict opts: Dictionary of options from the commandline
 	"""
-	# Load the manifest
-	from manifest import Manifest
-	manifest = Manifest(manifest_path)
-
 	# Get the tasklist
 	from tasklist import load_tasks
 	from tasklist import TaskList
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 8151bdd..969dfbf 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -2,6 +2,7 @@
 to determine which tasks should be added to the tasklist, what arguments various
 invocations should have etc..
 """
+from bootstrapvz.common.exceptions import ManifestError
 from bootstrapvz.common.tools import load_data
 import logging
 log = logging.getLogger(__name__)
@@ -14,32 +15,44 @@ class Manifest(object):
 	Currently, immutability is not enforced and it would require a fair amount of code
 	to enforce it, instead we just rely on tasks behaving properly.
 	"""
-	def __init__(self, path):
+	def __init__(self, path=None, data=None):
 		"""Initializer: Given a path we load, validate and parse the manifest.
+		To create the manifest from dynamic data instead of the contents of a file,
+		provide a properly constructed dict as the data argument.
 
-		:param str path: The path to the manifest
+		:param str path: The path to the manifest (ignored, when `data' is provided)
+		:param str data: The manifest data, if it is not None, it will be used instead of the contents of `path'
 		"""
+		if path is None and data is None:
+			raise ManifestError('`path\' or `data\' must be provided')
 		self.path = path
-		self.load()
+		self.load(data)
+		self.initialize()
 		self.validate()
 		self.parse()
 
-	def load(self):
+	def load(self, data=None):
 		"""Loads the manifest and performs a basic validation.
-		This function reads the manifest, loads the specified provider and plugins, and performs
-		some basic validation of the manifest itself to ensure that the properties
-		required for initalization are accessible
+		This function reads the manifest and performs some basic validation of
+		the manifest itself to ensure that the properties required for initalization are accessible
 		(otherwise the user would be presented with some cryptic error messages).
-		Once the provider and plugins are loaded,
-		the initialize() function is called on each of them (if it exists).
-		The provider must have an initialize function.
 		"""
-		self.data = load_data(self.path)
+		if data is None:
+			self.data = load_data(self.path)
+		else:
+			self.data = data
 
 		from . import validate_manifest
 		# Validate the manifest with the base validation function in __init__
 		validate_manifest(self.data, self.schema_validator, self.validation_error)
 
+	def initialize(self):
+		"""Initializes the provider and the plugins.
+		This function loads the specified provider and plugins.
+		Once the provider and plugins are loaded,
+		the initialize() function is called on each of them (if it exists).
+		The provider must have an initialize function.
+		"""
 		# Get the provider name from the manifest and load the corresponding module
 		provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name']
 		log.debug('Loading provider ' + provider_modname)
@@ -116,5 +129,4 @@ class Manifest(object):
 		:param list data_path: A path to the location in the manifest where the error occurred
 		:raises ManifestError: With absolute certainty
 		"""
-		from bootstrapvz.common.exceptions import ManifestError
 		raise ManifestError(message, self.path, data_path)
diff --git a/tests/integration/manifests_tests.py b/tests/integration/manifests_tests.py
index 74a89d7..fd2b4b6 100644
--- a/tests/integration/manifests_tests.py
+++ b/tests/integration/manifests_tests.py
@@ -29,7 +29,7 @@ def validate_manifests(path):
     and checks that all the data values have successfully
     been created.
     """
-    manifest = Manifest(path)
+    manifest = Manifest(path=path)
 
     assert_true(manifest.data)
     assert_true(manifest.data['provider'])

From 2fb6f344eebf615b2f9344dfd24cdb46d50ab9a4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 23:22:39 +0200
Subject: [PATCH 036/345] Update CHANGELOG

---
 CHANGELOG | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/CHANGELOG b/CHANGELOG
index 31efe54..4a0b246 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,21 @@
+2014-07-09:
+	Anders Ingemann:
+		* Allow passing data into the manifest
+		* Refactor logging setup to be more modular
+		* Convert every JSON file to YAML
+		* Convert "provider" into provider specific section 
+2014-07-02:
+	Vladimir Vitkov:
+		* Improve grub options to work better with virtual machines
+2014-06-30:
+	Tomasz Rybak:
+		* Return information about created image
+2014-06-22:
+	Victor Marmol:
+		* Enable the memory cgroup for the Docker plugin
+2014-06-19:
+	Vladimir Vitkov:
+		* Improve ami listing performance
 2014-06-06:
 	Ilya Margolin:
 		* pip_install plugin

From cca31b642f09c1ad66d54e09c9a7bce2b9704c7d Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 9 Jul 2014 23:31:16 +0200
Subject: [PATCH 037/345] Add documentation for the run() function. Also: Move
 the return statement in run() to the bottom

---
 bootstrapvz/base/main.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 882ac5f..764cfbd 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -75,7 +75,10 @@ def setup_loggers(opts):
 def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	"""Runs the bootstrapping process
 
-	:params dict opts: Dictionary of options from the commandline
+	:params Manifest manifest: The manifest to run the bootstrapping process for
+	:params bool debug: Whether to turn debugging mode on
+	:params bool pause_on_error: Whether to pause on error, before rollback
+	:params bool dry_run: Don't actually run the tasks
 	"""
 	# Get the tasklist
 	from tasklist import load_tasks
@@ -93,7 +96,6 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 		tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
-		return bootstrap_info
 	except (Exception, KeyboardInterrupt) as e:
 		# When an error occurs, log it and begin rollback
 		log.exception(e)
@@ -124,3 +126,4 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 		rollback_tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		log.info('Successfully completed rollback')
 		raise e
+	return bootstrap_info

From 210999ff5d494430a6f4cfe3a75b4dc39729174e Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 12 Jul 2014 13:28:46 -0300
Subject: [PATCH 038/345] Remove redundant code from GCE's `tasks.apt`

GCE's `tasks.apt.SetPackageRepositories` was adding duplicated entries
to apt `info.source_lists`, even ignoring the package mirror specified
on manifest.
---
 bootstrapvz/providers/gce/tasks/apt.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/bootstrapvz/providers/gce/tasks/apt.py b/bootstrapvz/providers/gce/tasks/apt.py
index 733c1c6..4d620a3 100644
--- a/bootstrapvz/providers/gce/tasks/apt.py
+++ b/bootstrapvz/providers/gce/tasks/apt.py
@@ -6,19 +6,12 @@ import os
 
 
 class SetPackageRepositories(Task):
-	description = 'Adding apt sources'
+	description = 'Adding goog apt source'
 	phase = phases.preparation
 	successors = [apt.AddManifestSources]
 
 	@classmethod
 	def run(cls, info):
-		components = 'main'
-		if 'components' in info.manifest.system:
-			components = ' '.join(info.manifest.system['components'])
-		info.source_lists.add('main', 'deb     http://http.debian.net/debian {system.release} ' + components)
-		info.source_lists.add('main', 'deb-src http://http.debian.net/debian {system.release} ' + components)
-		info.source_lists.add('backports', 'deb     http://http.debian.net/debian {system.release}-backports ' + components)
-		info.source_lists.add('backports', 'deb-src http://http.debian.net/debian {system.release}-backports ' + components)
 		info.source_lists.add('goog', 'deb http://goog-repo.appspot.com/debian pigeon main')
 
 

From d6fe85e124a83f1a428705c6b97fcfa1ae188717 Mon Sep 17 00:00:00 2001
From: Tomasz Rybak <tomasz.rybak@post.pl>
Date: Sat, 12 Jul 2014 19:40:53 +0200
Subject: [PATCH 039/345] Do not crash with KeyError when there is no
 credentials in manifest.

---
 bootstrapvz/providers/ec2/tasks/connection.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/providers/ec2/tasks/connection.py b/bootstrapvz/providers/ec2/tasks/connection.py
index e68cd6d..5ace8bc 100644
--- a/bootstrapvz/providers/ec2/tasks/connection.py
+++ b/bootstrapvz/providers/ec2/tasks/connection.py
@@ -18,10 +18,11 @@ class GetCredentials(Task):
 	def get_credentials(cls, manifest, keys):
 		from os import getenv
 		creds = {}
-		if all(key in manifest.provider['credentials'] for key in keys):
-			for key in keys:
-				creds[key] = manifest.provider['credentials'][key]
-			return creds
+		if 'credentials' in manifest.provider:
+			if all(key in manifest.provider['credentials'] for key in keys):
+				for key in keys:
+					creds[key] = manifest.provider['credentials'][key]
+				return creds
 
 		def env_key(key):
 			return ('aws-' + key).upper().replace('-', '_')

From 8f43ee1dc710fb07e1721dcefb2ad63e8dc4fc61 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 12 Jul 2014 14:42:52 -0300
Subject: [PATCH 040/345] Move `AddBackports` to `common.tasks.apt`

The `AddBackports` task is needed by many different plugins, so is
better if it is available as a common task.

This closes #96.
---
 bootstrapvz/common/tasks/apt.py               | 17 +++++++++++++++++
 bootstrapvz/plugins/cloud_init/__init__.py    |  4 ++--
 bootstrapvz/plugins/cloud_init/tasks.py       | 16 +---------------
 bootstrapvz/plugins/docker_daemon/__init__.py |  7 +++++--
 bootstrapvz/plugins/opennebula/__init__.py    |  4 ++--
 bootstrapvz/plugins/opennebula/tasks.py       | 17 +----------------
 bootstrapvz/providers/gce/__init__.py         |  3 ++-
 7 files changed, 30 insertions(+), 38 deletions(-)

diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 87732f2..e757394 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -2,9 +2,26 @@ from bootstrapvz.base import Task
 from .. import phases
 from ..tools import log_check_call
 import locale
+import logging
 import os
 
 
+class AddBackports(Task):
+	description = 'Adding backports to the apt sources'
+	phase = phases.preparation
+
+	@classmethod
+	def run(cls, info):
+		if info.source_lists.target_exists('{system.release}-backports'):
+			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
+			logging.getLogger(__name__).info(msg)
+		elif info.release_codename == 'sid':
+			logging.getLogger(__name__).info('There are no backports for sid/unstable')
+		else:
+			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
+			info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main')
+
+
 class AddManifestSources(Task):
 	description = 'Adding sources from the manifest'
 	phase = phases.preparation
diff --git a/bootstrapvz/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py
index 0268d61..db3571f 100644
--- a/bootstrapvz/plugins/cloud_init/__init__.py
+++ b/bootstrapvz/plugins/cloud_init/__init__.py
@@ -7,14 +7,14 @@ def validate_manifest(data, validator, error):
 
 
 def resolve_tasks(taskset, manifest):
-	import tasks
 	import bootstrapvz.providers.ec2.tasks.initd as initd_ec2
+	from bootstrapvz.common.tasks import apt
 	from bootstrapvz.common.tasks import initd
 	from bootstrapvz.common.tasks import ssh
 
 	from bootstrapvz.common.tools import get_codename
 	if get_codename(manifest.system['release']) == 'wheezy':
-		taskset.add(tasks.AddBackports)
+		taskset.add(apt.AddBackports)
 
 	taskset.update([tasks.SetMetadataSource,
 	                tasks.AddCloudInitPackages,
diff --git a/bootstrapvz/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py
index 6f64c38..39bb7be 100644
--- a/bootstrapvz/plugins/cloud_init/tasks.py
+++ b/bootstrapvz/plugins/cloud_init/tasks.py
@@ -7,24 +7,10 @@ import logging
 import os.path
 
 
-class AddBackports(Task):
-	description = 'Adding backports to the apt sources'
-	phase = phases.preparation
-
-	@classmethod
-	def run(cls, info):
-		if info.source_lists.target_exists('{system.release}-backports'):
-			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
-			logging.getLogger(__name__).info(msg)
-		else:
-			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
-			info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main')
-
-
 class AddCloudInitPackages(Task):
 	description = 'Adding cloud-init package and sudo'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources, AddBackports]
+	predecessors = [apt.AddDefaultSources, apt.AddBackports]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index b85e32f..ef3b320 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -1,11 +1,12 @@
-import tasks
 import os.path
+import tasks
+from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tools import get_codename
 
 
 def validate_manifest(data, validator, error):
 	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
-	from bootstrapvz.common.tools import get_codename
 	if get_codename(data['system']['release']) == 'wheezy':
 		# prefs is a generator of apt preferences across files in the manifest
 		prefs = (item for vals in data.get('packages', {}).get('preferences', {}).values() for item in vals)
@@ -15,6 +16,8 @@ def validate_manifest(data, validator, error):
 
 
 def resolve_tasks(taskset, manifest):
+	if get_codename(manifest.system['release']) == 'wheezy':
+		taskset.add(apt.AddBackports)
 	taskset.add(tasks.AddDockerDeps)
 	taskset.add(tasks.AddDockerBinary)
 	taskset.add(tasks.AddDockerInit)
diff --git a/bootstrapvz/plugins/opennebula/__init__.py b/bootstrapvz/plugins/opennebula/__init__.py
index bb90773..d138c68 100644
--- a/bootstrapvz/plugins/opennebula/__init__.py
+++ b/bootstrapvz/plugins/opennebula/__init__.py
@@ -1,8 +1,8 @@
-import tasks
 
 
 def resolve_tasks(taskset, manifest):
+	from bootstrapvz.common.tasks import apt
 	from bootstrapvz.common.tools import get_codename
 	if get_codename(manifest.system['release']) == 'wheezy':
-		taskset.add(tasks.AddBackports)
+		taskset.add(apt.AddBackports)
 	taskset.update([tasks.AddONEContextPackage])
diff --git a/bootstrapvz/plugins/opennebula/tasks.py b/bootstrapvz/plugins/opennebula/tasks.py
index 4e5c48b..d37e9be 100644
--- a/bootstrapvz/plugins/opennebula/tasks.py
+++ b/bootstrapvz/plugins/opennebula/tasks.py
@@ -3,25 +3,10 @@ from bootstrapvz.common.tasks import apt
 from bootstrapvz.common import phases
 
 
-class AddBackports(Task):
-	description = 'Adding backports to the apt sources'
-	phase = phases.preparation
-
-	@classmethod
-	def run(cls, info):
-		if info.source_lists.target_exists('{system.release}-backports'):
-			import logging
-			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
-			logging.getLogger(__name__).info(msg)
-		else:
-			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
-			info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main')
-
-
 class AddONEContextPackage(Task):
 	description = 'Adding the OpenNebula context package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources, AddBackports]
+	predecessors = [apt.AddDefaultSources, apt.AddBackports]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 710e893..8278c72 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -5,6 +5,7 @@ import tasks.configuration
 import tasks.image
 import tasks.host
 import tasks.packages
+from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import ssh
@@ -25,7 +26,7 @@ def validate_manifest(data, validator, error):
 def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
 
-	taskset.update([bootstrapvz.plugins.cloud_init.tasks.AddBackports,
+	taskset.update([apt.AddBackports,
 	                loopback.AddRequiredCommands,
 	                loopback.Create,
 	                tasks.apt.SetPackageRepositories,

From f925045e59711d54d88f94b5971e860d2fde54c3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 12 Jul 2014 19:53:17 +0200
Subject: [PATCH 041/345] 8f43ee1d removed a used import in the cloud-init and
 opennebula plugins, readded

---
 bootstrapvz/plugins/cloud_init/__init__.py | 1 +
 bootstrapvz/plugins/opennebula/__init__.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/bootstrapvz/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py
index db3571f..f01b02f 100644
--- a/bootstrapvz/plugins/cloud_init/__init__.py
+++ b/bootstrapvz/plugins/cloud_init/__init__.py
@@ -7,6 +7,7 @@ def validate_manifest(data, validator, error):
 
 
 def resolve_tasks(taskset, manifest):
+	import tasks
 	import bootstrapvz.providers.ec2.tasks.initd as initd_ec2
 	from bootstrapvz.common.tasks import apt
 	from bootstrapvz.common.tasks import initd
diff --git a/bootstrapvz/plugins/opennebula/__init__.py b/bootstrapvz/plugins/opennebula/__init__.py
index d138c68..f41e21b 100644
--- a/bootstrapvz/plugins/opennebula/__init__.py
+++ b/bootstrapvz/plugins/opennebula/__init__.py
@@ -1,6 +1,7 @@
 
 
 def resolve_tasks(taskset, manifest):
+	import tasks
 	from bootstrapvz.common.tasks import apt
 	from bootstrapvz.common.tools import get_codename
 	if get_codename(manifest.system['release']) == 'wheezy':

From af3cca26447bee8dd9a1644ad17494ff69d66354 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 12 Jul 2014 19:53:56 +0200
Subject: [PATCH 042/345] Remove unused import

---
 bootstrapvz/providers/gce/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 8278c72..3d2f341 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -10,7 +10,6 @@ from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import ssh
 from bootstrapvz.common.tasks import volume
-import bootstrapvz.plugins.cloud_init.tasks
 
 
 def initialize():

From a1e17841d364063ed3417ea4cda9cac8765bb77f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 12 Jul 2014 19:58:25 +0200
Subject: [PATCH 043/345] Fix usage of virtualization reference in manifests

---
 .../ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml | 2 +-
 manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml      | 2 +-
 .../ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml | 2 +-
 manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml      | 2 +-
 manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml       | 2 +-
 manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml       | 2 +-
 manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml      | 2 +-
 .../ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml    | 2 +-
 manifests/ec2-ebs-partitioned.manifest.yml                    | 4 ++--
 manifests/ec2-ebs-single.manifest.yml                         | 4 ++--
 .../ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml  | 2 +-
 manifests/ec2-s3.manifest.yml                                 | 2 +-
 .../squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml    | 2 +-
 .../squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml     | 2 +-
 14 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
index 7ec2d57..83917a8 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: wheezy
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
index ad5a030..ed10508 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: wheezy
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
index 9de6156..2160e59 100644
--- a/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm-cn-north-1.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: wheezy
diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
index 78d414e..1729377 100644
--- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: wheezy
diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
index d6aabc8..360ed6d 100644
--- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: wheezy
diff --git a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
index 6f66f07..7488073 100644
--- a/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-testing-amd64-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: testing
diff --git a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
index f7bb3ca..cf417a2 100644
--- a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: unstable
diff --git a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
index 9394c3c..f7f4935 100644
--- a/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-unstable-contrib-amd64-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: unstable
diff --git a/manifests/ec2-ebs-partitioned.manifest.yml b/manifests/ec2-ebs-partitioned.manifest.yml
index 9cfc2fe..2cf1652 100644
--- a/manifests/ec2-ebs-partitioned.manifest.yml
+++ b/manifests/ec2-ebs-partitioned.manifest.yml
@@ -8,8 +8,8 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
-  description: Debian {system.release} {system.architecture} AMI ({virtualization})
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture} AMI ({provider.virtualization})
 system:
   release: wheezy
   architecture: amd64
diff --git a/manifests/ec2-ebs-single.manifest.yml b/manifests/ec2-ebs-single.manifest.yml
index d017eae..25f1b1a 100644
--- a/manifests/ec2-ebs-single.manifest.yml
+++ b/manifests/ec2-ebs-single.manifest.yml
@@ -8,8 +8,8 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
-  description: Debian {system.release} {system.architecture} AMI ({virtualization})
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture} AMI ({provider.virtualization})
 system:
   release: wheezy
   architecture: amd64
diff --git a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
index 708e6eb..6268883 100644
--- a/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
+++ b/manifests/ec2-s3-debian-official-amd64-pvm-cn-north-1.manifest.yml
@@ -11,7 +11,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}
   description: Debian {system.release} {system.architecture} AMI
   bucket: debian-amis-cn-north-1
   region: cn-north-1
diff --git a/manifests/ec2-s3.manifest.yml b/manifests/ec2-s3.manifest.yml
index 464fefe..bb837ff 100644
--- a/manifests/ec2-s3.manifest.yml
+++ b/manifests/ec2-s3.manifest.yml
@@ -11,7 +11,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%y}{%m}{%d}
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%y}{%m}{%d}
   description: Debian {system.release} {system.architecture} AMI
   bucket: debian-amis
   region: us-west-1
diff --git a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
index c175565..b36e832 100644
--- a/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
+++ b/manifests/squeeze-ec2-ebs-debian-official-amd64-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: squeeze
diff --git a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
index 1fb07b6..539c2d3 100644
--- a/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
+++ b/manifests/squeeze-ec2-ebs-debian-official-i386-pvm.manifest.yml
@@ -8,7 +8,7 @@ provider:
 bootstrapper:
   workspace: /target
 image:
-  name: debian-{system.release}-{system.architecture}-{virtualization}-{%Y}-{%m}-{%d}-ebs
+  name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-ebs
   description: Debian {system.release} {system.architecture}
 system:
   release: squeeze

From cc95f146b12219587229e2adabf15856048a0378 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sun, 13 Jul 2014 02:57:37 -0300
Subject: [PATCH 044/345] Add AddDefaultSources as AddBackports predecessor

---
 bootstrapvz/common/tasks/apt.py | 38 ++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index e757394..2cda135 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -1,27 +1,11 @@
 from bootstrapvz.base import Task
-from .. import phases
-from ..tools import log_check_call
+from bootstrapvz.common import phases
+from bootstrapvz.common.tools import log_check_call
 import locale
 import logging
 import os
 
 
-class AddBackports(Task):
-	description = 'Adding backports to the apt sources'
-	phase = phases.preparation
-
-	@classmethod
-	def run(cls, info):
-		if info.source_lists.target_exists('{system.release}-backports'):
-			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
-			logging.getLogger(__name__).info(msg)
-		elif info.release_codename == 'sid':
-			logging.getLogger(__name__).info('There are no backports for sid/unstable')
-		else:
-			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
-			info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main')
-
-
 class AddManifestSources(Task):
 	description = 'Adding sources from the manifest'
 	phase = phases.preparation
@@ -50,6 +34,23 @@ class AddDefaultSources(Task):
 			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates ' + components)
 
 
+class AddBackports(Task):
+	description = 'Adding backports to the apt sources'
+	phase = phases.preparation
+	predecessors = [AddDefaultSources]
+
+	@classmethod
+	def run(cls, info):
+		if info.source_lists.target_exists('{system.release}-backports'):
+			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
+			logging.getLogger(__name__).info(msg)
+		elif info.release_codename == 'sid':
+			logging.getLogger(__name__).info('There are no backports for sid/unstable')
+		else:
+			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
+			info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main')
+
+
 class AddManifestPreferences(Task):
 	description = 'Adding preferences from the manifest'
 	phase = phases.preparation
@@ -155,7 +156,6 @@ class AptUpgrade(Task):
 			                           '--assume-yes'])
 		except CalledProcessError as e:
 			if e.returncode == 100:
-				import logging
 				msg = ('apt exited with status code 100. '
 				       'This can sometimes occur when package retrieval times out or a package extraction failed. '
 				       'apt might succeed if you try bootstrapping again.')

From 5420b7e546f45b05d0166a800804cded0f7ce4e7 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sun, 13 Jul 2014 12:19:43 -0300
Subject: [PATCH 045/345] Update CHANGELOG

---
 CHANGELOG | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG b/CHANGELOG
index 4a0b246..2f12d11 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,9 +1,12 @@
+2014-07-12:
+	Tiago Ilieve:
+		* Fixes #96: AddBackports is now a common task
 2014-07-09:
 	Anders Ingemann:
 		* Allow passing data into the manifest
 		* Refactor logging setup to be more modular
 		* Convert every JSON file to YAML
-		* Convert "provider" into provider specific section 
+		* Convert "provider" into provider specific section
 2014-07-02:
 	Vladimir Vitkov:
 		* Improve grub options to work better with virtual machines
@@ -14,11 +17,20 @@
 	Victor Marmol:
 		* Enable the memory cgroup for the Docker plugin
 2014-06-19:
+	Tiago Ilieve:
+		* Fixes #94: allow stable/oldstable as release name on manifest
 	Vladimir Vitkov:
 		* Improve ami listing performance
+2014-06-07:
+	Tiago Ilieve:
+		* Download `gsutil` tarball to workspace instead of working directory
+		* Fixes #97: remove raw disk image created by GCE after build
 2014-06-06:
 	Ilya Margolin:
 		* pip_install plugin
+2014-05-23:
+	Tiago Ilieve:
+		* Fixes #95: check if the specified APT proxy server can be reached
 2014-05-04:
 	Dhananjay Balan:
 		* Salt minion installation & configuration plugin

From e41dbdd80759556d34b0dda1a4e7203de997b5d7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 16 Jul 2014 22:05:54 +0200
Subject: [PATCH 046/345] Rename variable in load_yaml to match load_json

---
 bootstrapvz/common/tools.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 777a111..79d5202 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -72,8 +72,8 @@ def load_json(path):
 
 def load_yaml(path):
 	import yaml
-	with open(path, 'r') as fobj:
-		return yaml.safe_load(fobj)
+	with open(path, 'r') as stream:
+		return yaml.safe_load(stream)
 
 
 def load_data(path):

From 9e61ac94d2d0428214f5f9037d1d61d51643cc58 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 16 Jul 2014 22:06:42 +0200
Subject: [PATCH 047/345] Add a word, to make error message about nbd more
 understable

---
 bootstrapvz/common/fs/qemuvolume.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/common/fs/qemuvolume.py b/bootstrapvz/common/fs/qemuvolume.py
index 605e77c..2f9adfe 100644
--- a/bootstrapvz/common/fs/qemuvolume.py
+++ b/bootstrapvz/common/fs/qemuvolume.py
@@ -23,7 +23,8 @@ class QEMUVolume(LoopbackVolume):
 			num_partitions = len(self.partition_map.partitions)
 			if not self._module_loaded('nbd'):
 				msg = ('The kernel module `nbd\' must be loaded '
-				       '(`modprobe nbd max_part={num_partitions}\') to attach .{extension} images'
+				       '(run `modprobe nbd max_part={num_partitions}\') '
+				       'to attach .{extension} images'
 				       .format(num_partitions=num_partitions, extension=self.extension))
 				raise VolumeError(msg)
 			nbd_max_part = int(self._module_param('nbd', 'max_part'))

From 719a6c31b00778343f61800064c52122be592a96 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Fri, 25 Jul 2014 00:08:07 -0300
Subject: [PATCH 048/345] Reverting 210999f, as asked by @jkaplowitz

---
 bootstrapvz/providers/gce/tasks/apt.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/gce/tasks/apt.py b/bootstrapvz/providers/gce/tasks/apt.py
index 4d620a3..733c1c6 100644
--- a/bootstrapvz/providers/gce/tasks/apt.py
+++ b/bootstrapvz/providers/gce/tasks/apt.py
@@ -6,12 +6,19 @@ import os
 
 
 class SetPackageRepositories(Task):
-	description = 'Adding goog apt source'
+	description = 'Adding apt sources'
 	phase = phases.preparation
 	successors = [apt.AddManifestSources]
 
 	@classmethod
 	def run(cls, info):
+		components = 'main'
+		if 'components' in info.manifest.system:
+			components = ' '.join(info.manifest.system['components'])
+		info.source_lists.add('main', 'deb     http://http.debian.net/debian {system.release} ' + components)
+		info.source_lists.add('main', 'deb-src http://http.debian.net/debian {system.release} ' + components)
+		info.source_lists.add('backports', 'deb     http://http.debian.net/debian {system.release}-backports ' + components)
+		info.source_lists.add('backports', 'deb-src http://http.debian.net/debian {system.release}-backports ' + components)
 		info.source_lists.add('goog', 'deb http://goog-repo.appspot.com/debian pigeon main')
 
 

From c412c4cdcf0663bd00dff5b1a4207d578d68c5b7 Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Tue, 20 May 2014 13:00:55 -0700
Subject: [PATCH 049/345] Fix list of tasks and their ordering

- GCE provider wasn't including the GCE SetHostname task, without which
  https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=604883 was
  preventing the hostname from getting set after reboot.
- During the GCE build, one of the GCE cleaning tasks was trying to run
  an apt-get update after the build-time resolv.conf file was removed.
  Fix this ordering by moving the network.Remove* tasks to the
  system_cleaning phase as they should have been all along, and adding
  an appropriate ordering rule for the GCE cleaning task.
- Add the fallback http.debian.net mirror after, not before, our mirror.
- The puppet plugin's ApplyPuppetManifest task specified that it should
  run before the network.Remove* tasks within the system_modification
  phase. Now that those tasks have been moved to a later phase
  (system_cleaning), remove this dependency. I have no puppet manifest
  to test this change, but am including it in hopes of avoiding a
  breakage there. Hopefully someone who uses puppet can test this or at
  least confirm that it's correct.

Change-Id: Ieca97f288f456bab119989f4cbc4c3993a755830
---
 bootstrapvz/common/tasks/network.py    | 4 ++--
 bootstrapvz/plugins/puppet/tasks.py    | 2 --
 bootstrapvz/providers/gce/__init__.py  | 1 +
 bootstrapvz/providers/gce/tasks/apt.py | 5 +++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/common/tasks/network.py b/bootstrapvz/common/tasks/network.py
index 858b158..6bec727 100644
--- a/bootstrapvz/common/tasks/network.py
+++ b/bootstrapvz/common/tasks/network.py
@@ -5,7 +5,7 @@ import os
 
 class RemoveDNSInfo(Task):
 	description = 'Removing resolv.conf'
-	phase = phases.system_modification
+	phase = phases.system_cleaning
 
 	@classmethod
 	def run(cls, info):
@@ -15,7 +15,7 @@ class RemoveDNSInfo(Task):
 
 class RemoveHostname(Task):
 	description = 'Removing the hostname file'
-	phase = phases.system_modification
+	phase = phases.system_cleaning
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/puppet/tasks.py b/bootstrapvz/plugins/puppet/tasks.py
index 2627e69..02d98ab 100644
--- a/bootstrapvz/plugins/puppet/tasks.py
+++ b/bootstrapvz/plugins/puppet/tasks.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
-from bootstrapvz.common.tasks import network
 from bootstrapvz.common.tools import sed_i
 import os
 
@@ -62,7 +61,6 @@ class ApplyPuppetManifest(Task):
 	description = 'Applying puppet manifest'
 	phase = phases.system_modification
 	predecessors = [CopyPuppetAssets]
-	successors = [network.RemoveHostname, network.RemoveDNSInfo]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 3d2f341..11f225f 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -37,6 +37,7 @@ def resolve_tasks(taskset, manifest):
 	                tasks.configuration.GatherReleaseInformation,
 
 	                tasks.host.DisableIPv6,
+	                tasks.host.SetHostname,
 	                tasks.boot.ConfigureGrub,
 	                initd.InstallInitScripts,
 	                ssh.AddSSHKeyGeneration,
diff --git a/bootstrapvz/providers/gce/tasks/apt.py b/bootstrapvz/providers/gce/tasks/apt.py
index 733c1c6..1e85808 100644
--- a/bootstrapvz/providers/gce/tasks/apt.py
+++ b/bootstrapvz/providers/gce/tasks/apt.py
@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tasks import network
 from bootstrapvz.common.tools import log_check_call
 import os
 
@@ -8,7 +9,7 @@ import os
 class SetPackageRepositories(Task):
 	description = 'Adding apt sources'
 	phase = phases.preparation
-	successors = [apt.AddManifestSources]
+	predecessors = [apt.AddManifestSources]
 
 	@classmethod
 	def run(cls, info):
@@ -39,7 +40,7 @@ class ImportGoogleKey(Task):
 class CleanGoogleRepositoriesAndKeys(Task):
 	description = 'Removing Google key and apt source files'
 	phase = phases.system_cleaning
-	successors = [apt.AptClean]
+	successors = [apt.AptClean, network.RemoveDNSInfo]
 
 	@classmethod
 	def run(cls, info):

From e8f04d0baf384203fa785ffc70abd67e96890359 Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Fri, 25 Jul 2014 09:27:27 -0700
Subject: [PATCH 050/345] Install ca-certificates on GCE

Needed to fetch GCE startup scripts over HTTPS, among other reasons.

Change-Id: I89e3afb44f786539b5e3275b5f1f3b7201ab44fd
---
 bootstrapvz/providers/gce/tasks/packages.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index dc964e5..e38715a 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -20,6 +20,7 @@ class DefaultPackages(Task):
 		info.packages.add('openssh-client')
 		info.packages.add('openssh-server')
 		info.packages.add('dhcpd')
+		info.packages.add('ca-certificates')
 
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get

From a352f4d576ce0d8275856331c32e690d3be3b993 Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Fri, 1 Aug 2014 20:05:23 -0700
Subject: [PATCH 051/345] Installing Cloud SDK which includes gcutil and
 gsutil. We still don't have a package for it so laying out the tarball for
 now.

Change-Id: If66f0f1c074e6077e1ca57375cac9c4832bbd7fc
---
 .../plugins/google_cloud_sdk/__init__.py      |  6 ++
 bootstrapvz/plugins/google_cloud_sdk/tasks.py | 74 +++++++++++++++++++
 bootstrapvz/providers/gce/__init__.py         |  1 -
 bootstrapvz/providers/gce/tasks/packages.py   | 19 -----
 manifests/gce-backports.manifest.yml          |  1 +
 manifests/gce.manifest.yml                    |  1 +
 6 files changed, 82 insertions(+), 20 deletions(-)
 create mode 100644 bootstrapvz/plugins/google_cloud_sdk/__init__.py
 create mode 100644 bootstrapvz/plugins/google_cloud_sdk/tasks.py

diff --git a/bootstrapvz/plugins/google_cloud_sdk/__init__.py b/bootstrapvz/plugins/google_cloud_sdk/__init__.py
new file mode 100644
index 0000000..7a7ba47
--- /dev/null
+++ b/bootstrapvz/plugins/google_cloud_sdk/__init__.py
@@ -0,0 +1,6 @@
+import tasks
+
+
+def resolve_tasks(taskset, manifest):
+	taskset.add(tasks.InstallCloudSDK)
+	taskset.add(tasks.RemoveCloudSDKTarball)
diff --git a/bootstrapvz/plugins/google_cloud_sdk/tasks.py b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
new file mode 100644
index 0000000..9b3b08d
--- /dev/null
+++ b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
@@ -0,0 +1,74 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import workspace
+from bootstrapvz.common.tools import log_check_call
+import os
+
+
+class InstallCloudSDK(Task):
+	description = 'Install Cloud SDK, not yet packaged'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		import contextlib
+		import re
+		import urllib
+		import urlparse
+
+		# The current download URL needs to be determined dynamically via a sha1sum file. Here's the
+		# necessary logic.
+
+		cloudsdk_download_site = 'https://dl.google.com/dl/cloudsdk/release/'
+		cloudsdk_filelist_url = urlparse.urljoin(cloudsdk_download_site, 'sha1.txt')
+		cloudsdk_pathname_regexp = r'^packages/google-cloud-sdk-coretools-linux-[0-9]+\.tar\.gz$'
+		cloudsdk_filename = ''  # This is set in the 'with' block below.
+
+		with contextlib.closing(urllib.urlopen(cloudsdk_filelist_url)) as cloudsdk_filelist:
+			# cloudsdk_filelist is in sha1sum format, so <hash><whitespace><pathname>
+			# pathname is a suffix relative to cloudsdk_download_site
+			#
+			# Retrieve the pathname which matches cloudsdk_pathname_regexp. It's currently safe to
+			# assume that only one pathname will match.
+			for cloudsdk_filelist_line in cloudsdk_filelist:
+				_, pathname = cloudsdk_filelist_line.split()
+				if re.match(cloudsdk_pathname_regexp, pathname):
+					# Don't use os.path.basename since we're actually parsing a URL
+					# suffix, not a path. Same probable result, but wrong semantics.
+					#
+					# The format of pathname is already known to match
+					# cloudsdk_pathname_regexp, so this is safe.
+					_, cloudsdk_filename = pathname.rsplit('/', 1)
+					break
+
+		cloudsdk_download_dest = os.path.join(info.workspace, cloudsdk_filename)
+
+		cloudsdk_url = urlparse.urljoin(cloudsdk_download_site, pathname)
+
+		urllib.urlretrieve(cloudsdk_url, cloudsdk_download_dest)
+
+		# Make a "mental note" of which file to remove in the system cleaning phase.
+		info._google_cloud_sdk['tarball_pathname'] = cloudsdk_download_dest
+
+		cloudsdk_directory = os.path.join(info.root, 'usr/local/share/google')
+		os.makedirs(cloudsdk_directory)
+		log_check_call(['file', cloudsdk_download_dest, cloudsdk_directory])
+		log_check_call(['tar', 'xaf', cloudsdk_download_dest, '-C', cloudsdk_directory])
+
+		# We need to symlink certain programs from the Cloud SDK bin directory into /usr/local/bin.
+		# Keep a list and do it in a unified way. Naturally this will go away with proper packaging.
+		gcloud_programs = ['bq', 'gsutil', 'gcutil', 'gcloud', 'git-credential-gcloud.sh']
+		for prog in gcloud_programs:
+			src = os.path.join('..', 'share', 'google', 'google-cloud-sdk', 'bin', prog)
+			dest = os.path.join(info.root, 'usr', 'local', 'bin', prog)
+			os.symlink(src, dest)
+
+
+class RemoveCloudSDKTarball(Task):
+	description = 'Remove tarball for Cloud SDK'
+	phase = phases.system_cleaning
+	successors = [workspace.DeleteWorkspace]
+
+	@classmethod
+	def run(cls, info):
+		os.remove(info._google_cloud_sdk['tarball_pathname'])
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 3d2f341..b0abfe2 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -32,7 +32,6 @@ def resolve_tasks(taskset, manifest):
 	                tasks.apt.ImportGoogleKey,
 	                tasks.packages.DefaultPackages,
 	                tasks.packages.GooglePackages,
-	                tasks.packages.InstallGSUtil,
 
 	                tasks.configuration.GatherReleaseInformation,
 
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index e38715a..d680cac 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
-from bootstrapvz.common.tools import log_check_call
 import os
 
 
@@ -39,21 +38,3 @@ class GooglePackages(Task):
 		info.packages.add('google-compute-daemon')
 		info.packages.add('google-startup-scripts')
 		info.packages.add('python-gcimagebundle')
-		info.packages.add('gcutil')
-
-
-class InstallGSUtil(Task):
-	description = 'Install gsutil, not yet packaged'
-	phase = phases.package_installation
-
-	@classmethod
-	def run(cls, info):
-		gsutil_tarball = os.path.join(info.manifest.bootstrapper['workspace'], 'gsutil.tar.gz')
-		log_check_call(['wget', '--output-document', gsutil_tarball,
-		                'http://storage.googleapis.com/pub/gsutil.tar.gz'])
-		gsutil_directory = os.path.join(info.root, 'usr/local/share/google')
-		gsutil_binary = os.path.join(os.path.join(info.root, 'usr/local/bin'), 'gsutil')
-		os.makedirs(gsutil_directory)
-		log_check_call(['tar', 'xaf', gsutil_tarball, '-C', gsutil_directory])
-		os.remove(gsutil_tarball)
-		log_check_call(['ln', '-s', '../share/google/gsutil/gsutil', gsutil_binary])
diff --git a/manifests/gce-backports.manifest.yml b/manifests/gce-backports.manifest.yml
index 76cf7f9..7c3ed8c 100644
--- a/manifests/gce-backports.manifest.yml
+++ b/manifests/gce-backports.manifest.yml
@@ -36,6 +36,7 @@ packages:
       pin: release n=wheezy-backports
       pin-priority: 500
 plugins:
+  google_cloud_sdk: {}
   ntp:
     servers:
     - metadata.google.internal
diff --git a/manifests/gce.manifest.yml b/manifests/gce.manifest.yml
index db26dd1..bf9ba52 100644
--- a/manifests/gce.manifest.yml
+++ b/manifests/gce.manifest.yml
@@ -27,6 +27,7 @@ packages:
   - contrib
   - non-free
 plugins:
+  google_cloud_sdk: {}
   ntp:
     servers:
     - metadata.google.internal

From 3bfe9dddf98f129889efeb6cf669091ac9b1eb01 Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Tue, 5 Aug 2014 16:55:14 -0700
Subject: [PATCH 052/345] Reinstate hostname hook for GCE

This DHCP exit hook to shorten the system hostname on GCE was previously
installed by build-debian-cloud and bootstrap-vz, but seems to have been
inadvertently removed in commit c81045cc6e0f6a2c90002ab50219a15c337630f1
as part of a broad cross-cloud cleanup. Again, this was caught by our
validation tests, and we might be done with the fixes at this point.

In this commit, I'm reinstating the hook with a name change and an
explanatory comment, to reduce the risk of this vanishing accidentally
in the future.

Change-Id: I4e7268f8b9ab3b2a7fc8b510898c6fbdd685aa53
---
 bootstrapvz/providers/gce/__init__.py   |  2 +-
 bootstrapvz/providers/gce/tasks/host.py | 17 +++++++++++++++--
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index d6a187d..4fbe5b1 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -36,7 +36,7 @@ def resolve_tasks(taskset, manifest):
 	                tasks.configuration.GatherReleaseInformation,
 
 	                tasks.host.DisableIPv6,
-	                tasks.host.SetHostname,
+	                tasks.host.InstallHostnameHook,
 	                tasks.boot.ConfigureGrub,
 	                initd.InstallInitScripts,
 	                ssh.AddSSHKeyGeneration,
diff --git a/bootstrapvz/providers/gce/tasks/host.py b/bootstrapvz/providers/gce/tasks/host.py
index bd61878..aabd017 100644
--- a/bootstrapvz/providers/gce/tasks/host.py
+++ b/bootstrapvz/providers/gce/tasks/host.py
@@ -17,12 +17,25 @@ class DisableIPv6(Task):
 			print >>config_file, "net.ipv6.conf.all.disable_ipv6 = 1"
 
 
-class SetHostname(Task):
-	description = "Setting hostname"
+class InstallHostnameHook(Task):
+	description = "Installing hostname hook"
 	phase = phases.system_modification
 
 	@classmethod
 	def run(cls, info):
+		# There's a surprising amount of software out there which doesn't react well to the system
+		# hostname being set to a potentially long the fully qualified domain name, including Java 7
+		# and lower, quite relevant to a lot of cloud use cases such as Hadoop. Since Google Compute
+		# Engine's out-of-the-box domain names are long but predictable based on project name, we
+		# install this hook to set the hostname to the short hostname but add a suitable /etc/hosts
+		# entry.
+		#
+		# Since not all operating systems which Google supports on Compute Engine work with the
+		# /etc/dhcp/dhclient-exit-hooks.d directory, Google's internally-built packaging uses the
+		# consistent install path of /usr/share/google/set-hostname, and OS-specific build steps are
+		# used to activate the DHCP hook. In any future Debian-maintained distro-specific packaging,
+		# the updated deb could handle installing the below symlink or the script itself into
+		# /etc/dhcp/dhclient-exit-hooks.d.
 		log_check_call(['chroot', info.root, 'ln', '-s',
 		                '/usr/share/google/set-hostname',
 		                '/etc/dhcp/dhclient-exit-hooks.d/set-hostname'])

From 89e1a701ebb211f61edfd1b54b24114f8a5d32dc Mon Sep 17 00:00:00 2001
From: Jimmy Kaplowitz <jkaplowitz@google.com>
Date: Wed, 13 Aug 2014 11:51:50 -0700
Subject: [PATCH 053/345] Disable SSH password auth on GCE

This change was unintentional but occurred as part of GCE's transition
from build-debian-cloud to bootstrap-vz.

Might be replaced later with a similar change that applies to all
bootstrap-vz providers, based on the opinion of the debian-cloud list.

Change-Id: I72a694c49f32df06252d9cc01b1d5c7cfc015347
---
 bootstrapvz/providers/gce/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 4fbe5b1..627cc9c 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -40,6 +40,7 @@ def resolve_tasks(taskset, manifest):
 	                tasks.boot.ConfigureGrub,
 	                initd.InstallInitScripts,
 	                ssh.AddSSHKeyGeneration,
+	                ssh.DisableSSHPasswordAuthentication,
 	                tasks.apt.CleanGoogleRepositoriesAndKeys,
 
 	                loopback.MoveImage,

From 3ff1bfb7d96ca97a1263c751c91bd2b04a269cfa Mon Sep 17 00:00:00 2001
From: Max Illfelder <illfelder@google.com>
Date: Wed, 13 Aug 2014 16:58:37 -0700
Subject: [PATCH 054/345] Removed a check to verify file type

The file command is not installed by bootstrap-vz by default.

Change-Id: I59d5f04145b7db517c28bfec1d4d758be468b398
---
 bootstrapvz/plugins/google_cloud_sdk/tasks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bootstrapvz/plugins/google_cloud_sdk/tasks.py b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
index 9b3b08d..c59cf97 100644
--- a/bootstrapvz/plugins/google_cloud_sdk/tasks.py
+++ b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
@@ -52,7 +52,6 @@ class InstallCloudSDK(Task):
 
 		cloudsdk_directory = os.path.join(info.root, 'usr/local/share/google')
 		os.makedirs(cloudsdk_directory)
-		log_check_call(['file', cloudsdk_download_dest, cloudsdk_directory])
 		log_check_call(['tar', 'xaf', cloudsdk_download_dest, '-C', cloudsdk_directory])
 
 		# We need to symlink certain programs from the Cloud SDK bin directory into /usr/local/bin.

From 84cf497c667078af2df7b462f08c47dafe2a5b12 Mon Sep 17 00:00:00 2001
From: Filipe Brandenburger <filbranden@google.com>
Date: Thu, 21 Aug 2014 14:40:11 -0700
Subject: [PATCH 055/345] Support --color option to indicate whether to use
 colors in the terminal

Mimic the behavior of the --color=<auto|always|never> found in tools
like `ls' and `grep'. Default to `auto' which checks whether stderr is a
tty to define whether colors are used.

Tested:
- Ran ./boostrap-vz --color=never and --color=always, confirmed
  colorization worked as expected.
- Ran ./boostrap-vz --color=auto ${manifest_file} 2>bootstrap.log,
  confirmed colors were not added to boostrap.log by default, repeated
  the test with --color=always and confirmed escape sequences were
  output.

Signed-off-by: Filipe Brandenburger <filbranden@google.com>
---
 bootstrapvz/base/log.py  |  8 +++++---
 bootstrapvz/base/main.py | 12 +++++++++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py
index 5699dcc..b6096e3 100644
--- a/bootstrapvz/base/log.py
+++ b/bootstrapvz/base/log.py
@@ -4,18 +4,20 @@ both to a file and to the console.
 import logging
 
 
-def get_console_handler(debug):
+def get_console_handler(debug, colorize):
 	"""Returns a log handler for the console
 	The handler color codes the different log levels
 
 	:params bool debug: Whether to set the log level to DEBUG (otherwise INFO)
+	:params bool colorize: Whether to colorize console output
 	:return: The console logging handler
 	"""
 	# Create a console log handler
 	import sys
 	console_handler = logging.StreamHandler(sys.stderr)
-	# We want to colorize the output to the console, so we add a formatter
-	console_handler.setFormatter(ConsoleFormatter())
+	if colorize:
+		# We want to colorize the output to the console, so we add a formatter
+		console_handler.setFormatter(ConsoleFormatter())
 	# Set the log level depending on the debug argument
 	if debug:
 		console_handler.setLevel(logging.DEBUG)
diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 764cfbd..227f5b9 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -44,6 +44,8 @@ Options:
                      If <path> is `-' file logging will be disabled.
   --pause-on-error   Pause on error, before rollback
   --dry-run          Don't actually run the tasks
+  --color=auto|always|never
+                     Colorize the console output [default: auto]
   --debug            Print debugging information
   -h, --help         show this help
 	"""
@@ -68,7 +70,15 @@ def setup_loggers(opts):
 		file_handler = log.get_file_handler(path=logpath, debug=True)
 		root.addHandler(file_handler)
 
-	console_handler = log.get_console_handler(debug=opts['--debug'])
+	if opts['--color'] == 'never':
+		colorize = False
+	elif opts['--color'] == 'always':
+		colorize = True
+	else:
+		# If --color=auto (default), decide whether to colorize by whether stderr is a tty.
+		import os
+		colorize = os.isatty(2)
+	console_handler = log.get_console_handler(debug=opts['--debug'], colorize=colorize)
 	root.addHandler(console_handler)
 
 

From 1fe10207b8f13f6eb0805a756184cf6a2fb5639e Mon Sep 17 00:00:00 2001
From: Filipe Brandenburger <filbranden@google.com>
Date: Sun, 24 Aug 2014 10:50:07 -0700
Subject: [PATCH 056/345] Check the value of the --color argument

Make sure it's either `auto' (the default), `always' or `never'. If it
does not match any of the values, raise a Docopt exception that causes
it to print usage and exit.

Tested:
- $ sudo ./bootstrap-vz --color=invalid manifests/gce.manifest.yml
  Value of --color must be one of auto, always or never.
  Usage: bootstrap-vz [options] MANIFEST

Signed-off-by: Filipe Brandenburger <filbranden@google.com>
---
 bootstrapvz/base/main.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 227f5b9..7e1c054 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -34,7 +34,7 @@ def main():
 def get_opts():
 	"""Creates an argument parser and returns the arguments it has parsed
 	"""
-	from docopt import docopt
+	import docopt
 	usage = """bootstrap-vz
 
 Usage: bootstrap-vz [options] MANIFEST
@@ -49,7 +49,10 @@ Options:
   --debug            Print debugging information
   -h, --help         show this help
 	"""
-	return docopt(usage)
+	opts = docopt.docopt(usage)
+	if opts['--color'] not in ('auto', 'always', 'never'):
+		raise docopt.DocoptExit('Value of --color must be one of auto, always or never.')
+	return opts
 
 
 def setup_loggers(opts):

From 89a74a33c8b48457ba888d8870252932ba9d8a1e Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Tue, 2 Sep 2014 19:53:10 -0300
Subject: [PATCH 057/345] Fix `linux-headers` package version detection

The `uname -r` command returns the version of the running kernel running
on the host machine, as the chroot environment doesn't load a new one.
This prevents the proper version of the `linux-headers-*` package from
being added when the target has a different kernel version or
architecure.

This closes #121.
---
 .../providers/virtualbox/tasks/guest_additions.py      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
index 726ef29..e4e3890 100644
--- a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
+++ b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
@@ -27,11 +27,11 @@ class AddGuestAdditionsPackages(Task):
 		info.packages.add('bzip2')
 		info.packages.add('build-essential')
 		info.packages.add('dkms')
-
-		from bootstrapvz.common.tools import log_check_call
-		[kernel_version] = log_check_call(['chroot', info.root,
-		                                   'uname', '-r'])
-		kernel_headers_pkg = 'linux-headers-' + kernel_version
+		kernel_headers_pkg = 'linux-headers-'
+		if info.manifest.system['architecture'] == 'i386':
+			kernel_headers_pkg += '686-pae'
+		else:
+			kernel_headers_pkg += 'amd64'
 		info.packages.add(kernel_headers_pkg)
 
 

From 96a1683c26f7f4b174ca5a4ca0d6a2b82609d8d8 Mon Sep 17 00:00:00 2001
From: Rick Wright <rickw@google.com>
Date: Fri, 5 Sep 2014 13:56:50 -0700
Subject: [PATCH 058/345] Fix task ordering to better support customizations

This ensures that preferences are written before apt update and also ensures
that in GCE the backports are added to the sources before the provider-specific
SetPackageRepositories happens.

Change-Id: I3c85f922c49c2a6fbd3c0f2bad1072eff0d098c8
---
 bootstrapvz/common/tasks/apt.py        | 2 +-
 bootstrapvz/providers/gce/tasks/apt.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 2cda135..298a956 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -128,7 +128,7 @@ class DisableDaemonAutostart(Task):
 class AptUpdate(Task):
 	description = 'Updating the package cache'
 	phase = phases.package_installation
-	predecessors = [locale.GenerateLocale, WriteSources]
+	predecessors = [locale.GenerateLocale, WriteSources, WritePreferences]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/gce/tasks/apt.py b/bootstrapvz/providers/gce/tasks/apt.py
index 1e85808..229656a 100644
--- a/bootstrapvz/providers/gce/tasks/apt.py
+++ b/bootstrapvz/providers/gce/tasks/apt.py
@@ -9,7 +9,7 @@ import os
 class SetPackageRepositories(Task):
 	description = 'Adding apt sources'
 	phase = phases.preparation
-	predecessors = [apt.AddManifestSources]
+	predecessors = [apt.AddManifestSources, apt.AddBackports]
 
 	@classmethod
 	def run(cls, info):

From 0f5de13bae8edaf3452ccf92e3bf9e3edb3a9af9 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.ilieve@sisqualis.com.br>
Date: Sat, 13 Sep 2014 15:08:52 -0300
Subject: [PATCH 059/345] Add version option to docker_daemon plugin

As requested on #147, there is now an option to specify the Docker
version to be installed when using the `docker_daemon` plugin. The
version string is validated against a pattern extracted from the
Docker's CHANGELOG. If the version is not present, it will just download
the latest available.

The download method was also changed from `urllib` to `wget`, so we can
see its progress if needed.

This closes #147.
---
 bootstrapvz/plugins/docker_daemon/manifest-schema.yml |  9 +++++++++
 bootstrapvz/plugins/docker_daemon/tasks.py            | 11 ++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
index 0746d90..b98a28d 100644
--- a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
+++ b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
@@ -3,6 +3,15 @@ $schema: http://json-schema.org/draft-04/schema#
 title: Install Docker plugin manifest
 type: object
 properties:
+  plugins:
+    type: object
+    properties:
+      docker_daemon:
+        type: object
+        properties:
+          version:
+            pattern: '^\d\.\d{1,2}\.\d$'
+            type: string
   system:
     type: object
     properties:
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index e82aad2..a27a2ca 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -25,13 +25,18 @@ class AddDockerDeps(Task):
 class AddDockerBinary(Task):
 	description = 'Add docker binary'
 	phase = phases.system_modification
-	DOCKER_URL = 'https://get.docker.io/builds/Linux/x86_64/docker-latest'
 
 	@classmethod
 	def run(cls, info):
-		import urllib
+		from bootstrapvz.common.tools import log_check_call
+		docker_version = info.manifest.plugins['docker_daemon'].get('version', False)
+		docker_url = 'https://get.docker.io/builds/Linux/x86_64/docker-'
+		if docker_version:
+			docker_url += docker_version
+		else:
+			docker_url += 'latest'
 		bin_docker = os.path.join(info.root, 'usr/bin/docker')
-		urllib.urlretrieve(cls.DOCKER_URL, bin_docker)
+		log_check_call(['wget', '-O', bin_docker, docker_url])
 		os.chmod(bin_docker, 0755)
 
 

From 107577cb160eb36d1894cf754d4c72238242b6da Mon Sep 17 00:00:00 2001
From: Dave Bailey <davebailey@google.com>
Date: Wed, 17 Sep 2014 18:12:49 +0000
Subject: [PATCH 060/345] allow custom ulimit -n for Docker

---
 bootstrapvz/plugins/docker_daemon/assets/default/docker | 3 +++
 bootstrapvz/plugins/docker_daemon/assets/init.d/docker  | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/bootstrapvz/plugins/docker_daemon/assets/default/docker b/bootstrapvz/plugins/docker_daemon/assets/default/docker
index 14e6601..c1dc5c4 100644
--- a/bootstrapvz/plugins/docker_daemon/assets/default/docker
+++ b/bootstrapvz/plugins/docker_daemon/assets/default/docker
@@ -6,6 +6,9 @@
 # Use DOCKER_OPTS to modify the daemon startup options.
 #DOCKER_OPTS="--dns 8.8.8.8 --dns 8.8.4.4"
 
+# Use DOCKER_NOFILE to set ulimit -n before starting Docker.
+#DOCKER_NOFILE=65536
+
 # If you need Docker to use an HTTP proxy, it can also be specified here.
 #export http_proxy="http://127.0.0.1:3128/"
 
diff --git a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
index 67f0d28..c722568 100644
--- a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
+++ b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
@@ -83,6 +83,10 @@ case "$1" in
 		touch "$DOCKER_LOGFILE"
 		chgrp docker "$DOCKER_LOGFILE"
 
+		if [ -n $DOCKER_NOFILE ]; then
+			ulimit -n $DOCKER_NOFILE
+		fi
+
 		log_begin_msg "Starting $DOCKER_DESC: $BASE"
 		start-stop-daemon --start --background \
 			--no-close \

From bad378e28bb281f5485d6a082118ac31cca6ad2d Mon Sep 17 00:00:00 2001
From: Dave Bailey <davebailey@google.com>
Date: Mon, 22 Sep 2014 17:11:26 +0000
Subject: [PATCH 061/345] fix test for DOCKER_NOFILE

---
 bootstrapvz/plugins/docker_daemon/assets/init.d/docker | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
index c722568..825b4dd 100644
--- a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
+++ b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
@@ -83,7 +83,7 @@ case "$1" in
 		touch "$DOCKER_LOGFILE"
 		chgrp docker "$DOCKER_LOGFILE"
 
-		if [ -n $DOCKER_NOFILE ]; then
+		if [ -n "$DOCKER_NOFILE" ]; then
 			ulimit -n $DOCKER_NOFILE
 		fi
 

From 341011825044a19f011b56f33252e122bf740878 Mon Sep 17 00:00:00 2001
From: Tim Smith <timbot@google.com>
Date: Thu, 9 Oct 2014 21:05:48 -0700
Subject: [PATCH 062/345] Fix typo in AdjustExpandRootScript

This commit fixes a typo in common.tasks.initd.AdjustExpandRootScript (a
missing double-quote in the search regex) that rendered it ineffective.
---
 bootstrapvz/common/tasks/initd.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py
index ccf9119..c145465 100644
--- a/bootstrapvz/common/tasks/initd.py
+++ b/bootstrapvz/common/tasks/initd.py
@@ -61,4 +61,4 @@ class AdjustExpandRootScript(Task):
 		script = os.path.join(info.root, 'etc/init.d/expand-root')
 		root_idx = info.volume.partition_map.root.get_index()
 		device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx)
-		sed_i(script, '^device_path="/dev/xvda$', device_path)
+		sed_i(script, '^device_path="/dev/xvda"$', device_path)

From 7944db886f0b3586cca24f59bd3829976a175f64 Mon Sep 17 00:00:00 2001
From: Tim Smith <timbot@google.com>
Date: Thu, 9 Oct 2014 21:06:15 -0700
Subject: [PATCH 063/345] Enable auto-rootdisk growth on gce-backports.

This commit adds the cloud-initramfs-growroot package to the
installation list for GCE images with backports enabled, and updates
the gce tasklist to add the /etc/init.d/expand-root script
(with provider-appropriate touch-ups) to the image.
---
 bootstrapvz/providers/gce/__init__.py       |  7 +++++++
 bootstrapvz/providers/gce/tasks/initd.py    | 16 ++++++++++++++++
 bootstrapvz/providers/gce/tasks/packages.py | 16 ++++++++++++++++
 3 files changed, 39 insertions(+)
 create mode 100644 bootstrapvz/providers/gce/tasks/initd.py

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 627cc9c..88477a8 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -3,6 +3,7 @@ import tasks.apt
 import tasks.boot
 import tasks.configuration
 import tasks.image
+import tasks.initd
 import tasks.host
 import tasks.packages
 from bootstrapvz.common.tasks import apt
@@ -31,6 +32,7 @@ def resolve_tasks(taskset, manifest):
 	                tasks.apt.SetPackageRepositories,
 	                tasks.apt.ImportGoogleKey,
 	                tasks.packages.DefaultPackages,
+	                tasks.packages.ReleasePackages,
 	                tasks.packages.GooglePackages,
 
 	                tasks.configuration.GatherReleaseInformation,
@@ -38,6 +40,8 @@ def resolve_tasks(taskset, manifest):
 	                tasks.host.DisableIPv6,
 	                tasks.host.InstallHostnameHook,
 	                tasks.boot.ConfigureGrub,
+	                initd.AddExpandRoot,
+	                tasks.initd.AdjustExpandRootDev,
 	                initd.InstallInitScripts,
 	                ssh.AddSSHKeyGeneration,
 	                ssh.DisableSSHPasswordAuthentication,
@@ -48,6 +52,9 @@ def resolve_tasks(taskset, manifest):
 	                volume.Delete,
 	                ])
 
+	if manifest.volume['partitions']['type'] != 'none':
+		taskset.add(initd.AdjustExpandRootScript)
+
 	if 'gcs_destination' in manifest.image:
 		taskset.add(tasks.image.UploadImage)
 		if 'gce_project' in manifest.image:
diff --git a/bootstrapvz/providers/gce/tasks/initd.py b/bootstrapvz/providers/gce/tasks/initd.py
new file mode 100644
index 0000000..c826bfe
--- /dev/null
+++ b/bootstrapvz/providers/gce/tasks/initd.py
@@ -0,0 +1,16 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import initd
+
+
+class AdjustExpandRootDev(Task):
+	description = 'Adjusting the expand-root device'
+	phase = phases.system_modification
+	predecessors = [initd.AddExpandRoot, initd.AdjustExpandRootScript]
+
+	@classmethod
+	def run(cls, info):
+		import os.path
+		from bootstrapvz.common.tools import sed_i
+		script = os.path.join(info.root, 'etc/init.d/expand-root')
+		sed_i(script, '/dev/xvda', '/dev/sda')
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index d680cac..d746af0 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
+import logging
 import os
 
 
@@ -28,6 +29,21 @@ class DefaultPackages(Task):
 		info.packages.add(kernel_package)
 
 
+class ReleasePackages(Task):
+	description = 'Adding release-specific packages required for GCE'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources, apt.AddBackports, DefaultPackages]
+
+	@classmethod
+	def run(cls, info):
+		# Add release-specific packages, if available.
+		if info.source_lists.target_exists('wheezy-backports'):
+			info.packages.add('cloud-initramfs-growroot')
+		else:
+			msg = ('No release-specific packages found for {system.release}').format(**info.manifest_vars)
+			logging.getLogger(__name__).warning(msg)
+
+
 class GooglePackages(Task):
 	description = 'Adding image packages required for GCE from Google repositories'
 	phase = phases.preparation

From f3a9a1b1ec6662da9540985507ce4390c8f060ac Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Fri, 31 Oct 2014 05:32:15 -0700
Subject: [PATCH 064/345] docker_daemon: add pull_images option

---
 bootstrapvz/plugins/docker_daemon/__init__.py |  1 +
 bootstrapvz/plugins/docker_daemon/pull.py     | 43 +++++++++++++++++++
 bootstrapvz/plugins/docker_daemon/tasks.py    | 14 ++++++
 3 files changed, 58 insertions(+)
 create mode 100644 bootstrapvz/plugins/docker_daemon/pull.py

diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index ef3b320..5be1baa 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -22,3 +22,4 @@ def resolve_tasks(taskset, manifest):
 	taskset.add(tasks.AddDockerBinary)
 	taskset.add(tasks.AddDockerInit)
 	taskset.add(tasks.EnableMemoryCgroup)
+	taskset.add(tasks.PullDockerImages)
diff --git a/bootstrapvz/plugins/docker_daemon/pull.py b/bootstrapvz/plugins/docker_daemon/pull.py
new file mode 100644
index 0000000..ed56169
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/pull.py
@@ -0,0 +1,43 @@
+import os
+import subprocess
+import time
+import logging
+
+
+def pull(info, images, retries=10):
+  if len(images) == 0:
+    return
+
+  bin_docker = os.path.join(info.root, 'usr/bin/docker')
+  graph_dir = os.path.join(info.root, 'var/lib/docker')
+  socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
+  pidfile = os.path.join(info.workspace, 'docker.pid')
+  try:
+    daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
+    for _ in range(retries):
+      if subprocess.call([bin_docker, '-H', socket, 'version']) == 0:
+        break
+      time.sleep(1)
+    for img in images:
+      if img.endswith('.tar.gz') or img.endswith('.tgz'):
+        cmd = [bin_docker, '-H', socket, 'load', '-i', img]
+        logging.debug(' '.join(cmd))
+        if subprocess.call(cmd) != 0:
+          msg = 'error loading docker image {img}.'.format(img=img)
+          raise Exception(msg)
+        continue
+      cmd = [bin_docker, '-H', socket, 'pull', img]
+      logging.debug('running: %s', ' '.join(cmd))
+      if subprocess.call(cmd) != 0:
+        msg = 'error pulling docker image {img}.'.format(img=img)
+        raise Exception(msg)
+  finally:
+    daemon.terminate()
+
+if __name__ == '__main__':
+  class Info(object):
+    root = '/tmp/bootstrap-vz/root'
+    workspace = '/tmp/bootstrap-vz/workspace'
+
+  pull_images = ['/usr/local/google/home/proppy/bootstrap-vz/busybox.tar.gz', 'golang:1.3']
+  pull(Info(), pull_images)
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index a27a2ca..c003880 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -3,6 +3,7 @@ from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.providers.gce.tasks import boot as gceboot
+from bootstrapvz.plugins.docker_daemon.pull import pull
 import os
 import os.path
 import shutil
@@ -65,3 +66,16 @@ class EnableMemoryCgroup(Task):
 		from bootstrapvz.common.tools import sed_i
 		grub_config = os.path.join(info.root, 'etc/default/grub')
 		sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1 cgroup_enable=memory"')
+
+class PullDockerImages(Task):
+	description = 'Pull docker images'
+	phase = phases.system_modification
+	predecessors = [AddDockerBinary]
+
+	@classmethod
+	def run(cls, info):
+                pull_images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
+                if len(pull_images) == 0:
+                  return
+                pull_images_retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
+                pull(info, pull_images, pull_images_retries)

From 962532065c45b06753c91be38e8e1cb3894bd798 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 11:49:17 -0800
Subject: [PATCH 065/345] bootstrapvz/plugins/docker_daemon: flake8

---
 bootstrapvz/plugins/docker_daemon/pull.py  | 61 +++++++++-------------
 bootstrapvz/plugins/docker_daemon/tasks.py |  1 +
 2 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/pull.py b/bootstrapvz/plugins/docker_daemon/pull.py
index ed56169..8edd134 100644
--- a/bootstrapvz/plugins/docker_daemon/pull.py
+++ b/bootstrapvz/plugins/docker_daemon/pull.py
@@ -1,43 +1,32 @@
 import os
 import subprocess
 import time
-import logging
 
 
 def pull(info, images, retries=10):
-  if len(images) == 0:
-    return
+        if len(images) == 0:
+                return
 
-  bin_docker = os.path.join(info.root, 'usr/bin/docker')
-  graph_dir = os.path.join(info.root, 'var/lib/docker')
-  socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
-  pidfile = os.path.join(info.workspace, 'docker.pid')
-  try:
-    daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
-    for _ in range(retries):
-      if subprocess.call([bin_docker, '-H', socket, 'version']) == 0:
-        break
-      time.sleep(1)
-    for img in images:
-      if img.endswith('.tar.gz') or img.endswith('.tgz'):
-        cmd = [bin_docker, '-H', socket, 'load', '-i', img]
-        logging.debug(' '.join(cmd))
-        if subprocess.call(cmd) != 0:
-          msg = 'error loading docker image {img}.'.format(img=img)
-          raise Exception(msg)
-        continue
-      cmd = [bin_docker, '-H', socket, 'pull', img]
-      logging.debug('running: %s', ' '.join(cmd))
-      if subprocess.call(cmd) != 0:
-        msg = 'error pulling docker image {img}.'.format(img=img)
-        raise Exception(msg)
-  finally:
-    daemon.terminate()
-
-if __name__ == '__main__':
-  class Info(object):
-    root = '/tmp/bootstrap-vz/root'
-    workspace = '/tmp/bootstrap-vz/workspace'
-
-  pull_images = ['/usr/local/google/home/proppy/bootstrap-vz/busybox.tar.gz', 'golang:1.3']
-  pull(Info(), pull_images)
+        bin_docker = os.path.join(info.root, 'usr/bin/docker')
+        graph_dir = os.path.join(info.root, 'var/lib/docker')
+        socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
+        pidfile = os.path.join(info.workspace, 'docker.pid')
+        try:
+                daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
+                for _ in range(retries):
+                        if subprocess.call([bin_docker, '-H', socket, 'version']) == 0:
+                                break
+                        time.sleep(1)
+                for img in images:
+                        if img.endswith('.tar.gz') or img.endswith('.tgz'):
+                                cmd = [bin_docker, '-H', socket, 'load', '-i', img]
+                                if subprocess.call(cmd) != 0:
+                                        msg = 'error loading docker image {img}.'.format(img=img)
+                                        raise Exception(msg)
+                                continue
+                        cmd = [bin_docker, '-H', socket, 'pull', img]
+                        if subprocess.call(cmd) != 0:
+                                msg = 'error pulling docker image {img}.'.format(img=img)
+                                raise Exception(msg)
+        finally:
+                daemon.terminate()
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index c003880..75f8c10 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -67,6 +67,7 @@ class EnableMemoryCgroup(Task):
 		grub_config = os.path.join(info.root, 'etc/default/grub')
 		sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1 cgroup_enable=memory"')
 
+
 class PullDockerImages(Task):
 	description = 'Pull docker images'
 	phase = phases.system_modification

From feb4d093c9ab99690ee0a2a30de0e82e5635e029 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 11:53:23 -0800
Subject: [PATCH 066/345] bootstrapvz/plugins/docker_daemon: spaces to tabs

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 75f8c10..9172187 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -15,7 +15,7 @@ class AddDockerDeps(Task):
 	description = 'Add packages for docker deps'
 	phase = phases.package_installation
 	DOCKER_DEPS = ['aufs-tools', 'btrfs-tools', 'git', 'iptables',
-	               'procps', 'xz-utils', 'ca-certificates']
+	   	       'procps', 'xz-utils', 'ca-certificates']
 
 	@classmethod
 	def run(cls, info):
@@ -75,8 +75,8 @@ class PullDockerImages(Task):
 
 	@classmethod
 	def run(cls, info):
-                pull_images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
-                if len(pull_images) == 0:
-                  return
-                pull_images_retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
-                pull(info, pull_images, pull_images_retries)
+   	   	pull_images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
+   	   	if len(pull_images) == 0:
+   	   	  return
+   	   	pull_images_retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
+   	   	pull(info, pull_images, pull_images_retries)

From 31ba98821cfeab2c0509542d41b19494e528dbb4 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 16:14:41 -0800
Subject: [PATCH 067/345] plugins/docker_daemon: logcheckcall, inline pull and
 indent

---
 bootstrapvz/plugins/docker_daemon/pull.py  | 32 ----------------------
 bootstrapvz/plugins/docker_daemon/tasks.py | 32 +++++++++++++++++++---
 2 files changed, 28 insertions(+), 36 deletions(-)
 delete mode 100644 bootstrapvz/plugins/docker_daemon/pull.py

diff --git a/bootstrapvz/plugins/docker_daemon/pull.py b/bootstrapvz/plugins/docker_daemon/pull.py
deleted file mode 100644
index 8edd134..0000000
--- a/bootstrapvz/plugins/docker_daemon/pull.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import os
-import subprocess
-import time
-
-
-def pull(info, images, retries=10):
-        if len(images) == 0:
-                return
-
-        bin_docker = os.path.join(info.root, 'usr/bin/docker')
-        graph_dir = os.path.join(info.root, 'var/lib/docker')
-        socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
-        pidfile = os.path.join(info.workspace, 'docker.pid')
-        try:
-                daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
-                for _ in range(retries):
-                        if subprocess.call([bin_docker, '-H', socket, 'version']) == 0:
-                                break
-                        time.sleep(1)
-                for img in images:
-                        if img.endswith('.tar.gz') or img.endswith('.tgz'):
-                                cmd = [bin_docker, '-H', socket, 'load', '-i', img]
-                                if subprocess.call(cmd) != 0:
-                                        msg = 'error loading docker image {img}.'.format(img=img)
-                                        raise Exception(msg)
-                                continue
-                        cmd = [bin_docker, '-H', socket, 'pull', img]
-                        if subprocess.call(cmd) != 0:
-                                msg = 'error pulling docker image {img}.'.format(img=img)
-                                raise Exception(msg)
-        finally:
-                daemon.terminate()
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 9172187..2d061da 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -2,6 +2,7 @@ from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
+from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.providers.gce.tasks import boot as gceboot
 from bootstrapvz.plugins.docker_daemon.pull import pull
 import os
@@ -29,7 +30,6 @@ class AddDockerBinary(Task):
 
 	@classmethod
 	def run(cls, info):
-		from bootstrapvz.common.tools import log_check_call
 		docker_version = info.manifest.plugins['docker_daemon'].get('version', False)
 		docker_url = 'https://get.docker.io/builds/Linux/x86_64/docker-'
 		if docker_version:
@@ -75,8 +75,32 @@ class PullDockerImages(Task):
 
 	@classmethod
 	def run(cls, info):
-   	   	pull_images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
+   	   	images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
    	   	if len(pull_images) == 0:
    	   	  return
-   	   	pull_images_retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
-   	   	pull(info, pull_images, pull_images_retries)
+   	   	retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
+
+   	   	bin_docker = os.path.join(info.root, 'usr/bin/docker')
+   	   	graph_dir = os.path.join(info.root, 'var/lib/docker')
+   	   	socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
+   	   	pidfile = os.path.join(info.workspace, 'docker.pid')
+
+   	   	try:
+   	   	   	daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
+   	   	   	for _ in range(retries):
+   	   	   	   	if log_check_call([bin_docker, '-H', socket, 'version']) == 0:
+   	   	   	   	   	break
+   	   	   	   	time.sleep(1)
+   	   	   	for img in images:
+   	   	   	   	if img.endswith('.tar.gz') or img.endswith('.tgz'):
+   	   	   	   	   	cmd = [bin_docker, '-H', socket, 'load', '-i', img]
+   	   	   	   	   	if lock_check_call(cmd) != 0:
+   	   	   	   	   	   	msg = 'error loading docker image {img}.'.format(img=img)
+   	   	   	   	   	   	raise Exception(msg)
+   	   	   	   	else: # regular docker image
+   	   	   	   	   	cmd = [bin_docker, '-H', socket, 'pull', img]
+   	   	   	   	   	if lock_check_call(cmd) != 0:
+   	   	   	   	   	   	msg = 'error pulling docker image {img}.'.format(img=img)
+   	   	   	   	   	   	raise Exception(msg)
+   	   	finally:
+   	   	   	daemon.terminate()

From 6fdf0cc403268a88ed08bd6ba091cb9b23be2191 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 16:16:26 -0800
Subject: [PATCH 068/345] plugins/docker_daemon: align with space

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 2d061da..9620f8d 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -16,7 +16,7 @@ class AddDockerDeps(Task):
 	description = 'Add packages for docker deps'
 	phase = phases.package_installation
 	DOCKER_DEPS = ['aufs-tools', 'btrfs-tools', 'git', 'iptables',
-	   	       'procps', 'xz-utils', 'ca-certificates']
+                       'procps', 'xz-utils', 'ca-certificates']
 
 	@classmethod
 	def run(cls, info):

From e53e727c9cd6c75883e7a67aa51223f0bf5a88d2 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 16:21:38 -0800
Subject: [PATCH 069/345] plugins/docker_daemon: only add PullDockerImages
 tasks when pull_images is set

---
 bootstrapvz/plugins/docker_daemon/__init__.py | 3 ++-
 bootstrapvz/plugins/docker_daemon/tasks.py    | 2 --
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index 5be1baa..637163e 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -22,4 +22,5 @@ def resolve_tasks(taskset, manifest):
 	taskset.add(tasks.AddDockerBinary)
 	taskset.add(tasks.AddDockerInit)
 	taskset.add(tasks.EnableMemoryCgroup)
-	taskset.add(tasks.PullDockerImages)
+        if len(manifest.plugins['docker_daemon'].get('pull_images', [])) > 0:
+                taskset.add(tasks.PullDockerImages)
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 9620f8d..c2bb25e 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -76,8 +76,6 @@ class PullDockerImages(Task):
 	@classmethod
 	def run(cls, info):
    	   	images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
-   	   	if len(pull_images) == 0:
-   	   	  return
    	   	retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
 
    	   	bin_docker = os.path.join(info.root, 'usr/bin/docker')

From 99786539c7ba3edfc26d21cc74f310943a29c312 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Wed, 19 Nov 2014 16:25:27 -0800
Subject: [PATCH 070/345] plugins/docker_daemon: flake8

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index c2bb25e..d317f66 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -4,10 +4,11 @@ from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.providers.gce.tasks import boot as gceboot
-from bootstrapvz.plugins.docker_daemon.pull import pull
 import os
 import os.path
 import shutil
+import subprocess
+import time
 
 ASSETS_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets'))
 
@@ -16,7 +17,7 @@ class AddDockerDeps(Task):
 	description = 'Add packages for docker deps'
 	phase = phases.package_installation
 	DOCKER_DEPS = ['aufs-tools', 'btrfs-tools', 'git', 'iptables',
-                       'procps', 'xz-utils', 'ca-certificates']
+	               'procps', 'xz-utils', 'ca-certificates']
 
 	@classmethod
 	def run(cls, info):
@@ -92,12 +93,12 @@ class PullDockerImages(Task):
    	   	   	for img in images:
    	   	   	   	if img.endswith('.tar.gz') or img.endswith('.tgz'):
    	   	   	   	   	cmd = [bin_docker, '-H', socket, 'load', '-i', img]
-   	   	   	   	   	if lock_check_call(cmd) != 0:
+   	   	   	   	   	if log_check_call(cmd) != 0:
    	   	   	   	   	   	msg = 'error loading docker image {img}.'.format(img=img)
    	   	   	   	   	   	raise Exception(msg)
-   	   	   	   	else: # regular docker image
+   	   	   	   	else:  # regular docker image
    	   	   	   	   	cmd = [bin_docker, '-H', socket, 'pull', img]
-   	   	   	   	   	if lock_check_call(cmd) != 0:
+   	   	   	   	   	if log_check_call(cmd) != 0:
    	   	   	   	   	   	msg = 'error pulling docker image {img}.'.format(img=img)
    	   	   	   	   	   	raise Exception(msg)
    	   	finally:

From 57909eb9bc22064a877357aebcb025d5e6660795 Mon Sep 17 00:00:00 2001
From: jbergler <jbergler@meraki.net>
Date: Wed, 19 Nov 2014 21:57:24 +0000
Subject: [PATCH 071/345] Allow mixed kernel/userspace architecture.

This is to allow building an image for a 64bit machine but with 32bit userspace.

Probably not a common usecase but ideal if you need to address more higher quantities of memory but cant migrate to a full 64bit userspace due to something like ruby eating twice as much memory.
---
 bootstrapvz/base/manifest-schema.yml  | 2 ++
 bootstrapvz/common/tasks/bootstrap.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/manifest-schema.yml b/bootstrapvz/base/manifest-schema.yml
index 7110215..a49d2fa 100644
--- a/bootstrapvz/base/manifest-schema.yml
+++ b/bootstrapvz/base/manifest-schema.yml
@@ -40,6 +40,8 @@ properties:
     properties:
       architecture:
         enum: [i386, amd64]
+      userspace_architecture:
+        enum: [i386]
       bootloader:
         enum:
         - pvgrub
diff --git a/bootstrapvz/common/tasks/bootstrap.py b/bootstrapvz/common/tasks/bootstrap.py
index 5e99248..be9803d 100644
--- a/bootstrapvz/common/tasks/bootstrap.py
+++ b/bootstrapvz/common/tasks/bootstrap.py
@@ -19,7 +19,8 @@ class AddRequiredCommands(Task):
 
 def get_bootstrap_args(info):
 	executable = ['debootstrap']
-	options = ['--arch=' + info.manifest.system['architecture']]
+	arch = info.manifest.system.get('userspace_architecture', info.manifest.system.get('architecture'))
+	options = ['--arch=' + arch]
 	if len(info.include_packages) > 0:
 		options.append('--include=' + ','.join(info.include_packages))
 	if len(info.exclude_packages) > 0:

From 141e6399f34a12f828d0799e0ee759b10c9b7be3 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Fri, 21 Nov 2014 10:51:03 -0800
Subject: [PATCH 072/345] fix ident, raise TaskError

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 51 +++++++++++-----------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index d317f66..00046e4 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -76,30 +76,31 @@ class PullDockerImages(Task):
 
 	@classmethod
 	def run(cls, info):
-   	   	images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
-   	   	retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
+		from bootstrapvz.common.exceptions import TaskError
+		images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
+		retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
 
-   	   	bin_docker = os.path.join(info.root, 'usr/bin/docker')
-   	   	graph_dir = os.path.join(info.root, 'var/lib/docker')
-   	   	socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
-   	   	pidfile = os.path.join(info.workspace, 'docker.pid')
+		bin_docker = os.path.join(info.root, 'usr/bin/docker')
+		graph_dir = os.path.join(info.root, 'var/lib/docker')
+		socket = 'unix://' + os.path.join(info.workspace, 'docker.sock')
+		pidfile = os.path.join(info.workspace, 'docker.pid')
 
-   	   	try:
-   	   	   	daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
-   	   	   	for _ in range(retries):
-   	   	   	   	if log_check_call([bin_docker, '-H', socket, 'version']) == 0:
-   	   	   	   	   	break
-   	   	   	   	time.sleep(1)
-   	   	   	for img in images:
-   	   	   	   	if img.endswith('.tar.gz') or img.endswith('.tgz'):
-   	   	   	   	   	cmd = [bin_docker, '-H', socket, 'load', '-i', img]
-   	   	   	   	   	if log_check_call(cmd) != 0:
-   	   	   	   	   	   	msg = 'error loading docker image {img}.'.format(img=img)
-   	   	   	   	   	   	raise Exception(msg)
-   	   	   	   	else:  # regular docker image
-   	   	   	   	   	cmd = [bin_docker, '-H', socket, 'pull', img]
-   	   	   	   	   	if log_check_call(cmd) != 0:
-   	   	   	   	   	   	msg = 'error pulling docker image {img}.'.format(img=img)
-   	   	   	   	   	   	raise Exception(msg)
-   	   	finally:
-   	   	   	daemon.terminate()
+		try:
+			daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
+			for _ in range(retries):
+				if log_check_call([bin_docker, '-H', socket, 'version']) == 0:
+					break
+				time.sleep(1)
+			for img in images:
+				if img.endswith('.tar.gz') or img.endswith('.tgz'):
+					cmd = [bin_docker, '-H', socket, 'load', '-i', img]
+					if log_check_call(cmd) != 0:
+						msg = 'error loading docker image {img}.'.format(img=img)
+						raise TaskError(msg)
+				else:  # regular docker image
+					cmd = [bin_docker, '-H', socket, 'pull', img]
+					if log_check_call(cmd) != 0:
+						msg = 'error pulling docker image {img}.'.format(img=img)
+						raise TaskError(msg)
+		finally:
+			daemon.terminate()

From e4663f4fbe4b4692265c83811dd8b5c526607b05 Mon Sep 17 00:00:00 2001
From: Johan Euphrosine <proppy@google.com>
Date: Fri, 21 Nov 2014 11:22:11 -0800
Subject: [PATCH 073/345] docker_daemon: add comments

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 00046e4..269e143 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -86,21 +86,26 @@ class PullDockerImages(Task):
 		pidfile = os.path.join(info.workspace, 'docker.pid')
 
 		try:
+			# start docker daemon temporarly.
 			daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
+			# wait for docker daemon to start.
 			for _ in range(retries):
 				if log_check_call([bin_docker, '-H', socket, 'version']) == 0:
 					break
 				time.sleep(1)
 			for img in images:
+				# docker load if tarball.
 				if img.endswith('.tar.gz') or img.endswith('.tgz'):
 					cmd = [bin_docker, '-H', socket, 'load', '-i', img]
 					if log_check_call(cmd) != 0:
 						msg = 'error loading docker image {img}.'.format(img=img)
 						raise TaskError(msg)
-				else:  # regular docker image
+				# docker pull if image name.
+				else:
 					cmd = [bin_docker, '-H', socket, 'pull', img]
 					if log_check_call(cmd) != 0:
 						msg = 'error pulling docker image {img}.'.format(img=img)
 						raise TaskError(msg)
 		finally:
+			# shutdown docker daemon.
 			daemon.terminate()

From 170ece369149b1fb3ce1af0d1e82686ef4c59ab0 Mon Sep 17 00:00:00 2001
From: Dan Lorenc <lorenc.d@gmail.com>
Date: Fri, 21 Nov 2014 12:26:56 -0800
Subject: [PATCH 074/345] Add opts parameter to docker_daemon.

---
 bootstrapvz/plugins/docker_daemon/manifest-schema.yml | 2 ++
 bootstrapvz/plugins/docker_daemon/tasks.py            | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
index b98a28d..2e5f1f0 100644
--- a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
+++ b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
@@ -12,6 +12,8 @@ properties:
           version:
             pattern: '^\d\.\d{1,2}\.\d$'
             type: string
+          docker_opts:
+            type: string
   system:
     type: object
     properties:
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 269e143..af58af1 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -3,6 +3,7 @@ from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tools import log_check_call
+from bootstrapvz.common.tools import sed_i
 from bootstrapvz.providers.gce.tasks import boot as gceboot
 import os
 import os.path
@@ -54,6 +55,9 @@ class AddDockerInit(Task):
 		default_src = os.path.join(ASSETS_DIR, 'default/docker')
 		default_dest = os.path.join(info.root, 'etc/default/docker')
 		shutil.copy(default_src, default_dest)
+		docker_opts = info.manifest.plugins['docker_daemon'].get('docker_opts')
+		if docker_opts:
+			sed_i(default_dest, r'^#*DOCKER_OPTS=.*$', 'DOCKER_OPTS="%s"' % docker_opts)
 
 
 class EnableMemoryCgroup(Task):
@@ -64,7 +68,6 @@ class EnableMemoryCgroup(Task):
 
 	@classmethod
 	def run(cls, info):
-		from bootstrapvz.common.tools import sed_i
 		grub_config = os.path.join(info.root, 'etc/default/grub')
 		sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1 cgroup_enable=memory"')
 

From 718ef7ed26f9ec0c6d64e535715e5a9ec74a8d67 Mon Sep 17 00:00:00 2001
From: Dan Lorenc <lorenc.d@gmail.com>
Date: Sun, 23 Nov 2014 21:48:09 -0800
Subject: [PATCH 075/345] Fix log_check_call in docker_daemon.

---
 bootstrapvz/plugins/docker_daemon/tasks.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index af58af1..93fcb16 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -80,6 +80,7 @@ class PullDockerImages(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.common.exceptions import TaskError
+		from subprocess import CalledProcessError
 		images = info.manifest.plugins['docker_daemon'].get('pull_images', [])
 		retries = info.manifest.plugins['docker_daemon'].get('pull_images_retries', 10)
 
@@ -93,22 +94,29 @@ class PullDockerImages(Task):
 			daemon = subprocess.Popen([bin_docker, '-d', '--graph', graph_dir, '-H', socket, '-p', pidfile])
 			# wait for docker daemon to start.
 			for _ in range(retries):
-				if log_check_call([bin_docker, '-H', socket, 'version']) == 0:
+				try:
+					log_check_call([bin_docker, '-H', socket, 'version'])
 					break
-				time.sleep(1)
+				except CalledProcessError:
+					time.sleep(1)
 			for img in images:
 				# docker load if tarball.
 				if img.endswith('.tar.gz') or img.endswith('.tgz'):
 					cmd = [bin_docker, '-H', socket, 'load', '-i', img]
-					if log_check_call(cmd) != 0:
-						msg = 'error loading docker image {img}.'.format(img=img)
+					try:
+						log_check_call(cmd)
+					except CalledProcessError as e:
+						msg = 'error {e} loading docker image {img}.'.format(img=img, e=e)
 						raise TaskError(msg)
 				# docker pull if image name.
 				else:
 					cmd = [bin_docker, '-H', socket, 'pull', img]
-					if log_check_call(cmd) != 0:
-						msg = 'error pulling docker image {img}.'.format(img=img)
+					try:
+						log_check_call(cmd)
+					except CalledProcessError as e:
+						msg = 'error {e} pulling docker image {img}.'.format(img=img, e=e)
 						raise TaskError(msg)
 		finally:
 			# shutdown docker daemon.
 			daemon.terminate()
+			os.remove(os.path.join(info.workspace, 'docker.sock'))

From be5da54b464c9af9e5a193bb56fa1d703795fb02 Mon Sep 17 00:00:00 2001
From: Dan Merino <dmerino@vendasta.com>
Date: Thu, 27 Nov 2014 16:28:56 -0600
Subject: [PATCH 076/345] Use DOCKER_LOCKEDMEMORY to set ulimit -l before
 starting Docker. Very important when applications inside the containers need
 to lock to large amounts of memory.

---
 bootstrapvz/plugins/docker_daemon/assets/default/docker | 3 +++
 bootstrapvz/plugins/docker_daemon/assets/init.d/docker  | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/bootstrapvz/plugins/docker_daemon/assets/default/docker b/bootstrapvz/plugins/docker_daemon/assets/default/docker
index c1dc5c4..b2fc34d 100644
--- a/bootstrapvz/plugins/docker_daemon/assets/default/docker
+++ b/bootstrapvz/plugins/docker_daemon/assets/default/docker
@@ -9,6 +9,9 @@
 # Use DOCKER_NOFILE to set ulimit -n before starting Docker.
 #DOCKER_NOFILE=65536
 
+# Use DOCKER_LOCKEDMEMORY to set ulimit -l before starting Docker.
+#DOCKER_LOCKEDMEMORY=unlimited
+
 # If you need Docker to use an HTTP proxy, it can also be specified here.
 #export http_proxy="http://127.0.0.1:3128/"
 
diff --git a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
index 825b4dd..829cab5 100644
--- a/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
+++ b/bootstrapvz/plugins/docker_daemon/assets/init.d/docker
@@ -87,6 +87,10 @@ case "$1" in
 			ulimit -n $DOCKER_NOFILE
 		fi
 
+		if [ -n "$DOCKER_LOCKEDMEMORY" ]; then
+			ulimit -l $DOCKER_LOCKEDMEMORY
+		fi
+
 		log_begin_msg "Starting $DOCKER_DESC: $BASE"
 		start-stop-daemon --start --background \
 			--no-close \

From 4093693c2eae701f6eda6cb44a94679f002cfbfc Mon Sep 17 00:00:00 2001
From: Noah Fontes <nfontes@cynigram.com>
Date: Sat, 29 Nov 2014 13:46:57 -0800
Subject: [PATCH 077/345] Add support for enhanced networking on EC2.

This change adds a provider option, enhanced_networking,
which installs the Intel virtual networking driver for
SR-IOV using DKMS. It also modifies the EC2 AMI registration
to include support for SR-IOV.
---
 CHANGELOG                                     |  3 ++
 bootstrapvz/common/tasks/kernel.py            | 26 +++++++++++
 bootstrapvz/providers/ec2/__init__.py         | 11 +++++
 bootstrapvz/providers/ec2/manifest-schema.yml |  4 ++
 bootstrapvz/providers/ec2/tasks/ami.py        |  3 ++
 bootstrapvz/providers/ec2/tasks/network.py    | 44 ++++++++++++-------
 ...official-amd64-hvm-cn-north-1.manifest.yml |  1 +
 ...ebs-debian-official-amd64-hvm.manifest.yml |  1 +
 8 files changed, 76 insertions(+), 17 deletions(-)
 create mode 100644 bootstrapvz/common/tasks/kernel.py

diff --git a/CHANGELOG b/CHANGELOG
index 2f12d11..9d5b2de 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,6 @@
+2014-11-23:
+	Noah Fontes:
+		* Add support for enhanced networking on EC2 images
 2014-07-12:
 	Tiago Ilieve:
 		* Fixes #96: AddBackports is now a common task
diff --git a/bootstrapvz/common/tasks/kernel.py b/bootstrapvz/common/tasks/kernel.py
new file mode 100644
index 0000000..be978f7
--- /dev/null
+++ b/bootstrapvz/common/tasks/kernel.py
@@ -0,0 +1,26 @@
+from bootstrapvz.base import Task
+from .. import phases
+from ..tasks import packages
+
+
+class AddDKMSPackages(Task):
+	description = 'Adding DKMS and kernel header packages'
+	phase = phases.package_installation
+	successors = [packages.InstallPackages]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('dkms')
+		kernel_pkg_arch = {'i386': '686-pae', 'amd64': 'amd64'}[info.manifest.system['architecture']]
+		info.packages.add('linux-headers-' + kernel_pkg_arch)
+
+
+class UpdateInitramfs(Task):
+	description = 'Rebuilding initramfs'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import log_check_call
+	        # Update initramfs (-u) for all currently installed kernel versions (-k all)
+		log_check_call(['chroot', info.root, 'update-initramfs', '-u', '-k', 'all'])
diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index ed761d1..d58ff04 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -13,6 +13,7 @@ from bootstrapvz.common.tasks import filesystem
 from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import loopback
+from bootstrapvz.common.tasks import kernel
 
 
 def initialize():
@@ -41,6 +42,7 @@ def validate_manifest(data, validator, error):
 	virtualization = data['provider']['virtualization']
 	backing = data['volume']['backing']
 	partition_type = data['volume']['partitions']['type']
+	enhanced_networking = data['provider']['enhanced_networking'] if 'enhanced_networking' in data['provider'] else None
 
 	if virtualization == 'pvm' and bootloader != 'pvgrub':
 		error('Paravirtualized AMIs only support pvgrub as a bootloader', ['system', 'bootloader'])
@@ -58,6 +60,10 @@ def validate_manifest(data, validator, error):
 		if partition_type != 'none':
 			error('S3 backed AMIs currently only work with unpartitioned volumes', ['system', 'bootloader'])
 
+	if enhanced_networking == 'simple':
+		if virtualization != 'hvm':
+			error('Enhanced networking currently only works with HVM virtualization', ['provider', 'virtualization'])
+
 
 def resolve_tasks(taskset, manifest):
 	taskset.update(task_groups.get_standard_groups(manifest))
@@ -106,6 +112,11 @@ def resolve_tasks(taskset, manifest):
 		                ])
 		taskset.discard(filesystem.FStab)
 
+	if 'enhanced_networking' in manifest.provider and manifest.provider['enhanced_networking'] == 'simple':
+		taskset.update([kernel.AddDKMSPackages,
+		                tasks.network.InstallEnhancedNetworking,
+		                kernel.UpdateInitramfs])
+
 	taskset.update([filesystem.Format,
 	                volume.Delete,
 	                ])
diff --git a/bootstrapvz/providers/ec2/manifest-schema.yml b/bootstrapvz/providers/ec2/manifest-schema.yml
index 7bd28df..40fc332 100644
--- a/bootstrapvz/providers/ec2/manifest-schema.yml
+++ b/bootstrapvz/providers/ec2/manifest-schema.yml
@@ -19,6 +19,10 @@ properties:
         enum:
         - pvm
         - hvm
+      enhanced_networking:
+        enum:
+        - none
+        - simple
     required: [virtualization]
   system:
     type: object
diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 80feda3..7a39f2e 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -122,4 +122,7 @@ class RegisterAMI(Task):
 			registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'],
 			                                                          info.manifest.system['architecture']])
 
+		if 'enhanced_networking' in info.manifest.provider and info.manifest.provider['enhanced_networking'] == 'simple':
+			registration_params['sriov_net_support'] = 'simple'
+
 		info._ec2['image'] = info._ec2['connection'].register_image(**registration_params)
diff --git a/bootstrapvz/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py
index 100f9c8..d358f2b 100644
--- a/bootstrapvz/providers/ec2/tasks/network.py
+++ b/bootstrapvz/providers/ec2/tasks/network.py
@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tasks import kernel
 import os.path
 
 
@@ -29,30 +30,39 @@ class AddBuildEssentialPackage(Task):
 
 
 class InstallEnhancedNetworking(Task):
-	description = 'Installing network drivers for SR-IOV support'
-	phase = phases.package_installation
+	description = 'Installing enhanced networking kernel driver using DKMS'
+	phase = phases.system_modification
+	successors = [kernel.UpdateInitramfs]
 
 	@classmethod
 	def run(cls, info):
-		drivers_url = 'http://downloads.sourceforge.net/project/e1000/ixgbevf stable/2.11.3/ixgbevf-2.11.3.tar.gz'
-		archive = os.path.join(info.root, 'tmp', 'ixgbevf-2.11.3.tar.gz')
+		version = '2.15.3'
+		drivers_url = 'http://downloads.sourceforge.net/project/e1000/ixgbevf stable/%s/ixgbevf-%s.tar.gz' % (version, version)
+		archive = os.path.join(info.root, 'tmp', 'ixgbevf-%s.tar.gz' % (version))
+		module_path = os.path.join(info.root, 'usr', 'src', 'ixgbevf-%s' % (version))
 
 		import urllib
 		urllib.urlretrieve(drivers_url, archive)
 
 		from bootstrapvz.common.tools import log_check_call
-		log_check_call('tar', '--ungzip',
-		                      '--extract',
-		                      '--file', archive,
-		                      '--directory', os.path.join(info.root, 'tmp'))
+		log_check_call(['tar', '--ungzip',
+		                       '--extract',
+		                       '--file', archive,
+		                       '--directory', os.path.join(info.root, 'usr', 'src')])
 
-		src_dir = os.path.join('/tmp', os.path.basename(drivers_url), 'src')
-		log_check_call(['chroot', info.root,
-		                'make', '--directory', src_dir])
-		log_check_call(['chroot', info.root,
-		                'make', 'install',
-		                        '--directory', src_dir])
+		with open(os.path.join(module_path, 'dkms.conf'), 'w') as dkms_conf:
+			dkms_conf.write("""PACKAGE_NAME="ixgbevf"
+PACKAGE_VERSION="%s"
+CLEAN="cd src/; make clean"
+MAKE="cd src/; make BUILD_KERNEL=${kernelver}"
+BUILT_MODULE_LOCATION[0]="src/"
+BUILT_MODULE_NAME[0]="ixgbevf"
+DEST_MODULE_LOCATION[0]="/updates"
+DEST_MODULE_NAME[0]="ixgbevf"
+AUTOINSTALL="yes"
+""" % (version))
 
-		ixgbevf_conf_path = os.path.join(info.root, 'etc/modprobe.d/ixgbevf.conf')
-		with open(ixgbevf_conf_path, 'w') as ixgbevf_conf:
-			ixgbevf_conf.write('options ixgbevf InterruptThrottleRate=1,1,1,1,1,1,1,1')
+		for task in ['add', 'build', 'install']:
+			# Invoke DKMS task using specified kernel module (-m) and version (-v)
+			log_check_call(['chroot', info.root,
+			                'dkms', task, '-m', 'ixgbevf', '-v', version])
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
index 83917a8..5fa8688 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm-cn-north-1.manifest.yml
@@ -2,6 +2,7 @@
 provider:
   name: ec2
   virtualization: hvm
+  enhanced_networking: simple
   # credentials:
   #   access-key: AFAKEACCESSKEYFORAWS
   #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
index ed10508..7b7f23e 100644
--- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
+++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.yml
@@ -2,6 +2,7 @@
 provider:
   name: ec2
   virtualization: hvm
+  enhanced_networking: simple
   # credentials:
   #   access-key: AFAKEACCESSKEYFORAWS
   #   secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva

From d34aeab6e0af3314cdb5bb7a786044526ab70f63 Mon Sep 17 00:00:00 2001
From: Andrew Bogott <abogott@wikimedia.org>
Date: Wed, 10 Dec 2014 10:54:25 +0000
Subject: [PATCH 078/345] Add the file_copy plugin

File_copy supports creation of dirs and copying of files
from the host system into the image.

bug 168
---
 bootstrapvz/plugins/file_copy/__init__.py     | 16 +++++
 .../plugins/file_copy/manifest-schema.yml     | 44 +++++++++++++
 bootstrapvz/plugins/file_copy/tasks.py        | 65 +++++++++++++++++++
 3 files changed, 125 insertions(+)
 create mode 100644 bootstrapvz/plugins/file_copy/__init__.py
 create mode 100644 bootstrapvz/plugins/file_copy/manifest-schema.yml
 create mode 100644 bootstrapvz/plugins/file_copy/tasks.py

diff --git a/bootstrapvz/plugins/file_copy/__init__.py b/bootstrapvz/plugins/file_copy/__init__.py
new file mode 100644
index 0000000..aea8352
--- /dev/null
+++ b/bootstrapvz/plugins/file_copy/__init__.py
@@ -0,0 +1,16 @@
+import tasks
+
+
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+	validator(data, schema_path)
+
+
+def resolve_tasks(taskset, manifest):
+	taskset.add(tasks.ValidateSourcePaths)
+
+	if ('mkdirs' in manifest.plugins['file_copy']):
+		taskset.add(tasks.MkdirCommand)
+	if ('files' in manifest.plugins['file_copy']):
+		taskset.add(tasks.FileCopyCommand)
diff --git a/bootstrapvz/plugins/file_copy/manifest-schema.yml b/bootstrapvz/plugins/file_copy/manifest-schema.yml
new file mode 100644
index 0000000..25bdb12
--- /dev/null
+++ b/bootstrapvz/plugins/file_copy/manifest-schema.yml
@@ -0,0 +1,44 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+properties:
+  plugins:
+    properties:
+      file_copy:
+        properties:
+          mkdirs:
+            items:
+              dir:
+                $ref: '#/definitions/absolute_path'
+              permissions:
+                type: string
+              owner:
+                type: string
+              group:
+                type: string
+          files:
+            items:
+              src:
+                $ref: '#/definitions/absolute_path'
+              dst:
+                $ref: '#/definitions/absolute_path'
+              permissions:
+                type: string
+              owner:
+                type: string
+              group:
+                type: string
+            minItems: 1
+            type: array
+            required:
+              - src
+              - dst
+        required:
+        - files
+        type: object
+    required:
+    - file_copy
+    type: object
+required:
+- plugins
+title: File copy plugin manifest
+type: object
diff --git a/bootstrapvz/plugins/file_copy/tasks.py b/bootstrapvz/plugins/file_copy/tasks.py
new file mode 100644
index 0000000..06c5dd8
--- /dev/null
+++ b/bootstrapvz/plugins/file_copy/tasks.py
@@ -0,0 +1,65 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+
+import os
+import shutil
+
+
+class ValidateSourcePaths(Task):
+	description = 'Check whether the files to be copied exist'
+	phase = phases.preparation
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.exceptions import TaskError
+		for file_entry in info.manifest.plugins['file_copy']['files']:
+			srcfile = file_entry['src']
+			if not os.path.isfile(srcfile):
+				msg = 'The source file %s does not exist.' % srcfile
+				raise TaskError(msg)
+
+
+def modify_path(info, path, entry):
+	from bootstrapvz.common.tools import log_check_call
+	if 'permissions' in entry:
+		# We wrap the permissions string in str() in case
+		#  the user specified a numeric bitmask
+		chmod_command = ['chroot', info.root, 'chmod', str(entry['permissions']), path]
+		log_check_call(chmod_command)
+
+	if 'owner' in entry:
+		chown_command = ['chroot', info.root, 'chown', entry['owner'], path]
+		log_check_call(chown_command)
+
+	if 'group' in entry:
+		chgrp_command = ['chroot', info.root, 'chgrp', entry['group'], path]
+		log_check_call(chgrp_command)
+
+
+class MkdirCommand(Task):
+	description = 'copy files into the image'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import log_check_call
+
+		for dir_entry in info.manifest.plugins['file_copy']['mkdirs']:
+			mkdir_command = ['chroot', info.root, 'mkdir', '-p', dir_entry['dir']]
+			log_check_call(mkdir_command)
+			modify_path(info, dir_entry['dir'], dir_entry)
+
+
+class FileCopyCommand(Task):
+	description = 'copy files into the image'
+	phase = phases.system_modification
+	predecessors = [MkdirCommand]
+
+	@classmethod
+	def run(cls, info):
+		for file_entry in info.manifest.plugins['file_copy']['files']:
+			# note that we don't use os.path.join because it can't
+			#  handle absolute paths, which 'dst' most likely is.
+			final_destination = os.path.normpath("%s/%s" % (info.root, file_entry['dst']))
+			shutil.copy(file_entry['src'], final_destination)
+			modify_path(info, file_entry['dst'], file_entry)

From 81cb82a335fa9521f1168a3717e7dd6795810736 Mon Sep 17 00:00:00 2001
From: Andrew Bogott <abogott@wikimedia.org>
Date: Thu, 11 Dec 2014 07:55:07 +0000
Subject: [PATCH 079/345] Fix a slightly broken
 plugins/puppet/manifest-schema.yml

---
 bootstrapvz/plugins/puppet/manifest-schema.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/puppet/manifest-schema.yml b/bootstrapvz/plugins/puppet/manifest-schema.yml
index e874dac..3e4ff76 100644
--- a/bootstrapvz/plugins/puppet/manifest-schema.yml
+++ b/bootstrapvz/plugins/puppet/manifest-schema.yml
@@ -4,8 +4,8 @@ title: Puppet plugin manifest
 type: object
 properties:
   plugins:
-    properties:
     type: object
+    properties:
       puppet:
         type: object
         properties:

From 1ecc7df1d94fe14b12f6a4ce681f486a909b82ca Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 15 Jan 2015 21:17:54 +0100
Subject: [PATCH 080/345] Fixes #124

---
 README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/README.md b/README.md
index 71ea499..8eb57fe 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,20 @@ There, you can discover [what the dependencies](http://andsens.github.io/bootstr
 for a specific cloud provider are, [see a list of available plugins](http://andsens.github.io/bootstrap-vz/plugins.html)
 and learn [how you create a manifest](http://andsens.github.io/bootstrap-vz/manifest.html).
 
+Installation
+------------
+
+bootstrap-vz has a master branch for stable releases and a development for, well, development.  
+After checking out the branch of your choice you can install the python dependencies by running
+`python setup.py install`. However, depending on what kind of image you'd like to bootstrap,
+there are other debian package dependencies as well, at the very least you will need `debootstrap`.  
+[The documentation](http://andsens.github.io/bootstrap-vz/) explains this in more detail.
+
+Note that bootstrap-vz will tell you which tools it requires when they aren't
+present (the different packages are mentioned in the error message), so you can
+simply run bootstrap-vz once to get a list of the packages, install them,
+and then re-run.
+
 Developers
 ----------
 The API documentation, development guidelines and an explanation of bootstrap-vz internals

From 4bd71a2dbee666fa25873ac125c5c74987ac866a Mon Sep 17 00:00:00 2001
From: Brian Mattern <bmattern@google.com>
Date: Sat, 24 Jan 2015 08:17:34 +0000
Subject: [PATCH 081/345] Blacklist floppy module to speed up boot by several
 seconds.

Add UpdateInitramfs task which is needed for this to take effect.
Enable both tasks for GCE.

Console output before
=====================
[    1.877142] sd 0:0:1:0: [sda] Attached SCSI disk
[    1.880163] sd 0:0:1:0: Attached scsi generic sg0 type 0
[    2.684132] tsc: Refined TSC clocksource calibration: 2500.000 MHz
[    4.824081] floppy0: no floppy controllers found
[    5.103671] work still pending
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done.
Begin: Running /scripts/local-premount ... done.
[    5.313107] EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts: (null)
...
[    7.751955] alg: No test for crc32 (crc32-pclmul)
[   10.728078] floppy0: no floppy controllers found
[   11.006680] work still pending
[....] Activating swap... done
[   11.258954] EXT4-fs (sda1): re-mounted. Opts: (null)

Console output after
====================
[    1.829785] sd 0:0:1:0: [sda] Attached SCSI disk
[    1.832806] sd 0:0:1:0: Attached scsi generic sg0 type 0
Begin: Loading essential drivers ... done.
Begin: Running /scripts/init-premount ... done.
Begin: Mounting root file system ... Begin: Running /scripts/local-top ... done.
Begin: Running /scripts/local-premount ... done.
[    1.969862] EXT4-fs (sda1): mounted filesystem with ordered data mode. Opts: (null)
...
[    2.878920] alg: No test for crc32 (crc32-pclmul)
[....] Activating swap... done
[    2.986642] EXT4-fs (sda1): re-mounted. Opts: (null)

Delint.

Delint
---
 bootstrapvz/common/tasks/boot.py      | 16 ++++++++++++++--
 bootstrapvz/providers/gce/__init__.py |  3 +++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index c0a4d4d..1749603 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -7,16 +7,28 @@ from bootstrapvz.base.fs import partitionmaps
 import os.path
 
 
+class UpdateInitramfs(Task):
+	description = 'Updating initramfs'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		from ..tools import log_check_call
+		log_check_call(['chroot', info.root, 'update-initramfs', '-u'])
+
+
 class BlackListModules(Task):
 	description = 'Blacklisting kernel modules'
 	phase = phases.system_modification
+	successors = [UpdateInitramfs]
 
 	@classmethod
 	def run(cls, info):
 		blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf')
 		with open(blacklist_path, 'a') as blacklist:
-			blacklist.write(('# disable pc speaker\n'
-			                 'blacklist pcspkr'))
+			blacklist.write(('# disable pc speaker and floppy\n'
+			                 'blacklist pcspkr\n'
+			                 'blacklist floppy\n'))
 
 
 class DisableGetTTYs(Task):
diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 88477a8..4988ce8 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -7,6 +7,7 @@ import tasks.initd
 import tasks.host
 import tasks.packages
 from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tasks import boot
 from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import ssh
@@ -43,6 +44,8 @@ def resolve_tasks(taskset, manifest):
 	                initd.AddExpandRoot,
 	                tasks.initd.AdjustExpandRootDev,
 	                initd.InstallInitScripts,
+	                boot.BlackListModules,
+	                boot.UpdateInitramfs,
 	                ssh.AddSSHKeyGeneration,
 	                ssh.DisableSSHPasswordAuthentication,
 	                tasks.apt.CleanGoogleRepositoriesAndKeys,

From 8418090018edca365b8f95b564115ee5a97efe77 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Tue, 27 Jan 2015 21:12:39 -0200
Subject: [PATCH 082/345] Recover file permissions after shrink a vmdk image

After vmware-vdiskmanager runs, the image is left with a 600
permission. This commit fixes it by preserving the file permission
before the shrink operation.
---
 bootstrapvz/plugins/minimize_size/tasks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bootstrapvz/plugins/minimize_size/tasks.py b/bootstrapvz/plugins/minimize_size/tasks.py
index ef7ff97..5cba758 100644
--- a/bootstrapvz/plugins/minimize_size/tasks.py
+++ b/bootstrapvz/plugins/minimize_size/tasks.py
@@ -81,4 +81,6 @@ class ShrinkVolume(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.common.tools import log_check_call
+		perm = os.stat(info.volume.image_path).st_mode & 0777
 		log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path])
+		os.chmod(info.volume.image_path, perm)

From 0d494fb49ef4858e099243edbb5940816623dd81 Mon Sep 17 00:00:00 2001
From: Rick Wright <rickw@google.com>
Date: Thu, 29 Jan 2015 15:58:46 -0800
Subject: [PATCH 083/345] Disable resize on disks larger than 2TB

Change-Id: I9764fe2a06cf47e8c0daf38df41c288c280bd6f7
---
 bootstrapvz/providers/gce/__init__.py         |  5 ++
 .../local-premount/gce-disable-growroot       | 52 +++++++++++++++++++
 bootstrapvz/providers/gce/tasks/__init__.py   |  3 ++
 bootstrapvz/providers/gce/tasks/initd.py      | 24 ++++++++-
 4 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100755 bootstrapvz/providers/gce/assets/initramfs-tools/scripts/local-premount/gce-disable-growroot

diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py
index 88477a8..1b49d1c 100644
--- a/bootstrapvz/providers/gce/__init__.py
+++ b/bootstrapvz/providers/gce/__init__.py
@@ -9,6 +9,7 @@ import tasks.packages
 from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import initd
+from bootstrapvz.common.tasks import kernel
 from bootstrapvz.common.tasks import ssh
 from bootstrapvz.common.tasks import volume
 
@@ -55,6 +56,10 @@ def resolve_tasks(taskset, manifest):
 	if manifest.volume['partitions']['type'] != 'none':
 		taskset.add(initd.AdjustExpandRootScript)
 
+	if manifest.volume['partitions']['type'] != 'mbr':
+		taskset.update([tasks.initd.AddGrowRootDisable,
+		                kernel.UpdateInitramfs])
+
 	if 'gcs_destination' in manifest.image:
 		taskset.add(tasks.image.UploadImage)
 		if 'gce_project' in manifest.image:
diff --git a/bootstrapvz/providers/gce/assets/initramfs-tools/scripts/local-premount/gce-disable-growroot b/bootstrapvz/providers/gce/assets/initramfs-tools/scripts/local-premount/gce-disable-growroot
new file mode 100755
index 0000000..0e7d7d9
--- /dev/null
+++ b/bootstrapvz/providers/gce/assets/initramfs-tools/scripts/local-premount/gce-disable-growroot
@@ -0,0 +1,52 @@
+# Selectively disable growroot               -*- shell-script -*-
+set -e
+
+message() { echo "DISABLE-GROWROOT:" "$@" ; }
+error_exit() { message "$@"; exit 1; }
+
+. /scripts/functions
+
+# initramfs-tools exports the following variables, used below:
+# $ROOT - Generally "/dev/disk/by-uuid/<id>" which is a link to /dev/sda1
+# $ROOTFLAGS - Generally empty
+# $ROOTFSTYPE - Generally empty
+# $rootmnt - Set to "/root"
+
+# Follow link to get real root location
+if [ ! -L "${ROOT}" ]; then
+  real_root=${ROOT}
+else
+  real_root=$(readlink -f "${ROOT}")
+fi
+
+# Remove partition number to get disk
+disk=$(echo ${real_root} | sed 's/[0-9]*$//')
+
+# Determine number of 512-byte sectors in 2TB
+two_tb=$((2*(1024**4)))
+max_sectors=$((${two_tb}/512))
+
+# Determine number of sectors on disk
+geometry=$(sfdisk ${disk} --show-pt-geometry)
+cyl=$(echo $geometry | cut -d " " -f 2)
+heads=$(echo $geometry | cut -d " " -f 4)
+secs=$(echo $geometry | cut -d " " -f 6)
+sectors=$((${cyl}*${heads}*${secs}))
+
+# If disk is >2TB, disable growroot
+if [ "$sectors" -gt "$max_sectors" ]; then
+  message "Disk size >2TB - Not expanding root partition"
+  # Temporarily mount filesystem
+  if [ -z "${ROOTFSTYPE}" ]; then
+    fstype=$(get_fstype "${ROOT}")
+  else
+    fstype=${ROOTFSTYPE}
+  fi
+  mount -w ${fstype:+-t ${fstype} }${ROOTFLAGS} ${ROOT} ${rootmnt} ||
+      error_exit "failed to mount ${ROOT}."
+  # Disable growroot
+  touch "${rootmnt}/etc/growroot-disabled"
+  # Unmount filesystem
+  umount "${rootmnt}" || error_exit "failed to umount ${rootmnt}";
+fi
+
diff --git a/bootstrapvz/providers/gce/tasks/__init__.py b/bootstrapvz/providers/gce/tasks/__init__.py
index e69de29..494ec79 100644
--- a/bootstrapvz/providers/gce/tasks/__init__.py
+++ b/bootstrapvz/providers/gce/tasks/__init__.py
@@ -0,0 +1,3 @@
+import os.path
+
+assets = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets'))
diff --git a/bootstrapvz/providers/gce/tasks/initd.py b/bootstrapvz/providers/gce/tasks/initd.py
index c826bfe..f2b20bc 100644
--- a/bootstrapvz/providers/gce/tasks/initd.py
+++ b/bootstrapvz/providers/gce/tasks/initd.py
@@ -1,6 +1,9 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import initd
+from bootstrapvz.common.tasks import kernel
+from . import assets
+import os.path
 
 
 class AdjustExpandRootDev(Task):
@@ -10,7 +13,26 @@ class AdjustExpandRootDev(Task):
 
 	@classmethod
 	def run(cls, info):
-		import os.path
 		from bootstrapvz.common.tools import sed_i
 		script = os.path.join(info.root, 'etc/init.d/expand-root')
 		sed_i(script, '/dev/xvda', '/dev/sda')
+
+
+class AddGrowRootDisable(Task):
+	description = 'Add script to selectively disable growroot'
+	phase = phases.system_modification
+	successors = [kernel.UpdateInitramfs]
+
+	@classmethod
+	def run(cls, info):
+		import stat
+		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+		             stat.S_IRGRP                | stat.S_IXGRP |
+		             stat.S_IROTH                | stat.S_IXOTH)
+		from shutil import copy
+		script_src = os.path.join(assets,
+                            'initramfs-tools/scripts/local-premount/gce-disable-growroot')
+		script_dst = os.path.join(info.root,
+                            'etc/initramfs-tools/scripts/local-premount/gce-disable-growroot')
+		copy(script_src, script_dst)
+		os.chmod(script_dst, rwxr_xr_x)

From ea21dcfcec7f128e35164f59d0571bfe1de2b131 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Thu, 5 Feb 2015 10:45:31 -0200
Subject: [PATCH 084/345] Don't boot quietly on extlinux (aws)

So that we are able to see the System Log through EC2 console
or command line tools.
---
 bootstrapvz/common/task_groups.py |  1 +
 bootstrapvz/common/tasks/boot.py  | 13 +++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index 96e125c..2ae5e50 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -138,6 +138,7 @@ def get_bootloader_group(manifest):
 			group.append(boot.InstallGrub_2)
 	if manifest.system['bootloader'] == 'extlinux':
 		group.extend([boot.AddExtlinuxPackage,
+		              boot.ConfigureExtLinux,
 		              boot.InstallExtLinux])
 	return group
 
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 1749603..71c6452 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -148,6 +148,19 @@ class AddExtlinuxPackage(Task):
 			info.packages.add('syslinux-common')
 
 
+class ConfigureExtLinux(Task):
+	description = 'Configuring extlinux'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import sed_i
+		extlinux_def = os.path.join(info.root, 'etc/default/extlinux')
+		sed_i(extlinux_def, '^EXTLINUX_PARAMETERS="ro quiet"',
+		                    'EXTLINUX_PARAMETERS="ro console=ttyS0"')
+
+
 class InstallExtLinux(Task):
 	description = 'Installing extlinux'
 	phase = phases.system_modification

From c3727571047f0ea97c7f7f4c277a42078048247b Mon Sep 17 00:00:00 2001
From: Rory Finnegan <rory.finnegan@invenia.ca>
Date: Mon, 22 Dec 2014 23:01:35 -0600
Subject: [PATCH 085/345] Added an Ansible plugin, which runs a playbook on the
 chroot before before build completion.

NOTE: I'm not doing any validation on the opt_flags param and I don't recommend using for more then adding a -vvvv.
Also, I'm purposely excluding the vault flags (which also pretty commonly used) because you shouldn't be baking
private keys and certs into your images. Instead, just avoid running the vault specific code or use the
opt_flags if absolutely necessary.
---
 bootstrapvz/plugins/ansible/__init__.py       | 13 +++
 .../plugins/ansible/manifest-schema.yml       | 29 ++++++
 bootstrapvz/plugins/ansible/tasks.py          | 98 +++++++++++++++++++
 3 files changed, 140 insertions(+)
 create mode 100644 bootstrapvz/plugins/ansible/__init__.py
 create mode 100644 bootstrapvz/plugins/ansible/manifest-schema.yml
 create mode 100644 bootstrapvz/plugins/ansible/tasks.py

diff --git a/bootstrapvz/plugins/ansible/__init__.py b/bootstrapvz/plugins/ansible/__init__.py
new file mode 100644
index 0000000..f060ff3
--- /dev/null
+++ b/bootstrapvz/plugins/ansible/__init__.py
@@ -0,0 +1,13 @@
+import tasks
+
+
+def validate_manifest(data, validator, error):
+    import os.path
+    schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+    validator(data, schema_path)
+
+
+def resolve_tasks(taskset, manifest):
+    taskset.add(tasks.AddPackages)
+    taskset.add(tasks.CheckPlaybookPath)
+    taskset.add(tasks.RunAnsiblePlaybook)
diff --git a/bootstrapvz/plugins/ansible/manifest-schema.yml b/bootstrapvz/plugins/ansible/manifest-schema.yml
new file mode 100644
index 0000000..0fc2f57
--- /dev/null
+++ b/bootstrapvz/plugins/ansible/manifest-schema.yml
@@ -0,0 +1,29 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Ansible plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      ansible:
+        type: object
+        properties:
+          extra_vars: {type: string}
+          tags: {type: string}
+          skip_tags: {type: string}
+          opt_flags:
+            type: array
+            flag: {type: string}
+            minItems: 1
+          hosts:
+            type: array
+            host: {type: string}
+            minItems: 1
+          playbook: {$ref: '#/definitions/absolute_path'}
+        required:
+        - playbook
+definitions:
+  absolute_path:
+    pattern: ^/[^\0]+$
+    type: string
diff --git a/bootstrapvz/plugins/ansible/tasks.py b/bootstrapvz/plugins/ansible/tasks.py
new file mode 100644
index 0000000..9d1c893
--- /dev/null
+++ b/bootstrapvz/plugins/ansible/tasks.py
@@ -0,0 +1,98 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import apt
+import os
+
+
+class CheckPlaybookPath(Task):
+    description = 'Checking whether the playbook path exist'
+    phase = phases.preparation
+
+    @classmethod
+    def run(cls, info):
+        from bootstrapvz.common.exceptions import TaskError
+        playbook = info.manifest.plugins['ansible']['playbook']
+        if not os.path.exists(playbook):
+            msg = 'The playbook file {playbook} does not exist.'.format(playbook=playbook)
+            raise TaskError(msg)
+        if not os.path.isfile(playbook):
+            msg = 'The playbook path {playbook} does not point to a file.'.format(playbook=playbook)
+            raise TaskError(msg)
+
+
+class AddPackages(Task):
+    description = 'Making sure python is installed'
+    phase = phases.preparation
+    predecessors = [apt.AddDefaultSources]
+
+    @classmethod
+    def run(cls, info):
+        info.packages.add('python')
+
+
+class RunAnsiblePlaybook(Task):
+    description = 'Running ansible playbooks'
+    phase = phases.system_modification
+
+    @classmethod
+    def run(cls, info):
+        from bootstrapvz.common.tools import log_check_call
+
+        # Extract playbook and directory
+        playbook = info.manifest.plugins['ansible']['playbook']
+        playbook_dir = os.path.dirname(os.path.realpath(playbook))
+
+        # Check for hosts
+        hosts = None
+        if 'hosts' in info.manifest.plugins['ansible']:
+            hosts = info.manifest.plugins['ansible']['hosts']
+
+        # Check for extra vars
+        extra_vars = None
+        if 'extra_vars' in info.manifest.plugins['ansible']:
+            extra_vars = info.manifest.plugins['ansible']['extra_vars']
+
+        tags = None
+        if 'tags' in info.manifest.plugins['ansible']:
+            tags = info.manifest.plugins['ansible']['tags']
+
+        skip_tags = None
+        if 'skip_tags' in info.manifest.plugins['ansible']:
+            skip_tags = info.manifest.plugins['ansible']['skip_tags']
+
+        opt_flags = None
+        if 'opt_flags' in info.manifest.plugins['ansible']:
+            opt_flags = info.manifest.plugins['ansible']['opt_flags']
+
+        # build the inventory file
+        inventory = os.path.join(info.root, 'tmp/bootstrap-inventory')
+        with open(inventory, 'w') as handle:
+            conn = '{} ansible_connection=chroot'.format(info.root)
+	    content = ""
+
+    	    if hosts:
+                for host in hosts:
+                    content += '[{}]\n{}\n'.format(host, conn)
+	    else:
+		content = conn
+
+            handle.write(content)
+
+        # build the ansible command
+        cmd = ['ansible-playbook', '-i', inventory, os.path.basename(playbook)]
+        if extra_vars:
+            tmp_cmd = ['--extra-vars', '\"{}\"'.format(extra_vars)]
+            cmd.extend(tmp_cmd)
+        if tags:
+            tmp_cmd = ['--tags={}'.format(tags)]
+            cmd.extend(tmp_cmd)
+        if skip_tags:
+            tmp_cmd = ['--skip_tags={}'.format(skip_tags)]
+            cmd.extend(tmp_cmd)
+        if opt_flags:
+            # Should probably do proper validation on these, but I don't think it should be used very often.
+            cmd.extend(opt_flags)
+
+        # Run and remove the inventory file
+        log_check_call(cmd, cwd=playbook_dir)
+        os.remove(inventory)

From f48d392df8f8b70d96b94da9b0411df4685c517e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 22 Feb 2015 16:44:58 +0100
Subject: [PATCH 086/345] Fix pep8 error

---
 bootstrapvz/base/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index b76d520..be224ce 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -1,8 +1,9 @@
-__all__ = ['Phase', 'Task', 'main']
 from phase import Phase
 from task import Task
 from main import main
 
+__all__ = ['Phase', 'Task', 'main']
+
 
 def validate_manifest(data, validator, error):
 	"""Validates the manifest using the base manifest

From 31d61e778bfe4affbf92c489d06724ad74d0e52c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 22 Feb 2015 16:49:52 +0100
Subject: [PATCH 087/345] Fix some more pep8 errors

---
 bootstrapvz/common/tasks/packages.py | 6 ++----
 bootstrapvz/plugins/vagrant/tasks.py | 3 +--
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/common/tasks/packages.py b/bootstrapvz/common/tasks/packages.py
index 602a2a0..a5b41b0 100644
--- a/bootstrapvz/common/tasks/packages.py
+++ b/bootstrapvz/common/tasks/packages.py
@@ -49,8 +49,7 @@ class InstallPackages(Task):
 			log_check_call(['chroot', info.root,
 			                'apt-get', 'install',
 			                           '--no-install-recommends',
-			                           '--assume-yes']
-			               + map(str, remote_packages),
+			                           '--assume-yes'] + map(str, remote_packages),
 			               env=env)
 		except CalledProcessError as e:
 			import logging
@@ -91,8 +90,7 @@ class InstallPackages(Task):
 		env = os.environ.copy()
 		env['DEBIAN_FRONTEND'] = 'noninteractive'
 		log_check_call(['chroot', info.root,
-		                'dpkg', '--install']
-		               + chrooted_package_paths,
+		                'dpkg', '--install'] + chrooted_package_paths,
 		               env=env)
 
 		for path in absolute_package_paths:
diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py
index c98f0f2..1810146 100644
--- a/bootstrapvz/plugins/vagrant/tasks.py
+++ b/bootstrapvz/plugins/vagrant/tasks.py
@@ -144,8 +144,7 @@ class PackageBox(Task):
 		box_files = os.listdir(info._vagrant['folder'])
 		log_check_call(['tar', '--create', '--gzip', '--dereference',
 		                '--file', info._vagrant['box_path'],
-		                '--directory', info._vagrant['folder']]
-		               + box_files
+		                '--directory', info._vagrant['folder']] + box_files
 		               )
 		import logging
 		logging.getLogger(__name__).info('The vagrant box has been placed at ' + info._vagrant['box_path'])

From 1290694f9a6fc65ecc00ed6748a3b61442c489c6 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Tue, 3 Mar 2015 19:39:58 -0300
Subject: [PATCH 088/345] Add the manifest "include-source-type" key for
 packages object

It controls whether to include the 'deb-src' lines in image's
source.list.

Currently they are always included. This patch changes this
behavior by not including them by default; the user must set
this new config to true in order to include them.

This saves a bit of bandwidth in default installations. Also,
the use of src packages is not so usual in ordinary installations.
---
 bootstrapvz/base/manifest-schema.yml |  1 +
 bootstrapvz/common/tasks/apt.py      | 10 +++++++---
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/base/manifest-schema.yml b/bootstrapvz/base/manifest-schema.yml
index a49d2fa..229feb6 100644
--- a/bootstrapvz/base/manifest-schema.yml
+++ b/bootstrapvz/base/manifest-schema.yml
@@ -121,6 +121,7 @@ properties:
         items:
           $ref: '#/definitions/absolute_path'
         minItems: 1
+      include-source-type: {type: boolean}
     additionalProperties: false
   plugins:
     type: object
diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 298a956..f009469 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -24,14 +24,18 @@ class AddDefaultSources(Task):
 
 	@classmethod
 	def run(cls, info):
+		include_src = info.manifest.packages.get('include-source-type', False)
 		components = ' '.join(info.manifest.packages.get('components', ['main']))
 		info.source_lists.add('main', 'deb     {apt_mirror} {system.release} ' + components)
-		info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} ' + components)
+		if include_src:
+			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} ' + components)
 		if info.release_codename != 'sid':
 			info.source_lists.add('main', 'deb     http://security.debian.org/  {system.release}/updates ' + components)
-			info.source_lists.add('main', 'deb-src http://security.debian.org/  {system.release}/updates ' + components)
+			if include_src:
+				info.source_lists.add('main', 'deb-src http://security.debian.org/  {system.release}/updates ' + components)
 			info.source_lists.add('main', 'deb     {apt_mirror} {system.release}-updates ' + components)
-			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates ' + components)
+			if include_src:
+				info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates ' + components)
 
 
 class AddBackports(Task):

From 8ee0af1a3b8ed726d089779ac01ec8d0fee2dc22 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Fri, 27 Mar 2015 11:23:58 -0300
Subject: [PATCH 089/345] Execute entries with a single string as shell
 commands

Documentation says about that, but it's not implemented like that.

If the command to be executed is an array with just one string, it's
likely the user passed the entire command inside the string.

Plus, they benefit from shell expansion of wildcards like '*'.
---
 bootstrapvz/plugins/image_commands/README.md | 6 ++++++
 bootstrapvz/plugins/image_commands/tasks.py  | 3 ++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/image_commands/README.md b/bootstrapvz/plugins/image_commands/README.md
index 61583d5..e4ffb8b 100644
--- a/bootstrapvz/plugins/image_commands/README.md
+++ b/bootstrapvz/plugins/image_commands/README.md
@@ -10,6 +10,12 @@ Plugin is defined in the manifest file, plugin section with:
 
 The *commands* element is an array of commands. Each command is an array describing the executable and its arguments.
 
+If you need shell expansion of wildcards, like __\*__, just put the entire command as a single entry:
+
+    "image_commands": {
+        "commands": [ [ "rm -f /tmp/*" ]],
+    }
+
 Command is executed in current context. It is possible to use variables to access the image or execute chroot commands in the image.
 
 Available variables are:
diff --git a/bootstrapvz/plugins/image_commands/tasks.py b/bootstrapvz/plugins/image_commands/tasks.py
index 70ec370..2ea50e9 100644
--- a/bootstrapvz/plugins/image_commands/tasks.py
+++ b/bootstrapvz/plugins/image_commands/tasks.py
@@ -11,4 +11,5 @@ class ImageExecuteCommand(Task):
 		from bootstrapvz.common.tools import log_check_call
 		for raw_command in info.manifest.plugins['image_commands']['commands']:
 			command = map(lambda part: part.format(root=info.root, **info.manifest_vars), raw_command)
-			log_check_call(command)
+			shell = len(command) == 1
+			log_check_call(command, shell=shell)

From af68be15c6022da1095c36accd057c39acc84b5d Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Mon, 30 Mar 2015 15:36:33 -0300
Subject: [PATCH 090/345] New plugin: ec2_launch

It adds the ability to automatically launch an EC2 instance
after AMI registration.

It has no mandatory configuration, only optional ones, like
instance type, security groups, etc. They should be documented in
later documentation patches.
---
 bootstrapvz/plugins/ec2_launch/__init__.py    | 11 +++
 .../plugins/ec2_launch/manifest-schema.yml    | 18 +++++
 bootstrapvz/plugins/ec2_launch/tasks.py       | 70 +++++++++++++++++++
 3 files changed, 99 insertions(+)
 create mode 100644 bootstrapvz/plugins/ec2_launch/__init__.py
 create mode 100644 bootstrapvz/plugins/ec2_launch/manifest-schema.yml
 create mode 100644 bootstrapvz/plugins/ec2_launch/tasks.py

diff --git a/bootstrapvz/plugins/ec2_launch/__init__.py b/bootstrapvz/plugins/ec2_launch/__init__.py
new file mode 100644
index 0000000..c1e8caf
--- /dev/null
+++ b/bootstrapvz/plugins/ec2_launch/__init__.py
@@ -0,0 +1,11 @@
+def validate_manifest(data, validator, error):
+    import os.path
+    schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+    validator(data, schema_path)
+
+
+def resolve_tasks(taskset, manifest):
+    import tasks
+    taskset.add(tasks.LaunchEC2Instance)
+    if 'print_public_ip' in manifest.plugins['ec2_launch']:
+        taskset.add(tasks.PrintPublicIPAddress)
diff --git a/bootstrapvz/plugins/ec2_launch/manifest-schema.yml b/bootstrapvz/plugins/ec2_launch/manifest-schema.yml
new file mode 100644
index 0000000..faba1e8
--- /dev/null
+++ b/bootstrapvz/plugins/ec2_launch/manifest-schema.yml
@@ -0,0 +1,18 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: EC2-launch plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      ec2_launch:
+        type: object
+        properties:
+          security_group_ids:
+            type: array
+            items: {type: string}
+            uniqueItems: true
+          instance_type: {type: string}
+          print_public_ip: {type: string}
+          tags: {type: object}
diff --git a/bootstrapvz/plugins/ec2_launch/tasks.py b/bootstrapvz/plugins/ec2_launch/tasks.py
new file mode 100644
index 0000000..53190ef
--- /dev/null
+++ b/bootstrapvz/plugins/ec2_launch/tasks.py
@@ -0,0 +1,70 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.providers.ec2.tasks import ami
+import time
+import logging
+
+
+class LaunchEC2Instance(Task):
+    description = 'Launching EC2 instance'
+    phase = phases.image_registration
+    predecessors = [ami.RegisterAMI]
+
+    @classmethod
+    def run(cls, info):
+        conn = info._ec2['connection']
+        r = conn.run_instances(info._ec2['image'],
+                               security_group_ids=info.manifest.plugins['ec2_launch'].get('security_group_ids'),
+                               instance_type=info.manifest.plugins['ec2_launch'].get('instance_type', 't2.micro'))
+        info._ec2['instance_id'] = r.instances[0].id
+
+        if 'tags' in info.manifest.plugins['ec2_launch']:
+            def apply_format(v):
+                return v.format(**info.manifest_vars)
+            tags = info.manifest.plugins['ec2_launch']['tags']
+            r = {k: apply_format(v) for k, v in tags.items()}
+            conn.create_tags([info._ec2['instance_id']], r)
+
+
+class PrintPublicIPAddress(Task):
+    description = 'Waiting for the instance to launch'
+    phase = phases.image_registration
+    predecessors = [LaunchEC2Instance]
+
+    @classmethod
+    def run(cls, info):
+        ec2 = info._ec2
+        logger = logging.getLogger(__name__)
+        filename = info.manifest.plugins['ec2_launch']['print_public_ip']
+        if not filename:
+            filename = '/dev/null'
+        f = open(filename, 'w')
+
+        i = 0
+        instance = None
+        while True:
+            logger.debug('Waiting a bit to get instance metadata...')
+            time.sleep(5)
+
+            i += 1
+            if i > 10:
+                logger.error('Waited too much, giving up')
+                break
+
+            r = ec2['connection'].get_only_instances([ec2['instance_id']])
+            if not r and not r[0]:
+                logger.error('Could not get instance metadata')
+                break
+
+            instance = r[0]
+            if instance.ip_address:
+                break
+
+        if instance and instance.ip_address:
+            logger.info('******* EC2 IP ADDRESS: %s *******' % instance.ip_address)
+            f.write(instance.ip_address)
+        else:
+            logger.error('Could not get IP address for the instance')
+            f.write('')
+
+        f.close()

From b934808cce62361c1ec9f829507373301de73287 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Mon, 6 Apr 2015 11:47:15 -0300
Subject: [PATCH 091/345] ec2_launch: Simplify the logic for getting the IP
 address

Use the function waituntil() available in the integration-test branch
to simplify the code that retrieves the IP of the instance.

A 'TODO' note was also added to remember us to merge this function
once it gets merged.
---
 bootstrapvz/plugins/ec2_launch/tasks.py | 53 +++++++++++++------------
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/bootstrapvz/plugins/ec2_launch/tasks.py b/bootstrapvz/plugins/ec2_launch/tasks.py
index 53190ef..df159db 100644
--- a/bootstrapvz/plugins/ec2_launch/tasks.py
+++ b/bootstrapvz/plugins/ec2_launch/tasks.py
@@ -1,10 +1,20 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.providers.ec2.tasks import ami
-import time
 import logging
 
 
+# TODO: Merge with the method available in wip-integration-tests branch
+def waituntil(predicate, timeout=5, interval=0.05):
+	import time
+	threshhold = time.time() + timeout
+	while time.time() < threshhold:
+		if predicate():
+			return True
+		time.sleep(interval)
+	return False
+
+
 class LaunchEC2Instance(Task):
     description = 'Launching EC2 instance'
     phase = phases.image_registration
@@ -40,31 +50,22 @@ class PrintPublicIPAddress(Task):
             filename = '/dev/null'
         f = open(filename, 'w')
 
-        i = 0
-        instance = None
-        while True:
-            logger.debug('Waiting a bit to get instance metadata...')
-            time.sleep(5)
-
-            i += 1
-            if i > 10:
-                logger.error('Waited too much, giving up')
-                break
-
-            r = ec2['connection'].get_only_instances([ec2['instance_id']])
-            if not r and not r[0]:
-                logger.error('Could not get instance metadata')
-                break
-
-            instance = r[0]
-            if instance.ip_address:
-                break
-
-        if instance and instance.ip_address:
-            logger.info('******* EC2 IP ADDRESS: %s *******' % instance.ip_address)
-            f.write(instance.ip_address)
-        else:
-            logger.error('Could not get IP address for the instance')
+        r = ec2['connection'].get_only_instances([ec2['instance_id']])
+        if not r and not r[0]:
+            logger.error('Could not get instance metadata')
             f.write('')
+        else:
+            instance = r[0]
+
+            def instance_has_ip():
+                instance.update()
+                return instance.ip_address
+
+            if waituntil(instance_has_ip, timeout=120, interval=5):
+                logger.info('******* EC2 IP ADDRESS: %s *******' % instance.ip_address)
+                f.write(instance.ip_address)
+            else:
+                logger.error('Could not get IP address for the instance')
+                f.write('')
 
         f.close()

From 50d61c735dfc0b390997c3f9384866b7636b3716 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Mon, 6 Apr 2015 14:18:14 -0300
Subject: [PATCH 092/345] ec2_launch: Store the instance object directly in the
 info dictionary

Instead of storing just its ID.

This gives quick access to the recently created instance, which
allows us to simplify the code that needs to fetch the instance object
every time it was necessary.
---
 bootstrapvz/plugins/ec2_launch/tasks.py | 29 ++++++++++---------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/bootstrapvz/plugins/ec2_launch/tasks.py b/bootstrapvz/plugins/ec2_launch/tasks.py
index df159db..c597254 100644
--- a/bootstrapvz/plugins/ec2_launch/tasks.py
+++ b/bootstrapvz/plugins/ec2_launch/tasks.py
@@ -26,14 +26,14 @@ class LaunchEC2Instance(Task):
         r = conn.run_instances(info._ec2['image'],
                                security_group_ids=info.manifest.plugins['ec2_launch'].get('security_group_ids'),
                                instance_type=info.manifest.plugins['ec2_launch'].get('instance_type', 't2.micro'))
-        info._ec2['instance_id'] = r.instances[0].id
+        info._ec2['instance'] = r.instances[0]
 
         if 'tags' in info.manifest.plugins['ec2_launch']:
             def apply_format(v):
                 return v.format(**info.manifest_vars)
             tags = info.manifest.plugins['ec2_launch']['tags']
             r = {k: apply_format(v) for k, v in tags.items()}
-            conn.create_tags([info._ec2['instance_id']], r)
+            conn.create_tags([info._ec2['instance'].id], r)
 
 
 class PrintPublicIPAddress(Task):
@@ -50,22 +50,15 @@ class PrintPublicIPAddress(Task):
             filename = '/dev/null'
         f = open(filename, 'w')
 
-        r = ec2['connection'].get_only_instances([ec2['instance_id']])
-        if not r and not r[0]:
-            logger.error('Could not get instance metadata')
-            f.write('')
+        def instance_has_ip():
+            ec2['instance'].update()
+            return ec2['instance'].ip_address
+
+        if waituntil(instance_has_ip, timeout=120, interval=5):
+            logger.info('******* EC2 IP ADDRESS: %s *******' % ec2['instance'].ip_address)
+            f.write(ec2['instance'].ip_address)
         else:
-            instance = r[0]
-
-            def instance_has_ip():
-                instance.update()
-                return instance.ip_address
-
-            if waituntil(instance_has_ip, timeout=120, interval=5):
-                logger.info('******* EC2 IP ADDRESS: %s *******' % instance.ip_address)
-                f.write(instance.ip_address)
-            else:
-                logger.error('Could not get IP address for the instance')
-                f.write('')
+            logger.error('Could not get IP address for the instance')
+            f.write('')
 
         f.close()

From a5cd6e077d57ccace73af65404c7acc38d8bfea9 Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Mon, 6 Apr 2015 14:23:36 -0300
Subject: [PATCH 093/345] ec2_launch: Allow to deregister the AMI after
 launching image

If all you want is to test an image or product and to achieve this
you need to generate several images a day, you will end up with
lots of AMI's and snapshots that have no use in the end of the day.

This commit adds the new boolean manifest option "deregister_ami"
that, if True, deletes the recently created AMI and snapshot. So, the
final result will be only the running instance, nothing else.
---
 bootstrapvz/plugins/ec2_launch/__init__.py    |  2 ++
 .../plugins/ec2_launch/manifest-schema.yml    |  2 ++
 bootstrapvz/plugins/ec2_launch/tasks.py       | 21 +++++++++++++++++++
 3 files changed, 25 insertions(+)

diff --git a/bootstrapvz/plugins/ec2_launch/__init__.py b/bootstrapvz/plugins/ec2_launch/__init__.py
index c1e8caf..69c29c7 100644
--- a/bootstrapvz/plugins/ec2_launch/__init__.py
+++ b/bootstrapvz/plugins/ec2_launch/__init__.py
@@ -9,3 +9,5 @@ def resolve_tasks(taskset, manifest):
     taskset.add(tasks.LaunchEC2Instance)
     if 'print_public_ip' in manifest.plugins['ec2_launch']:
         taskset.add(tasks.PrintPublicIPAddress)
+    if manifest.plugins['ec2_launch'].get('deregister_ami', False):
+        taskset.add(tasks.DeregisterAMI)
diff --git a/bootstrapvz/plugins/ec2_launch/manifest-schema.yml b/bootstrapvz/plugins/ec2_launch/manifest-schema.yml
index faba1e8..9d7992e 100644
--- a/bootstrapvz/plugins/ec2_launch/manifest-schema.yml
+++ b/bootstrapvz/plugins/ec2_launch/manifest-schema.yml
@@ -16,3 +16,5 @@ properties:
           instance_type: {type: string}
           print_public_ip: {type: string}
           tags: {type: object}
+          deregister_ami: {type: boolean}
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/ec2_launch/tasks.py b/bootstrapvz/plugins/ec2_launch/tasks.py
index c597254..5d4abc5 100644
--- a/bootstrapvz/plugins/ec2_launch/tasks.py
+++ b/bootstrapvz/plugins/ec2_launch/tasks.py
@@ -62,3 +62,24 @@ class PrintPublicIPAddress(Task):
             f.write('')
 
         f.close()
+
+
+class DeregisterAMI(Task):
+    description = 'Deregistering AMI'
+    phase = phases.image_registration
+    predecessors = [LaunchEC2Instance]
+
+    @classmethod
+    def run(cls, info):
+        ec2 = info._ec2
+        logger = logging.getLogger(__name__)
+
+        def instance_running():
+            ec2['instance'].update()
+            return ec2['instance'].state == 'running'
+
+        if waituntil(instance_running, timeout=120, interval=5):
+            info._ec2['connection'].deregister_image(info._ec2['image'])
+            info._ec2['snapshot'].delete()
+        else:
+            logger.error('Timeout while booting instance')

From d3f306a6d9f5702a7e8aa4fe58c06bd03e00ecbe Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 14:12:00 +0200
Subject: [PATCH 094/345] Build docs through tox

---
 docs/Makefile | 177 --------------------------------------------------
 tox.ini       |   7 ++
 2 files changed, 7 insertions(+), 177 deletions(-)
 delete mode 100644 docs/Makefile

diff --git a/docs/Makefile b/docs/Makefile
deleted file mode 100644
index 66d46ad..0000000
--- a/docs/Makefile
+++ /dev/null
@@ -1,177 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = _build
-
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
-
-help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  dirhtml    to make HTML files named index.html in directories"
-	@echo "  singlehtml to make a single large HTML file"
-	@echo "  pickle     to make pickle files"
-	@echo "  json       to make JSON files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
-	@echo "  qthelp     to make HTML files and a qthelp project"
-	@echo "  devhelp    to make HTML files and a Devhelp project"
-	@echo "  epub       to make an epub"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
-	@echo "  texinfo    to make Texinfo files"
-	@echo "  info       to make Texinfo files and run them through makeinfo"
-	@echo "  gettext    to make PO message catalogs"
-	@echo "  changes    to make an overview of all changed/added/deprecated items"
-	@echo "  xml        to make Docutils-native XML files"
-	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
-	@echo "  linkcheck  to check all external links for integrity"
-	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
-
-clean:
-	rm -rf $(BUILDDIR)/*
-
-html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
-	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bootstrap-vz.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bootstrap-vz.qhc"
-
-devhelp:
-	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
-	@echo
-	@echo "Build finished."
-	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/bootstrap-vz"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bootstrap-vz"
-	@echo "# devhelp"
-
-epub:
-	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
-	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make' in that directory to run these through (pdf)latex" \
-	      "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through pdflatex..."
-	$(MAKE) -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-latexpdfja:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through platex and dvipdfmx..."
-	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
-	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
-	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
-	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
-	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo
-	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
-	@echo "Run \`make' in that directory to run these through makeinfo" \
-	      "(use \`make info' here to do that automatically)."
-
-info:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo "Running Texinfo files through makeinfo..."
-	make -C $(BUILDDIR)/texinfo info
-	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
-	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
-	@echo
-	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
-xml:
-	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
-	@echo
-	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
-
-pseudoxml:
-	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
-	@echo
-	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/tox.ini b/tox.ini
index b4a76b4..d147ed8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,3 +14,10 @@ deps =
     nose
     nose-cov
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
+
+[testenv:docs]
+changedir = docs
+deps =
+    sphinx
+commands =
+    sphinx-build -b html -d _build/html/doctrees . _build/html
\ No newline at end of file

From 6f87287c31df9b04a0c9aab7c3a5cc1095a08a92 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 14:13:55 +0200
Subject: [PATCH 095/345] Also build task graph when building docs

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index d147ed8..d428d52 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,8 +16,8 @@ deps =
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:docs]
-changedir = docs
 deps =
     sphinx
 commands =
-    sphinx-build -b html -d _build/html/doctrees . _build/html
\ No newline at end of file
+    sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html
+    ./taskoverview.py --output docs/_static/graph.json

From 0276b7091042e7478b783d12a3ed43a5a3936572 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 14:35:39 +0200
Subject: [PATCH 096/345] Fix taskoverview graph

---
 docs/_static/graph.json          | 2 +-
 docs/_static/taskoverview.coffee | 6 +++---
 tox.ini                          | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/_static/graph.json b/docs/_static/graph.json
index 3caaa2b..3485725 100644
--- a/docs/_static/graph.json
+++ b/docs/_static/graph.json
@@ -1 +1 @@
-{"phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}], "modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "nodes": [{"phase": 0, "name": "AddDefaultSources", "module": 0}, {"phase": 0, "name": "AddManifestPreferences", "module": 0}, {"phase": 0, "name": "AddManifestSources", "module": 0}, {"phase": 7, "name": "AptClean", "module": 0}, {"phase": 5, "name": "AptUpdate", "module": 0}, {"phase": 5, "name": "AptUpgrade", "module": 0}, {"phase": 5, "name": "DisableDaemonAutostart", "module": 0}, {"phase": 7, "name": "EnableDaemonAutostart", "module": 0}, {"phase": 5, "name": "InstallTrustedKeys", "module": 0}, {"phase": 7, "name": "PurgeUnusedPackages", "module": 0}, {"phase": 5, "name": "WritePreferences", "module": 0}, {"phase": 5, "name": "WriteSources", "module": 0}, {"phase": 0, "name": "AddExtlinuxPackage", "module": 1}, {"phase": 0, "name": "AddGrubPackage", "module": 1}, {"phase": 6, "name": "BlackListModules", "module": 1}, {"phase": 6, "name": "ConfigureGrub", "module": 1}, {"phase": 6, "name": "DisableGetTTYs", "module": 1}, {"phase": 6, "name": "InstallExtLinux", "module": 1}, {"phase": 6, "name": "InstallGrub", "module": 1}, {"phase": 0, "name": "AddRequiredCommands", "module": 2}, {"phase": 4, "name": "Bootstrap", "module": 2}, {"phase": 0, "name": "ExcludePackagesInBootstrap", "module": 2}, {"phase": 0, "name": "IncludePackagesInBootstrap", "module": 2}, {"phase": 4, "name": "MakeTarball", "module": 2}, {"phase": 7, "name": "CleanTMP", "module": 3}, {"phase": 7, "name": "ClearMOTD", "module": 3}, {"phase": 10, "name": "TriggerRollback", "module": 4}, {"phase": 0, "name": "AddRequiredCommands", "module": 5}, {"phase": 0, "name": "AddXFSProgs", "module": 5}, {"phase": 3, "name": "CreateBootMountDir", "module": 5}, {"phase": 3, "name": "CreateMountDir", "module": 5}, {"phase": 8, "name": "DeleteMountDir", "module": 5}, {"phase": 6, "name": "FStab", "module": 5}, {"phase": 2, "name": "Format", "module": 5}, {"phase": 3, "name": "MountBoot", "module": 5}, {"phase": 3, "name": "MountRoot", "module": 5}, {"phase": 4, "name": "MountSpecials", "module": 5}, {"phase": 2, "name": "TuneVolumeFS", "module": 5}, {"phase": 8, "name": "UnmountRoot", "module": 5}, {"phase": 0, "name": "CheckExternalCommands", "module": 6}, {"phase": 6, "name": "AddExpandRoot", "module": 7}, {"phase": 6, "name": "AdjustExpandRootScript", "module": 7}, {"phase": 6, "name": "InstallInitScripts", "module": 7}, {"phase": 6, "name": "RemoveHWClock", "module": 7}, {"phase": 5, "name": "GenerateLocale", "module": 8}, {"phase": 0, "name": "LocaleBootstrapPackage", "module": 8}, {"phase": 6, "name": "SetTimezone", "module": 8}, {"phase": 0, "name": "AddRequiredCommands", "module": 9}, {"phase": 1, "name": "Create", "module": 9}, {"phase": 9, "name": "MoveImage", "module": 9}, {"phase": 6, "name": "ConfigureNetworkIF", "module": 10}, {"phase": 6, "name": "RemoveDNSInfo", "module": 10}, {"phase": 6, "name": "RemoveHostname", "module": 10}, {"phase": 6, "name": "SetHostname", "module": 10}, {"phase": 0, "name": "AddManifestPackages", "module": 11}, {"phase": 5, "name": "AddTaskselStandardPackages", "module": 11}, {"phase": 5, "name": "InstallPackages", "module": 11}, {"phase": 0, "name": "AddRequiredCommands", "module": 12}, {"phase": 2, "name": "MapPartitions", "module": 12}, {"phase": 2, "name": "PartitionVolume", "module": 12}, {"phase": 8, "name": "UnmapPartitions", "module": 12}, {"phase": 6, "name": "EnableShadowConfig", "module": 13}, {"phase": 0, "name": "AddOpenSSHPackage", "module": 14}, {"phase": 6, "name": "AddSSHKeyGeneration", "module": 14}, {"phase": 6, "name": "DisableSSHDNSLookup", "module": 14}, {"phase": 6, "name": "DisableSSHPasswordAuthentication", "module": 14}, {"phase": 7, "name": "ShredHostkeys", "module": 14}, {"phase": 1, "name": "Attach", "module": 15}, {"phase": 10, "name": "Delete", "module": 15}, {"phase": 8, "name": "Detach", "module": 15}, {"phase": 0, "name": "CreateWorkspace", "module": 16}, {"phase": 10, "name": "DeleteWorkspace", "module": 16}, {"phase": 0, "name": "AddSudoPackage", "module": 17}, {"phase": 6, "name": "AdminUserCredentials", "module": 17}, {"phase": 6, "name": "CreateAdminUser", "module": 17}, {"phase": 6, "name": "DisableRootLogin", "module": 17}, {"phase": 6, "name": "PasswordlessSudo", "module": 17}, {"phase": 7, "name": "RemoveAptProxy", "module": 18}, {"phase": 5, "name": "SetAptProxy", "module": 18}, {"phase": 0, "name": "AddPackages", "module": 19}, {"phase": 0, "name": "CheckAssetsPath", "module": 19}, {"phase": 6, "name": "CopyChefAssets", "module": 19}, {"phase": 0, "name": "AddBackports", "module": 20}, {"phase": 0, "name": "AddCloudInitPackages", "module": 20}, {"phase": 6, "name": "DisableModules", "module": 20}, {"phase": 5, "name": "SetMetadataSource", "module": 20}, {"phase": 6, "name": "SetUsername", "module": 20}, {"phase": 6, "name": "ImageExecuteCommand", "module": 21}, {"phase": 4, "name": "AddFolderMounts", "module": 22}, {"phase": 0, "name": "AddRequiredCommands", "module": 22}, {"phase": 7, "name": "RemoveFolderMounts", "module": 22}, {"phase": 8, "name": "ShrinkVolume", "module": 22}, {"phase": 8, "name": "Zerofree", "module": 22}, {"phase": 5, "name": "AddNtpPackage", "module": 23}, {"phase": 6, "name": "SetNtpServers", "module": 23}, {"phase": 0, "name": "AddBackports", "module": 24}, {"phase": 0, "name": "AddONEContextPackage", "module": 24}, {"phase": 5, "name": "CopyImage", "module": 25}, {"phase": 1, "name": "CreateFromImage", "module": 25}, {"phase": 1, "name": "CreateFromSnapshot", "module": 25}, {"phase": 5, "name": "Snapshot", "module": 25}, {"phase": 0, "name": "AddPackages", "module": 26}, {"phase": 6, "name": "ApplyPuppetManifest", "module": 26}, {"phase": 0, "name": "CheckAssetsPath", "module": 26}, {"phase": 0, "name": "CheckManifestPath", "module": 26}, {"phase": 6, "name": "CopyPuppetAssets", "module": 26}, {"phase": 6, "name": "EnableAgent", "module": 26}, {"phase": 6, "name": "SetRootPassword", "module": 27}, {"phase": 5, "name": "BootstrapSaltMinion", "module": 28}, {"phase": 5, "name": "InstallSaltDependencies", "module": 28}, {"phase": 6, "name": "SetSaltGrains", "module": 28}, {"phase": 0, "name": "AddUnattendedUpgradesPackage", "module": 29}, {"phase": 6, "name": "EnablePeriodicUpgrades", "module": 29}, {"phase": 6, "name": "AddInsecurePublicKey", "module": 30}, {"phase": 0, "name": "AddPackages", "module": 30}, {"phase": 0, "name": "CheckBoxPath", "module": 30}, {"phase": 0, "name": "CreateVagrantBoxDir", "module": 30}, {"phase": 6, "name": "CreateVagrantUser", "module": 30}, {"phase": 9, "name": "PackageBox", "module": 30}, {"phase": 6, "name": "PasswordlessSudo", "module": 30}, {"phase": 10, "name": "RemoveVagrantBoxDir", "module": 30}, {"phase": 6, "name": "SetRootPassword", "module": 30}, {"phase": 6, "name": "ConfigureGrub", "module": 31}, {"phase": 9, "name": "ConvertToVhd", "module": 32}, {"phase": 0, "name": "DefaultPackages", "module": 33}, {"phase": 5, "name": "Waagent", "module": 33}, {"phase": 0, "name": "AMIName", "module": 34}, {"phase": 9, "name": "BundleImage", "module": 34}, {"phase": 9, "name": "RegisterAMI", "module": 34}, {"phase": 10, "name": "RemoveBundle", "module": 34}, {"phase": 9, "name": "UploadImage", "module": 34}, {"phase": 6, "name": "ConfigurePVGrub", "module": 35}, {"phase": 0, "name": "Connect", "module": 36}, {"phase": 0, "name": "GetCredentials", "module": 36}, {"phase": 1, "name": "Attach", "module": 37}, {"phase": 1, "name": "Create", "module": 37}, {"phase": 9, "name": "Snapshot", "module": 37}, {"phase": 6, "name": "S3FStab", "module": 38}, {"phase": 0, "name": "AddExternalCommands", "module": 39}, {"phase": 0, "name": "GetInstanceMetadata", "module": 39}, {"phase": 0, "name": "SetRegion", "module": 39}, {"phase": 6, "name": "AddEC2InitScripts", "module": 40}, {"phase": 0, "name": "AddBuildEssentialPackage", "module": 41}, {"phase": 6, "name": "EnableDHCPCDDNS", "module": 41}, {"phase": 5, "name": "InstallEnhancedNetworking", "module": 41}, {"phase": 0, "name": "DefaultPackages", "module": 42}, {"phase": 7, "name": "CleanGoogleRepositoriesAndKeys", "module": 43}, {"phase": 5, "name": "ImportGoogleKey", "module": 43}, {"phase": 0, "name": "SetPackageRepositories", "module": 43}, {"phase": 6, "name": "ConfigureGrub", "module": 44}, {"phase": 6, "name": "GatherReleaseInformation", "module": 45}, {"phase": 6, "name": "DisableIPv6", "module": 46}, {"phase": 6, "name": "SetHostname", "module": 46}, {"phase": 9, "name": "CreateTarball", "module": 47}, {"phase": 9, "name": "RegisterImage", "module": 47}, {"phase": 9, "name": "UploadImage", "module": 47}, {"phase": 0, "name": "DefaultPackages", "module": 48}, {"phase": 0, "name": "GooglePackages", "module": 48}, {"phase": 5, "name": "InstallGSUtil", "module": 48}, {"phase": 0, "name": "DefaultPackages", "module": 49}, {"phase": 6, "name": "VirtIO", "module": 50}, {"phase": 5, "name": "AddGuestAdditionsPackages", "module": 51}, {"phase": 0, "name": "CheckGuestAdditionsPath", "module": 51}, {"phase": 5, "name": "InstallGuestAdditions", "module": 51}, {"phase": 0, "name": "DefaultPackages", "module": 52}], "links": [{"source": 19, "target": 39, "definer": 19}, {"source": 21, "target": 20, "definer": 21}, {"source": 22, "target": 20, "definer": 22}, {"source": 27, "target": 39, "definer": 27}, {"source": 38, "target": 69, "definer": 38}, {"source": 40, "target": 42, "definer": 40}, {"source": 43, "target": 42, "definer": 43}, {"source": 47, "target": 39, "definer": 47}, {"source": 48, "target": 67, "definer": 48}, {"source": 55, "target": 56, "definer": 55}, {"source": 57, "target": 39, "definer": 57}, {"source": 58, "target": 33, "definer": 58}, {"source": 60, "target": 69, "definer": 60}, {"source": 63, "target": 42, "definer": 63}, {"source": 68, "target": 71, "definer": 68}, {"source": 78, "target": 4, "definer": 78}, {"source": 85, "target": 4, "definer": 85}, {"source": 89, "target": 39, "definer": 89}, {"source": 90, "target": 3, "definer": 90}, {"source": 92, "target": 60, "definer": 92}, {"source": 92, "target": 69, "definer": 92}, {"source": 93, "target": 56, "definer": 93}, {"source": 98, "target": 67, "definer": 98}, {"source": 99, "target": 134, "definer": 99}, {"source": 102, "target": 52, "definer": 102}, {"source": 102, "target": 51, "definer": 102}, {"source": 120, "target": 71, "definer": 120}, {"source": 122, "target": 18, "definer": 122}, {"source": 129, "target": 71, "definer": 129}, {"source": 138, "target": 39, "definer": 138}, {"source": 141, "target": 42, "definer": 141}, {"source": 146, "target": 3, "definer": 146}, {"source": 147, "target": 11, "definer": 147}, {"source": 148, "target": 2, "definer": 148}, {"source": 149, "target": 18, "definer": 149}, {"source": 161, "target": 56, "definer": 161}, {"source": 2, "target": 0, "definer": 0}, {"source": 44, "target": 4, "definer": 4}, {"source": 11, "target": 4, "definer": 4}, {"source": 4, "target": 5, "definer": 5}, {"source": 6, "target": 5, "definer": 5}, {"source": 11, "target": 10, "definer": 10}, {"source": 8, "target": 11, "definer": 11}, {"source": 0, "target": 12, "definer": 12}, {"source": 0, "target": 13, "definer": 13}, {"source": 32, "target": 15, "definer": 15}, {"source": 32, "target": 17, "definer": 17}, {"source": 32, "target": 18, "definer": 18}, {"source": 23, "target": 20, "definer": 20}, {"source": 0, "target": 28, "definer": 28}, {"source": 35, "target": 29, "definer": 29}, {"source": 38, "target": 31, "definer": 31}, {"source": 29, "target": 34, "definer": 34}, {"source": 30, "target": 35, "definer": 35}, {"source": 20, "target": 36, "definer": 36}, {"source": 33, "target": 37, "definer": 37}, {"source": 42, "target": 41, "definer": 41}, {"source": 0, "target": 54, "definer": 54}, {"source": 4, "target": 55, "definer": 55}, {"source": 5, "target": 56, "definer": 56}, {"source": 59, "target": 58, "definer": 58}, {"source": 38, "target": 60, "definer": 60}, {"source": 0, "target": 62, "definer": 62}, {"source": 0, "target": 72, "definer": 72}, {"source": 42, "target": 73, "definer": 73}, {"source": 0, "target": 79, "definer": 79}, {"source": 0, "target": 83, "definer": 83}, {"source": 82, "target": 83, "definer": 83}, {"source": 44, "target": 85, "definer": 85}, {"source": 20, "target": 88, "definer": 88}, {"source": 69, "target": 91, "definer": 91}, {"source": 38, "target": 92, "definer": 92}, {"source": 0, "target": 96, "definer": 96}, {"source": 95, "target": 96, "definer": 96}, {"source": 56, "target": 97, "definer": 97}, {"source": 163, "target": 97, "definer": 97}, {"source": 56, "target": 100, "definer": 100}, {"source": 163, "target": 100, "definer": 100}, {"source": 0, "target": 101, "definer": 101}, {"source": 105, "target": 102, "definer": 102}, {"source": 56, "target": 108, "definer": 108}, {"source": 0, "target": 109, "definer": 109}, {"source": 0, "target": 111, "definer": 111}, {"source": 117, "target": 113, "definer": 113}, {"source": 0, "target": 114, "definer": 114}, {"source": 70, "target": 116, "definer": 116}, {"source": 115, "target": 116, "definer": 116}, {"source": 0, "target": 124, "definer": 124}, {"source": 56, "target": 125, "definer": 125}, {"source": 132, "target": 126, "definer": 126}, {"source": 136, "target": 128, "definer": 128}, {"source": 130, "target": 128, "definer": 128}, {"source": 127, "target": 130, "definer": 130}, {"source": 133, "target": 132, "definer": 132}, {"source": 139, "target": 132, "definer": 132}, {"source": 140, "target": 132, "definer": 132}, {"source": 135, "target": 134, "definer": 134}, {"source": 0, "target": 142, "definer": 142}, {"source": 0, "target": 145, "definer": 145}, {"source": 8, "target": 147, "definer": 147}, {"source": 50, "target": 151, "definer": 151}, {"source": 49, "target": 153, "definer": 153}, {"source": 155, "target": 154, "definer": 154}, {"source": 153, "target": 155, "definer": 155}, {"source": 0, "target": 156, "definer": 156}, {"source": 156, "target": 157, "definer": 157}, {"source": 0, "target": 159, "definer": 159}, {"source": 56, "target": 163, "definer": 163}, {"source": 0, "target": 164, "definer": 164}]}
\ No newline at end of file
+{"modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.extlinux"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.grub"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.kernel"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.docker_daemon.tasks"}, {"name": "bootstrapvz.plugins.file_copy.tasks"}, {"name": "bootstrapvz.plugins.google_cloud_sdk.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.pip_install.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.initd"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "links": [{"target": 46, "definer": 15, "source": 15}, {"target": 53, "definer": 27, "source": 27}, {"target": 46, "definer": 28, "source": 28}, {"target": 41, "definer": 39, "source": 39}, {"target": 79, "definer": 41, "source": 41}, {"target": 53, "definer": 45, "source": 45}, {"target": 49, "definer": 47, "source": 47}, {"target": 49, "definer": 50, "source": 50}, {"target": 66, "definer": 51, "source": 51}, {"target": 46, "definer": 57, "source": 57}, {"target": 77, "definer": 58, "source": 58}, {"target": 66, "definer": 65, "source": 65}, {"target": 46, "definer": 67, "source": 67}, {"target": 35, "definer": 68, "source": 68}, {"target": 79, "definer": 70, "source": 70}, {"target": 49, "definer": 73, "source": 73}, {"target": 81, "definer": 78, "source": 78}, {"target": 5, "definer": 89, "source": 89}, {"target": 5, "definer": 95, "source": 95}, {"target": 49, "definer": 99, "source": 99}, {"target": 44, "definer": 100, "source": 100}, {"target": 45, "definer": 100, "source": 100}, {"target": 46, "definer": 109, "source": 109}, {"target": 4, "definer": 110, "source": 110}, {"target": 70, "definer": 112, "source": 112}, {"target": 79, "definer": 112, "source": 112}, {"target": 66, "definer": 113, "source": 113}, {"target": 77, "definer": 119, "source": 119}, {"target": 155, "definer": 120, "source": 120}, {"target": 81, "definer": 141, "source": 141}, {"target": 44, "definer": 143, "source": 143}, {"target": 45, "definer": 143, "source": 143}, {"target": 81, "definer": 150, "source": 150}, {"target": 46, "definer": 159, "source": 159}, {"target": 49, "definer": 162, "source": 162}, {"target": 53, "definer": 165, "source": 165}, {"target": 4, "definer": 167, "source": 167}, {"target": 61, "definer": 167, "source": 167}, {"target": 12, "definer": 168, "source": 168}, {"target": 44, "definer": 170, "source": 170}, {"target": 45, "definer": 170, "source": 170}, {"target": 66, "definer": 183, "source": 183}, {"target": 0, "definer": 0, "source": 1}, {"target": 1, "definer": 1, "source": 3}, {"target": 5, "definer": 5, "source": 54}, {"target": 5, "definer": 5, "source": 12}, {"target": 5, "definer": 5, "source": 11}, {"target": 6, "definer": 6, "source": 5}, {"target": 6, "definer": 6, "source": 7}, {"target": 11, "definer": 11, "source": 12}, {"target": 12, "definer": 12, "source": 9}, {"target": 16, "definer": 16, "source": 19}, {"target": 23, "definer": 23, "source": 1}, {"target": 26, "definer": 26, "source": 34}, {"target": 26, "definer": 26, "source": 24}, {"target": 27, "definer": 27, "source": 34}, {"target": 27, "definer": 27, "source": 25}, {"target": 29, "definer": 29, "source": 1}, {"target": 30, "definer": 30, "source": 38}, {"target": 31, "definer": 31, "source": 37}, {"target": 33, "definer": 33, "source": 41}, {"target": 36, "definer": 36, "source": 31}, {"target": 37, "definer": 37, "source": 32}, {"target": 38, "definer": 38, "source": 16}, {"target": 40, "definer": 40, "source": 35}, {"target": 42, "definer": 42, "source": 1}, {"target": 43, "definer": 43, "source": 34}, {"target": 44, "definer": 44, "source": 34}, {"target": 45, "definer": 45, "source": 34}, {"target": 48, "definer": 48, "source": 49}, {"target": 52, "definer": 52, "source": 66}, {"target": 64, "definer": 64, "source": 1}, {"target": 65, "definer": 65, "source": 5}, {"target": 66, "definer": 66, "source": 6}, {"target": 68, "definer": 68, "source": 69}, {"target": 70, "definer": 70, "source": 41}, {"target": 72, "definer": 72, "source": 1}, {"target": 82, "definer": 82, "source": 1}, {"target": 83, "definer": 83, "source": 49}, {"target": 90, "definer": 90, "source": 1}, {"target": 93, "definer": 93, "source": 1}, {"target": 93, "definer": 93, "source": 0}, {"target": 95, "definer": 95, "source": 54}, {"target": 100, "definer": 100, "source": 43}, {"target": 100, "definer": 100, "source": 170}, {"target": 101, "definer": 101, "source": 97}, {"target": 102, "definer": 102, "source": 103}, {"target": 108, "definer": 108, "source": 16}, {"target": 111, "definer": 111, "source": 79}, {"target": 112, "definer": 112, "source": 41}, {"target": 115, "definer": 115, "source": 1}, {"target": 115, "definer": 115, "source": 0}, {"target": 116, "definer": 116, "source": 1}, {"target": 118, "definer": 118, "source": 66}, {"target": 118, "definer": 118, "source": 185}, {"target": 121, "definer": 121, "source": 66}, {"target": 121, "definer": 121, "source": 185}, {"target": 122, "definer": 122, "source": 1}, {"target": 123, "definer": 123, "source": 126}, {"target": 129, "definer": 129, "source": 66}, {"target": 130, "definer": 130, "source": 1}, {"target": 132, "definer": 132, "source": 1}, {"target": 134, "definer": 134, "source": 138}, {"target": 135, "definer": 135, "source": 1}, {"target": 137, "definer": 137, "source": 80}, {"target": 137, "definer": 137, "source": 136}, {"target": 145, "definer": 145, "source": 1}, {"target": 146, "definer": 146, "source": 66}, {"target": 147, "definer": 147, "source": 153}, {"target": 149, "definer": 149, "source": 157}, {"target": 149, "definer": 149, "source": 151}, {"target": 151, "definer": 151, "source": 148}, {"target": 153, "definer": 153, "source": 154}, {"target": 153, "definer": 153, "source": 160}, {"target": 153, "definer": 153, "source": 161}, {"target": 155, "definer": 155, "source": 156}, {"target": 163, "definer": 163, "source": 1}, {"target": 166, "definer": 166, "source": 1}, {"target": 168, "definer": 168, "source": 9}, {"target": 169, "definer": 169, "source": 3}, {"target": 169, "definer": 169, "source": 0}, {"target": 172, "definer": 172, "source": 60}, {"target": 174, "definer": 174, "source": 59}, {"target": 175, "definer": 175, "source": 176}, {"target": 176, "definer": 176, "source": 174}, {"target": 177, "definer": 177, "source": 47}, {"target": 177, "definer": 177, "source": 48}, {"target": 178, "definer": 178, "source": 1}, {"target": 179, "definer": 179, "source": 178}, {"target": 180, "definer": 180, "source": 1}, {"target": 180, "definer": 180, "source": 0}, {"target": 180, "definer": 180, "source": 178}, {"target": 181, "definer": 181, "source": 1}, {"target": 185, "definer": 185, "source": 66}, {"target": 186, "definer": 186, "source": 1}], "nodes": [{"name": "AddBackports", "module": 0, "phase": 0}, {"name": "AddDefaultSources", "module": 0, "phase": 0}, {"name": "AddManifestPreferences", "module": 0, "phase": 0}, {"name": "AddManifestSources", "module": 0, "phase": 0}, {"name": "AptClean", "module": 0, "phase": 7}, {"name": "AptUpdate", "module": 0, "phase": 5}, {"name": "AptUpgrade", "module": 0, "phase": 5}, {"name": "DisableDaemonAutostart", "module": 0, "phase": 5}, {"name": "EnableDaemonAutostart", "module": 0, "phase": 7}, {"name": "InstallTrustedKeys", "module": 0, "phase": 5}, {"name": "PurgeUnusedPackages", "module": 0, "phase": 7}, {"name": "WritePreferences", "module": 0, "phase": 5}, {"name": "WriteSources", "module": 0, "phase": 5}, {"name": "BlackListModules", "module": 1, "phase": 6}, {"name": "DisableGetTTYs", "module": 1, "phase": 6}, {"name": "AddRequiredCommands", "module": 2, "phase": 0}, {"name": "Bootstrap", "module": 2, "phase": 4}, {"name": "ExcludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "IncludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "MakeTarball", "module": 2, "phase": 4}, {"name": "CleanTMP", "module": 3, "phase": 7}, {"name": "ClearMOTD", "module": 3, "phase": 7}, {"name": "TriggerRollback", "module": 4, "phase": 10}, {"name": "AddExtlinuxPackage", "module": 5, "phase": 0}, {"name": "ConfigureExtlinux", "module": 5, "phase": 6}, {"name": "ConfigureExtlinuxJessie", "module": 5, "phase": 6}, {"name": "InstallExtlinux", "module": 5, "phase": 6}, {"name": "InstallExtlinuxJessie", "module": 5, "phase": 6}, {"name": "AddRequiredCommands", "module": 6, "phase": 0}, {"name": "AddXFSProgs", "module": 6, "phase": 0}, {"name": "CopyMountTable", "module": 6, "phase": 4}, {"name": "CreateBootMountDir", "module": 6, "phase": 3}, {"name": "CreateMountDir", "module": 6, "phase": 3}, {"name": "DeleteMountDir", "module": 6, "phase": 8}, {"name": "FStab", "module": 6, "phase": 6}, {"name": "Format", "module": 6, "phase": 2}, {"name": "MountBoot", "module": 6, "phase": 3}, {"name": "MountRoot", "module": 6, "phase": 3}, {"name": "MountSpecials", "module": 6, "phase": 4}, {"name": "RemoveMountTable", "module": 6, "phase": 8}, {"name": "TuneVolumeFS", "module": 6, "phase": 2}, {"name": "UnmountRoot", "module": 6, "phase": 8}, {"name": "AddGrubPackage", "module": 7, "phase": 0}, {"name": "ConfigureGrub", "module": 7, "phase": 6}, {"name": "InstallGrub_1_99", "module": 7, "phase": 6}, {"name": "InstallGrub_2", "module": 7, "phase": 6}, {"name": "CheckExternalCommands", "module": 8, "phase": 0}, {"name": "AddExpandRoot", "module": 9, "phase": 6}, {"name": "AdjustExpandRootScript", "module": 9, "phase": 6}, {"name": "InstallInitScripts", "module": 9, "phase": 6}, {"name": "RemoveHWClock", "module": 9, "phase": 6}, {"name": "AddDKMSPackages", "module": 10, "phase": 5}, {"name": "DetermineKernelVersion", "module": 10, "phase": 5}, {"name": "UpdateInitramfs", "module": 10, "phase": 6}, {"name": "GenerateLocale", "module": 11, "phase": 5}, {"name": "LocaleBootstrapPackage", "module": 11, "phase": 0}, {"name": "SetTimezone", "module": 11, "phase": 6}, {"name": "AddRequiredCommands", "module": 12, "phase": 0}, {"name": "Create", "module": 12, "phase": 1}, {"name": "MoveImage", "module": 12, "phase": 9}, {"name": "ConfigureNetworkIF", "module": 13, "phase": 6}, {"name": "RemoveDNSInfo", "module": 13, "phase": 7}, {"name": "RemoveHostname", "module": 13, "phase": 7}, {"name": "SetHostname", "module": 13, "phase": 6}, {"name": "AddManifestPackages", "module": 14, "phase": 0}, {"name": "AddTaskselStandardPackages", "module": 14, "phase": 5}, {"name": "InstallPackages", "module": 14, "phase": 5}, {"name": "AddRequiredCommands", "module": 15, "phase": 0}, {"name": "MapPartitions", "module": 15, "phase": 2}, {"name": "PartitionVolume", "module": 15, "phase": 2}, {"name": "UnmapPartitions", "module": 15, "phase": 8}, {"name": "EnableShadowConfig", "module": 16, "phase": 6}, {"name": "AddOpenSSHPackage", "module": 17, "phase": 0}, {"name": "AddSSHKeyGeneration", "module": 17, "phase": 6}, {"name": "DisableSSHDNSLookup", "module": 17, "phase": 6}, {"name": "DisableSSHPasswordAuthentication", "module": 17, "phase": 6}, {"name": "ShredHostkeys", "module": 17, "phase": 7}, {"name": "Attach", "module": 18, "phase": 1}, {"name": "Delete", "module": 18, "phase": 10}, {"name": "Detach", "module": 18, "phase": 8}, {"name": "CreateWorkspace", "module": 19, "phase": 0}, {"name": "DeleteWorkspace", "module": 19, "phase": 10}, {"name": "AddSudoPackage", "module": 20, "phase": 0}, {"name": "AdminUserCredentials", "module": 20, "phase": 6}, {"name": "CreateAdminUser", "module": 20, "phase": 6}, {"name": "DisableRootLogin", "module": 20, "phase": 6}, {"name": "PasswordlessSudo", "module": 20, "phase": 6}, {"name": "CheckAptProxy", "module": 21, "phase": 0}, {"name": "RemoveAptProxy", "module": 21, "phase": 7}, {"name": "SetAptProxy", "module": 21, "phase": 5}, {"name": "AddPackages", "module": 22, "phase": 0}, {"name": "CheckAssetsPath", "module": 22, "phase": 0}, {"name": "CopyChefAssets", "module": 22, "phase": 6}, {"name": "AddCloudInitPackages", "module": 23, "phase": 0}, {"name": "DisableModules", "module": 23, "phase": 6}, {"name": "SetMetadataSource", "module": 23, "phase": 5}, {"name": "SetUsername", "module": 23, "phase": 6}, {"name": "AddDockerBinary", "module": 24, "phase": 6}, {"name": "AddDockerDeps", "module": 24, "phase": 5}, {"name": "AddDockerInit", "module": 24, "phase": 6}, {"name": "EnableMemoryCgroup", "module": 24, "phase": 6}, {"name": "PullDockerImages", "module": 24, "phase": 6}, {"name": "FileCopyCommand", "module": 25, "phase": 6}, {"name": "MkdirCommand", "module": 25, "phase": 6}, {"name": "ValidateSourcePaths", "module": 25, "phase": 0}, {"name": "InstallCloudSDK", "module": 26, "phase": 6}, {"name": "RemoveCloudSDKTarball", "module": 26, "phase": 7}, {"name": "ImageExecuteCommand", "module": 27, "phase": 6}, {"name": "AddFolderMounts", "module": 28, "phase": 4}, {"name": "AddRequiredCommands", "module": 28, "phase": 0}, {"name": "RemoveFolderMounts", "module": 28, "phase": 7}, {"name": "ShrinkVolume", "module": 28, "phase": 8}, {"name": "Zerofree", "module": 28, "phase": 8}, {"name": "AddNtpPackage", "module": 29, "phase": 5}, {"name": "SetNtpServers", "module": 29, "phase": 6}, {"name": "AddONEContextPackage", "module": 30, "phase": 0}, {"name": "AddPipPackage", "module": 31, "phase": 0}, {"name": "PipInstallCommand", "module": 31, "phase": 6}, {"name": "CopyImage", "module": 32, "phase": 5}, {"name": "CreateFromImage", "module": 32, "phase": 1}, {"name": "CreateFromSnapshot", "module": 32, "phase": 1}, {"name": "Snapshot", "module": 32, "phase": 5}, {"name": "AddPackages", "module": 33, "phase": 0}, {"name": "ApplyPuppetManifest", "module": 33, "phase": 6}, {"name": "CheckAssetsPath", "module": 33, "phase": 0}, {"name": "CheckManifestPath", "module": 33, "phase": 0}, {"name": "CopyPuppetAssets", "module": 33, "phase": 6}, {"name": "EnableAgent", "module": 33, "phase": 6}, {"name": "SetRootPassword", "module": 34, "phase": 6}, {"name": "BootstrapSaltMinion", "module": 35, "phase": 5}, {"name": "InstallSaltDependencies", "module": 35, "phase": 0}, {"name": "SetSaltGrains", "module": 35, "phase": 6}, {"name": "AddUnattendedUpgradesPackage", "module": 36, "phase": 0}, {"name": "EnablePeriodicUpgrades", "module": 36, "phase": 6}, {"name": "AddInsecurePublicKey", "module": 37, "phase": 6}, {"name": "AddPackages", "module": 37, "phase": 0}, {"name": "CheckBoxPath", "module": 37, "phase": 0}, {"name": "CreateVagrantBoxDir", "module": 37, "phase": 0}, {"name": "CreateVagrantUser", "module": 37, "phase": 6}, {"name": "PackageBox", "module": 37, "phase": 9}, {"name": "PasswordlessSudo", "module": 37, "phase": 6}, {"name": "RemoveVagrantBoxDir", "module": 37, "phase": 10}, {"name": "SetRootPassword", "module": 37, "phase": 6}, {"name": "ConfigureGrub", "module": 38, "phase": 6}, {"name": "ConvertToVhd", "module": 39, "phase": 9}, {"name": "DefaultPackages", "module": 40, "phase": 0}, {"name": "Waagent", "module": 40, "phase": 5}, {"name": "AMIName", "module": 41, "phase": 0}, {"name": "BundleImage", "module": 41, "phase": 9}, {"name": "RegisterAMI", "module": 41, "phase": 9}, {"name": "RemoveBundle", "module": 41, "phase": 10}, {"name": "UploadImage", "module": 41, "phase": 9}, {"name": "ConfigurePVGrub", "module": 42, "phase": 6}, {"name": "Connect", "module": 43, "phase": 0}, {"name": "GetCredentials", "module": 43, "phase": 0}, {"name": "Attach", "module": 44, "phase": 1}, {"name": "Create", "module": 44, "phase": 1}, {"name": "Snapshot", "module": 44, "phase": 9}, {"name": "S3FStab", "module": 45, "phase": 6}, {"name": "AddExternalCommands", "module": 46, "phase": 0}, {"name": "GetInstanceMetadata", "module": 46, "phase": 0}, {"name": "SetRegion", "module": 46, "phase": 0}, {"name": "AddEC2InitScripts", "module": 47, "phase": 6}, {"name": "AddBuildEssentialPackage", "module": 48, "phase": 0}, {"name": "EnableDHCPCDDNS", "module": 48, "phase": 6}, {"name": "InstallEnhancedNetworking", "module": 48, "phase": 6}, {"name": "DefaultPackages", "module": 49, "phase": 0}, {"name": "CleanGoogleRepositoriesAndKeys", "module": 50, "phase": 7}, {"name": "ImportGoogleKey", "module": 50, "phase": 5}, {"name": "SetPackageRepositories", "module": 50, "phase": 0}, {"name": "ConfigureGrub", "module": 51, "phase": 6}, {"name": "GatherReleaseInformation", "module": 52, "phase": 6}, {"name": "DisableIPv6", "module": 53, "phase": 6}, {"name": "InstallHostnameHook", "module": 53, "phase": 6}, {"name": "CreateTarball", "module": 54, "phase": 9}, {"name": "RegisterImage", "module": 54, "phase": 9}, {"name": "UploadImage", "module": 54, "phase": 9}, {"name": "AdjustExpandRootDev", "module": 55, "phase": 6}, {"name": "DefaultPackages", "module": 56, "phase": 0}, {"name": "GooglePackages", "module": 56, "phase": 0}, {"name": "ReleasePackages", "module": 56, "phase": 0}, {"name": "DefaultPackages", "module": 57, "phase": 0}, {"name": "VirtIO", "module": 58, "phase": 6}, {"name": "AddGuestAdditionsPackages", "module": 59, "phase": 5}, {"name": "CheckGuestAdditionsPath", "module": 59, "phase": 0}, {"name": "InstallGuestAdditions", "module": 59, "phase": 5}, {"name": "DefaultPackages", "module": 60, "phase": 0}], "phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}]}
\ No newline at end of file
diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee
index d9d3b94..4be9fe3 100644
--- a/docs/_static/taskoverview.coffee
+++ b/docs/_static/taskoverview.coffee
@@ -1,10 +1,10 @@
 class window.TaskOverview
 	viewBoxHeight = 800
-	viewBoxWidth = 200
+	viewBoxWidth = 800
 	margins =
-		top:    100
+		top:    200
 		left:   50
-		bottom: 100
+		bottom: 200
 		right:  50
 	gravity =
 		lateral: .1
diff --git a/tox.ini b/tox.ini
index d428d52..0d736c1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,5 +19,5 @@ commands = nosetests -v tests/integration --with-coverage --cover-package=bootst
 deps =
     sphinx
 commands =
-    sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html
     ./taskoverview.py --output docs/_static/graph.json
+    sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html

From be5590f411d197a65d6a3cfb89648c0cd42bfe7b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 15:06:41 +0200
Subject: [PATCH 097/345] Add highlighting of tasks in same module in
 taskoverview and a docs-serve tox target

---
 docs/_static/taskoverview.coffee | 14 +++++++++++---
 docs/_static/taskoverview.less   |  5 +++++
 tox.ini                          |  4 ++++
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee
index 4be9fe3..ccae87d 100644
--- a/docs/_static/taskoverview.coffee
+++ b/docs/_static/taskoverview.coffee
@@ -118,15 +118,23 @@ class window.TaskOverview
 		            .selectAll('line').data(layout.links()).enter()
 		            .append('line').attr('marker-end', 'url(#right-arrowhead)')
 
+		mouseOver = (d) ->
+			labels.classed 'hover', (l) -> d is l
+			nodes.classed 'highlight', (n) -> d.module is n.module
+
+		mouseOut = (d) ->
+			labels.classed 'hover', no
+			nodes.classed 'highlight', no
+
 		nodes = @svg.append('g').attr('class', 'nodes')
 		            .selectAll('g.partition').data(groups).enter()
 		            .append('g').attr('class', 'partition')
 		            .selectAll('circle').data((d) -> d.values).enter()
 		            .append('circle').attr('r', (d) -> d.radius)
-		                             .style('fill', (d, i) -> nodeColors(d[nodeColorKey]))
+		                             .style('fill', (d) -> nodeColors(d[nodeColorKey]))
 		                             .call(layout.drag)
-		                             .on('mouseover', (d) -> (labels.filter (l) -> d is l).classed 'hover', true)
-		                             .on('mouseout', (d) -> (labels.filter (l) -> d is l).classed 'hover', false)
+		                             .on('mouseover', mouseOver)
+		                             .on('mouseout', mouseOut)
 
 		labels = @svg.append('g').attr('class', 'node-labels')
 		             .selectAll('g.partition').data(groups).enter()
diff --git a/docs/_static/taskoverview.less b/docs/_static/taskoverview.less
index 2903816..8c1e67c 100644
--- a/docs/_static/taskoverview.less
+++ b/docs/_static/taskoverview.less
@@ -6,6 +6,11 @@
 	}
 	g.nodes circle {
 		stroke: #000000;
+		&.highlight {
+			stroke: #555599;
+			stroke-width: 2.5px;
+			fill: #EEAAAA !important;
+		}
 		opacity: .9;
 		stroke-width: 1.5px;
 	}
diff --git a/tox.ini b/tox.ini
index 0d736c1..17db141 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,3 +21,7 @@ deps =
 commands =
     ./taskoverview.py --output docs/_static/graph.json
     sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html
+
+[testenv:docs-serve]
+changedir = docs/_build/html
+commands = python -m SimpleHTTPServer 8080

From d06fbb4a2d9278328cfa0693326e33828f5c5c26 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 15:11:50 +0200
Subject: [PATCH 098/345] Turn on nazi option for sphinx and fix warnings

---
 bootstrapvz/base/fs/volume.py | 1 +
 docs/base/pkg.rst             | 2 +-
 tox.ini                       | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/fs/volume.py b/bootstrapvz/base/fs/volume.py
index 0a17f61..043094a 100644
--- a/bootstrapvz/base/fs/volume.py
+++ b/bootstrapvz/base/fs/volume.py
@@ -70,6 +70,7 @@ class Volume(FSMProxy):
 		rather than a loopback device or a network block device.
 
 		:param _e_obj e: Event object containing arguments to create()
+
 		Keyword arguments to link_dm_node() are:
 
 		:param int logical_start_sector: The sector the volume should start at in the new volume
diff --git a/docs/base/pkg.rst b/docs/base/pkg.rst
index 1a8db8b..4625358 100644
--- a/docs/base/pkg.rst
+++ b/docs/base/pkg.rst
@@ -16,7 +16,7 @@ Sources list
     :private-members:
 
 Preferences list
-------------
+----------------
 .. automodule:: bootstrapvz.base.pkg.preferenceslist
     :members:
     :private-members:
diff --git a/tox.ini b/tox.ini
index 17db141..e229a8d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,7 +20,7 @@ deps =
     sphinx
 commands =
     ./taskoverview.py --output docs/_static/graph.json
-    sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html
+    sphinx-build -W -b html -d docs/_build/html/doctrees docs docs/_build/html
 
 [testenv:docs-serve]
 changedir = docs/_build/html

From 73972e1b60e8c41a4a09c02d7572c1bb53552bc7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 15:16:18 +0200
Subject: [PATCH 099/345] Move taskoverview exec into docs/

---
 taskoverview.py => docs/taskoverview.py | 4 ++++
 tox.ini                                 | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)
 rename taskoverview.py => docs/taskoverview.py (95%)

diff --git a/taskoverview.py b/docs/taskoverview.py
similarity index 95%
rename from taskoverview.py
rename to docs/taskoverview.py
index f9811b1..c1b142f 100755
--- a/taskoverview.py
+++ b/docs/taskoverview.py
@@ -1,4 +1,8 @@
 #!/usr/bin/python
+import sys
+import os.path
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 
 def main(opts):
diff --git a/tox.ini b/tox.ini
index e229a8d..0ec323d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,11 +16,12 @@ deps =
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:docs]
+changedir = docs
 deps =
     sphinx
 commands =
-    ./taskoverview.py --output docs/_static/graph.json
-    sphinx-build -W -b html -d docs/_build/html/doctrees docs docs/_build/html
+    ./taskoverview.py --output _static/graph.json
+    sphinx-build -W -b html -d _build/html/doctrees . _build/html
 
 [testenv:docs-serve]
 changedir = docs/_build/html

From 953987fddf8f9dd5d3e7b82ffb76be2ce382fe8f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:15:49 +0200
Subject: [PATCH 100/345] Combine documentation from github wiki with sphinx
 This is the first step in the effort of combining all documentation about
 bootstrap-vz into sphinx while still being able to access it from github
 (github can parse rst as well)

---
 bootstrapvz/plugins/admin_user/README.rst     | 12 ++++
 bootstrapvz/plugins/apt_proxy/README.rst      | 21 ++++++
 bootstrapvz/plugins/cloud_init/README.rst     | 23 ++++++
 bootstrapvz/plugins/docker_daemon/README.rst  | 18 +++++
 bootstrapvz/plugins/image_commands/README.rst | 19 +++++
 bootstrapvz/plugins/minimize_size/README.rst  | 37 ++++++++++
 bootstrapvz/plugins/ntp/README.rst            | 12 ++++
 bootstrapvz/plugins/opennebula/README.md      | 13 ----
 bootstrapvz/plugins/opennebula/README.rst     | 27 +++++++
 bootstrapvz/plugins/pip_install/README.rst    | 14 ++++
 .../plugins/prebootstrapped/README.rst        | 26 +++++++
 bootstrapvz/plugins/puppet/README.rst         | 24 +++++++
 bootstrapvz/plugins/root_password/README.rst  | 11 +++
 bootstrapvz/plugins/salt/README.rst           | 26 +++++++
 .../plugins/unattended_upgrades/README.rst    | 18 +++++
 bootstrapvz/plugins/vagrant/README.rst        | 12 ++++
 .../providers/azure/{README.md => README.rst} | 28 +++++---
 bootstrapvz/providers/ec2/README.rst          | 71 +++++++++++++++++++
 bootstrapvz/providers/gce/README.rst          |  9 +++
 bootstrapvz/providers/kvm/README.rst          |  8 +++
 bootstrapvz/providers/virtualbox/README.rst   | 12 ++++
 docs/conf.py                                  | 20 ++++++
 docs/plugins/.gitignore                       |  3 +
 docs/plugins/index.rst                        | 16 +++++
 docs/providers/.gitignore                     |  3 +
 docs/providers/index.rst                      | 16 +++++
 26 files changed, 477 insertions(+), 22 deletions(-)
 create mode 100644 bootstrapvz/plugins/admin_user/README.rst
 create mode 100644 bootstrapvz/plugins/apt_proxy/README.rst
 create mode 100644 bootstrapvz/plugins/cloud_init/README.rst
 create mode 100644 bootstrapvz/plugins/docker_daemon/README.rst
 create mode 100644 bootstrapvz/plugins/image_commands/README.rst
 create mode 100644 bootstrapvz/plugins/minimize_size/README.rst
 create mode 100644 bootstrapvz/plugins/ntp/README.rst
 delete mode 100644 bootstrapvz/plugins/opennebula/README.md
 create mode 100644 bootstrapvz/plugins/opennebula/README.rst
 create mode 100644 bootstrapvz/plugins/pip_install/README.rst
 create mode 100644 bootstrapvz/plugins/prebootstrapped/README.rst
 create mode 100644 bootstrapvz/plugins/puppet/README.rst
 create mode 100644 bootstrapvz/plugins/root_password/README.rst
 create mode 100644 bootstrapvz/plugins/salt/README.rst
 create mode 100644 bootstrapvz/plugins/unattended_upgrades/README.rst
 create mode 100644 bootstrapvz/plugins/vagrant/README.rst
 rename bootstrapvz/providers/azure/{README.md => README.rst} (67%)
 create mode 100644 bootstrapvz/providers/ec2/README.rst
 create mode 100644 bootstrapvz/providers/gce/README.rst
 create mode 100644 bootstrapvz/providers/kvm/README.rst
 create mode 100644 bootstrapvz/providers/virtualbox/README.rst
 create mode 100644 docs/plugins/.gitignore
 create mode 100644 docs/providers/.gitignore

diff --git a/bootstrapvz/plugins/admin_user/README.rst b/bootstrapvz/plugins/admin_user/README.rst
new file mode 100644
index 0000000..3506f92
--- /dev/null
+++ b/bootstrapvz/plugins/admin_user/README.rst
@@ -0,0 +1,12 @@
+Admin user
+----------
+
+This plugin creates a user with passwordless sudo privileges. It also
+disables the SSH root login. If the EC2 init scripts are installed, the
+script for fetching the SSH authorized keys will be adjust to match the
+username specified.
+
+Settings
+~~~~~~~~
+
+-  ``username``: The username of the account to create. *``required``*
diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst
new file mode 100644
index 0000000..33711a7
--- /dev/null
+++ b/bootstrapvz/plugins/apt_proxy/README.rst
@@ -0,0 +1,21 @@
+APT Proxy
+---------
+
+This plugin creates a proxy configuration file for APT, so you could
+enjoy the benefits of using cached packages instead of downloading them
+from the mirror every time. You could just install ``apt-cacher-ng`` on
+the host machine and then add ``"address": "127.0.0.1"`` and
+``"port": 3142`` to the manifest file.
+
+Settings
+~~~~~~~~
+
+-  ``address``: The IP or host of the proxy server.
+    *``required``*
+-  ``port``: The port (integer) of the proxy server.
+    *``required``*
+-  ``persistent``: Whether the proxy configuration file should remain on
+   the machine or not.
+   Valid values: true, false
+   Default: ``false``
+   *``optional``*
diff --git a/bootstrapvz/plugins/cloud_init/README.rst b/bootstrapvz/plugins/cloud_init/README.rst
new file mode 100644
index 0000000..fa36e1f
--- /dev/null
+++ b/bootstrapvz/plugins/cloud_init/README.rst
@@ -0,0 +1,23 @@
+cloud-init
+----------
+
+This plugin installs and configures
+`cloud-init <https://packages.debian.org/wheezy-backports/cloud-init>`__
+on the system. Depending on the release it installs it from either
+backports or the main repository.
+
+cloud-init is only compatible with Debian wheezy and upwards.
+
+Settings
+~~~~~~~~
+
+-  ``username``: The username of the account to create.
+   *``required``*
+-  ``disable_modules``: A list of strings specifying which cloud-init
+   modules should be disabled.
+   *``optional``*
+-  ``metadata_sources``: A string that sets the
+   `datasources <http://cloudinit.readthedocs.org/en/latest/topics/datasources.html>`__
+   that cloud-init should try fetching metadata from. The source is
+   automatically set when using the ec2 provider.
+   *``optional``*
diff --git a/bootstrapvz/plugins/docker_daemon/README.rst b/bootstrapvz/plugins/docker_daemon/README.rst
new file mode 100644
index 0000000..e210435
--- /dev/null
+++ b/bootstrapvz/plugins/docker_daemon/README.rst
@@ -0,0 +1,18 @@
+Docker daemon
+-------------
+
+Install `docker <http://www.docker.io/>`__ daemon in the image. Uses
+init scripts for the official repository.
+
+This plugin can only be used if the distribution being bootstrapped is
+at least ``wheezy``, as Docker needs a kernel version ``3.8`` or higher,
+which is available at the ``wheezy-backports`` repository. There's also
+an architecture requirement, as it runs only on ``amd64``.
+
+Settings
+~~~~~~~~
+
+-  ``version``: Selects the docker version to install. To select the
+   latest version simply omit this setting.
+   Default: ``latest``
+   *``optional``*
diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/image_commands/README.rst
new file mode 100644
index 0000000..106ad22
--- /dev/null
+++ b/bootstrapvz/plugins/image_commands/README.rst
@@ -0,0 +1,19 @@
+Image commands
+--------------
+
+The image commands plugin allows you to run arbitrary commands during
+the bootstrap process. The commands are run at an indeterminate point
+*after* packages have been installed, but *before* the volume has been
+unmounted.
+
+Settings
+~~~~~~~~
+
+-  ``commands``: A list of lists containing strings. Each top-level item
+   is a single command, while the strings inside each list comprise
+   parts of a command. This allows for proper shell argument escaping
+   (to circumvent this, simply put the entire command in a single
+   string). In addition to the manifest variables ``{root}`` is also
+   available. It points at the root of the image volume.
+   *``required``*
+   *``manifest vars``*
diff --git a/bootstrapvz/plugins/minimize_size/README.rst b/bootstrapvz/plugins/minimize_size/README.rst
new file mode 100644
index 0000000..e275237
--- /dev/null
+++ b/bootstrapvz/plugins/minimize_size/README.rst
@@ -0,0 +1,37 @@
+minimize size
+-------------
+
+This plugin can be used to reduce the size of the resulting image. Often
+virtual volumes are much smaller than their reported size until any data
+is written to them. During the bootstrapping process temporary data like
+the aptitude cache is written to the volume only to be removed again.
+
+The minimize size plugin employs three different strategies to keep a
+low volume footprint:
+
+-  Mount folders from the host into key locations of the image volume to
+   avoid any unneccesary disk writes.
+-  Use `zerofree <http://intgat.tigress.co.uk/rmy/uml/index.html>`__ to
+   deallocate unused sectors on the volume. On an unpartitioned volume
+   this will be done for the entire volume, while it will only happen on
+   the root partition for partitioned volumes.
+-  Use
+   `vmware-vdiskmanager <https://www.vmware.com/support/ws45/doc/disks_vdiskmanager_eg_ws.html>`__
+   to shrink the real volume size (only applicable when using vmdk
+   backing). The tool is part of the `VMWare
+   Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
+   package.
+
+Settings
+~~~~~~~~
+
+-  ``zerofree``: Specifies if it should mark unallocated blocks as
+   zeroes, so the volume could be better shrunk after this.
+   Valid values: true, false
+   Default: false
+   *``optional``*
+-  ``shrink``: Whether the volume should be shrunk. This setting works
+   best in conjunction with the zerofree tool.
+   Valid values: true, false
+   Default: false
+   *``optional``*
diff --git a/bootstrapvz/plugins/ntp/README.rst b/bootstrapvz/plugins/ntp/README.rst
new file mode 100644
index 0000000..40421b1
--- /dev/null
+++ b/bootstrapvz/plugins/ntp/README.rst
@@ -0,0 +1,12 @@
+NTP
+---
+
+This plugins installs the Network Time Protocol daemon and optionally
+defines which time servers it should use.
+
+Settings
+~~~~~~~~
+
+-  ``servers``: A list of strings specifying which servers should be
+   used to synchronize the machine clock.
+   *``optional``*
diff --git a/bootstrapvz/plugins/opennebula/README.md b/bootstrapvz/plugins/opennebula/README.md
deleted file mode 100644
index 49ab0c9..0000000
--- a/bootstrapvz/plugins/opennebula/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Open Nebula provider
-
-This provider adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong).
-
-It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like:
-
-    ETH0_DNS            $NETWORK[DNS, NETWORK_ID=2]
-    ETH0_GATEWAY        $NETWORK[GATEWAY, NETWORK_ID=2]
-    ETH0_IP             $NIC[IP, NETWORK_ID=2]
-    ETH0_MASK	        $NETWORK[MASK, NETWORK_ID=2]
-    ETH0_NETWORK        $NETWORK[NETWORK, NETWORK_ID=2]
-    FILES               path_to_my_ssh_public_key.pub
-
diff --git a/bootstrapvz/plugins/opennebula/README.rst b/bootstrapvz/plugins/opennebula/README.rst
new file mode 100644
index 0000000..a29d7ae
--- /dev/null
+++ b/bootstrapvz/plugins/opennebula/README.rst
@@ -0,0 +1,27 @@
+Open Nebula
+-----------
+
+This plugin adds `OpenNebula
+contextualization <http://opennebula.org/documentation:rel4.2:cong>`__
+to the image, which sets up the network configuration and SSH keys.
+
+The virtual machine context should be configured as follows:
+
+::
+
+    ETH0_DNS      $NETWORK[DNS, NETWORK_ID=2]
+    ETH0_GATEWAY  $NETWORK[GATEWAY, NETWORK_ID=2]
+    ETH0_IP       $NIC[IP, NETWORK_ID=2]
+    ETH0_MASK     $NETWORK[MASK, NETWORK_ID=2]
+    ETH0_NETWORK  $NETWORK[NETWORK, NETWORK_ID=2]
+    FILES         path_to_my_ssh_public_key.pub
+
+The plugin will install all *.pub* files in the root authorized\_keys
+file. When using the ec2 provider, the USER\_EC2\_DATA will be executed
+if present.
+
+Settings
+~~~~~~~~
+
+This plugin has no settings. To enable it add ``"opennebula":{}`` to the
+plugin section of the manifest.
diff --git a/bootstrapvz/plugins/pip_install/README.rst b/bootstrapvz/plugins/pip_install/README.rst
new file mode 100644
index 0000000..3bed629
--- /dev/null
+++ b/bootstrapvz/plugins/pip_install/README.rst
@@ -0,0 +1,14 @@
+Pip install
+-----------
+
+Install packages from the Python Package Index via pip.
+
+Installs ``build-essential`` and ``python-dev`` debian packages, so
+Python extension modules can be built.
+
+Settings
+~~~~~~~~
+
+-  ``packages``: Python packages to install, a list of strings. The list
+   can contain anything that ``pip install`` would accept as an
+   argument, for example ``awscli==1.3.13``.
diff --git a/bootstrapvz/plugins/prebootstrapped/README.rst b/bootstrapvz/plugins/prebootstrapped/README.rst
new file mode 100644
index 0000000..bc30de8
--- /dev/null
+++ b/bootstrapvz/plugins/prebootstrapped/README.rst
@@ -0,0 +1,26 @@
+prebootstrapped
+---------------
+
+When developing for bootstrap-vz, testing can be quite tedious since the
+bootstrapping process can take a while. The prebootstrapped plugin
+solves that problem by creating a snapshot of your volume right after
+all the software has been installed. The next time bootstrap-vz is run,
+the plugin replaces all volume preparation and bootstrapping tasks and
+recreates the volume from the snapshot instead.
+
+The plugin assumes that the users knows what he is doing (e.g. it
+doesn't check whether bootstrap-vz is being run with a partitioned
+volume configuration, while the snapshot is unpartitioned).
+
+When no snapshot or image is specified the plugin creates one and
+outputs its ID/path. Specifying an ID/path enables the second mode of
+operation which recreates the volume from the specified snapshot instead
+of creating it from scratch.
+
+Settings
+~~~~~~~~
+
+-  ``snapshot``: ID of the EBS snapshot to use. This setting only works
+   with EBS backed EC2 configurations.
+-  ``image``: Path to the loopbackvolume snapshot. This setting works
+   with all configurable volume backings except EBS.
diff --git a/bootstrapvz/plugins/puppet/README.rst b/bootstrapvz/plugins/puppet/README.rst
new file mode 100644
index 0000000..203e0b3
--- /dev/null
+++ b/bootstrapvz/plugins/puppet/README.rst
@@ -0,0 +1,24 @@
+Puppet
+------
+
+Installs `puppet <http://puppetlabs.com/>`__ and optionally applies a
+manifest inside the chroot. You can also have it copy your puppet
+configuration into the image so it is readily available once the image
+is booted.
+
+Keep in mind that when applying a manifest, the system is in a chrooted
+environment. This can prevent daemons from running properly (e.g.
+listening to ports), they will also need to be shut down gracefully
+(which bootstrap-vz cannot do) before unmounting the volume. It is
+advisable to avoid starting any daemons inside the chroot at all.
+
+Settings
+~~~~~~~~
+
+-  ``manifest``: Path to the puppet manifest that should be applied.
+   *``optional``*
+-  ``assets``: Path to puppet assets. The contents will be copied into
+   ``/etc/puppet`` on the image. Any existing files will be overwritten.
+   *``optional``*
+-  ``enable_agent``: Whether the puppet agent daemon should be enabled.
+   *``optional``*
diff --git a/bootstrapvz/plugins/root_password/README.rst b/bootstrapvz/plugins/root_password/README.rst
new file mode 100644
index 0000000..a24d904
--- /dev/null
+++ b/bootstrapvz/plugins/root_password/README.rst
@@ -0,0 +1,11 @@
+root password
+-------------
+
+Sets the root password. This plugin removes the task that disables the
+SSH password authentication.
+
+Settings
+~~~~~~~~
+
+-  ``password``: The password for the root user.
+   *``required``*
diff --git a/bootstrapvz/plugins/salt/README.rst b/bootstrapvz/plugins/salt/README.rst
new file mode 100644
index 0000000..acb71ab
--- /dev/null
+++ b/bootstrapvz/plugins/salt/README.rst
@@ -0,0 +1,26 @@
+Salt
+----
+
+Install `salt <http://www.saltstack.com/>`__ minion in the image. Uses
+`salt-bootstrap <https://github.com/saltstack/salt-bootstrap>`__ script
+to install.
+
+Settings
+~~~~~~~~
+
+-  ``install_source``: Source to install salt codebase from. ``stable``
+   for current stable, ``daily`` for installing the daily build, and
+   ``git`` to install from git repository.
+   *``required``*
+-  ``version``: Only needed if you are installing from ``git``.
+   \ ``develop`` to install current development head, or provide any tag
+   name or commit hash from `salt
+   repo <https://github.com/saltstack/salt>`__
+   *``optional``*
+-  ``master``: Salt master FQDN or IP
+   *``optional``*
+-  ``grains``: Set `salt
+   grains <http://docs.saltstack.com/en/latest/topics/targeting/grains.html>`__
+   for this minion. Accepts a map with grain name as key and the grain
+   data as value.
+   *``optional``*
diff --git a/bootstrapvz/plugins/unattended_upgrades/README.rst b/bootstrapvz/plugins/unattended_upgrades/README.rst
new file mode 100644
index 0000000..3339b7b
--- /dev/null
+++ b/bootstrapvz/plugins/unattended_upgrades/README.rst
@@ -0,0 +1,18 @@
+Unattended upgrades
+-------------------
+
+Enables the `unattended update/upgrade
+feature <https://packages.debian.org/wheezy/unattended-upgrades>`__ in
+aptitude. Enable it to have your system automatically download and
+install security updates automatically with a set interval.
+
+Settings
+~~~~~~~~
+
+-  ``update_interval``: Days between running ``apt-get update``.
+   *``required``*
+-  ``download_interval``: Days between running
+   ``apt-get upgrade --download-only``
+   *``required``*
+-  ``upgrade_interval``: Days between installing any security upgrades.
+   *``required``*
diff --git a/bootstrapvz/plugins/vagrant/README.rst b/bootstrapvz/plugins/vagrant/README.rst
new file mode 100644
index 0000000..a40be4a
--- /dev/null
+++ b/bootstrapvz/plugins/vagrant/README.rst
@@ -0,0 +1,12 @@
+Vagrant
+-------
+
+Vagrant is a tool to quickly create virtualized environments. It uses
+"boxes" to make downloading and sharing those environments easier. A box
+is a tarball containing a virtual volumes accompanied by an `OVF
+specification <http://en.wikipedia.org/wiki/Open_Virtualization_Format>`__
+of the virtual machine.
+
+This plugin creates a vagrant box that is ready to be shared or
+deployed. At the moment it is only compatible with the VirtualBox
+provider and doesn't requires any additional settings.
diff --git a/bootstrapvz/providers/azure/README.md b/bootstrapvz/providers/azure/README.rst
similarity index 67%
rename from bootstrapvz/providers/azure/README.md
rename to bootstrapvz/providers/azure/README.rst
index c03009e..165b557 100644
--- a/bootstrapvz/providers/azure/README.md
+++ b/bootstrapvz/providers/azure/README.rst
@@ -1,22 +1,28 @@
-Azure provider
-===========
+Azure
+=====
 
-This provider generates raw images for Microsoft Azure computing platform.
+This provider generates raw images for Microsoft Azure computing
+platform.
 
 Setup
-=====
+-----
 
 qemu-img >= 1.7.0 required to convert raw image to vhd fixed size disk.
 This release is available in wheezy-backports.
 
 *wget* must be installed on local computer.
 
-
-Manifest must use the *raw* format, provider will automatically transform the disk to a vhd disk format.
+Manifest must use the *raw* format, provider will automatically
+transform the disk to a vhd disk format.
 
 Do not create swap space on the OS disk:
 
-The Windows Azure Linux Agent can automatically configure swap space using the local resource disk that is attached to the VM after provisioning on Azure. Modify the following parameters in /etc/waagent.conf appropriately:
+The Windows Azure Linux Agent can automatically configure swap space
+using the local resource disk that is attached to the VM after
+provisioning on Azure. Modify the following parameters in
+/etc/waagent.conf appropriately:
+
+::
 
     ResourceDisk.Format=y
     ResourceDisk.Filesystem=ext4
@@ -24,7 +30,10 @@ The Windows Azure Linux Agent can automatically configure swap space using the l
     ResourceDisk.EnableSwap=y
     ResourceDisk.SwapSizeMB=2048    ## NOTE: set this to whatever you need it to be.
 
-You can specify a waagent.conf file to replace the default one in the manifest in the azure/waagent section of the provider:
+You can specify a waagent.conf file to replace the default one in the
+manifest in the azure/waagent section of the provider:
+
+::
 
     "system" : { 
         "waagent" : {
@@ -33,4 +42,5 @@ You can specify a waagent.conf file to replace the default one in the manifest i
         }
     }, ...
 
-Waagent versions are available at: https://github.com/Azure/WALinuxAgent/releases
+Waagent versions are available at:
+https://github.com/Azure/WALinuxAgent/releases
diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst
new file mode 100644
index 0000000..7c79201
--- /dev/null
+++ b/bootstrapvz/providers/ec2/README.rst
@@ -0,0 +1,71 @@
+EC2
+===
+
+The `EC2 <http://aws.amazon.com/ec2/>`__ provider automatically creates
+a volume for bootstrapping (be it EBS or S3), makes a snapshot of it
+once it is done and registers it as an AMI. EBS volume backing only
+works on an EC2 host while S3 backed volumes *should* work locally (at
+this time however they do not, a fix is in the works).
+
+Unless `the cloud-init plugin <dev_Plugins#cloud_init>`__ is used,
+special startup scripts will be installed that automatically fetch the
+configured authorized\_key from the instance metadata and save or run
+any userdata supplied (if the userdata begins with ``#!`` it will be
+run).
+
+Credentials
+-----------
+
+The AWS credentials can be configured in two ways: Via the manifest or
+through environment variables. To bootstrap S3 backed instances you will
+need a user certificate and a private key in addition to the access key
+and secret key, which are needed for bootstraping EBS backed instances.
+
+The settings describes below should be placed in the ``credentials`` key
+under the ``provider`` section.
+
+-  **``access-key``**: AWS access-key.
+   May also be supplied via the environment variable
+   ``$AWS_ACCESS_KEY``
+   *``required for EBS & S3 backing``*
+-  **``secret-key``**: AWS secret-key.
+   May also be supplied via the environment variable
+   ``$AWS_SECRET_KEY``
+   *``required for EBS & S3 backing``*
+-  **``certificate``**: Path to the AWS user certificate. Used for
+   uploading the image to an S3 bucket.
+   May also be supplied via the environment variable
+   ``$AWS_CERTIFICATE``
+   *``required for S3 backing``*
+-  **``private-key``**: Path to the AWS private key. Used for uploading
+   the image to an S3 bucket.
+   May also be supplied via the environment variable
+   ``$AWS_PRIVATE_KEY``
+   *``required for S3 backing``*
+-  **``user-id``**: AWS user ID. Used for uploading the image to an S3
+   bucket.
+   May also be supplied via the environment variable ``$AWS_USER_ID``
+   *``required for S3 backing``*
+
+Example:
+
+.. code:: yaml
+
+    ---
+    provider:
+      name: ec2
+      virtualization: hvm
+      enhanced_networking: simple
+      credentials:
+        access-key: AFAKEACCESSKEYFORAWS
+        secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+
+Dependencies
+------------
+
+To communicate with the AWS API `boto <https://github.com/boto/boto>`__
+is required (version 2.14.0 or higher) you can install boto with
+``pip install boto`` (on wheezy, the packaged version is too low). S3
+images are chopped up and uploaded using
+`euca2ools <https://github.com/eucalyptus/euca2ools>`__ (install with
+``apt-get install euca2ools``).
diff --git a/bootstrapvz/providers/gce/README.rst b/bootstrapvz/providers/gce/README.rst
new file mode 100644
index 0000000..e760120
--- /dev/null
+++ b/bootstrapvz/providers/gce/README.rst
@@ -0,0 +1,9 @@
+Google Compute Engine
+---------------------
+
+The `GCE <https://cloud.google.com/products/compute-engine/>`__ provider
+can creates image as expected by GCE - i.e. raw disk image in \*.tar.gz
+file. It can upload created images to Google Storage Engine (to URI
+provided in manifest by ``gcs_destination``) and can register image to
+be used by Google Compute Engine to project provided in manifest by
+``gce_project``. Both of those functionalities are not fully tested yet.
diff --git a/bootstrapvz/providers/kvm/README.rst b/bootstrapvz/providers/kvm/README.rst
new file mode 100644
index 0000000..57e60d4
--- /dev/null
+++ b/bootstrapvz/providers/kvm/README.rst
@@ -0,0 +1,8 @@
+KVM
+---
+
+The `KVM <http://www.linux-kvm.org/page/Main_Page>`__ provider creates
+virtual images for Linux Kernel-based Virtual Machines. It supports the
+installation of `virtio kernel
+modules <http://www.linux-kvm.org/page/Virtio>`__ (paravirtualized
+drivers for IO operations).
diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst
new file mode 100644
index 0000000..b3e3d16
--- /dev/null
+++ b/bootstrapvz/providers/virtualbox/README.rst
@@ -0,0 +1,12 @@
+VirtualBox
+----------
+
+The `VirtualBox <https://www.virtualbox.org/>`__ provider can bootstrap
+to both .vdi and .vmdk images (raw images are also supported but do not
+run in VirtualBox). It's advisable to always use vmdk images for
+interoperability (e.g.
+`OVF <http://en.wikipedia.org/wiki/Open_Virtualization_Format>`__ files
+*should* support vdi files, but since they have no identifier URL not
+even VirtualBox itself can import them). VirtualBox Guest Additions can
+be installed automatically if the ISO is `provided in the
+manifest <dev_Manifest#bootstrapper>`__.
diff --git a/docs/conf.py b/docs/conf.py
index 6eb7717..5abdcfc 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -14,6 +14,8 @@
 
 import sys
 import os
+import glob
+import os.path
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -261,3 +263,21 @@ texinfo_documents = [('index', 'bootstrap-vz', u'bootstrap-vz Documentation',
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
 #texinfo_no_detailmenu = False
+
+
+# -- Link to rst files scattered throughout the project -------------------
+
+for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'):
+	provider_name = os.path.basename(os.path.dirname(readme_path))
+	include_path = os.path.join('providers', provider_name + '.rst')
+	path_to_readme = os.path.join('../../bootstrapvz/providers', provider_name, 'README.rst')
+	with open(include_path, 'w') as include:
+		include.write('.. include:: ' + path_to_readme)
+
+
+for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'):
+	plugin_name = os.path.basename(os.path.dirname(readme_path))
+	include_path = os.path.join('plugins', plugin_name + '.rst')
+	path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst')
+	with open(include_path, 'w') as include:
+		include.write('.. include:: ' + path_to_readme)
diff --git a/docs/plugins/.gitignore b/docs/plugins/.gitignore
new file mode 100644
index 0000000..50db996
--- /dev/null
+++ b/docs/plugins/.gitignore
@@ -0,0 +1,3 @@
+*
+!index.rst
+!.gitignore
diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst
index 690ccbc..208c1d3 100644
--- a/docs/plugins/index.rst
+++ b/docs/plugins/index.rst
@@ -1,3 +1,19 @@
 
 Plugins
 =======
+
+Plugins are a key feature of bootstrap-vz. Despite their small size
+(most plugins do not exceed 100 source lines of code) they can modify
+the behavior of bootstrapped systems to a great extent.
+
+Below you will find documentation for all plugins available for
+bootstrap-vz. If you cannot find what you are looking for, consider
+`developing it yourself <http://bootstrap-vz.readthedocs.org>`__ and
+contribute to this list!
+
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *
diff --git a/docs/providers/.gitignore b/docs/providers/.gitignore
new file mode 100644
index 0000000..50db996
--- /dev/null
+++ b/docs/providers/.gitignore
@@ -0,0 +1,3 @@
+*
+!index.rst
+!.gitignore
diff --git a/docs/providers/index.rst b/docs/providers/index.rst
index ef70c42..fb991f2 100644
--- a/docs/providers/index.rst
+++ b/docs/providers/index.rst
@@ -1,3 +1,19 @@
 
 Providers
 =========
+
+Plugins are a key feature of bootstrap-vz. Despite their small size
+(most plugins do not exceed 100 source lines of code) they can modify
+the behavior of bootstrapped systems to a great extent.
+
+Below you will find documentation for all plugins available for
+bootstrap-vz. If you cannot find what you are looking for, consider
+`developing it yourself <http://bootstrap-vz.readthedocs.org>`__ and
+contribute to this list!
+
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
+   *

From fc29266c9788ef3d9d9b4164da518eea790d5ca7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:17:50 +0200
Subject: [PATCH 101/345] Move API docs into api/

---
 docs/{ => api}/base/fs.rst            | 0
 docs/{ => api}/base/index.rst         | 4 ++--
 docs/{ => api}/base/pkg.rst           | 0
 docs/{ => api}/common/fs.rst          | 0
 docs/{ => api}/common/index.rst       | 0
 docs/{ => api}/common/tasks/index.rst | 0
 docs/api/index.rst                    | 9 +++++++++
 docs/index.rst                        | 6 +++---
 8 files changed, 14 insertions(+), 5 deletions(-)
 rename docs/{ => api}/base/fs.rst (100%)
 rename docs/{ => api}/base/index.rst (98%)
 rename docs/{ => api}/base/pkg.rst (100%)
 rename docs/{ => api}/common/fs.rst (100%)
 rename docs/{ => api}/common/index.rst (100%)
 rename docs/{ => api}/common/tasks/index.rst (100%)
 create mode 100644 docs/api/index.rst

diff --git a/docs/base/fs.rst b/docs/api/base/fs.rst
similarity index 100%
rename from docs/base/fs.rst
rename to docs/api/base/fs.rst
diff --git a/docs/base/index.rst b/docs/api/base/index.rst
similarity index 98%
rename from docs/base/index.rst
rename to docs/api/base/index.rst
index d80e702..c17d336 100644
--- a/docs/base/index.rst
+++ b/docs/api/base/index.rst
@@ -7,9 +7,9 @@ and handles the gather, sorting and running of tasks.
 
 .. toctree::
    :maxdepth: 2
+   :glob:
 
-   fs
-   pkg
+   *
 
 Bootstrap information
 ---------------------
diff --git a/docs/base/pkg.rst b/docs/api/base/pkg.rst
similarity index 100%
rename from docs/base/pkg.rst
rename to docs/api/base/pkg.rst
diff --git a/docs/common/fs.rst b/docs/api/common/fs.rst
similarity index 100%
rename from docs/common/fs.rst
rename to docs/api/common/fs.rst
diff --git a/docs/common/index.rst b/docs/api/common/index.rst
similarity index 100%
rename from docs/common/index.rst
rename to docs/api/common/index.rst
diff --git a/docs/common/tasks/index.rst b/docs/api/common/tasks/index.rst
similarity index 100%
rename from docs/common/tasks/index.rst
rename to docs/api/common/tasks/index.rst
diff --git a/docs/api/index.rst b/docs/api/index.rst
new file mode 100644
index 0000000..1c7c3c7
--- /dev/null
+++ b/docs/api/index.rst
@@ -0,0 +1,9 @@
+API
+===
+
+
+.. toctree::
+   :maxdepth: 2
+
+   base/index
+   common/index
diff --git a/docs/index.rst b/docs/index.rst
index 5413191..a27b43c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -11,10 +11,10 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
-   base/index
-   common/index
-   plugins/index
    providers/index
+   plugins/index
+
+   api/index
    guidelines
    taskoverview
    howitworks

From c089301f79dab9bdd35e2d0343a640adb36cf040 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:28:22 +0200
Subject: [PATCH 102/345] Generate task graph data through sphinx conf.py

---
 docs/__init__.py        |  0
 docs/_static/.gitignore |  1 +
 docs/_static/graph.json |  1 -
 docs/conf.py            | 13 +++++++++++--
 docs/taskoverview.py    |  9 ++++-----
 tox.ini                 |  1 -
 6 files changed, 16 insertions(+), 9 deletions(-)
 create mode 100644 docs/__init__.py
 create mode 100644 docs/_static/.gitignore
 delete mode 100644 docs/_static/graph.json

diff --git a/docs/__init__.py b/docs/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore
new file mode 100644
index 0000000..4dd81db
--- /dev/null
+++ b/docs/_static/.gitignore
@@ -0,0 +1 @@
+graph.json
diff --git a/docs/_static/graph.json b/docs/_static/graph.json
deleted file mode 100644
index 3485725..0000000
--- a/docs/_static/graph.json
+++ /dev/null
@@ -1 +0,0 @@
-{"modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.extlinux"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.grub"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.kernel"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.docker_daemon.tasks"}, {"name": "bootstrapvz.plugins.file_copy.tasks"}, {"name": "bootstrapvz.plugins.google_cloud_sdk.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.pip_install.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.initd"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "links": [{"target": 46, "definer": 15, "source": 15}, {"target": 53, "definer": 27, "source": 27}, {"target": 46, "definer": 28, "source": 28}, {"target": 41, "definer": 39, "source": 39}, {"target": 79, "definer": 41, "source": 41}, {"target": 53, "definer": 45, "source": 45}, {"target": 49, "definer": 47, "source": 47}, {"target": 49, "definer": 50, "source": 50}, {"target": 66, "definer": 51, "source": 51}, {"target": 46, "definer": 57, "source": 57}, {"target": 77, "definer": 58, "source": 58}, {"target": 66, "definer": 65, "source": 65}, {"target": 46, "definer": 67, "source": 67}, {"target": 35, "definer": 68, "source": 68}, {"target": 79, "definer": 70, "source": 70}, {"target": 49, "definer": 73, "source": 73}, {"target": 81, "definer": 78, "source": 78}, {"target": 5, "definer": 89, "source": 89}, {"target": 5, "definer": 95, "source": 95}, {"target": 49, "definer": 99, "source": 99}, {"target": 44, "definer": 100, "source": 100}, {"target": 45, "definer": 100, "source": 100}, {"target": 46, "definer": 109, "source": 109}, {"target": 4, "definer": 110, "source": 110}, {"target": 70, "definer": 112, "source": 112}, {"target": 79, "definer": 112, "source": 112}, {"target": 66, "definer": 113, "source": 113}, {"target": 77, "definer": 119, "source": 119}, {"target": 155, "definer": 120, "source": 120}, {"target": 81, "definer": 141, "source": 141}, {"target": 44, "definer": 143, "source": 143}, {"target": 45, "definer": 143, "source": 143}, {"target": 81, "definer": 150, "source": 150}, {"target": 46, "definer": 159, "source": 159}, {"target": 49, "definer": 162, "source": 162}, {"target": 53, "definer": 165, "source": 165}, {"target": 4, "definer": 167, "source": 167}, {"target": 61, "definer": 167, "source": 167}, {"target": 12, "definer": 168, "source": 168}, {"target": 44, "definer": 170, "source": 170}, {"target": 45, "definer": 170, "source": 170}, {"target": 66, "definer": 183, "source": 183}, {"target": 0, "definer": 0, "source": 1}, {"target": 1, "definer": 1, "source": 3}, {"target": 5, "definer": 5, "source": 54}, {"target": 5, "definer": 5, "source": 12}, {"target": 5, "definer": 5, "source": 11}, {"target": 6, "definer": 6, "source": 5}, {"target": 6, "definer": 6, "source": 7}, {"target": 11, "definer": 11, "source": 12}, {"target": 12, "definer": 12, "source": 9}, {"target": 16, "definer": 16, "source": 19}, {"target": 23, "definer": 23, "source": 1}, {"target": 26, "definer": 26, "source": 34}, {"target": 26, "definer": 26, "source": 24}, {"target": 27, "definer": 27, "source": 34}, {"target": 27, "definer": 27, "source": 25}, {"target": 29, "definer": 29, "source": 1}, {"target": 30, "definer": 30, "source": 38}, {"target": 31, "definer": 31, "source": 37}, {"target": 33, "definer": 33, "source": 41}, {"target": 36, "definer": 36, "source": 31}, {"target": 37, "definer": 37, "source": 32}, {"target": 38, "definer": 38, "source": 16}, {"target": 40, "definer": 40, "source": 35}, {"target": 42, "definer": 42, "source": 1}, {"target": 43, "definer": 43, "source": 34}, {"target": 44, "definer": 44, "source": 34}, {"target": 45, "definer": 45, "source": 34}, {"target": 48, "definer": 48, "source": 49}, {"target": 52, "definer": 52, "source": 66}, {"target": 64, "definer": 64, "source": 1}, {"target": 65, "definer": 65, "source": 5}, {"target": 66, "definer": 66, "source": 6}, {"target": 68, "definer": 68, "source": 69}, {"target": 70, "definer": 70, "source": 41}, {"target": 72, "definer": 72, "source": 1}, {"target": 82, "definer": 82, "source": 1}, {"target": 83, "definer": 83, "source": 49}, {"target": 90, "definer": 90, "source": 1}, {"target": 93, "definer": 93, "source": 1}, {"target": 93, "definer": 93, "source": 0}, {"target": 95, "definer": 95, "source": 54}, {"target": 100, "definer": 100, "source": 43}, {"target": 100, "definer": 100, "source": 170}, {"target": 101, "definer": 101, "source": 97}, {"target": 102, "definer": 102, "source": 103}, {"target": 108, "definer": 108, "source": 16}, {"target": 111, "definer": 111, "source": 79}, {"target": 112, "definer": 112, "source": 41}, {"target": 115, "definer": 115, "source": 1}, {"target": 115, "definer": 115, "source": 0}, {"target": 116, "definer": 116, "source": 1}, {"target": 118, "definer": 118, "source": 66}, {"target": 118, "definer": 118, "source": 185}, {"target": 121, "definer": 121, "source": 66}, {"target": 121, "definer": 121, "source": 185}, {"target": 122, "definer": 122, "source": 1}, {"target": 123, "definer": 123, "source": 126}, {"target": 129, "definer": 129, "source": 66}, {"target": 130, "definer": 130, "source": 1}, {"target": 132, "definer": 132, "source": 1}, {"target": 134, "definer": 134, "source": 138}, {"target": 135, "definer": 135, "source": 1}, {"target": 137, "definer": 137, "source": 80}, {"target": 137, "definer": 137, "source": 136}, {"target": 145, "definer": 145, "source": 1}, {"target": 146, "definer": 146, "source": 66}, {"target": 147, "definer": 147, "source": 153}, {"target": 149, "definer": 149, "source": 157}, {"target": 149, "definer": 149, "source": 151}, {"target": 151, "definer": 151, "source": 148}, {"target": 153, "definer": 153, "source": 154}, {"target": 153, "definer": 153, "source": 160}, {"target": 153, "definer": 153, "source": 161}, {"target": 155, "definer": 155, "source": 156}, {"target": 163, "definer": 163, "source": 1}, {"target": 166, "definer": 166, "source": 1}, {"target": 168, "definer": 168, "source": 9}, {"target": 169, "definer": 169, "source": 3}, {"target": 169, "definer": 169, "source": 0}, {"target": 172, "definer": 172, "source": 60}, {"target": 174, "definer": 174, "source": 59}, {"target": 175, "definer": 175, "source": 176}, {"target": 176, "definer": 176, "source": 174}, {"target": 177, "definer": 177, "source": 47}, {"target": 177, "definer": 177, "source": 48}, {"target": 178, "definer": 178, "source": 1}, {"target": 179, "definer": 179, "source": 178}, {"target": 180, "definer": 180, "source": 1}, {"target": 180, "definer": 180, "source": 0}, {"target": 180, "definer": 180, "source": 178}, {"target": 181, "definer": 181, "source": 1}, {"target": 185, "definer": 185, "source": 66}, {"target": 186, "definer": 186, "source": 1}], "nodes": [{"name": "AddBackports", "module": 0, "phase": 0}, {"name": "AddDefaultSources", "module": 0, "phase": 0}, {"name": "AddManifestPreferences", "module": 0, "phase": 0}, {"name": "AddManifestSources", "module": 0, "phase": 0}, {"name": "AptClean", "module": 0, "phase": 7}, {"name": "AptUpdate", "module": 0, "phase": 5}, {"name": "AptUpgrade", "module": 0, "phase": 5}, {"name": "DisableDaemonAutostart", "module": 0, "phase": 5}, {"name": "EnableDaemonAutostart", "module": 0, "phase": 7}, {"name": "InstallTrustedKeys", "module": 0, "phase": 5}, {"name": "PurgeUnusedPackages", "module": 0, "phase": 7}, {"name": "WritePreferences", "module": 0, "phase": 5}, {"name": "WriteSources", "module": 0, "phase": 5}, {"name": "BlackListModules", "module": 1, "phase": 6}, {"name": "DisableGetTTYs", "module": 1, "phase": 6}, {"name": "AddRequiredCommands", "module": 2, "phase": 0}, {"name": "Bootstrap", "module": 2, "phase": 4}, {"name": "ExcludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "IncludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "MakeTarball", "module": 2, "phase": 4}, {"name": "CleanTMP", "module": 3, "phase": 7}, {"name": "ClearMOTD", "module": 3, "phase": 7}, {"name": "TriggerRollback", "module": 4, "phase": 10}, {"name": "AddExtlinuxPackage", "module": 5, "phase": 0}, {"name": "ConfigureExtlinux", "module": 5, "phase": 6}, {"name": "ConfigureExtlinuxJessie", "module": 5, "phase": 6}, {"name": "InstallExtlinux", "module": 5, "phase": 6}, {"name": "InstallExtlinuxJessie", "module": 5, "phase": 6}, {"name": "AddRequiredCommands", "module": 6, "phase": 0}, {"name": "AddXFSProgs", "module": 6, "phase": 0}, {"name": "CopyMountTable", "module": 6, "phase": 4}, {"name": "CreateBootMountDir", "module": 6, "phase": 3}, {"name": "CreateMountDir", "module": 6, "phase": 3}, {"name": "DeleteMountDir", "module": 6, "phase": 8}, {"name": "FStab", "module": 6, "phase": 6}, {"name": "Format", "module": 6, "phase": 2}, {"name": "MountBoot", "module": 6, "phase": 3}, {"name": "MountRoot", "module": 6, "phase": 3}, {"name": "MountSpecials", "module": 6, "phase": 4}, {"name": "RemoveMountTable", "module": 6, "phase": 8}, {"name": "TuneVolumeFS", "module": 6, "phase": 2}, {"name": "UnmountRoot", "module": 6, "phase": 8}, {"name": "AddGrubPackage", "module": 7, "phase": 0}, {"name": "ConfigureGrub", "module": 7, "phase": 6}, {"name": "InstallGrub_1_99", "module": 7, "phase": 6}, {"name": "InstallGrub_2", "module": 7, "phase": 6}, {"name": "CheckExternalCommands", "module": 8, "phase": 0}, {"name": "AddExpandRoot", "module": 9, "phase": 6}, {"name": "AdjustExpandRootScript", "module": 9, "phase": 6}, {"name": "InstallInitScripts", "module": 9, "phase": 6}, {"name": "RemoveHWClock", "module": 9, "phase": 6}, {"name": "AddDKMSPackages", "module": 10, "phase": 5}, {"name": "DetermineKernelVersion", "module": 10, "phase": 5}, {"name": "UpdateInitramfs", "module": 10, "phase": 6}, {"name": "GenerateLocale", "module": 11, "phase": 5}, {"name": "LocaleBootstrapPackage", "module": 11, "phase": 0}, {"name": "SetTimezone", "module": 11, "phase": 6}, {"name": "AddRequiredCommands", "module": 12, "phase": 0}, {"name": "Create", "module": 12, "phase": 1}, {"name": "MoveImage", "module": 12, "phase": 9}, {"name": "ConfigureNetworkIF", "module": 13, "phase": 6}, {"name": "RemoveDNSInfo", "module": 13, "phase": 7}, {"name": "RemoveHostname", "module": 13, "phase": 7}, {"name": "SetHostname", "module": 13, "phase": 6}, {"name": "AddManifestPackages", "module": 14, "phase": 0}, {"name": "AddTaskselStandardPackages", "module": 14, "phase": 5}, {"name": "InstallPackages", "module": 14, "phase": 5}, {"name": "AddRequiredCommands", "module": 15, "phase": 0}, {"name": "MapPartitions", "module": 15, "phase": 2}, {"name": "PartitionVolume", "module": 15, "phase": 2}, {"name": "UnmapPartitions", "module": 15, "phase": 8}, {"name": "EnableShadowConfig", "module": 16, "phase": 6}, {"name": "AddOpenSSHPackage", "module": 17, "phase": 0}, {"name": "AddSSHKeyGeneration", "module": 17, "phase": 6}, {"name": "DisableSSHDNSLookup", "module": 17, "phase": 6}, {"name": "DisableSSHPasswordAuthentication", "module": 17, "phase": 6}, {"name": "ShredHostkeys", "module": 17, "phase": 7}, {"name": "Attach", "module": 18, "phase": 1}, {"name": "Delete", "module": 18, "phase": 10}, {"name": "Detach", "module": 18, "phase": 8}, {"name": "CreateWorkspace", "module": 19, "phase": 0}, {"name": "DeleteWorkspace", "module": 19, "phase": 10}, {"name": "AddSudoPackage", "module": 20, "phase": 0}, {"name": "AdminUserCredentials", "module": 20, "phase": 6}, {"name": "CreateAdminUser", "module": 20, "phase": 6}, {"name": "DisableRootLogin", "module": 20, "phase": 6}, {"name": "PasswordlessSudo", "module": 20, "phase": 6}, {"name": "CheckAptProxy", "module": 21, "phase": 0}, {"name": "RemoveAptProxy", "module": 21, "phase": 7}, {"name": "SetAptProxy", "module": 21, "phase": 5}, {"name": "AddPackages", "module": 22, "phase": 0}, {"name": "CheckAssetsPath", "module": 22, "phase": 0}, {"name": "CopyChefAssets", "module": 22, "phase": 6}, {"name": "AddCloudInitPackages", "module": 23, "phase": 0}, {"name": "DisableModules", "module": 23, "phase": 6}, {"name": "SetMetadataSource", "module": 23, "phase": 5}, {"name": "SetUsername", "module": 23, "phase": 6}, {"name": "AddDockerBinary", "module": 24, "phase": 6}, {"name": "AddDockerDeps", "module": 24, "phase": 5}, {"name": "AddDockerInit", "module": 24, "phase": 6}, {"name": "EnableMemoryCgroup", "module": 24, "phase": 6}, {"name": "PullDockerImages", "module": 24, "phase": 6}, {"name": "FileCopyCommand", "module": 25, "phase": 6}, {"name": "MkdirCommand", "module": 25, "phase": 6}, {"name": "ValidateSourcePaths", "module": 25, "phase": 0}, {"name": "InstallCloudSDK", "module": 26, "phase": 6}, {"name": "RemoveCloudSDKTarball", "module": 26, "phase": 7}, {"name": "ImageExecuteCommand", "module": 27, "phase": 6}, {"name": "AddFolderMounts", "module": 28, "phase": 4}, {"name": "AddRequiredCommands", "module": 28, "phase": 0}, {"name": "RemoveFolderMounts", "module": 28, "phase": 7}, {"name": "ShrinkVolume", "module": 28, "phase": 8}, {"name": "Zerofree", "module": 28, "phase": 8}, {"name": "AddNtpPackage", "module": 29, "phase": 5}, {"name": "SetNtpServers", "module": 29, "phase": 6}, {"name": "AddONEContextPackage", "module": 30, "phase": 0}, {"name": "AddPipPackage", "module": 31, "phase": 0}, {"name": "PipInstallCommand", "module": 31, "phase": 6}, {"name": "CopyImage", "module": 32, "phase": 5}, {"name": "CreateFromImage", "module": 32, "phase": 1}, {"name": "CreateFromSnapshot", "module": 32, "phase": 1}, {"name": "Snapshot", "module": 32, "phase": 5}, {"name": "AddPackages", "module": 33, "phase": 0}, {"name": "ApplyPuppetManifest", "module": 33, "phase": 6}, {"name": "CheckAssetsPath", "module": 33, "phase": 0}, {"name": "CheckManifestPath", "module": 33, "phase": 0}, {"name": "CopyPuppetAssets", "module": 33, "phase": 6}, {"name": "EnableAgent", "module": 33, "phase": 6}, {"name": "SetRootPassword", "module": 34, "phase": 6}, {"name": "BootstrapSaltMinion", "module": 35, "phase": 5}, {"name": "InstallSaltDependencies", "module": 35, "phase": 0}, {"name": "SetSaltGrains", "module": 35, "phase": 6}, {"name": "AddUnattendedUpgradesPackage", "module": 36, "phase": 0}, {"name": "EnablePeriodicUpgrades", "module": 36, "phase": 6}, {"name": "AddInsecurePublicKey", "module": 37, "phase": 6}, {"name": "AddPackages", "module": 37, "phase": 0}, {"name": "CheckBoxPath", "module": 37, "phase": 0}, {"name": "CreateVagrantBoxDir", "module": 37, "phase": 0}, {"name": "CreateVagrantUser", "module": 37, "phase": 6}, {"name": "PackageBox", "module": 37, "phase": 9}, {"name": "PasswordlessSudo", "module": 37, "phase": 6}, {"name": "RemoveVagrantBoxDir", "module": 37, "phase": 10}, {"name": "SetRootPassword", "module": 37, "phase": 6}, {"name": "ConfigureGrub", "module": 38, "phase": 6}, {"name": "ConvertToVhd", "module": 39, "phase": 9}, {"name": "DefaultPackages", "module": 40, "phase": 0}, {"name": "Waagent", "module": 40, "phase": 5}, {"name": "AMIName", "module": 41, "phase": 0}, {"name": "BundleImage", "module": 41, "phase": 9}, {"name": "RegisterAMI", "module": 41, "phase": 9}, {"name": "RemoveBundle", "module": 41, "phase": 10}, {"name": "UploadImage", "module": 41, "phase": 9}, {"name": "ConfigurePVGrub", "module": 42, "phase": 6}, {"name": "Connect", "module": 43, "phase": 0}, {"name": "GetCredentials", "module": 43, "phase": 0}, {"name": "Attach", "module": 44, "phase": 1}, {"name": "Create", "module": 44, "phase": 1}, {"name": "Snapshot", "module": 44, "phase": 9}, {"name": "S3FStab", "module": 45, "phase": 6}, {"name": "AddExternalCommands", "module": 46, "phase": 0}, {"name": "GetInstanceMetadata", "module": 46, "phase": 0}, {"name": "SetRegion", "module": 46, "phase": 0}, {"name": "AddEC2InitScripts", "module": 47, "phase": 6}, {"name": "AddBuildEssentialPackage", "module": 48, "phase": 0}, {"name": "EnableDHCPCDDNS", "module": 48, "phase": 6}, {"name": "InstallEnhancedNetworking", "module": 48, "phase": 6}, {"name": "DefaultPackages", "module": 49, "phase": 0}, {"name": "CleanGoogleRepositoriesAndKeys", "module": 50, "phase": 7}, {"name": "ImportGoogleKey", "module": 50, "phase": 5}, {"name": "SetPackageRepositories", "module": 50, "phase": 0}, {"name": "ConfigureGrub", "module": 51, "phase": 6}, {"name": "GatherReleaseInformation", "module": 52, "phase": 6}, {"name": "DisableIPv6", "module": 53, "phase": 6}, {"name": "InstallHostnameHook", "module": 53, "phase": 6}, {"name": "CreateTarball", "module": 54, "phase": 9}, {"name": "RegisterImage", "module": 54, "phase": 9}, {"name": "UploadImage", "module": 54, "phase": 9}, {"name": "AdjustExpandRootDev", "module": 55, "phase": 6}, {"name": "DefaultPackages", "module": 56, "phase": 0}, {"name": "GooglePackages", "module": 56, "phase": 0}, {"name": "ReleasePackages", "module": 56, "phase": 0}, {"name": "DefaultPackages", "module": 57, "phase": 0}, {"name": "VirtIO", "module": 58, "phase": 6}, {"name": "AddGuestAdditionsPackages", "module": 59, "phase": 5}, {"name": "CheckGuestAdditionsPath", "module": 59, "phase": 0}, {"name": "InstallGuestAdditions", "module": 59, "phase": 5}, {"name": "DefaultPackages", "module": 60, "phase": 0}], "phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}]}
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 5abdcfc..5e1d3e4 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -14,8 +14,6 @@
 
 import sys
 import os
-import glob
-import os.path
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -267,6 +265,9 @@ texinfo_documents = [('index', 'bootstrap-vz', u'bootstrap-vz Documentation',
 
 # -- Link to rst files scattered throughout the project -------------------
 
+import glob
+import os.path
+
 for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'):
 	provider_name = os.path.basename(os.path.dirname(readme_path))
 	include_path = os.path.join('providers', provider_name + '.rst')
@@ -281,3 +282,11 @@ for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'):
 	path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst')
 	with open(include_path, 'w') as include:
 		include.write('.. include:: ' + path_to_readme)
+
+
+# -- Create task overview graph data --------------------------------------
+
+from docs import taskoverview
+
+data = taskoverview.generate_graph_data()
+taskoverview.write_data(data, '_static/graph.json')
diff --git a/docs/taskoverview.py b/docs/taskoverview.py
index c1b142f..97ecbb0 100755
--- a/docs/taskoverview.py
+++ b/docs/taskoverview.py
@@ -5,7 +5,7 @@ import os.path
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 
-def main(opts):
+def generate_graph_data():
 	from bootstrapvz.base.tasklist import get_all_tasks
 	tasks = get_all_tasks()
 
@@ -49,13 +49,11 @@ def main(opts):
 			link[key] = tasks.index(link[key])
 		return link
 
-	data = {'phases': map(mk_phase, phases.order),
+	return {'phases': map(mk_phase, phases.order),
 	        'modules': map(mk_module, modules),
 	        'nodes': map(mk_node, tasks),
 	        'links': map(mk_link, task_links)}
 
-	write_data(data, opts.get('--output', None))
-
 
 def write_data(data, output_path=None):
 	import json
@@ -77,4 +75,5 @@ if __name__ == '__main__' and __package__ is None:
 """
 	opts = docopt(usage)
 
-	main(opts)
+	data = generate_graph_data()
+	write_data(data, opts.get('--output', None))
diff --git a/tox.ini b/tox.ini
index 0ec323d..e538b91 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,7 +20,6 @@ changedir = docs
 deps =
     sphinx
 commands =
-    ./taskoverview.py --output _static/graph.json
     sphinx-build -W -b html -d _build/html/doctrees . _build/html
 
 [testenv:docs-serve]

From f9b234e3e64037285fb06bdb8d9e6752d9b634de Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:28:39 +0200
Subject: [PATCH 103/345] Remove docs-serve testenv again It can cause some
 weird errors when running two tox invocations at the same time

---
 tox.ini | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index e538b91..64ef939 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,3 @@ deps =
     sphinx
 commands =
     sphinx-build -W -b html -d _build/html/doctrees . _build/html
-
-[testenv:docs-serve]
-changedir = docs/_build/html
-commands = python -m SimpleHTTPServer 8080

From 34a87acf1696081ae1f88a62496ff6ccc24e8b34 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:40:16 +0200
Subject: [PATCH 104/345] Add documentation about the manifest

---
 docs/index.rst       |   1 +
 docs/manifest.rst    |   1 +
 manifests/README.rst | 241 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 243 insertions(+)
 create mode 100644 docs/manifest.rst
 create mode 100644 manifests/README.rst

diff --git a/docs/index.rst b/docs/index.rst
index a27b43c..16a5568 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -11,6 +11,7 @@ Contents:
 .. toctree::
    :maxdepth: 2
 
+   manifest
    providers/index
    plugins/index
 
diff --git a/docs/manifest.rst b/docs/manifest.rst
new file mode 100644
index 0000000..d8b3cef
--- /dev/null
+++ b/docs/manifest.rst
@@ -0,0 +1 @@
+.. include:: ../manifests/README.rst
diff --git a/manifests/README.rst b/manifests/README.rst
new file mode 100644
index 0000000..3d5c1b5
--- /dev/null
+++ b/manifests/README.rst
@@ -0,0 +1,241 @@
+Manifest
+========
+
+| The manifest file is the primary way to interact with bootstrap-vz.
+| Every configuration and customization of a Debian installation is
+  specified in this file.
+
+The manifest format is YAML or JSON. It is near impossible to run the
+bootstrapper with an invalid configuration, since every part of the
+framework supplies a `json-schema <http://json-schema.org/>`__ that
+specifies exactly which configuration settings are valid in different
+situations.
+
+Manifest variables
+------------------
+
+Many of the settings in the example manifests use strings like
+``debian-{system.release}-{system.architecture}-{{"{%y"}}}{{"{%m"}}}{{"{%d"}}}``.
+These strings make use of manifest variables, which can cross reference
+other settings in the manifest or specific values supplied by the
+bootstrapper (e.g. all python date formatting variables are available).
+
+Any reference uses dots to specify a path to the desired manifest
+setting. Not all settings support this though, to see whether embedding
+a manifest variable in a setting is possible, look for the
+*``manifest vars``* label.
+
+Sections
+--------
+
+The manifest is split into 7 sections.
+
+Provider
+~~~~~~~~
+
+The provider section contains all provider specific settings and the
+name of the provider itself.
+
+-  **``name``**: target virtualization platform of the installation
+   *``required``*
+
+Consult the `providers <dev_Providers>`__ section of the documentation
+for a list of valid values.
+
+Bootstrapper
+~~~~~~~~~~~~
+
+This section concerns the bootstrapper itself and its behavior. There
+are 4 possible settings:
+
+-  **``workspace``**: Path to where the bootstrapper should place images
+   and intermediate files. Any volumes will be mounted under that path.
+   *``required``*
+-  **``tarball``**: debootstrap has the option to download all the
+   software and pack it up in a tarball. When starting the actual
+   bootstrapping process, debootstrap can then be pointed at that
+   tarball and use it instead of downloading anything from the internet.
+   If you plan on running the bootstrapper multiple times, this option
+   can save you a lot of bandwidth and time. This option just specifies
+   whether it should create a new tarball or not. It will search for and
+   use an available tarball if it already exists, regardless of this
+   setting.
+   *``optional``*
+   Valid values: ``true, false``
+   Default: ``false``
+-  **``mirror``**: The mirror debootstrap should download software from.
+   It is advisable to specify a mirror close to your location (or the
+   location of the host you are bootstrapping on), to decrease latency
+   and improve bandwidth. If not specified, `the configured aptitude
+   mirror URL <#packages>`__ is used.
+   *``optional``*
+-  **``include_packages``**: Extra packages to be installed during
+   bootstrap. Accepts a list of package names.
+   *``optional``*
+-  **``exclude_packages``**: Packages to exclude during bootstrap phase.
+   Accepts a list of package names.
+   *``optional``*
+-  **``guest_additions``**: This setting is only relevant for the
+   `virtualbox provider <dev_Providers#virtualbox>`__. It specifies the
+   path to the VirtualBox Guest Additions ISO, which, when specified,
+   will be mounted and used to install the VirtualBox Guest Additions.
+   *``optional``*
+
+Image
+~~~~~
+
+The image section configures anything pertaining directly to the image
+that will be created.
+
+-  **``name``**: The name of the resulting image.
+   When bootstrapping cloud images, this would be the name visible in
+   the interface when booting up new instances.
+   When bootstrapping for VirtualBox or kvm, it's the filename of the
+   image.
+   *``required``*
+   *``manifest vars``*
+-  **``description``**: Description of the image. Where this setting is
+   used depends highly on which provider is set. At the moment it is
+   only used for AWS images.
+   **``required for ec2 provider``**
+   *``manifest vars``*
+-  **``bucket``**: When bootstrapping an S3 backed image for AWS, this
+   will be the bucket where the image is uploaded to.
+   **``required for S3 backing``**
+-  **``region``**: Region in which the AMI should be registered.
+   **``required for S3 backing``**
+
+System
+~~~~~~
+
+This section defines anything that pertains directly to the bootstrapped
+system and does not fit under any other section.
+
+-  **``architecture``**: The architecture of the system.
+   Valid values: ``i386, amd64``
+   *``required``*
+-  **``bootloader``**: The bootloader for the system. Depending on the
+   bootmethod of the virtualization platform, the options may be
+   restricted.
+   Valid values: ``grub, extlinux, pv-grub``
+   *``required``*
+-  **``charmap``**: The default charmap of the system.
+   Valid values: Any valid charmap like ``UTF-8``, ``ISO-8859-*`` or
+   ``GBK``.
+   *``required``*
+-  **``hostname``**: hostname to preconfigure the system with.
+   *``optional``*
+-  **``locale``**: The default locale of the system.
+   Valid values: Any locale mentioned in ``/etc/locale.gen``
+   *``required``*
+-  **``release``**: Defines which debian release should be bootstrapped.
+   Valid values: ``squeeze``, ``wheezy``, ``jessie``, ``sid``,
+   ``oldstable``, ``stable``, ``testing``, ``unstable``
+   *``required``*
+-  **``timezone``**: Timezone of the system.
+   Valid values: Any filename from ``/usr/share/zoneinfo``
+   *``required``*
+
+Packages
+~~~~~~~~
+
+The packages section allows you to install custom packages from a
+variety of sources.
+
+-  **``install``**: A list of strings that specify which packages should
+   be installed. Valid values: package names optionally followed by a
+   ``/target`` or paths to local ``.deb`` files.
+-  **``install_standard``**: Defines if the packages of the
+   ``"Standard System Utilities"`` option of the Debian installer,
+   provided by `tasksel <https://wiki.debian.org/tasksel>`__, should be
+   installed or not. The problem is that with just ``debootstrap``, the
+   system ends up with very basic commands. This is not a problem for a
+   machine that will not be used interactively, but otherwise it is nice
+   to have at hand tools like ``bash-completion``, ``less``, ``locate``,
+   etc.
+   *``optional``*
+   Valid values: ``true``, ``false``
+   Default: ``false``
+-  **``mirror``**: The default aptitude mirror.
+   *``optional``*
+   Default: ``http://http.debian.net/debian``
+-  **``sources``**: A map of additional sources that should be added to
+   the aptitude sources list. The key becomes the filename in
+   ``/etc/apt/sources.list.d/`` (with ``.list`` appended to it), while
+   the value is an array with each entry being a line.
+   *``optional``*
+-  **``components``**: A list of components that should be added to the
+   default apt sources. For example ``contrib`` or ``non-free``
+   *``optional``*
+   Default: ``['main']``
+-  **``trusted-keys``**: List of paths to ``.gpg`` keyrings that should
+   be added to the aptitude keyring of trusted signatures for
+   repositories.
+   *``optional``*
+-  **``preferences``**: Allows you to pin packages through `apt
+   preferences <https://wiki.debian.org/AptPreferences>`__. The setting
+   is an object where the key is the preference filename in
+   ``/etc/apt/preferences.d/``. The key ``main`` is special and refers
+   to the file ``/etc/apt/preferences``, which will be overwritten if
+   specified.
+   *``optional``*
+   The values are objects with three keys:
+-  **``package``**: The package to pin (wildcards allowed)
+-  **``pin``**: The release to pin the package to.
+-  **``pin-priority``**: The priority of this pin.
+
+Volume
+~~~~~~
+
+bootstrap-vz allows a wide range of options for configuring the disk
+layout of the system. It can create unpartitioned as well as partitioned
+volumes using either the gpt or msdos scheme. At most, there are only
+three partitions with predefined roles configurable though. They are
+boot, root and swap.
+
+-  **``backing``**: Specifies the volume backing. This setting is very
+   provider specific.
+   Valid values: ``ebs``, ``s3``, ``vmdk``, ``vdi``, ``raw``
+   *``required``*
+-  **``partitions``**: A map of the partitions that should be created on
+   the volume.
+-  **``type``**: The partitioning scheme to use. When using ``none``,
+   only root can be specified as a partition.
+   Valid values: ``none``, ``gpt``, ``msdos``
+   *``required``*
+-  **``root``**: Configuration of the root partition. *``required``*
+
+   -  **``size``**: The size of the partition. Valid values: Any
+      datasize specification up to TB (e.g. 5KiB, 1MB, 6TB).
+      *``required``*
+   -  **``filesystem``**: The filesystem of the partition. When choosing
+      ``xfs``, the ``xfsprogs`` package will need to be installed.
+      Valid values: ``ext2``, ``ext3``, ``ext4``, ``xfs``
+      *``required``*
+   -  **``format_command``**: Command to format the partition with. This
+      optional setting overrides the command bootstrap-vz would normally
+      use to format the partition. The command is specified as a string
+      array where each option/argument is an item in that array (much
+      like the `image\_commands <dev_Plugins#image_commands>`__ plugin).
+      *``optional``* The following variables are available:
+   -  **``{fs}``**: The filesystem of the partition.
+   -  **``{device_path}``**: The device path of the partition.
+   -  **``{size}``**: The size of the partition.
+
+   The default command used by boostrap-vz is
+   ``['mkfs.{fs}', '{device_path}']``.
+
+   -  **``boot``**: Configuration of the boot partition. The three
+      settings equal those of the root partition.
+      *``optional``*
+   -  **``swap``**: Configuration of the swap partition. Since the swap
+      partition has its own filesystem you can only specify the size for
+      this partition.
+      *``optional``*
+
+Plugins
+~~~~~~~
+
+The plugins section is a map of plugin names to whatever configuration a
+plugin requires. Go to the `plugin section <dev_Plugins>`__ of the
+documentation, to see the configuration for a specific plugin.

From 79ba5577778a1df62f277885687753599d9f2ba7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 16:58:49 +0200
Subject: [PATCH 105/345] Convert README.md into rst and include from the docs

---
 README.md                |  45 ------------
 README.rst               | 145 +++++++++++++++++++++++++++++++++++++++
 docs/index.rst           |  16 ++---
 docs/intro.rst           |   1 +
 docs/providers/index.rst |   1 -
 5 files changed, 150 insertions(+), 58 deletions(-)
 delete mode 100644 README.md
 create mode 100644 README.rst
 create mode 100644 docs/intro.rst

diff --git a/README.md b/README.md
deleted file mode 100644
index 8eb57fe..0000000
--- a/README.md
+++ /dev/null
@@ -1,45 +0,0 @@
-bootstrap-vz
-===========================================
-bootstrap-vz is a bootstrapping framework for Debian.
-It is is specifically targeted at bootstrapping systems for virtualized environments.
-bootstrap-vz runs without any user intervention and generates ready-to-boot images for
-[a number of virtualization platforms](http://andsens.github.io/bootstrap-vz/providers.html).
-Its aim is to provide a reproducable bootstrapping process using [manifests](http://andsens.github.io/bootstrap-vz/manifest.html) as well as supporting a high degree of customizability through plugins.
-
-bootstrap-vz was coded from scratch in python once the bash script architecture that was used in the
-[build-debian-cloud](https://github.com/andsens/build-debian-cloud) bootstrapper reached its
-limits.
-
-Documentation
--------------
-The end-user documentation for bootstrap-vz is available
-at [andsens.github.io/bootstrap-vz](http://andsens.github.io/bootstrap-vz).
-There, you can discover [what the dependencies](http://andsens.github.io/bootstrap-vz/#dependencies)
-for a specific cloud provider are, [see a list of available plugins](http://andsens.github.io/bootstrap-vz/plugins.html)
-and learn [how you create a manifest](http://andsens.github.io/bootstrap-vz/manifest.html).
-
-Installation
-------------
-
-bootstrap-vz has a master branch for stable releases and a development for, well, development.  
-After checking out the branch of your choice you can install the python dependencies by running
-`python setup.py install`. However, depending on what kind of image you'd like to bootstrap,
-there are other debian package dependencies as well, at the very least you will need `debootstrap`.  
-[The documentation](http://andsens.github.io/bootstrap-vz/) explains this in more detail.
-
-Note that bootstrap-vz will tell you which tools it requires when they aren't
-present (the different packages are mentioned in the error message), so you can
-simply run bootstrap-vz once to get a list of the packages, install them,
-and then re-run.
-
-Developers
-----------
-The API documentation, development guidelines and an explanation of bootstrap-vz internals
-can be found at [bootstrap-vz.readthedocs.org](http://bootstrap-vz.readthedocs.org).
-
-Contributing
-------------
-
-Contribution guidelines are described on the [CONTRIBUTING](CONTRIBUTING.md) file. There's also a
-[topic on the documentation](http://bootstrap-vz.readthedocs.org/en/development/guidelines.html#coding-style)
-regarding the coding style.
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..93c699f
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,145 @@
+Introduction
+============
+
+bootstrap-vz is a bootstrapping framework for Debian.
+It is is specifically targeted at bootstrapping systems for virtualized environments.
+bootstrap-vz runs without any user intervention and
+generates ready-to-boot images for a number of virtualization
+platforms.
+Its aim is to provide a reproducable bootstrapping process using
+`manifests <http://andsens.github.io/bootstrap-vz/manifest.html>`__ as
+well as supporting a high degree of customizability through plugins.
+
+bootstrap-vz was coded from scratch in python once the bash script
+architecture that was used in the
+`build-debian-cloud <https://github.com/andsens/build-debian-cloud>`__
+bootstrapper reached its limits.
+
+Documentation
+-------------
+
+The end-user documentation for bootstrap-vz is available at
+`andsens.github.io/bootstrap-vz <http://andsens.github.io/bootstrap-vz>`__.
+There, you can discover `what the
+dependencies <http://andsens.github.io/bootstrap-vz/#dependencies>`__
+for a specific cloud provider are, `see a list of available
+plugins <http://andsens.github.io/bootstrap-vz/plugins.html>`__ and
+learn `how you create a
+manifest <http://andsens.github.io/bootstrap-vz/manifest.html>`__.
+
+Installation
+------------
+
+bootstrap-vz has a master branch for stable releases and a development
+for, well, development.
+
+After checking out the branch of your choice you can install the
+python dependencies by running ``python setup.py install``. However,
+depending on what kind of image you'd like to bootstrap, there are
+other debian package dependencies as well, at the very least you will
+need ``debootstrap``.
+`The documentation <http://andsens.github.io/bootstrap-vz/>`__
+explains this in more detail.
+
+Note that bootstrap-vz will tell you which tools it requires when they
+aren't present (the different packages are mentioned in the error
+message), so you can simply run bootstrap-vz once to get a list of the
+packages, install them, and then re-run.
+
+Quick start
+-----------
+
+Here are a few quickstart tutorials for the most common images.
+If you plan on partitioning your volume, you will need the ``parted``
+package and ``kpartx``:
+
+.. code:: sh
+
+    root@host:~# apt-get install parted kpartx
+
+VirtualBox Vagrant
+~~~~~~~~~~~~~~~~~~
+
+::
+
+    user@host:~$ sudo -i # become root
+    root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo
+    root@host:~# apt-get install qemu-utils debootstrap python-pip # Install dependencies from aptitude
+    root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies
+    root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml
+
+If you want to use the `minimize\_size <dev_Plugins#minimize_size>`__
+plugin, you will have to install the ``zerofree`` package and `VMWare
+Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
+as well.
+
+Amazon EC2 EBS backed AMI
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: sh
+
+    user@host:~$ sudo -i # become root
+    root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo
+    root@host:~# apt-get install debootstrap python-pip # Install dependencies from aptitude
+    root@host:~# pip install termcolor jsonschema fysom docopt pyyaml boto # Install python dependencies
+    root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml
+
+To bootstrap S3 backed AMIs, bootstrap-vz will also need the
+``euca2ools`` package. However, version 3.2.0 is required meaning you
+must however install it directly from the eucalyptus repository like
+this:
+
+.. code:: sh
+
+    apt-get install --no-install-recommends python-dev libxml2-dev libxslt-dev gcc
+    pip install git+git://github.com/eucalyptus/euca2ools.git@v3.2.0
+
+Cleanup
+-------
+
+bootstrap-vz tries very hard to clean up after itself both if a run was
+successful but also if it failed. This ensures that you are not left
+with volumes still attached to the host which are useless. If an error
+occurred you can simply correct the problem that caused it and rerun
+everything, there will be no leftovers from the previous run (as always
+there are of course rare/unlikely exceptions to that rule). The error
+messages should always give you a strong hint at what is wrong, if that
+is not the case please consider `opening an
+issue <https://github.com/andsens/bootstrap-vz/issues>`__ and attach
+both the error message and your manifest (preferably as a gist or
+similar).
+
+Dependencies
+------------
+
+bootstrap-vz has a number of dependencies depending on the target
+platform and `the selected plugins <dev_Plugins>`__. At a bare minimum
+the following python libraries are needed: \*
+`termcolor <https://pypi.python.org/pypi/termcolor>`__ \*
+`fysom <https://pypi.python.org/pypi/fysom>`__ \*
+`jsonschema <https://pypi.python.org/pypi/jsonschema>`__ \*
+`docopt <https://pypi.python.org/pypi/docopt>`__ \*
+`pyyaml <https://pypi.python.org/pypi/pyyaml>`__ To bootstrap Debian
+itself
+`debootstrap <https://packages.debian.org/wheezy/debootstrap>`__ is
+needed as well.
+
+Any other requirements are dependent upon the manifest configuration
+and are detailed in the corresponding sections of the documentation.
+bootstrap-vz will however warn you if a requirement has not been met,
+before the bootstrapping process begins.
+
+Developers
+----------
+
+The API documentation, development guidelines and an explanation of
+bootstrap-vz internals can be found at
+`bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org>`__.
+
+Contributing
+------------
+
+Contribution guidelines are described on the
+`CONTRIBUTING <CONTRIBUTING.md>`__ file. There's also a `topic on the
+documentation <http://bootstrap-vz.readthedocs.org/en/development/guidelines.html#coding-style>`__
+regarding the coding style.
diff --git a/docs/index.rst b/docs/index.rst
index 16a5568..9944ea4 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -3,14 +3,14 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to bootstrap-vz's documentation!
-========================================
 
-Contents:
+bootstrap-vz
+============
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
+   intro
    manifest
    providers/index
    plugins/index
@@ -21,11 +21,3 @@ Contents:
    howitworks
    switches
    logging
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-
diff --git a/docs/intro.rst b/docs/intro.rst
new file mode 100644
index 0000000..72a3355
--- /dev/null
+++ b/docs/intro.rst
@@ -0,0 +1 @@
+.. include:: ../README.rst
diff --git a/docs/providers/index.rst b/docs/providers/index.rst
index fb991f2..f5b187b 100644
--- a/docs/providers/index.rst
+++ b/docs/providers/index.rst
@@ -1,4 +1,3 @@
-
 Providers
 =========
 

From 619fed52754e38fc82f83b11dfb74c542118c36f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 17:04:09 +0200
Subject: [PATCH 106/345] Merge dev guidelines and contribution info into
 single doc

---
 CONTRIBUTING.md                         | 42 ---------------------
 docs/guidelines.rst => CONTRIBUTING.rst | 49 ++++++++++++++++++++++++-
 docs/contributing.rst                   |  1 +
 docs/index.rst                          |  2 +-
 4 files changed, 50 insertions(+), 44 deletions(-)
 delete mode 100644 CONTRIBUTING.md
 rename docs/guidelines.rst => CONTRIBUTING.rst (69%)
 create mode 100644 docs/contributing.rst

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
deleted file mode 100644
index 0cbba9d..0000000
--- a/CONTRIBUTING.md
+++ /dev/null
@@ -1,42 +0,0 @@
-Contributing
-============
-
-Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow:
-
-* Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html)
-* Fork this repository.
-* Make any changes you want/need.
-* Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8`
-  and fix any warnings that may appear.
-  This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz)
-  once you send a pull request, so it's better if you check this beforehand.
-* If the change is significant (e.g. a new plugin, manifest setting or security fix)
-  add your name and contribution to the [CHANGELOG](CHANGELOG).
-* Commit your changes.
-* Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
-  of work, but there's no reason to have many little commits just because of corrected typos.
-* Push to your fork, preferably on a topic branch.
-
-From here on there are two paths to consider:
-
-If your patch is a new feature, e.g.: plugin, provider, etc. then:
-
-* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make
-  sure that the code is stable.
-
-If it is a bug/security fix:
-
-* Send a pull request to the `master` branch.
-
---
-
-Please try to be very descriptive about your changes when you write a pull request, stating what it does, why
-it is needed, which use cases this change covers etc.
-You may be asked to rebase your work on the current branch state, so it can be merged cleanly.
-If you push a new commit to your pull request you will have to add a new comment to the PR,
-provided that you want us notified. Github will otherwise not send a notification.
-
-Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they
-concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`.
-
-Happy hacking! :-)
diff --git a/docs/guidelines.rst b/CONTRIBUTING.rst
similarity index 69%
rename from docs/guidelines.rst
rename to CONTRIBUTING.rst
index 0d4f81d..7554a35 100644
--- a/docs/guidelines.rst
+++ b/CONTRIBUTING.rst
@@ -1,6 +1,53 @@
+Contributing
+============
+
+Sending pull requests
+---------------------
+
+Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow:
+
++ Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html)
++ Fork this repository.
++ Make any changes you want/need.
++ Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8`
+  and fix any warnings that may appear.
+  This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz)
+  once you send a pull request, so it's better if you check this beforehand.
++ If the change is significant (e.g. a new plugin, manifest setting or security fix)
+  add your name and contribution to the [CHANGELOG](CHANGELOG).
++ Commit your changes.
++ Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
+  of work, but there's no reason to have many little commits just because of corrected typos.
++ Push to your fork, preferably on a topic branch.
+
+From here on there are two paths to consider:
+
+If your patch is a new feature, e.g.: plugin, provider, etc. then:
+
+* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make
+  sure that the code is stable.
+
+If it is a bug/security fix:
+
+* Send a pull request to the `master` branch.
+
+--
+
+Please try to be very descriptive about your changes when you write a pull request, stating what it does, why
+it is needed, which use cases this change covers etc.
+You may be asked to rebase your work on the current branch state, so it can be merged cleanly.
+If you push a new commit to your pull request you will have to add a new comment to the PR,
+provided that you want us notified. Github will otherwise not send a notification.
+
+Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they
+concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`.
+
+Happy hacking! :-)
+
 
 Development guidelines
-======================
+----------------------
+
 The following guidelines should serve as general advice when
 developing providers or plugins for bootstrap-vz. Keep in mind that
 these guidelines are not rules , they are advice on how to better add
diff --git a/docs/contributing.rst b/docs/contributing.rst
new file mode 100644
index 0000000..e582053
--- /dev/null
+++ b/docs/contributing.rst
@@ -0,0 +1 @@
+.. include:: ../CONTRIBUTING.rst
diff --git a/docs/index.rst b/docs/index.rst
index 9944ea4..d17042b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -16,7 +16,7 @@ bootstrap-vz
    plugins/index
 
    api/index
-   guidelines
+   contributing
    taskoverview
    howitworks
    switches

From c31715305a99d79a228dd9ff4e5da9b805f76e3c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 17:05:51 +0200
Subject: [PATCH 107/345] Add yml files to MANIFEST.in

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index a671734..e165705 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,3 +2,4 @@ include LICENSE
 include manifests/*
 recursive-include bootstrapvz assets/*
 recursive-include bootstrapvz *.json
+recursive-include bootstrapvz *.yml

From f27d622e2cfb68ed771ba6ffcefb1b7daffa83be Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 17:12:31 +0200
Subject: [PATCH 108/345] Include changelog in docs (converted it to rst)

---
 CHANGELOG          | 48 -----------------------------
 CHANGELOG.rst      | 77 ++++++++++++++++++++++++++++++++++++++++++++++
 docs/changelog.rst |  1 +
 docs/index.rst     |  1 +
 4 files changed, 79 insertions(+), 48 deletions(-)
 delete mode 100644 CHANGELOG
 create mode 100644 CHANGELOG.rst
 create mode 100644 docs/changelog.rst

diff --git a/CHANGELOG b/CHANGELOG
deleted file mode 100644
index 9d5b2de..0000000
--- a/CHANGELOG
+++ /dev/null
@@ -1,48 +0,0 @@
-2014-11-23:
-	Noah Fontes:
-		* Add support for enhanced networking on EC2 images
-2014-07-12:
-	Tiago Ilieve:
-		* Fixes #96: AddBackports is now a common task
-2014-07-09:
-	Anders Ingemann:
-		* Allow passing data into the manifest
-		* Refactor logging setup to be more modular
-		* Convert every JSON file to YAML
-		* Convert "provider" into provider specific section
-2014-07-02:
-	Vladimir Vitkov:
-		* Improve grub options to work better with virtual machines
-2014-06-30:
-	Tomasz Rybak:
-		* Return information about created image
-2014-06-22:
-	Victor Marmol:
-		* Enable the memory cgroup for the Docker plugin
-2014-06-19:
-	Tiago Ilieve:
-		* Fixes #94: allow stable/oldstable as release name on manifest
-	Vladimir Vitkov:
-		* Improve ami listing performance
-2014-06-07:
-	Tiago Ilieve:
-		* Download `gsutil` tarball to workspace instead of working directory
-		* Fixes #97: remove raw disk image created by GCE after build
-2014-06-06:
-	Ilya Margolin:
-		* pip_install plugin
-2014-05-23:
-	Tiago Ilieve:
-		* Fixes #95: check if the specified APT proxy server can be reached
-2014-05-04:
-	Dhananjay Balan:
-		* Salt minion installation & configuration plugin
-		* Expose debootstrap --include-packages and --exclude-packages options to manifest
-2014-05-03:
-	Anders Ingemann:
-		* Require hostname setting for vagrant plugin
-		* Fixes #14: S3 images can now be bootstrapped outside EC2.
-		* Added enable_agent option to puppet plugin
-2014-05-02:
-	Tomasz Rybak:
-		* Added Google Compute Engine Provider
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
new file mode 100644
index 0000000..420540c
--- /dev/null
+++ b/CHANGELOG.rst
@@ -0,0 +1,77 @@
+Changelog
+=========
+
+2014-11-23:
+
+Noah Fontes:
+	+ Add support for enhanced networking on EC2 images
+
+2014-07-12:
+
+Tiago Ilieve:
+	+ Fixes #96: AddBackports is now a common task
+
+2014-07-09:
+
+Anders Ingemann:
+	+ Allow passing data into the manifest
+	+ Refactor logging setup to be more modular
+	+ Convert every JSON file to YAML
+	+ Convert "provider" into provider specific section
+
+2014-07-02:
+
+Vladimir Vitkov:
+	+ Improve grub options to work better with virtual machines
+
+2014-06-30:
+
+Tomasz Rybak:
+	+ Return information about created image
+
+2014-06-22:
+
+Victor Marmol:
+	+ Enable the memory cgroup for the Docker plugin
+
+2014-06-19:
+
+Tiago Ilieve:
+	+ Fixes #94: allow stable/oldstable as release name on manifest
+
+Vladimir Vitkov:
+	+ Improve ami listing performance
+
+2014-06-07:
+
+Tiago Ilieve:
+	+ Download `gsutil` tarball to workspace instead of working directory
+	+ Fixes #97: remove raw disk image created by GCE after build
+
+2014-06-06:
+
+Ilya Margolin:
+	+ pip_install plugin
+
+2014-05-23:
+
+Tiago Ilieve:
+	+ Fixes #95: check if the specified APT proxy server can be reached
+
+2014-05-04:
+
+Dhananjay Balan:
+	+ Salt minion installation & configuration plugin
+	+ Expose debootstrap --include-packages and --exclude-packages options to manifest
+
+2014-05-03:
+
+Anders Ingemann:
+	+ Require hostname setting for vagrant plugin
+	+ Fixes #14: S3 images can now be bootstrapped outside EC2.
+	+ Added enable_agent option to puppet plugin
+
+2014-05-02:
+
+Tomasz Rybak:
+	+ Added Google Compute Engine Provider
diff --git a/docs/changelog.rst b/docs/changelog.rst
new file mode 100644
index 0000000..565b052
--- /dev/null
+++ b/docs/changelog.rst
@@ -0,0 +1 @@
+.. include:: ../CHANGELOG.rst
diff --git a/docs/index.rst b/docs/index.rst
index d17042b..f10ebe0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -14,6 +14,7 @@ bootstrap-vz
    manifest
    providers/index
    plugins/index
+   changelog
 
    api/index
    contributing

From 3e129b594b25546dfde22c10cf6e1851ea33dc3c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 11:15:27 +0200
Subject: [PATCH 109/345] Fix unit testing, don't try parsing README.rst as
 manifest

---
 tests/integration/manifests_tests.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/integration/manifests_tests.py b/tests/integration/manifests_tests.py
index fd2b4b6..a3e0152 100644
--- a/tests/integration/manifests_tests.py
+++ b/tests/integration/manifests_tests.py
@@ -1,12 +1,6 @@
-import os
 from nose.tools import assert_true
 from bootstrapvz.base.manifest import Manifest
 
-MANIFEST_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    '../../manifests'
-)
-
 
 def test_manifest_generator():
     """
@@ -15,10 +9,16 @@ def test_manifest_generator():
     Loops through the manifests directory and tests that
     each file can successfully be loaded and validated.
     """
-    for fobj in os.listdir(MANIFEST_DIR):
-        path = os.path.join(os.path.abspath(MANIFEST_DIR), fobj)
-
-        yield validate_manifests, path
+    import os.path
+    import glob
+    manifests = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        '../../manifests'
+    )
+    for manifest_path in glob.glob(manifests + '/*.yml'):
+        yield validate_manifests, manifest_path
+    for manifest_path in glob.glob(manifests + '/*.json'):
+        yield validate_manifests, manifest_path
 
 
 def validate_manifests(path):

From 62b87f22d599b458f83b4a2f0b3ca510e69170a7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 11:54:50 +0200
Subject: [PATCH 110/345] Link to code in github rather than embedding it

---
 docs/conf.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 55 insertions(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index 5e1d3e4..7f412db 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -30,7 +30,7 @@ sys.path.insert(0, os.path.abspath(os.pardir))
 # ones.
 extensions = ['sphinx.ext.coverage',
               'sphinx.ext.autodoc',
-              'sphinx.ext.viewcode',
+              'sphinx.ext.linkcode',
               ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -290,3 +290,57 @@ from docs import taskoverview
 
 data = taskoverview.generate_graph_data()
 taskoverview.write_data(data, '_static/graph.json')
+
+
+# -- Substitute links for github with relative links in readthedocs -------
+
+
+if on_rtd:
+	pass
+
+# Snatched from here:
+# https://sourcegraph.com/github.com/Gallopsled/pwntools@master/.PipPackage/pwntools/.def/docs/source/conf/linkcode_resolve/lines
+baseurl = 'https://github.com/andsens/bootstrap-vz'
+
+import subprocess
+try:
+	git_head = subprocess.check_output('git describe --tags 2>/dev/null', shell=True)
+except subprocess.CalledProcessError:
+	try:
+		git_head = subprocess.check_output('git rev-parse HEAD', shell=True).strip()[:10]
+	except subprocess.CalledProcessError:
+		pass
+
+
+def linkcode_resolve(domain, info):
+	if domain != 'py':
+		return None
+	if not info['module']:
+		return None
+
+	filepath = info['module'].replace('.', '/') + '.py'
+	fmt_args = {'baseurl': baseurl,
+	            'commit': git_head,
+	            'path': filepath}
+
+	import importlib
+	import inspect
+	import types
+	module = importlib.import_module(info['module'])
+	value = module
+	for part in info['fullname'].split('.'):
+		value = getattr(value, part, None)
+		if value is None:
+			break
+	valid_types = (types.ModuleType, types.ClassType, types.MethodType,
+	               types.FunctionType, types.TracebackType,
+	               types.FrameType, types.CodeType)
+	if isinstance(value, valid_types):
+		try:
+			lines, first = inspect.getsourcelines(value)
+			fmt_args['linestart'] = first
+			fmt_args['lineend'] = first + len(lines) - 1
+			return '{baseurl}/blob/{commit}/{path}#L{linestart}-L{lineend}'.format(**fmt_args)
+		except IOError:
+			pass
+	return '{baseurl}/blob/{commit}/{path}'.format(**fmt_args)

From 6980dd6517e0211380e89c91e23e11a20f90f7b4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 13:18:10 +0200
Subject: [PATCH 111/345] Add sphinx extension that replaces absolut RTD urls
 This makes it possible to add proper links when showing an rst on github,
 while also resolving it to relative links on readthedocs

---
 README.rst                |  4 ++--
 docs/conf.py              |  1 +
 docs/replace_rtd_links.py | 19 +++++++++++++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)
 create mode 100644 docs/replace_rtd_links.py

diff --git a/README.rst b/README.rst
index 93c699f..9a54e5c 100644
--- a/README.rst
+++ b/README.rst
@@ -7,8 +7,8 @@ bootstrap-vz runs without any user intervention and
 generates ready-to-boot images for a number of virtualization
 platforms.
 Its aim is to provide a reproducable bootstrapping process using
-`manifests <http://andsens.github.io/bootstrap-vz/manifest.html>`__ as
-well as supporting a high degree of customizability through plugins.
+`manifests <http://bootstrap-vz.readthedocs.org/en/master/manifest.html>`__
+as well as supporting a high degree of customizability through plugins.
 
 bootstrap-vz was coded from scratch in python once the bash script
 architecture that was used in the
diff --git a/docs/conf.py b/docs/conf.py
index 7f412db..f6deef4 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,6 +31,7 @@ sys.path.insert(0, os.path.abspath(os.pardir))
 extensions = ['sphinx.ext.coverage',
               'sphinx.ext.autodoc',
               'sphinx.ext.linkcode',
+              'docs.replace_rtd_links',
               ]
 
 # Add any paths that contain templates here, relative to this directory.
diff --git a/docs/replace_rtd_links.py b/docs/replace_rtd_links.py
new file mode 100644
index 0000000..c18d1c4
--- /dev/null
+++ b/docs/replace_rtd_links.py
@@ -0,0 +1,19 @@
+
+def setup(app):
+	app.connect('doctree-resolved', replace_rtd_links)
+
+	return {'version': '0.1'}
+
+
+def replace_rtd_links(app, doctree, fromdocname):
+	from docutils import nodes
+	import re
+
+	rtd_baseurl = 'http://bootstrap-vz.readthedocs.org/en/master/'
+	search = re.compile('^' + re.escape(rtd_baseurl) + '(.*)$')
+	for node in doctree.traverse(nodes.reference):
+		if 'refuri' not in node:
+			continue
+		if not node['refuri'].startswith(rtd_baseurl):
+			continue
+		node['refuri'] = re.sub(search, r'\1', node['refuri'])

From 65ddee99d735414812549ac82529b47d2eaa7005 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 14:32:47 +0200
Subject: [PATCH 112/345] Fix some links

---
 CONTRIBUTING.rst                            | 14 +++----
 README.rst                                  | 43 +++++++++++----------
 bootstrapvz/providers/ec2/README.rst        |  4 +-
 bootstrapvz/providers/virtualbox/README.rst |  2 +-
 manifests/README.rst                        | 12 +++---
 5 files changed, 39 insertions(+), 36 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 7554a35..9c853f5 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -6,15 +6,15 @@ Sending pull requests
 
 Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow:
 
-+ Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html)
++ Read the `development guidelines <http://bootstrap-vz.readthedocs.org/en/master/contributing.html#development-guidelines>`__
 + Fork this repository.
 + Make any changes you want/need.
-+ Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8`
++ Check the coding style of your changes using `tox <http://tox.readthedocs.org/>`__ by running `tox -e flake8`
   and fix any warnings that may appear.
-  This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz)
+  This check will be repeated by `Travis CI <https://travis-ci.org/andsens/bootstrap-vz>`__
   once you send a pull request, so it's better if you check this beforehand.
 + If the change is significant (e.g. a new plugin, manifest setting or security fix)
-  add your name and contribution to the [CHANGELOG](CHANGELOG).
+  add your name and contribution to the `changelog <http://bootstrap-vz.readthedocs.org/en/master/changelog.html>`__.
 + Commit your changes.
 + Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
   of work, but there's no reason to have many little commits just because of corrected typos.
@@ -61,7 +61,7 @@ value to the bootstrap-vz codebase.
   This allows others to easily reproduce any
   setup other people are running and makes it possible to share
   manifests. `The official debian EC2 images <https:/aws.amazon.com/marketplace/seller-
-  profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5>`_
+  profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5>`__
   for example can be reproduced using the manifests available
   in the manifest directory of bootstrap-vz.
 
@@ -107,7 +107,7 @@ value to the bootstrap-vz codebase.
   This allows other tasks to interleave with the control-flow and add extended
   functionality (e.g. because volume creation and mounting are two
   separate tasks, `the prebootstrapped plugin
-  <http://andsens.github.io/bootstrap-vz/plugins.html#prebootstrapped>`_
+  <http://bootstrap-vz.readthedocs.org/en/master/plugins/prebootstrapped.html>`__
   can replace the volume creation task with a task of its own that
   creates a volume from a snapshot instead, but still reuse the mount task).
 
@@ -152,7 +152,7 @@ guidelines. There however a few exceptions:
 + Ignore ``W191``: Indent with tabs not spaces
 
 The codebase can be checked for any violations quite easily, since those rules are already specified in the
-`tox <http://tox.readthedocs.org/>`_ configuration file.
+`tox <http://tox.readthedocs.org/>`__ configuration file.
 ::
 
     tox -e flake8
diff --git a/README.rst b/README.rst
index 9a54e5c..1e08507 100644
--- a/README.rst
+++ b/README.rst
@@ -18,14 +18,14 @@ bootstrapper reached its limits.
 Documentation
 -------------
 
-The end-user documentation for bootstrap-vz is available at
-`andsens.github.io/bootstrap-vz <http://andsens.github.io/bootstrap-vz>`__.
+The documentation for bootstrap-vz is available at
+`bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org/en/master>`__.
 There, you can discover `what the
-dependencies <http://andsens.github.io/bootstrap-vz/#dependencies>`__
+dependencies <http://bootstrap-vz.readthedocs.org/en/master/index.html#dependencies>`__
 for a specific cloud provider are, `see a list of available
-plugins <http://andsens.github.io/bootstrap-vz/plugins.html>`__ and
+plugins <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__ and
 learn `how you create a
-manifest <http://andsens.github.io/bootstrap-vz/manifest.html>`__.
+manifest <http://bootstrap-vz.readthedocs.org/en/master/manifest.html>`__.
 
 Installation
 ------------
@@ -38,7 +38,7 @@ python dependencies by running ``python setup.py install``. However,
 depending on what kind of image you'd like to bootstrap, there are
 other debian package dependencies as well, at the very least you will
 need ``debootstrap``.
-`The documentation <http://andsens.github.io/bootstrap-vz/>`__
+`The documentation <http://bootstrap-vz.readthedocs.org/en/master>`__
 explains this in more detail.
 
 Note that bootstrap-vz will tell you which tools it requires when they
@@ -68,7 +68,7 @@ VirtualBox Vagrant
     root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies
     root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml
 
-If you want to use the `minimize\_size <dev_Plugins#minimize_size>`__
+If you want to use the `minimize\_size <http://bootstrap-vz.readthedocs.org/en/master/plugins/minimize_size.html>`__
 plugin, you will have to install the ``zerofree`` package and `VMWare
 Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
 as well.
@@ -113,16 +113,18 @@ Dependencies
 ------------
 
 bootstrap-vz has a number of dependencies depending on the target
-platform and `the selected plugins <dev_Plugins>`__. At a bare minimum
-the following python libraries are needed: \*
-`termcolor <https://pypi.python.org/pypi/termcolor>`__ \*
-`fysom <https://pypi.python.org/pypi/fysom>`__ \*
-`jsonschema <https://pypi.python.org/pypi/jsonschema>`__ \*
-`docopt <https://pypi.python.org/pypi/docopt>`__ \*
-`pyyaml <https://pypi.python.org/pypi/pyyaml>`__ To bootstrap Debian
-itself
-`debootstrap <https://packages.debian.org/wheezy/debootstrap>`__ is
-needed as well.
+platform and `the selected plugins <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__.
+At a bare minimum the following python libraries are needed:
+
+* `termcolor <https://pypi.python.org/pypi/termcolor>`__
+* `fysom <https://pypi.python.org/pypi/fysom>`__
+* `jsonschema <https://pypi.python.org/pypi/jsonschema>`__
+* `docopt <https://pypi.python.org/pypi/docopt>`__
+* `pyyaml <https://pypi.python.org/pypi/pyyaml>`__
+
+To bootstrap Debian itself
+`debootstrap <https://packages.debian.org/wheezy/debootstrap>`__
+is needed as well.
 
 Any other requirements are dependent upon the manifest configuration
 and are detailed in the corresponding sections of the documentation.
@@ -139,7 +141,8 @@ bootstrap-vz internals can be found at
 Contributing
 ------------
 
-Contribution guidelines are described on the
-`CONTRIBUTING <CONTRIBUTING.md>`__ file. There's also a `topic on the
-documentation <http://bootstrap-vz.readthedocs.org/en/development/guidelines.html#coding-style>`__
+Contribution guidelines are described in the documentation under
+`Contributing <http://bootstrap-vz.readthedocs.org/en/master/contributing.html>`__.
+There's also
+`a topic <http://bootstrap-vz.readthedocs.org/en/master/contributing.html#coding-style>`__
 regarding the coding style.
diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst
index 7c79201..05b63e8 100644
--- a/bootstrapvz/providers/ec2/README.rst
+++ b/bootstrapvz/providers/ec2/README.rst
@@ -7,8 +7,8 @@ once it is done and registers it as an AMI. EBS volume backing only
 works on an EC2 host while S3 backed volumes *should* work locally (at
 this time however they do not, a fix is in the works).
 
-Unless `the cloud-init plugin <dev_Plugins#cloud_init>`__ is used,
-special startup scripts will be installed that automatically fetch the
+Unless `the cloud-init plugin <http://bootstrap-vz.readthedocs.org/en/master/plugins/cloud_init.html>`__
+is used, special startup scripts will be installed that automatically fetch the
 configured authorized\_key from the instance metadata and save or run
 any userdata supplied (if the userdata begins with ``#!`` it will be
 run).
diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst
index b3e3d16..154199b 100644
--- a/bootstrapvz/providers/virtualbox/README.rst
+++ b/bootstrapvz/providers/virtualbox/README.rst
@@ -9,4 +9,4 @@ interoperability (e.g.
 *should* support vdi files, but since they have no identifier URL not
 even VirtualBox itself can import them). VirtualBox Guest Additions can
 be installed automatically if the ISO is `provided in the
-manifest <dev_Manifest#bootstrapper>`__.
+manifest <http://bootstrap-vz.readthedocs.org/en/master/manifest.html#bootstrapper>`__.
diff --git a/manifests/README.rst b/manifests/README.rst
index 3d5c1b5..d644979 100644
--- a/manifests/README.rst
+++ b/manifests/README.rst
@@ -39,7 +39,7 @@ name of the provider itself.
 -  **``name``**: target virtualization platform of the installation
    *``required``*
 
-Consult the `providers <dev_Providers>`__ section of the documentation
+Consult the `providers <http://bootstrap-vz.readthedocs.org/en/master/providers>`__ section of the documentation
 for a list of valid values.
 
 Bootstrapper
@@ -76,8 +76,8 @@ are 4 possible settings:
    Accepts a list of package names.
    *``optional``*
 -  **``guest_additions``**: This setting is only relevant for the
-   `virtualbox provider <dev_Providers#virtualbox>`__. It specifies the
-   path to the VirtualBox Guest Additions ISO, which, when specified,
+   `virtualbox provider <http://bootstrap-vz.readthedocs.org/en/master/providers/virtualbox.html>`__.
+   It specifies the path to the VirtualBox Guest Additions ISO, which, when specified,
    will be mounted and used to install the VirtualBox Guest Additions.
    *``optional``*
 
@@ -216,7 +216,7 @@ boot, root and swap.
       optional setting overrides the command bootstrap-vz would normally
       use to format the partition. The command is specified as a string
       array where each option/argument is an item in that array (much
-      like the `image\_commands <dev_Plugins#image_commands>`__ plugin).
+      like the `image\_commands <http://bootstrap-vz.readthedocs.org/en/master/plugins/image_commands.html>`__ plugin).
       *``optional``* The following variables are available:
    -  **``{fs}``**: The filesystem of the partition.
    -  **``{device_path}``**: The device path of the partition.
@@ -237,5 +237,5 @@ Plugins
 ~~~~~~~
 
 The plugins section is a map of plugin names to whatever configuration a
-plugin requires. Go to the `plugin section <dev_Plugins>`__ of the
-documentation, to see the configuration for a specific plugin.
+plugin requires. Go to the `plugin section <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__
+of the documentation, to see the configuration for a specific plugin.

From 188672404bb90dc965e2dd3691566d0e623ff9c7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 15:12:37 +0200
Subject: [PATCH 113/345] Move developer doc stuff into developers/ subfolder

---
 README.rst                                    |  6 ++---
 docs/_static/taskoverview.coffee              |  2 +-
 docs/contributing.rst                         |  1 -
 docs/{ => developers}/api/base/fs.rst         |  0
 docs/{ => developers}/api/base/index.rst      |  0
 docs/{ => developers}/api/base/pkg.rst        |  0
 docs/{ => developers}/api/common/fs.rst       |  0
 docs/{ => developers}/api/common/index.rst    |  0
 .../api/common/tasks/index.rst                |  0
 docs/{ => developers}/api/index.rst           |  0
 docs/developers/contributing.rst              |  1 +
 docs/{howitworks.rst => developers/index.rst} | 27 +++++++++++++------
 docs/{ => developers}/switches.rst            |  0
 docs/{ => developers}/taskoverview.rst        |  4 +--
 docs/index.rst                                | 15 ++---------
 15 files changed, 28 insertions(+), 28 deletions(-)
 delete mode 100644 docs/contributing.rst
 rename docs/{ => developers}/api/base/fs.rst (100%)
 rename docs/{ => developers}/api/base/index.rst (100%)
 rename docs/{ => developers}/api/base/pkg.rst (100%)
 rename docs/{ => developers}/api/common/fs.rst (100%)
 rename docs/{ => developers}/api/common/index.rst (100%)
 rename docs/{ => developers}/api/common/tasks/index.rst (100%)
 rename docs/{ => developers}/api/index.rst (100%)
 create mode 100644 docs/developers/contributing.rst
 rename docs/{howitworks.rst => developers/index.rst} (80%)
 rename docs/{ => developers}/switches.rst (100%)
 rename docs/{ => developers}/taskoverview.rst (76%)

diff --git a/README.rst b/README.rst
index 1e08507..ac2ce78 100644
--- a/README.rst
+++ b/README.rst
@@ -136,13 +136,13 @@ Developers
 
 The API documentation, development guidelines and an explanation of
 bootstrap-vz internals can be found at
-`bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org>`__.
+`bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org/en/master/developers>`__.
 
 Contributing
 ------------
 
 Contribution guidelines are described in the documentation under
-`Contributing <http://bootstrap-vz.readthedocs.org/en/master/contributing.html>`__.
+`Contributing <http://bootstrap-vz.readthedocs.org/en/master/developers/contributing.html>`__.
 There's also
-`a topic <http://bootstrap-vz.readthedocs.org/en/master/contributing.html#coding-style>`__
+`a topic <http://bootstrap-vz.readthedocs.org/en/master/developers/contributing.html#coding-style>`__
 regarding the coding style.
diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee
index ccae87d..f4e1f25 100644
--- a/docs/_static/taskoverview.coffee
+++ b/docs/_static/taskoverview.coffee
@@ -26,7 +26,7 @@ class window.TaskOverview
 	constructor: ({@selector}) ->
 		@svg = d3.select(@selector)
 			.attr('viewBox', "0 0 #{viewBoxWidth} #{viewBoxHeight}")
-		d3.json '_static/graph.json', @buildGraph
+		d3.json '../_static/graph.json', @buildGraph
 
 	buildGraph: (error, @data) =>
 		@createDefinitions()
diff --git a/docs/contributing.rst b/docs/contributing.rst
deleted file mode 100644
index e582053..0000000
--- a/docs/contributing.rst
+++ /dev/null
@@ -1 +0,0 @@
-.. include:: ../CONTRIBUTING.rst
diff --git a/docs/api/base/fs.rst b/docs/developers/api/base/fs.rst
similarity index 100%
rename from docs/api/base/fs.rst
rename to docs/developers/api/base/fs.rst
diff --git a/docs/api/base/index.rst b/docs/developers/api/base/index.rst
similarity index 100%
rename from docs/api/base/index.rst
rename to docs/developers/api/base/index.rst
diff --git a/docs/api/base/pkg.rst b/docs/developers/api/base/pkg.rst
similarity index 100%
rename from docs/api/base/pkg.rst
rename to docs/developers/api/base/pkg.rst
diff --git a/docs/api/common/fs.rst b/docs/developers/api/common/fs.rst
similarity index 100%
rename from docs/api/common/fs.rst
rename to docs/developers/api/common/fs.rst
diff --git a/docs/api/common/index.rst b/docs/developers/api/common/index.rst
similarity index 100%
rename from docs/api/common/index.rst
rename to docs/developers/api/common/index.rst
diff --git a/docs/api/common/tasks/index.rst b/docs/developers/api/common/tasks/index.rst
similarity index 100%
rename from docs/api/common/tasks/index.rst
rename to docs/developers/api/common/tasks/index.rst
diff --git a/docs/api/index.rst b/docs/developers/api/index.rst
similarity index 100%
rename from docs/api/index.rst
rename to docs/developers/api/index.rst
diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst
new file mode 100644
index 0000000..ac7b6bc
--- /dev/null
+++ b/docs/developers/contributing.rst
@@ -0,0 +1 @@
+.. include:: ../../CONTRIBUTING.rst
diff --git a/docs/howitworks.rst b/docs/developers/index.rst
similarity index 80%
rename from docs/howitworks.rst
rename to docs/developers/index.rst
index cca75c9..5f1cdea 100644
--- a/docs/howitworks.rst
+++ b/docs/developers/index.rst
@@ -1,6 +1,17 @@
+Developers
+============
+
+.. toctree::
+   :maxdepth: 1
+
+   contributing
+   switches
+   api/index
+   taskoverview
+
 
 How bootstrap-vz works
-======================
+----------------------
 
 Tasks
 ~~~~~
@@ -15,14 +26,14 @@ via attributes. Here is an example:
 ::
 
     class MapPartitions(Task):
-    	description = 'Mapping volume partitions'
-    	phase = phases.volume_preparation
-    	predecessors = [PartitionVolume]
-    	successors = [filesystem.Format]
+      description = 'Mapping volume partitions'
+      phase = phases.volume_preparation
+      predecessors = [PartitionVolume]
+      successors = [filesystem.Format]
     
-    	@classmethod
-    	def run(cls, info):
-    		info.volume.partition_map.map(info.volume)
+      @classmethod
+      def run(cls, info):
+         info.volume.partition_map.map(info.volume)
 
 In this case the attributes define that the task at hand should run
 after the ``PartitionVolume`` task — i.e. after volume has been
diff --git a/docs/switches.rst b/docs/developers/switches.rst
similarity index 100%
rename from docs/switches.rst
rename to docs/developers/switches.rst
diff --git a/docs/taskoverview.rst b/docs/developers/taskoverview.rst
similarity index 76%
rename from docs/taskoverview.rst
rename to docs/developers/taskoverview.rst
index e84680e..2e76e19 100644
--- a/docs/taskoverview.rst
+++ b/docs/developers/taskoverview.rst
@@ -5,11 +5,11 @@ Taskoverview
 .. raw:: html
 
 	<svg id="taskoverview-graph"></svg>
-	<link rel="stylesheet/less" type="text/css" href="_static/taskoverview.less">
+	<link rel="stylesheet/less" type="text/css" href="../_static/taskoverview.less">
 	<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/less.js/1.5.0/less.min.js"></script>
 	<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/coffee-script/1.6.3/coffee-script.min.js"></script>
 	<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/d3/3.3.11/d3.js"></script>
-	<script type="text/coffeescript" src="_static/taskoverview.coffee"></script>
+	<script type="text/coffeescript" src="../_static/taskoverview.coffee"></script>
 	<script type="text/coffeescript">
 		overview = new window.TaskOverview selector: '#taskoverview-graph'
 	</script>
diff --git a/docs/index.rst b/docs/index.rst
index f10ebe0..4056a36 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,9 +1,3 @@
-.. bootstrap-vz documentation master file, created by
-   sphinx-quickstart on Sun Mar 23 16:17:28 2014.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-
 bootstrap-vz
 ============
 
@@ -14,11 +8,6 @@ bootstrap-vz
    manifest
    providers/index
    plugins/index
-   changelog
-
-   api/index
-   contributing
-   taskoverview
-   howitworks
-   switches
    logging
+   changelog
+   developers/index

From 2f3ee404d12047bcd47c8148c55fb05d62fbe0fd Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 15:42:43 +0200
Subject: [PATCH 114/345] Remove bogus intro text from providers/index.rst

---
 docs/providers/index.rst | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/docs/providers/index.rst b/docs/providers/index.rst
index f5b187b..7bf1ed5 100644
--- a/docs/providers/index.rst
+++ b/docs/providers/index.rst
@@ -1,16 +1,6 @@
 Providers
 =========
 
-Plugins are a key feature of bootstrap-vz. Despite their small size
-(most plugins do not exceed 100 source lines of code) they can modify
-the behavior of bootstrapped systems to a great extent.
-
-Below you will find documentation for all plugins available for
-bootstrap-vz. If you cannot find what you are looking for, consider
-`developing it yourself <http://bootstrap-vz.readthedocs.org>`__ and
-contribute to this list!
-
-
 .. toctree::
    :maxdepth: 1
    :glob:

From e82bdf4a84fa42dc6832232c9014e4619f998712 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 10 Jul 2014 20:29:31 +0200
Subject: [PATCH 115/345] Rename integration tests to unit tests, since they
 really only cover small parts of the system.

---
 tests/{integration => unit}/manifests_tests.py | 0
 tests/{integration => unit}/subprocess.sh      | 0
 tests/{integration => unit}/tools_tests.py     | 0
 tox.ini                                        | 4 ++--
 4 files changed, 2 insertions(+), 2 deletions(-)
 rename tests/{integration => unit}/manifests_tests.py (100%)
 rename tests/{integration => unit}/subprocess.sh (100%)
 rename tests/{integration => unit}/tools_tests.py (100%)

diff --git a/tests/integration/manifests_tests.py b/tests/unit/manifests_tests.py
similarity index 100%
rename from tests/integration/manifests_tests.py
rename to tests/unit/manifests_tests.py
diff --git a/tests/integration/subprocess.sh b/tests/unit/subprocess.sh
similarity index 100%
rename from tests/integration/subprocess.sh
rename to tests/unit/subprocess.sh
diff --git a/tests/integration/tools_tests.py b/tests/unit/tools_tests.py
similarity index 100%
rename from tests/integration/tools_tests.py
rename to tests/unit/tools_tests.py
diff --git a/tox.ini b/tox.ini
index 64ef939..999071f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,11 +9,11 @@ envlist = flake8, integration
 deps = flake8
 commands = flake8 bootstrapvz/ --exclude=minify_json.py
 
-[testenv:integration]
+[testenv:unit]
 deps =
     nose
     nose-cov
-commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
+commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:docs]
 changedir = docs

From e271f3e49a90182ea6215a200b22c4faad20394e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 31 Aug 2014 13:45:35 +0200
Subject: [PATCH 116/345] Initial work on integration testing started. The work
 consists of three parts: * Allow for bootstrapping remotely,   this makes it
 possible to run the tests   on e.g. OSX with VirtualBox installed * Make
 bootstrapping a fully automated process   where the manifests can be
 generated by the tests   and the tests can call the bootstrapper directly in
 python * Create a framework wherein instances can be booted up   using the
 bootstrapped images and subsequently tested

---
 bootstrap-vz                                  |   2 +-
 bootstrap-vz-remote                           |   5 +
 bootstrap-vz-server                           |   5 +
 bootstrapvz/base/__init__.py                  |   1 -
 bootstrapvz/base/main.py                      |   2 +
 bootstrapvz/base/remote/__init__.py           | 100 ++++++++++
 bootstrapvz/base/remote/callback.py           |  31 +++
 bootstrapvz/base/remote/log.py                |  24 +++
 bootstrapvz/base/remote/server.py             |  50 +++++
 bootstrapvz/base/remote/ssh_rpc_manager.py    |  67 +++++++
 build_servers.yml                             |  23 +++
 remote/client.py                              |  63 ++++++
 remote/server.py                              |  97 +++++++++
 remote/ssh_wrapper.py                         |  28 +++
 tests/integration/__init__.py                 |   9 +
 tests/integration/build/__init__.py           |   3 +
 tests/integration/build/local/__init__.py     |   0
 tests/integration/build/local/client.py       |   5 +
 tests/integration/build/remote/__init__.py    |   0
 tests/integration/build/remote/client.py      |   5 +
 tests/integration/build/remote/forward.py     | 186 ++++++++++++++++++
 tests/integration/build/remote/test.py        |  56 ++++++
 tests/integration/build/settings.yml          |   8 +
 tests/integration/image/__init__.py           |  21 ++
 tests/integration/manifests/base.yml          |  14 ++
 tests/integration/manifests/partitioned.yml   |  11 ++
 tests/integration/manifests/unpartitioned.yml |   7 +
 tests/integration/tools/__init__.py           |  56 ++++++
 tests/integration/virtualbox_tests.py         |  28 +++
 tox.ini                                       |   4 +
 30 files changed, 909 insertions(+), 2 deletions(-)
 create mode 100755 bootstrap-vz-remote
 create mode 100755 bootstrap-vz-server
 create mode 100644 bootstrapvz/base/remote/__init__.py
 create mode 100644 bootstrapvz/base/remote/callback.py
 create mode 100644 bootstrapvz/base/remote/log.py
 create mode 100644 bootstrapvz/base/remote/server.py
 create mode 100644 bootstrapvz/base/remote/ssh_rpc_manager.py
 create mode 100644 build_servers.yml
 create mode 100755 remote/client.py
 create mode 100755 remote/server.py
 create mode 100644 remote/ssh_wrapper.py
 create mode 100644 tests/integration/__init__.py
 create mode 100644 tests/integration/build/__init__.py
 create mode 100644 tests/integration/build/local/__init__.py
 create mode 100644 tests/integration/build/local/client.py
 create mode 100644 tests/integration/build/remote/__init__.py
 create mode 100644 tests/integration/build/remote/client.py
 create mode 100644 tests/integration/build/remote/forward.py
 create mode 100755 tests/integration/build/remote/test.py
 create mode 100644 tests/integration/build/settings.yml
 create mode 100644 tests/integration/image/__init__.py
 create mode 100644 tests/integration/manifests/base.yml
 create mode 100644 tests/integration/manifests/partitioned.yml
 create mode 100644 tests/integration/manifests/unpartitioned.yml
 create mode 100644 tests/integration/tools/__init__.py
 create mode 100644 tests/integration/virtualbox_tests.py

diff --git a/bootstrap-vz b/bootstrap-vz
index 46f8fbf..de62f32 100755
--- a/bootstrap-vz
+++ b/bootstrap-vz
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 
 if __name__ == '__main__':
-	from bootstrapvz.base import main
+	from bootstrapvz.base.main import main
 	main()
diff --git a/bootstrap-vz-remote b/bootstrap-vz-remote
new file mode 100755
index 0000000..81fa633
--- /dev/null
+++ b/bootstrap-vz-remote
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+if __name__ == '__main__':
+	from bootstrapvz.base.remote import main
+	main()
diff --git a/bootstrap-vz-server b/bootstrap-vz-server
new file mode 100755
index 0000000..a79f005
--- /dev/null
+++ b/bootstrap-vz-server
@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+
+if __name__ == '__main__':
+	from bootstrapvz.base.remote.server import main
+	main()
diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index be224ce..305d2e3 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -1,6 +1,5 @@
 from phase import Phase
 from task import Task
-from main import main
 
 __all__ = ['Phase', 'Task', 'main']
 
diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 7e1c054..acfacd3 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -86,6 +86,8 @@ def setup_loggers(opts):
 
 
 def run(manifest, debug=False, pause_on_error=False, dry_run=False):
+	log.info('test')
+	return 'derp'
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
new file mode 100644
index 0000000..27a7e05
--- /dev/null
+++ b/bootstrapvz/base/remote/__init__.py
@@ -0,0 +1,100 @@
+import Pyro4
+from threading import Thread
+"""Remote module containing methods to bootstrap remotely
+"""
+
+
+import logging
+log = logging.getLogger(__name__)
+
+stop = False
+
+
+def main():
+	"""Main function for invoking the bootstrap process remotely
+	"""
+	# Get the commandline arguments
+	opts = get_opts()
+
+	# Load the manifest
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(path=opts['MANIFEST'])
+
+	from bootstrapvz.common.tools import load_data
+	build_servers = load_data(opts['--servers'])
+
+	# Set up logging
+	from bootstrapvz.base.main import setup_loggers
+	setup_loggers(opts)
+
+	from ssh_rpc_manager import SSHRPCManager
+	manager = SSHRPCManager(build_servers[opts['SERVER']])
+	try:
+		manager.start()
+		server = manager.rpc_server
+
+		# Everything has been set up, begin the bootstrapping process
+		print('run')
+		server.run(None,
+		           debug=opts['--debug'],
+		           pause_on_error=False,
+		           dry_run=opts['--dry-run'])
+		print('hasrun')
+	finally:
+		manager.stop()
+
+
+def get_opts():
+	"""Creates an argument parser and returns the arguments it has parsed
+	"""
+	from docopt import docopt
+	usage = """bootstrap-vz-remote
+
+Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
+
+Options:
+  --servers <path>   Path to list of build servers
+  --log <path>       Log to given directory [default: /var/log/bootstrap-vz]
+                     If <path> is `-' file logging will be disabled.
+  --pause-on-error   Pause on error, before rollback
+  --dry-run          Don't actually run the tasks
+  --debug            Print debugging information
+  -h, --help         show this help
+	"""
+	return docopt(usage)
+
+
+def setup_interrupt_server(manager):
+
+	def on_error(e):
+		raw_input('Press Enter to commence rollback')
+		return True
+
+	daemon = Pyro4.Daemon()
+	daemon.register(on_error)
+
+	def serve():
+		daemon.requestLoop(loopCondition=lambda: manager.current == 'rpc_started')
+
+	thread = Thread(target=serve)
+	thread.start()
+	return (thread, on_error)
+
+
+def setup_log_server(manager):
+	from log import LogServer
+	log_server = LogServer()
+	daemon = Pyro4.Daemon()
+	daemon.register(log_server)
+
+	def serve():
+		def check():
+			import logging
+			log = logging.getLogger(__name__)
+			log.info(stop)
+			return not stop
+		daemon.requestLoop(loopCondition=check)
+
+	thread = Thread(target=serve)
+	thread.start()
+	return (thread, log_server)
diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
new file mode 100644
index 0000000..80fe22c
--- /dev/null
+++ b/bootstrapvz/base/remote/callback.py
@@ -0,0 +1,31 @@
+
+
+class CallbackServer(object):
+
+	def __init__(self, listen_port):
+		self.listen_port = listen_port
+		self.stop_serving = False
+
+		from log import LogServer
+		self.log_server = LogServer()
+
+	def start(self, rpc_server):
+		import Pyro4
+		Pyro4.config.COMMTIMEOUT = 0.5
+		daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
+		daemon.register(self.log_server)
+
+		def serve():
+			daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
+		from threading import Thread
+		self.thread = Thread(target=serve)
+		self.thread.start()
+
+		rpc_server.set_log_server(self.log_server)
+
+	def stop(self):
+		self.stop_serving = True
+		if hasattr(self, 'thread'):
+			print('joining')
+			self.thread.join()
+			print('joined')
diff --git a/bootstrapvz/base/remote/log.py b/bootstrapvz/base/remote/log.py
new file mode 100644
index 0000000..169ce52
--- /dev/null
+++ b/bootstrapvz/base/remote/log.py
@@ -0,0 +1,24 @@
+import logging
+import pickle
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			self.server.handle(pickle.dumps(record))
+
+
+class LogServer(object):
+
+	def handle(self, pickled_record):
+		import logging
+		log = logging.getLogger()
+		record = pickle.loads(pickled_record)
+		log.handle(record)
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
new file mode 100644
index 0000000..837349b
--- /dev/null
+++ b/bootstrapvz/base/remote/server.py
@@ -0,0 +1,50 @@
+
+
+def main():
+	opts = getopts()
+	log_forwarder = setup_logging()
+	serve(opts, log_forwarder)
+
+
+def setup_logging():
+	import logging
+	from log import LogForwarder
+	log_forwarder = LogForwarder()
+	root = logging.getLogger()
+	root.addHandler(log_forwarder)
+	root.setLevel(logging.NOTSET)
+	return log_forwarder
+
+
+def serve(opts, log_forwarder):
+	class Server(object):
+
+		def run(self, *args, **kwargs):
+			from bootstrapvz.base.main import run
+			return run(*args, **kwargs)
+
+		def set_log_server(self, server):
+			return log_forwarder.set_server(server)
+
+		def ping(self):
+			return 'pong'
+
+	server = Server()
+
+	import Pyro4
+	daemon = Pyro4.Daemon('localhost', port=int(opts['--listen']), unixsocket=None)
+	daemon.register(server, 'server')
+	daemon.requestLoop()
+
+
+def getopts():
+	from docopt import docopt
+	usage = """bootstrap-vz-server
+
+Usage: bootstrap-vz-server [options]
+
+Options:
+  --listen <port>    Serve on specified port [default: 46675]
+  -h, --help         show this help
+"""
+	return docopt(usage)
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
new file mode 100644
index 0000000..ef14317
--- /dev/null
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -0,0 +1,67 @@
+from fysom import Fysom
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class SSHRPCManager(object):
+
+	def __init__(self, settings):
+		self.settings = settings
+
+		import random
+		self.local_server_port = random.randrange(1024, 65535)
+		self.local_callback_port = random.randrange(1024, 65535)
+		# self.remote_server_port = random.randrange(1024, 65535)
+		# self.remote_callback_port = random.randrange(1024, 65535)
+		self.remote_server_port = self.local_server_port
+		self.remote_callback_port = self.local_callback_port
+
+	def start(self):
+		log.debug('Opening SSH connection')
+		import subprocess
+
+		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
+		                  '-p', str(self.settings['port']),
+		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
+		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
+		                  self.settings['username'] + '@' + self.settings['address'],
+		                  '--',
+		                  'sudo', self.settings['server-bin'],
+		                  '--listen', str(self.remote_server_port)]
+		import sys
+		self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
+
+		# Check that we can connect to the server
+		try:
+			import Pyro4
+			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
+			self.rpc_server = Pyro4.Proxy(server_uri)
+
+			log.debug('Connecting to PYRO')
+			remaining_retries = 5
+			while True:
+				try:
+					self.rpc_server.ping()
+					break
+				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
+					if remaining_retries > 0:
+						remaining_retries -= 1
+						from time import sleep
+						sleep(2)
+					else:
+						raise e
+		except (Exception, KeyboardInterrupt) as e:
+			print('terminateE')
+			self.process.terminate()
+			raise e
+
+		from callback import CallbackServer
+		self.callback_server = CallbackServer(self.local_callback_port)
+		self.callback_server.start(self.rpc_server)
+
+	def stop(self):
+		print('terminate')
+		self.process.terminate()
+		if hasattr(self, 'callback_server'):
+			self.callback_server.stop()
diff --git a/build_servers.yml b/build_servers.yml
new file mode 100644
index 0000000..dcfabd1
--- /dev/null
+++ b/build_servers.yml
@@ -0,0 +1,23 @@
+virtualbox:
+  can_bootstrap:
+    - virtualbox
+    - ec2-s3
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/bootstrap-vz-server
+ec2:
+  can_bootstrap:
+    - virtualbox
+    - ec2-ebs
+    - ec2-s3
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/bootstrap-vz-server
diff --git a/remote/client.py b/remote/client.py
new file mode 100755
index 0000000..da3cc35
--- /dev/null
+++ b/remote/client.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import random
+import Pyro4
+
+# We need to set either a socket communication timeout,
+# or use the select based server. Otherwise the daemon requestLoop
+# will block indefinitely and is never able to evaluate the loopCondition.
+Pyro4.config.COMMTIMEOUT = 0.5
+
+NUM_WORKERS = 5
+
+from ssh_wrapper import RemoteServer
+srv = RemoteServer()
+srv.start()
+
+
+class CallbackHandler(object):
+	workdone = 0
+
+	def done(self, number):
+		print("callback: worker %d reports work is done!" % number)
+		CallbackHandler.workdone += 1
+
+
+class LogServer(object):
+
+	def handle(self, record):
+		print('logging' + record)
+		# import logging
+		# log = logging.getLogger()
+		# (handler.handle(record) for handler in log.handlers)
+
+
+with Pyro4.Daemon('localhost', port=srv.client_port, unixsocket=None) as daemon:
+	# register our callback handler
+	callback = CallbackHandler()
+	daemon.register(callback)
+	logger = LogServer()
+	daemon.register(logger)
+
+	# contact the server and put it to work
+
+	def serve():
+		daemon.requestLoop(loopCondition=lambda: CallbackHandler.workdone < NUM_WORKERS)
+	from threading import Thread
+	thread = Thread(target=serve)
+	thread.start()
+
+	print("creating a bunch of workers")
+	with Pyro4.core.Proxy("PYRO:srv@localhost:" + str(srv.server_port)) as server:
+		server.set_log_server(logger)
+		for _ in range(NUM_WORKERS):
+			worker = server.addworker(callback)   # provide our callback handler!
+			# worker._pyroOneway.add("work")      # to be able to run in the background
+			worker.work(0.5)
+		server.stop()
+
+	print("waiting for all work complete...")
+	thread.join()
+	print("done!")
+
+srv.stop()
diff --git a/remote/server.py b/remote/server.py
new file mode 100755
index 0000000..081d66d
--- /dev/null
+++ b/remote/server.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+import time
+import Pyro4
+import logging
+Pyro4.config.COMMTIMEOUT = 5
+
+log = logging.getLogger(__name__)
+
+class Worker(object):
+	def __init__(self, number, callback):
+		self.number = number
+		self.callback = callback
+		log.info("Worker %d created" % self.number)
+
+	def work(self, amount):
+		print("Worker %d busy..." % self.number)
+		time.sleep(amount)
+		print("Worker %d done. Informing callback client." % self.number)
+		self._pyroDaemon.unregister(self)
+		self.callback.done(self.number)  # invoke the callback object
+
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			self.server.handle('hans')
+
+
+class CallbackServer(object):
+
+	def __init__(self):
+		self.number = 0
+		self.serve = True
+
+	def addworker(self, callback):
+		self.number += 1
+		print("server: adding worker %d" % self.number)
+		worker = Worker(self.number, callback)
+		self._pyroDaemon.register(worker)   # make it a Pyro object
+		return worker
+
+	def stop(self):
+		print('called stop()')
+		self.serve = False
+
+	def still_serve(self):
+		print('called still_serve()')
+		return self.serve
+
+	def set_log_server(self, server):
+		import logging
+		log_forwarder = LogForwarder()
+		root = logging.getLogger()
+		root.addHandler(log_forwarder)
+		root.setLevel(logging.NOTSET)
+		log_forwarder.set_server(server)
+
+	def test(self, msg):
+		import logging
+		root = logging.getLogger()
+		root.info(msg)
+
+
+def main():
+	opts = getopts()
+	with Pyro4.Daemon('localhost', port=int(opts['--listen-port']), unixsocket=None) as daemon:
+		obj = CallbackServer()
+		uri = daemon.register(obj, 'srv')
+		print uri
+		print("Server ready.")
+		daemon.requestLoop(loopCondition=lambda: obj.still_serve())
+
+
+def getopts():
+	from docopt import docopt
+	usage = """bootstrap-vz-server
+
+Usage: bootstrap-vz-server [options]
+
+Options:
+  --listen-port <port>   Serve on specified port [default: 46675]
+  --callback-port <port> Connect callback to specified port [default: 46674]
+  -h, --help          show this help
+"""
+	return docopt(usage)
+
+if __name__ == '__main__':
+	main()
diff --git a/remote/ssh_wrapper.py b/remote/ssh_wrapper.py
new file mode 100644
index 0000000..f1f9d5d
--- /dev/null
+++ b/remote/ssh_wrapper.py
@@ -0,0 +1,28 @@
+
+
+class RemoteServer(object):
+
+	def __init__(self):
+		import random
+		self.server_port = random.randrange(1024, 65535)
+		self.client_port = random.randrange(1024, 65535)
+
+	def start(self):
+		import subprocess
+
+		command = ['ssh', '-i', '/Users/anders/.vagrant.d/insecure_private_key',
+		                  '-t',  # Force pseudo-tty allocation so that server.py quits when we close the connection
+		                  '-p', '2222',
+		                  '-L' + str(self.server_port) + ':localhost:' + str(self.server_port),
+		                  '-R' + str(self.client_port) + ':localhost:' + str(self.client_port),
+		                  'vagrant@localhost',
+		                  '--',
+		                  'sudo', '/root/bootstrap/remote/server.py',
+		                  '--listen-port', str(self.server_port),
+		                  '--callback-port', str(self.client_port)]
+		self.process = subprocess.Popen(args=command)
+		import time
+		time.sleep(2)
+
+	def stop(self):
+		self.process.terminate()
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 0000000..ccce1cc
--- /dev/null
+++ b/tests/integration/__init__.py
@@ -0,0 +1,9 @@
+import os.path
+from bootstrapvz.common.tools import load_data
+
+combine_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests')
+manifests = {'base': load_data(os.path.join(combine_manifests_path, 'base.yml')),
+             'partitioned': load_data(os.path.join(combine_manifests_path, 'partitioned.yml')),
+             'unpartitioned': load_data(os.path.join(combine_manifests_path, 'unpartitioned.yml')),
+             }
+build_settings = load_data(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'build_settings.yml'))
diff --git a/tests/integration/build/__init__.py b/tests/integration/build/__init__.py
new file mode 100644
index 0000000..9416fa0
--- /dev/null
+++ b/tests/integration/build/__init__.py
@@ -0,0 +1,3 @@
+
+def get_client(build_settings):
+	pass
diff --git a/tests/integration/build/local/__init__.py b/tests/integration/build/local/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/build/local/client.py b/tests/integration/build/local/client.py
new file mode 100644
index 0000000..85473cc
--- /dev/null
+++ b/tests/integration/build/local/client.py
@@ -0,0 +1,5 @@
+from .. import BaseClient
+
+
+class Client(BaseClient):
+	pass
diff --git a/tests/integration/build/remote/__init__.py b/tests/integration/build/remote/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/build/remote/client.py b/tests/integration/build/remote/client.py
new file mode 100644
index 0000000..85473cc
--- /dev/null
+++ b/tests/integration/build/remote/client.py
@@ -0,0 +1,5 @@
+from .. import BaseClient
+
+
+class Client(BaseClient):
+	pass
diff --git a/tests/integration/build/remote/forward.py b/tests/integration/build/remote/forward.py
new file mode 100644
index 0000000..96e1700
--- /dev/null
+++ b/tests/integration/build/remote/forward.py
@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2003-2007  Robey Pointer <robeypointer@gmail.com>
+#
+# This file is part of paramiko.
+#
+# Paramiko is free software; you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+
+"""
+Sample script showing how to do local port forwarding over paramiko.
+
+This script connects to the requested SSH server and sets up local port
+forwarding (the openssh -L option) from a local port through a tunneled
+connection to a destination reachable from the SSH server machine.
+"""
+
+import getpass
+import os
+import socket
+import select
+try:
+    import SocketServer
+except ImportError:
+    import socketserver as SocketServer
+
+import sys
+from optparse import OptionParser
+
+import paramiko
+
+SSH_PORT = 22
+DEFAULT_PORT = 4000
+
+g_verbose = True
+
+
+class ForwardServer (SocketServer.ThreadingTCPServer):
+    daemon_threads = True
+    allow_reuse_address = True
+    
+
+class Handler (SocketServer.BaseRequestHandler):
+
+    def handle(self):
+        try:
+            chan = self.ssh_transport.open_channel('direct-tcpip',
+                                                   (self.chain_host, self.chain_port),
+                                                   self.request.getpeername())
+        except Exception as e:
+            verbose('Incoming request to %s:%d failed: %s' % (self.chain_host,
+                                                              self.chain_port,
+                                                              repr(e)))
+            return
+        if chan is None:
+            verbose('Incoming request to %s:%d was rejected by the SSH server.' %
+                    (self.chain_host, self.chain_port))
+            return
+
+        verbose('Connected!  Tunnel open %r -> %r -> %r' % (self.request.getpeername(),
+                                                            chan.getpeername(), (self.chain_host, self.chain_port)))
+        while True:
+            r, w, x = select.select([self.request, chan], [], [])
+            if self.request in r:
+                data = self.request.recv(1024)
+                if len(data) == 0:
+                    break
+                chan.send(data)
+            if chan in r:
+                data = chan.recv(1024)
+                if len(data) == 0:
+                    break
+                self.request.send(data)
+                
+        peername = self.request.getpeername()
+        chan.close()
+        self.request.close()
+        verbose('Tunnel closed from %r' % (peername,))
+
+
+def forward_tunnel(local_port, remote_host, remote_port, transport):
+    # this is a little convoluted, but lets me configure things for the Handler
+    # object.  (SocketServer doesn't give Handlers any way to access the outer
+    # server normally.)
+    class SubHander (Handler):
+        chain_host = remote_host
+        chain_port = remote_port
+        ssh_transport = transport
+    ForwardServer(('', local_port), SubHander).serve_forever()
+
+
+def verbose(s):
+    if g_verbose:
+        print(s)
+
+
+HELP = """\
+Set up a forward tunnel across an SSH server, using paramiko. A local port
+(given with -p) is forwarded across an SSH session to an address:port from
+the SSH server. This is similar to the openssh -L option.
+"""
+
+
+def get_host_port(spec, default_port):
+    "parse 'hostname:22' into a host and port, with the port optional"
+    args = (spec.split(':', 1) + [default_port])[:2]
+    args[1] = int(args[1])
+    return args[0], args[1]
+
+
+def parse_options():
+    global g_verbose
+    
+    parser = OptionParser(usage='usage: %prog [options] <ssh-server>[:<server-port>]',
+                          version='%prog 1.0', description=HELP)
+    parser.add_option('-q', '--quiet', action='store_false', dest='verbose', default=True,
+                      help='squelch all informational output')
+    parser.add_option('-p', '--local-port', action='store', type='int', dest='port',
+                      default=DEFAULT_PORT,
+                      help='local port to forward (default: %d)' % DEFAULT_PORT)
+    parser.add_option('-u', '--user', action='store', type='string', dest='user',
+                      default=getpass.getuser(),
+                      help='username for SSH authentication (default: %s)' % getpass.getuser())
+    parser.add_option('-K', '--key', action='store', type='string', dest='keyfile',
+                      default=None,
+                      help='private key file to use for SSH authentication')
+    parser.add_option('', '--no-key', action='store_false', dest='look_for_keys', default=True,
+                      help='don\'t look for or use a private key file')
+    parser.add_option('-P', '--password', action='store_true', dest='readpass', default=False,
+                      help='read password (for key or password auth) from stdin')
+    parser.add_option('-r', '--remote', action='store', type='string', dest='remote', default=None, metavar='host:port',
+                      help='remote host and port to forward to')
+    options, args = parser.parse_args()
+
+    if len(args) != 1:
+        parser.error('Incorrect number of arguments.')
+    if options.remote is None:
+        parser.error('Remote address required (-r).')
+    
+    g_verbose = options.verbose
+    server_host, server_port = get_host_port(args[0], SSH_PORT)
+    remote_host, remote_port = get_host_port(options.remote, SSH_PORT)
+    return options, (server_host, server_port), (remote_host, remote_port)
+
+
+def main():
+    options, server, remote = parse_options()
+    
+    password = None
+    if options.readpass:
+        password = getpass.getpass('Enter SSH password: ')
+    
+    client = paramiko.SSHClient()
+    client.load_system_host_keys()
+    client.set_missing_host_key_policy(paramiko.WarningPolicy())
+
+    verbose('Connecting to ssh host %s:%d ...' % (server[0], server[1]))
+    try:
+        client.connect(server[0], server[1], username=options.user, key_filename=options.keyfile,
+                       look_for_keys=options.look_for_keys, password=password)
+    except Exception as e:
+        print('*** Failed to connect to %s:%d: %r' % (server[0], server[1], e))
+        sys.exit(1)
+
+    verbose('Now forwarding port %d to %s:%d ...' % (options.port, remote[0], remote[1]))
+
+    try:
+        forward_tunnel(options.port, remote[0], remote[1], client.get_transport())
+    except KeyboardInterrupt:
+        print('C-c: Port forwarding stopped.')
+        sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/integration/build/remote/test.py b/tests/integration/build/remote/test.py
new file mode 100755
index 0000000..abe74a4
--- /dev/null
+++ b/tests/integration/build/remote/test.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+from remote.ssh_rpc_client import SSHRPCClient
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def main():
+	import os.path
+
+	settings_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'settings.yml'))
+	with open(settings_path, 'r') as stream:
+		import yaml
+		settings = yaml.safe_load(stream)
+
+	bootstrapvz_root = os.path.normpath(os.path.join(os.path.dirname(__file__), '../../../'))
+	import sys
+	sys.path.append(bootstrapvz_root)
+
+	from bootstrapvz.base.log import get_console_handler
+	console_handler = get_console_handler(debug=True)
+
+	root_logger = logging.getLogger()
+	root_logger.setLevel(logging.NOTSET)
+	root_logger.addHandler(console_handler)
+
+	rpc_server = SSHRPCClient(settings)
+	try:
+		rpc_server.start_server()
+		log.info('connecting to Pyro on remote')
+		import Pyro4
+		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
+		main = Pyro4.Proxy(main_uri)
+		log.info('running command')
+		remaining_retries = 5
+		while True:
+			try:
+				main.run('eogubhswg')
+				break
+			except Pyro4.errors.ConnectionClosedError as e:
+				if remaining_retries > 0:
+					remaining_retries -= 1
+					from time import sleep
+					sleep(2)
+				else:
+					raise e
+		log.info('stopping server')
+		rpc_server.stop_server()
+	except (Exception, KeyboardInterrupt) as e:
+		log.error(e.__class__.__name__ + ': ' + str(e))
+	finally:
+		print 'cleaning up'
+		rpc_server.cleanup()
+
+if __name__ == '__main__':
+	main()
diff --git a/tests/integration/build/settings.yml b/tests/integration/build/settings.yml
new file mode 100644
index 0000000..2788774
--- /dev/null
+++ b/tests/integration/build/settings.yml
@@ -0,0 +1,8 @@
+build_host:
+  address: 127.0.0.1
+  port: 2222
+  username: vagrant
+  password: vagrant
+  root_password: vagrant
+  keyfile: /Users/anders/.vagrant.d/insecure_private_key
+  server-bin: /root/bootstrap/tests/integration/build/server.py
diff --git a/tests/integration/image/__init__.py b/tests/integration/image/__init__.py
new file mode 100644
index 0000000..67bc2b1
--- /dev/null
+++ b/tests/integration/image/__init__.py
@@ -0,0 +1,21 @@
+
+
+class Image(object):
+
+	def create_instance(self):
+		return Instance()
+
+	def destroy(self):
+		pass
+
+
+class Instance(object):
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass
diff --git a/tests/integration/manifests/base.yml b/tests/integration/manifests/base.yml
new file mode 100644
index 0000000..849a31a
--- /dev/null
+++ b/tests/integration/manifests/base.yml
@@ -0,0 +1,14 @@
+---
+provider: {}
+bootstrapper:
+  workspace: /target
+image:
+  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  description: Debian {system.release} {system.architecture}
+system:
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  partitions: {}
+packages: {}
diff --git a/tests/integration/manifests/partitioned.yml b/tests/integration/manifests/partitioned.yml
new file mode 100644
index 0000000..e6686c4
--- /dev/null
+++ b/tests/integration/manifests/partitioned.yml
@@ -0,0 +1,11 @@
+---
+volume:
+  partitions:
+    boot:
+      filesystem: ext2
+      size: 32MiB
+    root:
+      filesystem: ext4
+      size: 864MiB
+    swap:
+      size: 128MiB
diff --git a/tests/integration/manifests/unpartitioned.yml b/tests/integration/manifests/unpartitioned.yml
new file mode 100644
index 0000000..ef7f97d
--- /dev/null
+++ b/tests/integration/manifests/unpartitioned.yml
@@ -0,0 +1,7 @@
+---
+volume:
+  type: none
+  partitions:
+    root:
+      filesystem: ext4
+      size: 1GiB
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
new file mode 100644
index 0000000..3bdad63
--- /dev/null
+++ b/tests/integration/tools/__init__.py
@@ -0,0 +1,56 @@
+
+
+# Snatched from here: http://stackoverflow.com/a/7205107
+def merge_dicts(*args):
+	def merge(a, b, path=None):
+		if path is None:
+			path = []
+		for key in b:
+			if key in a:
+				if isinstance(a[key], dict) and isinstance(b[key], dict):
+					merge(a[key], b[key], path + [str(key)])
+				elif a[key] == b[key]:
+					pass
+				else:
+					raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
+			else:
+						a[key] = b[key]
+		return a
+	return reduce(merge, args, {})
+
+
+def bootstrap(manifest, build_settings):
+	# if 'build_host' in build_settings:
+	# 	run = get_remote_run(build_settings)
+	# else:
+	# 	run = __import__('bootstrapvz.base.run')
+	# run(manifest)
+	from ..image import Image
+	return Image()
+
+
+def test_instance(instance, build_settings):
+	pass
+
+
+def get_remote_run(build_settings):
+	from ..build.client import SSHRPCServer
+
+	rpc_server = SSHRPCServer(settings)
+	try:
+		rpc_server.start()
+		from time import sleep
+		sleep(2)
+		log.info('connection to Pyro on remote')
+		import Pyro4
+		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
+		main = Pyro4.Proxy(main_uri)
+		log.info('running command')
+		main.run('eogubhswg')
+		log.info('stopping server')
+		rpc_server.stop()
+	except (Exception, KeyboardInterrupt) as e:
+		log.error(e.__class__.__name__ + ': ' + str(e))
+	finally:
+		print 'cleaning up'
+		rpc_server.cleanup()
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
new file mode 100644
index 0000000..d3b317b
--- /dev/null
+++ b/tests/integration/virtualbox_tests.py
@@ -0,0 +1,28 @@
+import tools
+from . import manifests
+from . import build_settings
+
+
+def test_virtualbox_unpartitioned_extlinux():
+	specific_settings = {}
+	specific_settings['provider'] = {'name': 'virtualbox',
+	                                 'guest_additions': build_settings['virtualbox']['guest_additions']}
+	specific_settings['system'] = {'release': 'wheezy',
+	                               'architecture': 'amd64',
+	                               'bootloader': 'extlinux'}
+	specific_settings['volume'] = {'backing': 'vdi',
+	                               'partitions': {'type': 'msdos'}}
+	manifest = tools.merge_dicts(manifests['base'], manifests['unpartitioned'], specific_settings)
+
+	client = tools.get_client(build_settings['virtualbox'])
+
+	image = client.bootstrap(manifest, build_settings['virtualbox'])
+	instance = image.create_instance()
+	instance.boot()
+
+	tools.test_instance(instance, build_settings['virtualbox'])
+
+	instance.destroy()
+	image.destroy()
+
+	client.shutdown()
diff --git a/tox.ini b/tox.ini
index 999071f..122ee83 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,6 +15,10 @@ deps =
     nose-cov
 commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
+[testenv:integration]
+deps = nose
+commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
+
 [testenv:docs]
 changedir = docs
 deps =

From 16837b38c2bb5c397aa971eecf7801f6b0bfa2ea Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 17:26:13 +0100
Subject: [PATCH 117/345] Close thread on both ends!

---
 bootstrapvz/base/remote/__init__.py        | 38 ----------------------
 bootstrapvz/base/remote/callback.py        |  2 --
 bootstrapvz/base/remote/log.py             |  1 +
 bootstrapvz/base/remote/server.py          | 11 +++++--
 bootstrapvz/base/remote/ssh_rpc_manager.py |  5 ++-
 5 files changed, 12 insertions(+), 45 deletions(-)

diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index 27a7e05..c65b550 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -34,12 +34,10 @@ def main():
 		server = manager.rpc_server
 
 		# Everything has been set up, begin the bootstrapping process
-		print('run')
 		server.run(None,
 		           debug=opts['--debug'],
 		           pause_on_error=False,
 		           dry_run=opts['--dry-run'])
-		print('hasrun')
 	finally:
 		manager.stop()
 
@@ -62,39 +60,3 @@ Options:
   -h, --help         show this help
 	"""
 	return docopt(usage)
-
-
-def setup_interrupt_server(manager):
-
-	def on_error(e):
-		raw_input('Press Enter to commence rollback')
-		return True
-
-	daemon = Pyro4.Daemon()
-	daemon.register(on_error)
-
-	def serve():
-		daemon.requestLoop(loopCondition=lambda: manager.current == 'rpc_started')
-
-	thread = Thread(target=serve)
-	thread.start()
-	return (thread, on_error)
-
-
-def setup_log_server(manager):
-	from log import LogServer
-	log_server = LogServer()
-	daemon = Pyro4.Daemon()
-	daemon.register(log_server)
-
-	def serve():
-		def check():
-			import logging
-			log = logging.getLogger(__name__)
-			log.info(stop)
-			return not stop
-		daemon.requestLoop(loopCondition=check)
-
-	thread = Thread(target=serve)
-	thread.start()
-	return (thread, log_server)
diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index 80fe22c..9b77684 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -26,6 +26,4 @@ class CallbackServer(object):
 	def stop(self):
 		self.stop_serving = True
 		if hasattr(self, 'thread'):
-			print('joining')
 			self.thread.join()
-			print('joined')
diff --git a/bootstrapvz/base/remote/log.py b/bootstrapvz/base/remote/log.py
index 169ce52..76ee608 100644
--- a/bootstrapvz/base/remote/log.py
+++ b/bootstrapvz/base/remote/log.py
@@ -1,6 +1,7 @@
 import logging
 import pickle
 
+
 class LogForwarder(logging.Handler):
 
 	def __init__(self, level=logging.NOTSET):
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index 837349b..76f2832 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -19,6 +19,9 @@ def setup_logging():
 def serve(opts, log_forwarder):
 	class Server(object):
 
+		def __init__(self):
+			self.stop_serving = False
+
 		def run(self, *args, **kwargs):
 			from bootstrapvz.base.main import run
 			return run(*args, **kwargs)
@@ -29,12 +32,16 @@ def serve(opts, log_forwarder):
 		def ping(self):
 			return 'pong'
 
-	server = Server()
+		def stop(self):
+			self.stop_serving = True
 
 	import Pyro4
+	Pyro4.config.COMMTIMEOUT = 0.5
 	daemon = Pyro4.Daemon('localhost', port=int(opts['--listen']), unixsocket=None)
+
+	server = Server()
 	daemon.register(server, 'server')
-	daemon.requestLoop()
+	daemon.requestLoop(loopCondition=lambda: not server.stop_serving)
 
 
 def getopts():
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index ef14317..2110f00 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -52,7 +52,6 @@ class SSHRPCManager(object):
 					else:
 						raise e
 		except (Exception, KeyboardInterrupt) as e:
-			print('terminateE')
 			self.process.terminate()
 			raise e
 
@@ -61,7 +60,7 @@ class SSHRPCManager(object):
 		self.callback_server.start(self.rpc_server)
 
 	def stop(self):
-		print('terminate')
-		self.process.terminate()
 		if hasattr(self, 'callback_server'):
 			self.callback_server.stop()
+		self.rpc_server.stop()
+		self.process.terminate()

From aa9616f4a642d8a426c514e852a3555350ee28fa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 17:28:06 +0100
Subject: [PATCH 118/345] Stop callback server last

---
 bootstrapvz/base/remote/ssh_rpc_manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index 2110f00..f359655 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -60,7 +60,7 @@ class SSHRPCManager(object):
 		self.callback_server.start(self.rpc_server)
 
 	def stop(self):
+		self.rpc_server.stop()
 		if hasattr(self, 'callback_server'):
 			self.callback_server.stop()
-		self.rpc_server.stop()
 		self.process.terminate()

From 150b15bb4f2cdfa90acb8160a8bc3a7f9bfaec03 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 17:42:33 +0100
Subject: [PATCH 119/345] (g|s)etstate for manifest

---
 bootstrapvz/base/main.py            |  3 +--
 bootstrapvz/base/manifest.py        | 12 ++++++++++++
 bootstrapvz/base/remote/__init__.py |  9 +++++----
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index acfacd3..b32bf3c 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -86,8 +86,7 @@ def setup_loggers(opts):
 
 
 def run(manifest, debug=False, pause_on_error=False, dry_run=False):
-	log.info('test')
-	return 'derp'
+	return manifest
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 969dfbf..54eb5d6 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -15,6 +15,7 @@ class Manifest(object):
 	Currently, immutability is not enforced and it would require a fair amount of code
 	to enforce it, instead we just rely on tasks behaving properly.
 	"""
+
 	def __init__(self, path=None, data=None):
 		"""Initializer: Given a path we load, validate and parse the manifest.
 		To create the manifest from dynamic data instead of the contents of a file,
@@ -130,3 +131,14 @@ class Manifest(object):
 		:raises ManifestError: With absolute certainty
 		"""
 		raise ManifestError(message, self.path, data_path)
+
+	def __getstate__(self):
+		return {'path': self.path,
+		        'data': self.data}
+
+	def __setstate__(self, vals):
+		self.path = vals.path
+		self.load(vals.data)
+		self.initialize()
+		self.validate()
+		self.parse()
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index c65b550..7185097 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -34,10 +34,11 @@ def main():
 		server = manager.rpc_server
 
 		# Everything has been set up, begin the bootstrapping process
-		server.run(None,
-		           debug=opts['--debug'],
-		           pause_on_error=False,
-		           dry_run=opts['--dry-run'])
+		ret = server.run(manifest,
+		                 debug=opts['--debug'],
+		                 pause_on_error=False,
+		                 dry_run=opts['--dry-run'])
+		logging.getLogger(__name__).info(ret)
 	finally:
 		manager.stop()
 

From 922cabe80a5cd759848828edff75aebb527579f8 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 18:32:18 +0100
Subject: [PATCH 120/345] Serialize exception info by printing it

---
 bootstrapvz/base/remote/log.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bootstrapvz/base/remote/log.py b/bootstrapvz/base/remote/log.py
index 76ee608..443b3b4 100644
--- a/bootstrapvz/base/remote/log.py
+++ b/bootstrapvz/base/remote/log.py
@@ -13,6 +13,10 @@ class LogForwarder(logging.Handler):
 
 	def emit(self, record):
 		if self.server is not None:
+			if record.exc_info is not None:
+				import traceback
+				exc_type, exc_value, exc_traceback = record.exc_info
+				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
 			self.server.handle(pickle.dumps(record))
 
 

From 569e1246a54d57b79cd6b899a3ad4868cec84f2d Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 18:38:23 +0100
Subject: [PATCH 121/345] Deserialize manifest

---
 bootstrapvz/base/main.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index b32bf3c..bea8686 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -86,7 +86,6 @@ def setup_loggers(opts):
 
 
 def run(manifest, debug=False, pause_on_error=False, dry_run=False):
-	return manifest
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
@@ -94,6 +93,9 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	:params bool pause_on_error: Whether to pause on error, before rollback
 	:params bool dry_run: Don't actually run the tasks
 	"""
+	if isinstance(manifest, dict):
+		from manifest import Manifest
+		manifest = Manifest(path=manifest['path'], data=manifest['data'])
 	# Get the tasklist
 	from tasklist import load_tasks
 	from tasklist import TaskList

From 282e22c001d0ab3006cb1540cb20376fbb73b0ea Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 18:38:32 +0100
Subject: [PATCH 122/345] debugging

---
 bootstrapvz/base/manifest.py        | 7 -------
 bootstrapvz/base/remote/__init__.py | 9 ++++-----
 bootstrapvz/base/tasklist.py        | 2 ++
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 54eb5d6..576e31f 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -135,10 +135,3 @@ class Manifest(object):
 	def __getstate__(self):
 		return {'path': self.path,
 		        'data': self.data}
-
-	def __setstate__(self, vals):
-		self.path = vals.path
-		self.load(vals.data)
-		self.initialize()
-		self.validate()
-		self.parse()
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index 7185097..7367efb 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -34,11 +34,10 @@ def main():
 		server = manager.rpc_server
 
 		# Everything has been set up, begin the bootstrapping process
-		ret = server.run(manifest,
-		                 debug=opts['--debug'],
-		                 pause_on_error=False,
-		                 dry_run=opts['--dry-run'])
-		logging.getLogger(__name__).info(ret)
+		server.run(manifest,
+		           debug=opts['--debug'],
+		           pause_on_error=False,
+		           dry_run=opts['--dry-run'])
 	finally:
 		manager.stop()
 
diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 8e5dbb1..15cfd5d 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -21,6 +21,8 @@ class TaskList(object):
 		:param dict info: The bootstrap information object
 		:param bool dry_run: Whether to actually run the tasks or simply step through them
 		"""
+		logging.getLogger(__name__).debug('test')
+		return
 		# Create a list for us to run
 		task_list = create_list(self.tasks)
 		# Output the tasklist

From a35ae91b677b36793e95a559e3c1add72779ce2a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 18:54:31 +0100
Subject: [PATCH 123/345] Simplify? Hopefully...

---
 bootstrapvz/base/log.py                    | 29 ++++++++++++++++++++++
 bootstrapvz/base/remote/__init__.py        | 19 ++++++++++----
 bootstrapvz/base/remote/callback.py        |  9 ++-----
 bootstrapvz/base/remote/log.py             | 29 ----------------------
 bootstrapvz/base/remote/server.py          |  2 +-
 bootstrapvz/base/remote/ssh_rpc_manager.py |  6 -----
 6 files changed, 46 insertions(+), 48 deletions(-)
 delete mode 100644 bootstrapvz/base/remote/log.py

diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py
index b6096e3..b406763 100644
--- a/bootstrapvz/base/log.py
+++ b/bootstrapvz/base/log.py
@@ -88,3 +88,32 @@ class FileFormatter(logging.Formatter):
 	"""
 	def format(self, record):
 		return super(FileFormatter, self).format(record)
+
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			if record.exc_info is not None:
+				import traceback
+				exc_type, exc_value, exc_traceback = record.exc_info
+				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
+			# TODO: Use serpent instead
+			import pickle
+			self.server.handle(pickle.dumps(record))
+
+
+class LogServer(object):
+
+	def handle(self, pickled_record):
+		import pickle
+		record = pickle.loads(pickled_record)
+		log = logging.getLogger()
+		log.handle(record)
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index 7367efb..b7ba576 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -32,12 +32,21 @@ def main():
 	try:
 		manager.start()
 		server = manager.rpc_server
+		from callback import CallbackServer
+		callback_server = CallbackServer(manager.local_callback_port)
+		from bootstrapvz.base.log import LogServer
+		log_server = LogServer()
+		try:
+			callback_server.start(log_server)
+			server.set_log_server(log_server)
 
-		# Everything has been set up, begin the bootstrapping process
-		server.run(manifest,
-		           debug=opts['--debug'],
-		           pause_on_error=False,
-		           dry_run=opts['--dry-run'])
+			# Everything has been set up, begin the bootstrapping process
+			server.run(manifest,
+			           debug=opts['--debug'],
+			           pause_on_error=False,
+			           dry_run=opts['--dry-run'])
+		finally:
+			callback_server.stop()
 	finally:
 		manager.stop()
 
diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index 9b77684..82d9060 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -6,14 +6,11 @@ class CallbackServer(object):
 		self.listen_port = listen_port
 		self.stop_serving = False
 
-		from log import LogServer
-		self.log_server = LogServer()
-
-	def start(self, rpc_server):
+	def start(self, log_server):
 		import Pyro4
 		Pyro4.config.COMMTIMEOUT = 0.5
 		daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
-		daemon.register(self.log_server)
+		daemon.register(log_server)
 
 		def serve():
 			daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
@@ -21,8 +18,6 @@ class CallbackServer(object):
 		self.thread = Thread(target=serve)
 		self.thread.start()
 
-		rpc_server.set_log_server(self.log_server)
-
 	def stop(self):
 		self.stop_serving = True
 		if hasattr(self, 'thread'):
diff --git a/bootstrapvz/base/remote/log.py b/bootstrapvz/base/remote/log.py
deleted file mode 100644
index 443b3b4..0000000
--- a/bootstrapvz/base/remote/log.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import logging
-import pickle
-
-
-class LogForwarder(logging.Handler):
-
-	def __init__(self, level=logging.NOTSET):
-		self.server = None
-		super(LogForwarder, self).__init__(level)
-
-	def set_server(self, server):
-		self.server = server
-
-	def emit(self, record):
-		if self.server is not None:
-			if record.exc_info is not None:
-				import traceback
-				exc_type, exc_value, exc_traceback = record.exc_info
-				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
-			self.server.handle(pickle.dumps(record))
-
-
-class LogServer(object):
-
-	def handle(self, pickled_record):
-		import logging
-		log = logging.getLogger()
-		record = pickle.loads(pickled_record)
-		log.handle(record)
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index 76f2832..2a834db 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -8,7 +8,7 @@ def main():
 
 def setup_logging():
 	import logging
-	from log import LogForwarder
+	from bootstrapvz.base.log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
 	root.addHandler(log_forwarder)
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index f359655..7143916 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -55,12 +55,6 @@ class SSHRPCManager(object):
 			self.process.terminate()
 			raise e
 
-		from callback import CallbackServer
-		self.callback_server = CallbackServer(self.local_callback_port)
-		self.callback_server.start(self.rpc_server)
-
 	def stop(self):
 		self.rpc_server.stop()
-		if hasattr(self, 'callback_server'):
-			self.callback_server.stop()
 		self.process.terminate()

From 2b33561b8258626197090e1e7cc58284719c4438 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 22:45:05 +0100
Subject: [PATCH 124/345] Don't kill SSH, ask callback server to stop instead
 of commtimeout

---
 bootstrapvz/base/remote/callback.py        | 13 +++++--------
 bootstrapvz/base/remote/server.py          |  3 +++
 bootstrapvz/base/remote/ssh_rpc_manager.py |  3 ++-
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index 82d9060..865081d 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -4,21 +4,18 @@ class CallbackServer(object):
 
 	def __init__(self, listen_port):
 		self.listen_port = listen_port
-		self.stop_serving = False
 
 	def start(self, log_server):
 		import Pyro4
-		Pyro4.config.COMMTIMEOUT = 0.5
-		daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
-		daemon.register(log_server)
+		self.daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
+		self.daemon.register(log_server)
 
 		def serve():
-			daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
+			self.daemon.requestLoop()
 		from threading import Thread
 		self.thread = Thread(target=serve)
 		self.thread.start()
 
 	def stop(self):
-		self.stop_serving = True
-		if hasattr(self, 'thread'):
-			self.thread.join()
+		self.daemon.shutdown()
+		self.thread.join()
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index 2a834db..28b9d38 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -11,6 +11,9 @@ def setup_logging():
 	from bootstrapvz.base.log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
+	from bootstrapvz.base import log
+	file_handler = log.get_file_handler(path='/var/log/bootstrap-vz/remote.log', debug=True)
+	root.addHandler(file_handler)
 	root.addHandler(log_forwarder)
 	root.setLevel(logging.NOTSET)
 	return log_forwarder
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index 7143916..d9521cb 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -57,4 +57,5 @@ class SSHRPCManager(object):
 
 	def stop(self):
 		self.rpc_server.stop()
-		self.process.terminate()
+		self.rpc_server._pyroRelease()
+		self.process.wait()

From 3542406b91e8a9a4d549ee300a693bf4c80b1a08 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 24 Nov 2014 23:43:17 +0100
Subject: [PATCH 125/345] Fix (de-)serialization

---
 bootstrapvz/base/bootstrapinfo.py   | 15 +++++++++++
 bootstrapvz/base/manifest.py        |  3 ++-
 bootstrapvz/base/remote/__init__.py | 39 ++++++++++++++++++++---------
 bootstrapvz/base/remote/server.py   |  2 ++
 bootstrapvz/base/tasklist.py        |  2 --
 5 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 0313e3b..70dfa4d 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -90,6 +90,9 @@ class BootstrapInformation(object):
 			def __delattr__(self, name):
 				del self[name]
 
+			def __getstate__(self):
+				return self.__dict__
+
 		def set_manifest_vars(obj, data):
 			"""Runs through the manifest and creates DictClasses for every key
 
@@ -124,3 +127,15 @@ class BootstrapInformation(object):
 		# They are added last so that they may override previous variables
 		set_manifest_vars(manifest_vars, additional_vars)
 		return manifest_vars
+
+	def __getstate__(self):
+		state = self.__dict__.copy()
+		exclude_keys = ['volume', 'source_lists', 'preference_lists', 'packages']
+		for key in exclude_keys:
+			del state[key]
+		state['__class__'] = 'bootstrapvz.base.bootstrapinfo.BootstrapInformation'
+		return state
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 576e31f..2854068 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -133,5 +133,6 @@ class Manifest(object):
 		raise ManifestError(message, self.path, data_path)
 
 	def __getstate__(self):
-		return {'path': self.path,
+		return {'__class__': 'bootstrapvz.base.manifest.Manifest',
+		        'path': self.path,
 		        'data': self.data}
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index b7ba576..52e01c5 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -1,15 +1,7 @@
-import Pyro4
-from threading import Thread
 """Remote module containing methods to bootstrap remotely
 """
 
 
-import logging
-log = logging.getLogger(__name__)
-
-stop = False
-
-
 def main():
 	"""Main function for invoking the bootstrap process remotely
 	"""
@@ -27,6 +19,10 @@ def main():
 	from bootstrapvz.base.main import setup_loggers
 	setup_loggers(opts)
 
+	register_deserialization_handlers()
+
+	bootstrap_info = None
+
 	from ssh_rpc_manager import SSHRPCManager
 	manager = SSHRPCManager(build_servers[opts['SERVER']])
 	try:
@@ -41,14 +37,15 @@ def main():
 			server.set_log_server(log_server)
 
 			# Everything has been set up, begin the bootstrapping process
-			server.run(manifest,
-			           debug=opts['--debug'],
-			           pause_on_error=False,
-			           dry_run=opts['--dry-run'])
+			bootstrap_info = server.run(manifest,
+			                            debug=opts['--debug'],
+			                            pause_on_error=False,
+			                            dry_run=opts['--dry-run'])
 		finally:
 			callback_server.stop()
 	finally:
 		manager.stop()
+	return bootstrap_info
 
 
 def get_opts():
@@ -69,3 +66,21 @@ Options:
   -h, --help         show this help
 	"""
 	return docopt(usage)
+
+
+def register_deserialization_handlers():
+	from Pyro4.util import SerializerBase
+	SerializerBase.register_dict_to_class('bootstrapvz.base.manifest.Manifest', deserialize_manifest)
+	SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo)
+
+
+def deserialize_manifest(classname, state):
+	from bootstrapvz.base.manifest import Manifest
+	return Manifest(path=state['path'], data=state['data'])
+
+
+def deserialize_bootstrapinfo(classname, state):
+	from bootstrapvz.base.bootstrapinfo import BootstrapInformation
+	bootstrap_info = BootstrapInformation.__new__(BootstrapInformation)
+	bootstrap_info.__setstate__(state)
+	return bootstrap_info
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index 28b9d38..e9325a8 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -2,6 +2,8 @@
 
 def main():
 	opts = getopts()
+	from . import register_deserialization_handlers
+	register_deserialization_handlers()
 	log_forwarder = setup_logging()
 	serve(opts, log_forwarder)
 
diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 15cfd5d..8e5dbb1 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -21,8 +21,6 @@ class TaskList(object):
 		:param dict info: The bootstrap information object
 		:param bool dry_run: Whether to actually run the tasks or simply step through them
 		"""
-		logging.getLogger(__name__).debug('test')
-		return
 		# Create a list for us to run
 		task_list = create_list(self.tasks)
 		# Output the tasklist

From ed222d579f0c386f7aa4c2bb7205812a3ccdad70 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:14:56 +0100
Subject: [PATCH 126/345] Little refactor

---
 bootstrap-vz-remote                 |  2 +-
 bootstrapvz/base/remote/__init__.py | 66 -----------------------------
 bootstrapvz/base/remote/remote.py   | 66 +++++++++++++++++++++++++++++
 bootstrapvz/base/remote/server.py   | 63 ++++++++++++++-------------
 4 files changed, 98 insertions(+), 99 deletions(-)
 create mode 100644 bootstrapvz/base/remote/remote.py

diff --git a/bootstrap-vz-remote b/bootstrap-vz-remote
index 81fa633..b5050b8 100755
--- a/bootstrap-vz-remote
+++ b/bootstrap-vz-remote
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 
 if __name__ == '__main__':
-	from bootstrapvz.base.remote import main
+	from bootstrapvz.base.remote.remote import main
 	main()
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index 52e01c5..8e604e5 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -2,72 +2,6 @@
 """
 
 
-def main():
-	"""Main function for invoking the bootstrap process remotely
-	"""
-	# Get the commandline arguments
-	opts = get_opts()
-
-	# Load the manifest
-	from bootstrapvz.base.manifest import Manifest
-	manifest = Manifest(path=opts['MANIFEST'])
-
-	from bootstrapvz.common.tools import load_data
-	build_servers = load_data(opts['--servers'])
-
-	# Set up logging
-	from bootstrapvz.base.main import setup_loggers
-	setup_loggers(opts)
-
-	register_deserialization_handlers()
-
-	bootstrap_info = None
-
-	from ssh_rpc_manager import SSHRPCManager
-	manager = SSHRPCManager(build_servers[opts['SERVER']])
-	try:
-		manager.start()
-		server = manager.rpc_server
-		from callback import CallbackServer
-		callback_server = CallbackServer(manager.local_callback_port)
-		from bootstrapvz.base.log import LogServer
-		log_server = LogServer()
-		try:
-			callback_server.start(log_server)
-			server.set_log_server(log_server)
-
-			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = server.run(manifest,
-			                            debug=opts['--debug'],
-			                            pause_on_error=False,
-			                            dry_run=opts['--dry-run'])
-		finally:
-			callback_server.stop()
-	finally:
-		manager.stop()
-	return bootstrap_info
-
-
-def get_opts():
-	"""Creates an argument parser and returns the arguments it has parsed
-	"""
-	from docopt import docopt
-	usage = """bootstrap-vz-remote
-
-Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
-
-Options:
-  --servers <path>   Path to list of build servers
-  --log <path>       Log to given directory [default: /var/log/bootstrap-vz]
-                     If <path> is `-' file logging will be disabled.
-  --pause-on-error   Pause on error, before rollback
-  --dry-run          Don't actually run the tasks
-  --debug            Print debugging information
-  -h, --help         show this help
-	"""
-	return docopt(usage)
-
-
 def register_deserialization_handlers():
 	from Pyro4.util import SerializerBase
 	SerializerBase.register_dict_to_class('bootstrapvz.base.manifest.Manifest', deserialize_manifest)
diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/base/remote/remote.py
new file mode 100644
index 0000000..dc1cf6e
--- /dev/null
+++ b/bootstrapvz/base/remote/remote.py
@@ -0,0 +1,66 @@
+
+def main():
+	"""Main function for invoking the bootstrap process remotely
+	"""
+	# Get the commandline arguments
+	opts = get_opts()
+
+	# Load the manifest
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(path=opts['MANIFEST'])
+
+	from bootstrapvz.common.tools import load_data
+	build_servers = load_data(opts['--servers'])
+
+	# Set up logging
+	from bootstrapvz.base.main import setup_loggers
+	setup_loggers(opts)
+
+	from . import register_deserialization_handlers
+	register_deserialization_handlers()
+
+	bootstrap_info = None
+
+	from ssh_rpc_manager import SSHRPCManager
+	manager = SSHRPCManager(build_servers[opts['SERVER']])
+	try:
+		manager.start()
+		server = manager.rpc_server
+		from callback import CallbackServer
+		callback_server = CallbackServer(manager.local_callback_port)
+		from bootstrapvz.base.log import LogServer
+		log_server = LogServer()
+		try:
+			callback_server.start(log_server)
+			server.set_log_server(log_server)
+
+			# Everything has been set up, begin the bootstrapping process
+			bootstrap_info = server.run(manifest,
+			                            debug=opts['--debug'],
+			                            pause_on_error=False,
+			                            dry_run=opts['--dry-run'])
+		finally:
+			callback_server.stop()
+	finally:
+		manager.stop()
+	return bootstrap_info
+
+
+def get_opts():
+	"""Creates an argument parser and returns the arguments it has parsed
+	"""
+	from docopt import docopt
+	usage = """bootstrap-vz-remote
+
+Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
+
+Options:
+  --servers <path>   Path to list of build servers
+  --log <path>       Log to given directory [default: /var/log/bootstrap-vz]
+                     If <path> is `-' file logging will be disabled.
+  --pause-on-error   Pause on error, before rollback
+  --dry-run          Don't actually run the tasks
+  --debug            Print debugging information
+  -h, --help         show this help
+	"""
+	return docopt(usage)
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index e9325a8..fc638bc 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -5,7 +5,8 @@ def main():
 	from . import register_deserialization_handlers
 	register_deserialization_handlers()
 	log_forwarder = setup_logging()
-	serve(opts, log_forwarder)
+	server = Server(opts, log_forwarder)
+	server.start()
 
 
 def setup_logging():
@@ -13,42 +14,11 @@ def setup_logging():
 	from bootstrapvz.base.log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
-	from bootstrapvz.base import log
-	file_handler = log.get_file_handler(path='/var/log/bootstrap-vz/remote.log', debug=True)
-	root.addHandler(file_handler)
 	root.addHandler(log_forwarder)
 	root.setLevel(logging.NOTSET)
 	return log_forwarder
 
 
-def serve(opts, log_forwarder):
-	class Server(object):
-
-		def __init__(self):
-			self.stop_serving = False
-
-		def run(self, *args, **kwargs):
-			from bootstrapvz.base.main import run
-			return run(*args, **kwargs)
-
-		def set_log_server(self, server):
-			return log_forwarder.set_server(server)
-
-		def ping(self):
-			return 'pong'
-
-		def stop(self):
-			self.stop_serving = True
-
-	import Pyro4
-	Pyro4.config.COMMTIMEOUT = 0.5
-	daemon = Pyro4.Daemon('localhost', port=int(opts['--listen']), unixsocket=None)
-
-	server = Server()
-	daemon.register(server, 'server')
-	daemon.requestLoop(loopCondition=lambda: not server.stop_serving)
-
-
 def getopts():
 	from docopt import docopt
 	usage = """bootstrap-vz-server
@@ -60,3 +30,32 @@ Options:
   -h, --help         show this help
 """
 	return docopt(usage)
+
+
+class Server(object):
+
+	def __init__(self, opts, log_forwarder):
+		self.stop_serving = False
+		self.log_forwarder = log_forwarder
+		self.listen_port = opts['--listen']
+
+	def start(self):
+		import Pyro4
+		Pyro4.config.COMMTIMEOUT = 0.5
+		daemon = Pyro4.Daemon('localhost', port=int(self.listen_port), unixsocket=None)
+
+		daemon.register(self, 'server')
+		daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
+
+	def run(self, *args, **kwargs):
+		from bootstrapvz.base.main import run
+		return run(*args, **kwargs)
+
+	def set_log_server(self, server):
+		return self.log_forwarder.set_server(server)
+
+	def ping(self):
+		return 'pong'
+
+	def stop(self):
+		self.stop_serving = True

From fd079547642e050a0800eed73b80cdfbeee611e1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:16:07 +0100
Subject: [PATCH 127/345] Remove manual deserialization

---
 bootstrapvz/base/main.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index bea8686..7e1c054 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -93,9 +93,6 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	:params bool pause_on_error: Whether to pause on error, before rollback
 	:params bool dry_run: Don't actually run the tasks
 	"""
-	if isinstance(manifest, dict):
-		from manifest import Manifest
-		manifest = Manifest(path=manifest['path'], data=manifest['data'])
 	# Get the tasklist
 	from tasklist import load_tasks
 	from tasklist import TaskList

From 90f207897e326594f6acb50064d0d30584b1cebb Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:18:14 +0100
Subject: [PATCH 128/345] Ignore build_servers.yml

---
 .gitignore        |  1 +
 build_servers.yml | 23 -----------------------
 2 files changed, 1 insertion(+), 23 deletions(-)
 delete mode 100644 build_servers.yml

diff --git a/.gitignore b/.gitignore
index ac36a3d..d2b3547 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@
 # Testing
 /.coverage
 /.tox/
+/build_servers.yml
diff --git a/build_servers.yml b/build_servers.yml
deleted file mode 100644
index dcfabd1..0000000
--- a/build_servers.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-virtualbox:
-  can_bootstrap:
-    - virtualbox
-    - ec2-s3
-  address: 127.0.0.1
-  port: 2222
-  username: vagrant
-  password: vagrant
-  root_password: vagrant
-  keyfile: /Users/anders/.vagrant.d/insecure_private_key
-  server-bin: /root/bootstrap/bootstrap-vz-server
-ec2:
-  can_bootstrap:
-    - virtualbox
-    - ec2-ebs
-    - ec2-s3
-  address: 127.0.0.1
-  port: 2222
-  username: vagrant
-  password: vagrant
-  root_password: vagrant
-  keyfile: /Users/anders/.vagrant.d/insecure_private_key
-  server-bin: /root/bootstrap/bootstrap-vz-server

From d88d73c7f13ba98a2069d72db19e915faf708191 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:43:04 +0100
Subject: [PATCH 129/345] Allow listening on different ports on remote and
 local

---
 bootstrapvz/base/remote/callback.py        |  7 +++++--
 bootstrapvz/base/remote/remote.py          |  3 ++-
 bootstrapvz/base/remote/server.py          |  6 +++---
 bootstrapvz/base/remote/ssh_rpc_manager.py | 18 ++++++++++++------
 4 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index 865081d..b494e8f 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -2,12 +2,15 @@
 
 class CallbackServer(object):
 
-	def __init__(self, listen_port):
+	def __init__(self, listen_port, remote_port):
 		self.listen_port = listen_port
+		self.remote_port = remote_port
 
 	def start(self, log_server):
 		import Pyro4
-		self.daemon = Pyro4.Daemon('localhost', port=self.listen_port, unixsocket=None)
+		self.daemon = Pyro4.Daemon(host='localhost', port=self.listen_port,
+		                           nathost='localhost', natport=self.remote_port,
+		                           unixsocket=None)
 		self.daemon.register(log_server)
 
 		def serve():
diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/base/remote/remote.py
index dc1cf6e..1f1e96c 100644
--- a/bootstrapvz/base/remote/remote.py
+++ b/bootstrapvz/base/remote/remote.py
@@ -27,7 +27,8 @@ def main():
 		manager.start()
 		server = manager.rpc_server
 		from callback import CallbackServer
-		callback_server = CallbackServer(manager.local_callback_port)
+		callback_server = CallbackServer(listen_port=manager.local_callback_port,
+		                                 remote_port=manager.remote_callback_port)
 		from bootstrapvz.base.log import LogServer
 		log_server = LogServer()
 		try:
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index fc638bc..483bbb4 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -5,7 +5,7 @@ def main():
 	from . import register_deserialization_handlers
 	register_deserialization_handlers()
 	log_forwarder = setup_logging()
-	server = Server(opts, log_forwarder)
+	server = Server(opts['--listen'], log_forwarder)
 	server.start()
 
 
@@ -34,10 +34,10 @@ Options:
 
 class Server(object):
 
-	def __init__(self, opts, log_forwarder):
+	def __init__(self, listen_port, log_forwarder):
 		self.stop_serving = False
 		self.log_forwarder = log_forwarder
-		self.listen_port = opts['--listen']
+		self.listen_port = listen_port
 
 	def start(self):
 		import Pyro4
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index d9521cb..bb33a9b 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -9,13 +9,19 @@ class SSHRPCManager(object):
 	def __init__(self, settings):
 		self.settings = settings
 
+		[self.local_server_port, self.local_callback_port] = self.getNPorts(2)
+		[self.remote_server_port, self.remote_callback_port] = self.getNPorts(2)
+
+	def getNPorts(self, n, port_range=(1024, 65535)):
 		import random
-		self.local_server_port = random.randrange(1024, 65535)
-		self.local_callback_port = random.randrange(1024, 65535)
-		# self.remote_server_port = random.randrange(1024, 65535)
-		# self.remote_callback_port = random.randrange(1024, 65535)
-		self.remote_server_port = self.local_server_port
-		self.remote_callback_port = self.local_callback_port
+		ports = []
+		for i in range(0, n):
+			while True:
+				port = random.randrange(*port_range)
+				if port not in ports:
+					ports.append(port)
+					break
+		return ports
 
 	def start(self):
 		log.debug('Opening SSH connection')

From f8d79f06e3dea3cc338eba9b6780390d09ebb8aa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:43:23 +0100
Subject: [PATCH 130/345] More graceful CallbackServer shutdown

---
 bootstrapvz/base/remote/callback.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index b494e8f..a4fc6d8 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -20,5 +20,7 @@ class CallbackServer(object):
 		self.thread.start()
 
 	def stop(self):
-		self.daemon.shutdown()
-		self.thread.join()
+		if hasattr(self, 'daemon'):
+			self.daemon.shutdown()
+		if hasattr(self, 'thread'):
+			self.thread.join()

From 9fd30e2cc9fcc795fc6d667908f640c2010c01cd Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:44:00 +0100
Subject: [PATCH 131/345] Remove remote/

---
 remote/client.py      | 63 ----------------------------
 remote/server.py      | 97 -------------------------------------------
 remote/ssh_wrapper.py | 28 -------------
 3 files changed, 188 deletions(-)
 delete mode 100755 remote/client.py
 delete mode 100755 remote/server.py
 delete mode 100644 remote/ssh_wrapper.py

diff --git a/remote/client.py b/remote/client.py
deleted file mode 100755
index da3cc35..0000000
--- a/remote/client.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-
-import random
-import Pyro4
-
-# We need to set either a socket communication timeout,
-# or use the select based server. Otherwise the daemon requestLoop
-# will block indefinitely and is never able to evaluate the loopCondition.
-Pyro4.config.COMMTIMEOUT = 0.5
-
-NUM_WORKERS = 5
-
-from ssh_wrapper import RemoteServer
-srv = RemoteServer()
-srv.start()
-
-
-class CallbackHandler(object):
-	workdone = 0
-
-	def done(self, number):
-		print("callback: worker %d reports work is done!" % number)
-		CallbackHandler.workdone += 1
-
-
-class LogServer(object):
-
-	def handle(self, record):
-		print('logging' + record)
-		# import logging
-		# log = logging.getLogger()
-		# (handler.handle(record) for handler in log.handlers)
-
-
-with Pyro4.Daemon('localhost', port=srv.client_port, unixsocket=None) as daemon:
-	# register our callback handler
-	callback = CallbackHandler()
-	daemon.register(callback)
-	logger = LogServer()
-	daemon.register(logger)
-
-	# contact the server and put it to work
-
-	def serve():
-		daemon.requestLoop(loopCondition=lambda: CallbackHandler.workdone < NUM_WORKERS)
-	from threading import Thread
-	thread = Thread(target=serve)
-	thread.start()
-
-	print("creating a bunch of workers")
-	with Pyro4.core.Proxy("PYRO:srv@localhost:" + str(srv.server_port)) as server:
-		server.set_log_server(logger)
-		for _ in range(NUM_WORKERS):
-			worker = server.addworker(callback)   # provide our callback handler!
-			# worker._pyroOneway.add("work")      # to be able to run in the background
-			worker.work(0.5)
-		server.stop()
-
-	print("waiting for all work complete...")
-	thread.join()
-	print("done!")
-
-srv.stop()
diff --git a/remote/server.py b/remote/server.py
deleted file mode 100755
index 081d66d..0000000
--- a/remote/server.py
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env python
-
-import time
-import Pyro4
-import logging
-Pyro4.config.COMMTIMEOUT = 5
-
-log = logging.getLogger(__name__)
-
-class Worker(object):
-	def __init__(self, number, callback):
-		self.number = number
-		self.callback = callback
-		log.info("Worker %d created" % self.number)
-
-	def work(self, amount):
-		print("Worker %d busy..." % self.number)
-		time.sleep(amount)
-		print("Worker %d done. Informing callback client." % self.number)
-		self._pyroDaemon.unregister(self)
-		self.callback.done(self.number)  # invoke the callback object
-
-
-class LogForwarder(logging.Handler):
-
-	def __init__(self, level=logging.NOTSET):
-		self.server = None
-		super(LogForwarder, self).__init__(level)
-
-	def set_server(self, server):
-		self.server = server
-
-	def emit(self, record):
-		if self.server is not None:
-			self.server.handle('hans')
-
-
-class CallbackServer(object):
-
-	def __init__(self):
-		self.number = 0
-		self.serve = True
-
-	def addworker(self, callback):
-		self.number += 1
-		print("server: adding worker %d" % self.number)
-		worker = Worker(self.number, callback)
-		self._pyroDaemon.register(worker)   # make it a Pyro object
-		return worker
-
-	def stop(self):
-		print('called stop()')
-		self.serve = False
-
-	def still_serve(self):
-		print('called still_serve()')
-		return self.serve
-
-	def set_log_server(self, server):
-		import logging
-		log_forwarder = LogForwarder()
-		root = logging.getLogger()
-		root.addHandler(log_forwarder)
-		root.setLevel(logging.NOTSET)
-		log_forwarder.set_server(server)
-
-	def test(self, msg):
-		import logging
-		root = logging.getLogger()
-		root.info(msg)
-
-
-def main():
-	opts = getopts()
-	with Pyro4.Daemon('localhost', port=int(opts['--listen-port']), unixsocket=None) as daemon:
-		obj = CallbackServer()
-		uri = daemon.register(obj, 'srv')
-		print uri
-		print("Server ready.")
-		daemon.requestLoop(loopCondition=lambda: obj.still_serve())
-
-
-def getopts():
-	from docopt import docopt
-	usage = """bootstrap-vz-server
-
-Usage: bootstrap-vz-server [options]
-
-Options:
-  --listen-port <port>   Serve on specified port [default: 46675]
-  --callback-port <port> Connect callback to specified port [default: 46674]
-  -h, --help          show this help
-"""
-	return docopt(usage)
-
-if __name__ == '__main__':
-	main()
diff --git a/remote/ssh_wrapper.py b/remote/ssh_wrapper.py
deleted file mode 100644
index f1f9d5d..0000000
--- a/remote/ssh_wrapper.py
+++ /dev/null
@@ -1,28 +0,0 @@
-
-
-class RemoteServer(object):
-
-	def __init__(self):
-		import random
-		self.server_port = random.randrange(1024, 65535)
-		self.client_port = random.randrange(1024, 65535)
-
-	def start(self):
-		import subprocess
-
-		command = ['ssh', '-i', '/Users/anders/.vagrant.d/insecure_private_key',
-		                  '-t',  # Force pseudo-tty allocation so that server.py quits when we close the connection
-		                  '-p', '2222',
-		                  '-L' + str(self.server_port) + ':localhost:' + str(self.server_port),
-		                  '-R' + str(self.client_port) + ':localhost:' + str(self.client_port),
-		                  'vagrant@localhost',
-		                  '--',
-		                  'sudo', '/root/bootstrap/remote/server.py',
-		                  '--listen-port', str(self.server_port),
-		                  '--callback-port', str(self.client_port)]
-		self.process = subprocess.Popen(args=command)
-		import time
-		time.sleep(2)
-
-	def stop(self):
-		self.process.terminate()

From 1ddc0fbc32e9fc05ba74f549bd3d540a567a8ef6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:51:53 +0100
Subject: [PATCH 132/345] Add comment about random ports

---
 bootstrapvz/base/remote/ssh_rpc_manager.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index bb33a9b..ece1fed 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -9,6 +9,9 @@ class SSHRPCManager(object):
 	def __init__(self, settings):
 		self.settings = settings
 
+		# We can't use :0 because
+		# A: It's quite hard to retrieve the port on the remote after the daemon has started
+		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
 		[self.local_server_port, self.local_callback_port] = self.getNPorts(2)
 		[self.remote_server_port, self.remote_callback_port] = self.getNPorts(2)
 

From 08976ffe07b687db99a8ebeb29d62e88d99b1366 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 20:53:20 +0100
Subject: [PATCH 133/345] Remove unused import

---
 bootstrapvz/base/remote/ssh_rpc_manager.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index ece1fed..558dcfb 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -1,5 +1,3 @@
-from fysom import Fysom
-
 import logging
 log = logging.getLogger(__name__)
 

From 0f4c08e51d79b4ae05818e4aa8b9365ac277bdc2 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 21:12:37 +0100
Subject: [PATCH 134/345] Refactor, logging, comments

---
 bootstrapvz/base/main.py                   |  6 +-
 bootstrapvz/base/remote/__init__.py        |  6 ++
 bootstrapvz/base/remote/callback.py        |  3 +
 bootstrapvz/base/remote/remote.py          | 78 ++++++++++++++--------
 bootstrapvz/base/remote/server.py          |  6 +-
 bootstrapvz/base/remote/ssh_rpc_manager.py | 16 +++--
 6 files changed, 78 insertions(+), 37 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 7e1c054..fd3064d 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -1,9 +1,6 @@
 """Main module containing all the setup necessary for running the bootstrapping process
 """
 
-import logging
-log = logging.getLogger(__name__)
-
 
 def main():
 	"""Main function for invoking the bootstrap process
@@ -12,6 +9,7 @@ def main():
 	"""
 	# Get the commandline arguments
 	opts = get_opts()
+
 	# Require root privileges, except when doing a dry-run where they aren't needed
 	import os
 	if os.geteuid() != 0 and not opts['--dry-run']:
@@ -104,6 +102,8 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	from bootstrapinfo import BootstrapInformation
 	bootstrap_info = BootstrapInformation(manifest=manifest, debug=debug)
 
+	import logging
+	log = logging.getLogger(__name__)
 	try:
 		# Run all the tasks the tasklist has gathered
 		tasklist.run(info=bootstrap_info, dry_run=dry_run)
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/base/remote/__init__.py
index 8e604e5..3c61646 100644
--- a/bootstrapvz/base/remote/__init__.py
+++ b/bootstrapvz/base/remote/__init__.py
@@ -8,6 +8,12 @@ def register_deserialization_handlers():
 	SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo)
 
 
+def unregister_deserialization_handlers():
+	from Pyro4.util import SerializerBase
+	SerializerBase.unregister_dict_to_class('bootstrapvz.base.manifest.Manifest')
+	SerializerBase.unregister_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation')
+
+
 def deserialize_manifest(classname, state):
 	from bootstrapvz.base.manifest import Manifest
 	return Manifest(path=state['path'], data=state['data'])
diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/base/remote/callback.py
index a4fc6d8..25b197b 100644
--- a/bootstrapvz/base/remote/callback.py
+++ b/bootstrapvz/base/remote/callback.py
@@ -1,3 +1,5 @@
+import logging
+log = logging.getLogger(__name__)
 
 
 class CallbackServer(object):
@@ -17,6 +19,7 @@ class CallbackServer(object):
 			self.daemon.requestLoop()
 		from threading import Thread
 		self.thread = Thread(target=serve)
+		log.debug('Starting the callback server')
 		self.thread.start()
 
 	def stop(self):
diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/base/remote/remote.py
index 1f1e96c..aeffa31 100644
--- a/bootstrapvz/base/remote/remote.py
+++ b/bootstrapvz/base/remote/remote.py
@@ -1,6 +1,11 @@
+"""Main module containing all the setup necessary for running the remote bootstrapping process
+"""
+
 
 def main():
 	"""Main function for invoking the bootstrap process remotely
+
+
 	"""
 	# Get the commandline arguments
 	opts = get_opts()
@@ -9,6 +14,7 @@ def main():
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(path=opts['MANIFEST'])
 
+	# Load the build servers
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data(opts['--servers'])
 
@@ -16,35 +22,16 @@ def main():
 	from bootstrapvz.base.main import setup_loggers
 	setup_loggers(opts)
 
+	# Register deserialization handlers for objects
+	# that will pass between server and client
 	from . import register_deserialization_handlers
 	register_deserialization_handlers()
 
-	bootstrap_info = None
-
-	from ssh_rpc_manager import SSHRPCManager
-	manager = SSHRPCManager(build_servers[opts['SERVER']])
-	try:
-		manager.start()
-		server = manager.rpc_server
-		from callback import CallbackServer
-		callback_server = CallbackServer(listen_port=manager.local_callback_port,
-		                                 remote_port=manager.remote_callback_port)
-		from bootstrapvz.base.log import LogServer
-		log_server = LogServer()
-		try:
-			callback_server.start(log_server)
-			server.set_log_server(log_server)
-
-			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = server.run(manifest,
-			                            debug=opts['--debug'],
-			                            pause_on_error=False,
-			                            dry_run=opts['--dry-run'])
-		finally:
-			callback_server.stop()
-	finally:
-		manager.stop()
-	return bootstrap_info
+	# Everything has been set up, connect to the server and begin the bootstrapping process
+	run(manifest,
+	    build_servers[opts['SERVER']],
+	    debug=opts['--debug'],
+	    dry_run=opts['--dry-run'])
 
 
 def get_opts():
@@ -65,3 +52,42 @@ Options:
   -h, --help         show this help
 	"""
 	return docopt(usage)
+
+
+def run(manifest, server, debug=False, dry_run=False):
+	"""Connects to the remote build server, starts an RPC daemin
+	on the other side and initiates a remote bootstrapping procedure
+	"""
+	bootstrap_info = None
+
+	from ssh_rpc_manager import SSHRPCManager
+	manager = SSHRPCManager(server)
+	try:
+		# Connect to the build server and start the RPC daemon
+		manager.start()
+		server = manager.rpc_server
+		# Start a callback server on this side, so that we may receive log entries
+		from callback import CallbackServer
+		callback_server = CallbackServer(listen_port=manager.local_callback_port,
+		                                 remote_port=manager.remote_callback_port)
+		from bootstrapvz.base.log import LogServer
+		log_server = LogServer()
+		try:
+			# Start the callback server (in a background thread)
+			callback_server.start(log_server)
+			# Tell the RPC daemon about the callback server
+			server.set_log_server(log_server)
+
+			# Everything has been set up, begin the bootstrapping process
+			bootstrap_info = server.run(manifest,
+			                            debug=debug,
+			                            # We can't pause the bootstrapping process remotely, yet...
+			                            pause_on_error=False,
+			                            dry_run=dry_run)
+		finally:
+			# Stop the callback server
+			callback_server.stop()
+	finally:
+		# Stop the RPC daemon and close the SSH connection
+		manager.stop()
+	return bootstrap_info
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index 483bbb4..d41d7ea 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -1,3 +1,5 @@
+import logging
+log = logging.getLogger(__name__)
 
 
 def main():
@@ -10,7 +12,6 @@ def main():
 
 
 def setup_logging():
-	import logging
 	from bootstrapvz.base.log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
@@ -52,7 +53,8 @@ class Server(object):
 		return run(*args, **kwargs)
 
 	def set_log_server(self, server):
-		return self.log_forwarder.set_server(server)
+		self.log_forwarder.set_server(server)
+		log.debug('Successfully set the log forwarding server')
 
 	def ping(self):
 		return 'pong'
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index 558dcfb..8d280f6 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -37,7 +37,7 @@ class SSHRPCManager(object):
 		                  'sudo', self.settings['server-bin'],
 		                  '--listen', str(self.remote_server_port)]
 		import sys
-		self.process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
+		self.ssh_process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
 
 		# Check that we can connect to the server
 		try:
@@ -45,7 +45,7 @@ class SSHRPCManager(object):
 			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
 			self.rpc_server = Pyro4.Proxy(server_uri)
 
-			log.debug('Connecting to PYRO')
+			log.debug('Connecting to the RPC daemon')
 			remaining_retries = 5
 			while True:
 				try:
@@ -59,10 +59,14 @@ class SSHRPCManager(object):
 					else:
 						raise e
 		except (Exception, KeyboardInterrupt) as e:
-			self.process.terminate()
+			self.ssh_process.terminate()
 			raise e
 
 	def stop(self):
-		self.rpc_server.stop()
-		self.rpc_server._pyroRelease()
-		self.process.wait()
+		if hasattr(self, 'rpc_server'):
+			log.debug('Stopping the RPC daemon')
+			self.rpc_server.stop()
+			self.rpc_server._pyroRelease()
+		if hasattr(self, 'ssh_process'):
+			log.debug('Waiting for the SSH connection to terminate')
+			self.ssh_process.wait()

From 31c453070feab7adf957d9b9fa6a3dd8a3ab444b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 21:37:35 +0100
Subject: [PATCH 135/345] Add log source to log messages

---
 bootstrapvz/base/log.py | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py
index b406763..0083eb5 100644
--- a/bootstrapvz/base/log.py
+++ b/bootstrapvz/base/log.py
@@ -17,7 +17,7 @@ def get_console_handler(debug, colorize):
 	console_handler = logging.StreamHandler(sys.stderr)
 	if colorize:
 		# We want to colorize the output to the console, so we add a formatter
-		console_handler.setFormatter(ConsoleFormatter())
+		console_handler.setFormatter(ColorFormatter())
 	# Set the log level depending on the debug argument
 	if debug:
 		console_handler.setLevel(logging.DEBUG)
@@ -66,8 +66,23 @@ def get_log_filename(manifest_path):
 	return filename
 
 
-class ConsoleFormatter(logging.Formatter):
-	"""Formats log statements for the console
+class SourceFormatter(logging.Formatter):
+	"""Adds a [source] tag to the log message if it exists
+	The python docs suggest using a LoggingAdapter, but that would mean we'd
+	have to use it everywhere we log something (and only when called remotely),
+	which is not feasible.
+	"""
+
+	def format(self, record):
+		extra = getattr(record, 'extra', {})
+		if 'source' in extra:
+			record.msg = '[{source}] {message}'.format(source=record.extra['source'],
+			                                           message=record.msg)
+		return super(SourceFormatter, self).format(record)
+
+
+class ColorFormatter(SourceFormatter):
+	"""Colorizes log messages depending on the loglevel
 	"""
 	level_colors = {logging.ERROR: 'red',
 	                logging.WARNING: 'magenta',
@@ -75,14 +90,13 @@ class ConsoleFormatter(logging.Formatter):
 	                }
 
 	def format(self, record):
-		if(record.levelno in self.level_colors):
-			# Colorize the message if we have a color for it (DEBUG has no color)
-			from termcolor import colored
-			record.msg = colored(record.msg, self.level_colors[record.levelno])
-		return super(ConsoleFormatter, self).format(record)
+		# Colorize the message if we have a color for it (DEBUG has no color)
+		from termcolor import colored
+		record.msg = colored(record.msg, self.level_colors.get(record.levelno, None))
+		return super(ColorFormatter, self).format(record)
 
 
-class FileFormatter(logging.Formatter):
+class FileFormatter(SourceFormatter):
 	"""Formats log statements for output to file
 	Currently this is just a stub
 	"""
@@ -116,4 +130,6 @@ class LogServer(object):
 		import pickle
 		record = pickle.loads(pickled_record)
 		log = logging.getLogger()
+		record.extra = getattr(record, 'extra', {})
+		record.extra['source'] = 'remote'
 		log.handle(record)

From 6b693429f39ab704504958f4ff6633fc3e5482c3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 21:58:25 +0100
Subject: [PATCH 136/345] Remove old test code

---
 tests/integration/build/__init__.py        |   3 -
 tests/integration/build/local/__init__.py  |   0
 tests/integration/build/local/client.py    |   5 -
 tests/integration/build/remote/__init__.py |   0
 tests/integration/build/remote/client.py   |   5 -
 tests/integration/build/remote/forward.py  | 186 ---------------------
 tests/integration/build/remote/test.py     |  56 -------
 tests/integration/build/settings.yml       |   8 -
 tests/integration/tools/__init__.py        |  27 ---
 9 files changed, 290 deletions(-)
 delete mode 100644 tests/integration/build/__init__.py
 delete mode 100644 tests/integration/build/local/__init__.py
 delete mode 100644 tests/integration/build/local/client.py
 delete mode 100644 tests/integration/build/remote/__init__.py
 delete mode 100644 tests/integration/build/remote/client.py
 delete mode 100644 tests/integration/build/remote/forward.py
 delete mode 100755 tests/integration/build/remote/test.py
 delete mode 100644 tests/integration/build/settings.yml

diff --git a/tests/integration/build/__init__.py b/tests/integration/build/__init__.py
deleted file mode 100644
index 9416fa0..0000000
--- a/tests/integration/build/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-
-def get_client(build_settings):
-	pass
diff --git a/tests/integration/build/local/__init__.py b/tests/integration/build/local/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/integration/build/local/client.py b/tests/integration/build/local/client.py
deleted file mode 100644
index 85473cc..0000000
--- a/tests/integration/build/local/client.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from .. import BaseClient
-
-
-class Client(BaseClient):
-	pass
diff --git a/tests/integration/build/remote/__init__.py b/tests/integration/build/remote/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/integration/build/remote/client.py b/tests/integration/build/remote/client.py
deleted file mode 100644
index 85473cc..0000000
--- a/tests/integration/build/remote/client.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from .. import BaseClient
-
-
-class Client(BaseClient):
-	pass
diff --git a/tests/integration/build/remote/forward.py b/tests/integration/build/remote/forward.py
deleted file mode 100644
index 96e1700..0000000
--- a/tests/integration/build/remote/forward.py
+++ /dev/null
@@ -1,186 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (C) 2003-2007  Robey Pointer <robeypointer@gmail.com>
-#
-# This file is part of paramiko.
-#
-# Paramiko is free software; you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation; either version 2.1 of the License, or (at your option)
-# any later version.
-#
-# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
-# A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Paramiko; if not, write to the Free Software Foundation, Inc.,
-# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
-
-"""
-Sample script showing how to do local port forwarding over paramiko.
-
-This script connects to the requested SSH server and sets up local port
-forwarding (the openssh -L option) from a local port through a tunneled
-connection to a destination reachable from the SSH server machine.
-"""
-
-import getpass
-import os
-import socket
-import select
-try:
-    import SocketServer
-except ImportError:
-    import socketserver as SocketServer
-
-import sys
-from optparse import OptionParser
-
-import paramiko
-
-SSH_PORT = 22
-DEFAULT_PORT = 4000
-
-g_verbose = True
-
-
-class ForwardServer (SocketServer.ThreadingTCPServer):
-    daemon_threads = True
-    allow_reuse_address = True
-    
-
-class Handler (SocketServer.BaseRequestHandler):
-
-    def handle(self):
-        try:
-            chan = self.ssh_transport.open_channel('direct-tcpip',
-                                                   (self.chain_host, self.chain_port),
-                                                   self.request.getpeername())
-        except Exception as e:
-            verbose('Incoming request to %s:%d failed: %s' % (self.chain_host,
-                                                              self.chain_port,
-                                                              repr(e)))
-            return
-        if chan is None:
-            verbose('Incoming request to %s:%d was rejected by the SSH server.' %
-                    (self.chain_host, self.chain_port))
-            return
-
-        verbose('Connected!  Tunnel open %r -> %r -> %r' % (self.request.getpeername(),
-                                                            chan.getpeername(), (self.chain_host, self.chain_port)))
-        while True:
-            r, w, x = select.select([self.request, chan], [], [])
-            if self.request in r:
-                data = self.request.recv(1024)
-                if len(data) == 0:
-                    break
-                chan.send(data)
-            if chan in r:
-                data = chan.recv(1024)
-                if len(data) == 0:
-                    break
-                self.request.send(data)
-                
-        peername = self.request.getpeername()
-        chan.close()
-        self.request.close()
-        verbose('Tunnel closed from %r' % (peername,))
-
-
-def forward_tunnel(local_port, remote_host, remote_port, transport):
-    # this is a little convoluted, but lets me configure things for the Handler
-    # object.  (SocketServer doesn't give Handlers any way to access the outer
-    # server normally.)
-    class SubHander (Handler):
-        chain_host = remote_host
-        chain_port = remote_port
-        ssh_transport = transport
-    ForwardServer(('', local_port), SubHander).serve_forever()
-
-
-def verbose(s):
-    if g_verbose:
-        print(s)
-
-
-HELP = """\
-Set up a forward tunnel across an SSH server, using paramiko. A local port
-(given with -p) is forwarded across an SSH session to an address:port from
-the SSH server. This is similar to the openssh -L option.
-"""
-
-
-def get_host_port(spec, default_port):
-    "parse 'hostname:22' into a host and port, with the port optional"
-    args = (spec.split(':', 1) + [default_port])[:2]
-    args[1] = int(args[1])
-    return args[0], args[1]
-
-
-def parse_options():
-    global g_verbose
-    
-    parser = OptionParser(usage='usage: %prog [options] <ssh-server>[:<server-port>]',
-                          version='%prog 1.0', description=HELP)
-    parser.add_option('-q', '--quiet', action='store_false', dest='verbose', default=True,
-                      help='squelch all informational output')
-    parser.add_option('-p', '--local-port', action='store', type='int', dest='port',
-                      default=DEFAULT_PORT,
-                      help='local port to forward (default: %d)' % DEFAULT_PORT)
-    parser.add_option('-u', '--user', action='store', type='string', dest='user',
-                      default=getpass.getuser(),
-                      help='username for SSH authentication (default: %s)' % getpass.getuser())
-    parser.add_option('-K', '--key', action='store', type='string', dest='keyfile',
-                      default=None,
-                      help='private key file to use for SSH authentication')
-    parser.add_option('', '--no-key', action='store_false', dest='look_for_keys', default=True,
-                      help='don\'t look for or use a private key file')
-    parser.add_option('-P', '--password', action='store_true', dest='readpass', default=False,
-                      help='read password (for key or password auth) from stdin')
-    parser.add_option('-r', '--remote', action='store', type='string', dest='remote', default=None, metavar='host:port',
-                      help='remote host and port to forward to')
-    options, args = parser.parse_args()
-
-    if len(args) != 1:
-        parser.error('Incorrect number of arguments.')
-    if options.remote is None:
-        parser.error('Remote address required (-r).')
-    
-    g_verbose = options.verbose
-    server_host, server_port = get_host_port(args[0], SSH_PORT)
-    remote_host, remote_port = get_host_port(options.remote, SSH_PORT)
-    return options, (server_host, server_port), (remote_host, remote_port)
-
-
-def main():
-    options, server, remote = parse_options()
-    
-    password = None
-    if options.readpass:
-        password = getpass.getpass('Enter SSH password: ')
-    
-    client = paramiko.SSHClient()
-    client.load_system_host_keys()
-    client.set_missing_host_key_policy(paramiko.WarningPolicy())
-
-    verbose('Connecting to ssh host %s:%d ...' % (server[0], server[1]))
-    try:
-        client.connect(server[0], server[1], username=options.user, key_filename=options.keyfile,
-                       look_for_keys=options.look_for_keys, password=password)
-    except Exception as e:
-        print('*** Failed to connect to %s:%d: %r' % (server[0], server[1], e))
-        sys.exit(1)
-
-    verbose('Now forwarding port %d to %s:%d ...' % (options.port, remote[0], remote[1]))
-
-    try:
-        forward_tunnel(options.port, remote[0], remote[1], client.get_transport())
-    except KeyboardInterrupt:
-        print('C-c: Port forwarding stopped.')
-        sys.exit(0)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/tests/integration/build/remote/test.py b/tests/integration/build/remote/test.py
deleted file mode 100755
index abe74a4..0000000
--- a/tests/integration/build/remote/test.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-from remote.ssh_rpc_client import SSHRPCClient
-
-import logging
-log = logging.getLogger(__name__)
-
-
-def main():
-	import os.path
-
-	settings_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'settings.yml'))
-	with open(settings_path, 'r') as stream:
-		import yaml
-		settings = yaml.safe_load(stream)
-
-	bootstrapvz_root = os.path.normpath(os.path.join(os.path.dirname(__file__), '../../../'))
-	import sys
-	sys.path.append(bootstrapvz_root)
-
-	from bootstrapvz.base.log import get_console_handler
-	console_handler = get_console_handler(debug=True)
-
-	root_logger = logging.getLogger()
-	root_logger.setLevel(logging.NOTSET)
-	root_logger.addHandler(console_handler)
-
-	rpc_server = SSHRPCClient(settings)
-	try:
-		rpc_server.start_server()
-		log.info('connecting to Pyro on remote')
-		import Pyro4
-		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
-		main = Pyro4.Proxy(main_uri)
-		log.info('running command')
-		remaining_retries = 5
-		while True:
-			try:
-				main.run('eogubhswg')
-				break
-			except Pyro4.errors.ConnectionClosedError as e:
-				if remaining_retries > 0:
-					remaining_retries -= 1
-					from time import sleep
-					sleep(2)
-				else:
-					raise e
-		log.info('stopping server')
-		rpc_server.stop_server()
-	except (Exception, KeyboardInterrupt) as e:
-		log.error(e.__class__.__name__ + ': ' + str(e))
-	finally:
-		print 'cleaning up'
-		rpc_server.cleanup()
-
-if __name__ == '__main__':
-	main()
diff --git a/tests/integration/build/settings.yml b/tests/integration/build/settings.yml
deleted file mode 100644
index 2788774..0000000
--- a/tests/integration/build/settings.yml
+++ /dev/null
@@ -1,8 +0,0 @@
-build_host:
-  address: 127.0.0.1
-  port: 2222
-  username: vagrant
-  password: vagrant
-  root_password: vagrant
-  keyfile: /Users/anders/.vagrant.d/insecure_private_key
-  server-bin: /root/bootstrap/tests/integration/build/server.py
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 3bdad63..0c9f002 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -27,30 +27,3 @@ def bootstrap(manifest, build_settings):
 	# run(manifest)
 	from ..image import Image
 	return Image()
-
-
-def test_instance(instance, build_settings):
-	pass
-
-
-def get_remote_run(build_settings):
-	from ..build.client import SSHRPCServer
-
-	rpc_server = SSHRPCServer(settings)
-	try:
-		rpc_server.start()
-		from time import sleep
-		sleep(2)
-		log.info('connection to Pyro on remote')
-		import Pyro4
-		main_uri = 'PYRO:runner@localhost:{local_port}'.format(local_port=rpc_server.local_port)
-		main = Pyro4.Proxy(main_uri)
-		log.info('running command')
-		main.run('eogubhswg')
-		log.info('stopping server')
-		rpc_server.stop()
-	except (Exception, KeyboardInterrupt) as e:
-		log.error(e.__class__.__name__ + ': ' + str(e))
-	finally:
-		print 'cleaning up'
-		rpc_server.cleanup()

From 03a48b940772706595cba979e10844625e035b73 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 21:59:12 +0100
Subject: [PATCH 137/345] Simplify loading of partial manifests

---
 tests/integration/__init__.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index ccce1cc..04befef 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -1,9 +1,11 @@
 import os.path
 from bootstrapvz.common.tools import load_data
 
-combine_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests')
-manifests = {'base': load_data(os.path.join(combine_manifests_path, 'base.yml')),
-             'partitioned': load_data(os.path.join(combine_manifests_path, 'partitioned.yml')),
-             'unpartitioned': load_data(os.path.join(combine_manifests_path, 'unpartitioned.yml')),
-             }
+partial_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests')
+import glob
+partial_yaml = glob.glob(os.path.join(partial_manifests_path, '*.yml'))
+partial_json = glob.glob(os.path.join(partial_manifests_path, '*.json'))
+manifests = {}
+for path in partial_yaml + partial_json:
+	manifests[os.path.basename(path)] = load_data(path)
 build_settings = load_data(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'build_settings.yml'))

From 86afbaf929e53f218e649cf754250a2f83946cf0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 25 Nov 2014 22:45:03 +0100
Subject: [PATCH 138/345] Clean up integration testing

---
 build_settings.yml                       |  3 +++
 tests/integration/__init__.py            | 11 ++------
 tests/integration/manifests/__init__.py  | 15 +++++++++++
 tests/integration/manifests/stable64.yml |  4 +++
 tests/integration/tools/__init__.py      | 13 +++++++++-
 tests/integration/virtualbox_tests.py    | 33 ++++++++++++------------
 tox.ini                                  |  2 +-
 7 files changed, 54 insertions(+), 27 deletions(-)
 create mode 100644 build_settings.yml
 create mode 100644 tests/integration/manifests/__init__.py
 create mode 100644 tests/integration/manifests/stable64.yml

diff --git a/build_settings.yml b/build_settings.yml
new file mode 100644
index 0000000..87b0ba2
--- /dev/null
+++ b/build_settings.yml
@@ -0,0 +1,3 @@
+---
+virtualbox:
+  guest_additions: /root/images/VBoxGuestAdditions.iso
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index 04befef..ed30250 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -1,11 +1,4 @@
-import os.path
 from bootstrapvz.common.tools import load_data
 
-partial_manifests_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'manifests')
-import glob
-partial_yaml = glob.glob(os.path.join(partial_manifests_path, '*.yml'))
-partial_json = glob.glob(os.path.join(partial_manifests_path, '*.json'))
-manifests = {}
-for path in partial_yaml + partial_json:
-	manifests[os.path.basename(path)] = load_data(path)
-build_settings = load_data(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'build_settings.yml'))
+# tox makes sure that the cwd is the project root
+build_settings = load_data('build_settings.yml')
diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
new file mode 100644
index 0000000..7d84774
--- /dev/null
+++ b/tests/integration/manifests/__init__.py
@@ -0,0 +1,15 @@
+import os.path
+import glob
+from bootstrapvz.common.tools import load_data
+
+partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml'))
+partial_yaml = glob.glob(os.path.join(os.path.dirname(__file__), '*.json'))
+
+def dictkey(path):
+	return 
+# yaml overrides json
+partials = {}
+for path in partial_json + partial_yaml:
+	key = os.path.splitext(os.path.basename(path))[0]
+	partials[key] = load_data(path)
+	
diff --git a/tests/integration/manifests/stable64.yml b/tests/integration/manifests/stable64.yml
new file mode 100644
index 0000000..4489269
--- /dev/null
+++ b/tests/integration/manifests/stable64.yml
@@ -0,0 +1,4 @@
+---
+system:
+  release: wheezy
+  architecture: amd64
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 0c9f002..d057444 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,4 +1,6 @@
+from bootstrapvz.common.tools import load_data
 
+build_servers = load_data('build_servers.yml')
 
 # Snatched from here: http://stackoverflow.com/a/7205107
 def merge_dicts(*args):
@@ -19,11 +21,20 @@ def merge_dicts(*args):
 	return reduce(merge, args, {})
 
 
-def bootstrap(manifest, build_settings):
+def bootstrap(manifest):
 	# if 'build_host' in build_settings:
 	# 	run = get_remote_run(build_settings)
 	# else:
 	# 	run = __import__('bootstrapvz.base.run')
 	# run(manifest)
+	from bootstrapvz.base.remote.remote import run
+	run(manifest,
+	    build_servers['virtualbox'],
+	    debug=True,
+	    dry_run=True)
+
 	from ..image import Image
 	return Image()
+
+def test_instance(instance):
+	pass
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index d3b317b..3fc641a 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,28 +1,29 @@
 import tools
-from . import manifests
+from manifests import partials
 from . import build_settings
 
 
 def test_virtualbox_unpartitioned_extlinux():
-	specific_settings = {}
-	specific_settings['provider'] = {'name': 'virtualbox',
-	                                 'guest_additions': build_settings['virtualbox']['guest_additions']}
-	specific_settings['system'] = {'release': 'wheezy',
-	                               'architecture': 'amd64',
-	                               'bootloader': 'extlinux'}
-	specific_settings['volume'] = {'backing': 'vdi',
-	                               'partitions': {'type': 'msdos'}}
-	manifest = tools.merge_dicts(manifests['base'], manifests['unpartitioned'], specific_settings)
+	import yaml
+	specific_settings = yaml.load("""
+provider:
+  name: virtualbox
+  guest_additions: {guest_additions}
+system:
+  bootloader: extlinux
+volume:
+  backing: vdi
+  partitions:
+    type: msdos
+""".format(guest_additions=build_settings['virtualbox']['guest_additions']))
+	manifest = tools.merge_dicts(partials['base'], partials['stable64'],
+	                             partials['unpartitioned'], specific_settings)
 
-	client = tools.get_client(build_settings['virtualbox'])
-
-	image = client.bootstrap(manifest, build_settings['virtualbox'])
+	image = tools.bootstrap(manifest)
 	instance = image.create_instance()
 	instance.boot()
 
-	tools.test_instance(instance, build_settings['virtualbox'])
+	tools.test_instance(instance)
 
 	instance.destroy()
 	image.destroy()
-
-	client.shutdown()
diff --git a/tox.ini b/tox.ini
index 122ee83..660e716 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,7 +3,7 @@ ignore = E101,E221,E241,E501,W191
 max-line-length = 110
 
 [tox]
-envlist = flake8, integration
+envlist = flake8, unit
 
 [testenv:flake8]
 deps = flake8

From 0f8dbb7ac39a95d0a7f5c4be674e803d3b8d4aad Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 26 Nov 2014 01:00:41 +0100
Subject: [PATCH 139/345] Install Pyro4 dep when testing

---
 tox.ini | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 660e716..4529ea9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,7 +16,9 @@ deps =
 commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:integration]
-deps = nose
+deps =
+    nose
+    Pyro4 >= 4.26
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:docs]

From 65b9e10ce3a89a91b1fe68481b1cc08994695b00 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 00:33:42 +0100
Subject: [PATCH 140/345] VirtualBox integration

---
 bootstrapvz/base/remote/ssh_rpc_manager.py |  2 +-
 build_settings.yml                         |  3 -
 tests/integration/__init__.py              |  4 --
 tests/integration/image/__init__.py        | 21 -------
 tests/integration/tools/__init__.py        | 41 ++++++++-----
 tests/integration/tools/build_servers.py   | 40 +++++++++++++
 tests/integration/tools/images.py          | 25 ++++++++
 tests/integration/tools/instances.py       | 69 ++++++++++++++++++++++
 tests/integration/virtualbox_tests.py      | 35 ++++++++---
 tox.ini                                    |  1 +
 10 files changed, 188 insertions(+), 53 deletions(-)
 delete mode 100644 build_settings.yml
 delete mode 100644 tests/integration/image/__init__.py
 create mode 100644 tests/integration/tools/build_servers.py
 create mode 100644 tests/integration/tools/images.py
 create mode 100644 tests/integration/tools/instances.py

diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/base/remote/ssh_rpc_manager.py
index 8d280f6..39fd037 100644
--- a/bootstrapvz/base/remote/ssh_rpc_manager.py
+++ b/bootstrapvz/base/remote/ssh_rpc_manager.py
@@ -34,7 +34,7 @@ class SSHRPCManager(object):
 		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
 		                  self.settings['username'] + '@' + self.settings['address'],
 		                  '--',
-		                  'sudo', self.settings['server-bin'],
+		                  'sudo', self.settings['server_bin'],
 		                  '--listen', str(self.remote_server_port)]
 		import sys
 		self.ssh_process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
diff --git a/build_settings.yml b/build_settings.yml
deleted file mode 100644
index 87b0ba2..0000000
--- a/build_settings.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-virtualbox:
-  guest_additions: /root/images/VBoxGuestAdditions.iso
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index ed30250..e69de29 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -1,4 +0,0 @@
-from bootstrapvz.common.tools import load_data
-
-# tox makes sure that the cwd is the project root
-build_settings = load_data('build_settings.yml')
diff --git a/tests/integration/image/__init__.py b/tests/integration/image/__init__.py
deleted file mode 100644
index 67bc2b1..0000000
--- a/tests/integration/image/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-
-
-class Image(object):
-
-	def create_instance(self):
-		return Instance()
-
-	def destroy(self):
-		pass
-
-
-class Instance(object):
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index d057444..dbe0446 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,6 +1,7 @@
 from bootstrapvz.common.tools import load_data
+from build_servers import LocalBuildServer
+from build_servers import RemoteBuildServer
 
-build_servers = load_data('build_servers.yml')
 
 # Snatched from here: http://stackoverflow.com/a/7205107
 def merge_dicts(*args):
@@ -21,20 +22,30 @@ def merge_dicts(*args):
 	return reduce(merge, args, {})
 
 
-def bootstrap(manifest):
-	# if 'build_host' in build_settings:
-	# 	run = get_remote_run(build_settings)
-	# else:
-	# 	run = __import__('bootstrapvz.base.run')
-	# run(manifest)
-	from bootstrapvz.base.remote.remote import run
-	run(manifest,
-	    build_servers['virtualbox'],
-	    debug=True,
-	    dry_run=True)
+def pick_build_server(manifest):
+	if manifest['provider']['name'] == 'ec2':
+		img_type = 'ec2-' + manifest['volume']['backing']
+	else:
+		img_type = manifest['provider']['name']
 
-	from ..image import Image
-	return Image()
+	# tox makes sure that the cwd is the project root
+	build_servers = load_data('build_servers.yml')
+	settings = next((server for name, server in build_servers.iteritems() if img_type in server['can_bootstrap']), None)
+	if settings['type'] == 'local':
+		return LocalBuildServer(settings)
+	else:
+		return RemoteBuildServer(settings)
 
-def test_instance(instance):
+
+def bootstrap(manifest, build_server):
+	if isinstance(build_server, LocalBuildServer):
+		from bootstrapvz.base.main import run
+		bootstrap_info = run(manifest)
+	else:
+		from bootstrapvz.base.remote.remote import run
+		bootstrap_info = run(manifest, build_server.settings)
+	return bootstrap_info
+
+
+def test(instance):
 	pass
diff --git a/tests/integration/tools/build_servers.py b/tests/integration/tools/build_servers.py
new file mode 100644
index 0000000..c103624
--- /dev/null
+++ b/tests/integration/tools/build_servers.py
@@ -0,0 +1,40 @@
+from bootstrapvz.common.tools import log_check_call
+
+
+class BuildServer(object):
+
+	def __init__(self, settings):
+		self.settings = settings
+		self.build_settings = settings.get('build_settings', None)
+		self.can_bootstrap = settings['can_bootstrap']
+		self.release = settings.get('release', None)
+
+
+class LocalBuildServer(BuildServer):
+	pass
+
+
+class RemoteBuildServer(BuildServer):
+
+	def __init__(self, settings):
+		self.address = settings['address']
+		self.port = settings['port']
+		self.username = settings['username']
+		self.password = settings['password']
+		self.root_password = settings['root_password']
+		self.keyfile = settings['keyfile']
+		self.server_bin = settings['server_bin']
+		super(RemoteBuildServer, self).__init__(settings)
+
+	def download(self, src, dst):
+		src_arg = '{user}@{host}:{path}'.format(self.username, self.address, src)
+		log_check_call(['scp', '-i', self.keyfile, '-P', self.port,
+		                src_arg, dst])
+
+	def delete(self, path):
+		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
+		                  '-p', str(self.settings['port']),
+		                  self.username + '@' + self.address,
+		                  '--',
+		                  'sudo', 'rm', path]
+		log_check_call(ssh_cmd)
diff --git a/tests/integration/tools/images.py b/tests/integration/tools/images.py
new file mode 100644
index 0000000..928338b
--- /dev/null
+++ b/tests/integration/tools/images.py
@@ -0,0 +1,25 @@
+
+
+class Image(object):
+
+	def __init__(self, manifest):
+		self.manifest = manifest
+
+	def destroy(self):
+		pass
+
+
+class VirtualBoxImage(Image):
+
+	def __init__(self, manifest, image_path):
+		super(VirtualBoxImage, self).__init__(manifest)
+		self.image_path = image_path
+		self.medium = self.vbox.open_medium(location=self.image.image_path,
+		                                    decive_type=self.vbox.library.DeviceType.HardDisk,
+		                                    access_mode=self.vbox.library.AccessMode.read_only,
+		                                    force_new_uuid=False)
+
+	def destroy(self):
+		self.medium.delete_storage()
+		import os
+		os.remove(self.image_path)
diff --git a/tests/integration/tools/instances.py b/tests/integration/tools/instances.py
new file mode 100644
index 0000000..075fc20
--- /dev/null
+++ b/tests/integration/tools/instances.py
@@ -0,0 +1,69 @@
+from bootstrapvz.common.tools import log_check_call
+
+
+class Instance(object):
+
+	def __init__(self, name, image):
+		self.name = name
+		self.image = image
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass
+
+
+class VirtualBoxInstance(Instance):
+
+	cpus = 1
+	memory = 256
+
+	def __init__(self, name, image):
+		super(VirtualBoxInstance, self).__init__(name, image)
+		import virtualbox
+		self.vbox = virtualbox.VirtualBox()
+
+	def create(self):
+		if self.image.manifest['system']['architecture'] == 'x86':
+			os_type = 'Debian'
+		else:
+			os_type = 'Debian_64'
+		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
+		                                        groups=[], os_type_id=os_type, flags='')
+		self.machine.save_settings()
+		self.machine.cpu_count = self.cpus
+		self.machine.memory_size = self.memory
+		self.machine.attach_device(name='root', controller_port=0, device=0,
+		                           type_p=self.vbox.library.DeviceType.HardDisk,
+		                           medium=self.image.medium)
+		self.vbox.register_machine(self.machine)
+		# [self.uuid] = log_check_call(['VBoxManage', 'createvm'
+		#                               '--name', self.name])
+		# log_check_call(['VBoxManage', 'modifyvm', self.uuid,
+		#                 '--cpus', self.cpus,
+		#                 '--memory', self.memory])
+		# log_check_call(['VBoxManage', 'storageattach', self.uuid,
+		#                 '--storagectl', '"SATA Controller"',
+		#                 '--device', '0',
+		#                 '--port', '0',
+		#                 '--type', 'hdd',
+		#                 '--medium', self.image.image_path])
+
+	def boot(self):
+		self.session = self.vbox.Session()
+		self.machine.launch_vm_process(self.session, 'headless')
+		# log_check_call(['VBoxManage', 'startvm', self.uuid,
+		#                 '--type', 'headless'])
+
+	def shutdown(self):
+		self.session.console.power_down()
+		log_check_call(['VBoxManage', 'stopvm', self.uuid,
+		                '--type', 'headless'])
+
+	def destroy(self):
+		self.machine.unregister(self.vbox.CleanupMode.full)
+		self.machine.remove(delete=True)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 3fc641a..21bc847 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,6 +1,5 @@
 import tools
 from manifests import partials
-from . import build_settings
 
 
 def test_virtualbox_unpartitioned_extlinux():
@@ -8,22 +7,40 @@ def test_virtualbox_unpartitioned_extlinux():
 	specific_settings = yaml.load("""
 provider:
   name: virtualbox
-  guest_additions: {guest_additions}
 system:
   bootloader: extlinux
 volume:
   backing: vdi
   partitions:
     type: msdos
-""".format(guest_additions=build_settings['virtualbox']['guest_additions']))
+""")
 	manifest = tools.merge_dicts(partials['base'], partials['stable64'],
 	                             partials['unpartitioned'], specific_settings)
 
-	image = tools.bootstrap(manifest)
-	instance = image.create_instance()
-	instance.boot()
+	build_server = tools.pick_build_server(manifest)
+	manifest['provider']['guest_additions'] = build_server.build_settings['guest_additions']
 
-	tools.test_instance(instance)
+	bootstrap_info = tools.bootstrap(manifest, build_server)
 
-	instance.destroy()
-	image.destroy()
+	if isinstance(build_server, tools.build_servers.LocalBuildServer):
+		image_path = bootstrap_info.volume.image_path
+	else:
+		import tempfile
+		handle, image_path = tempfile.mkstemp()
+		handle.close()
+		build_server.download(bootstrap_info.volume.image_path, image_path)
+		build_server.delete(bootstrap_info.volume.image_path)
+
+	try:
+		image = tools.images.VirtualBoxImage(manifest, image_path)
+
+		instance = tools.instances.VirtualBoxInstance(image)
+		instance.create()
+		instance.boot()
+
+		tools.test(instance)
+	finally:
+		if 'instance' in locals():
+			instance.destroy()
+		if 'image' in locals():
+			image.destroy()
diff --git a/tox.ini b/tox.ini
index 4529ea9..0b5ef44 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,6 +19,7 @@ commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz -
 deps =
     nose
     Pyro4 >= 4.26
+    pyvbox >= 0.2.0
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
 
 [testenv:docs]

From c5bc45218eaef64682619fb1dab1160f1e055138 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 13:00:21 +0100
Subject: [PATCH 141/345] Fix documentation

---
 bootstrapvz/base/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index fd3064d..d949bcd 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -120,8 +120,8 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 		# Create a useful little function for the provider and plugins to use,
 		# when figuring out what tasks should be added to the rollback list.
 		def counter_task(taskset, task, counter):
-			"""counter_task() adds the second argument to the rollback tasklist
-			if the first argument is present in the list of completed tasks
+			"""counter_task() adds the third argument to the rollback tasklist
+			if the second argument is present in the list of completed tasks
 
 			:param set taskset: The taskset to add the rollback task to
 			:param Task task: The task to look for in the completed tasks list

From a7a3161d66d9387d39098f22bf7d66395e932e28 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 13:58:10 +0100
Subject: [PATCH 142/345] Require Pyro4 >= 4.30

---
 bootstrapvz/base/remote/server.py | 8 +++++++-
 tox.ini                           | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/base/remote/server.py
index d41d7ea..045ecac 100644
--- a/bootstrapvz/base/remote/server.py
+++ b/bootstrapvz/base/remote/server.py
@@ -1,4 +1,7 @@
+import Pyro4
 import logging
+
+Pyro4.config.REQUIRE_EXPOSE = True
 log = logging.getLogger(__name__)
 
 
@@ -41,23 +44,26 @@ class Server(object):
 		self.listen_port = listen_port
 
 	def start(self):
-		import Pyro4
 		Pyro4.config.COMMTIMEOUT = 0.5
 		daemon = Pyro4.Daemon('localhost', port=int(self.listen_port), unixsocket=None)
 
 		daemon.register(self, 'server')
 		daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
 
+	@Pyro4.expose
 	def run(self, *args, **kwargs):
 		from bootstrapvz.base.main import run
 		return run(*args, **kwargs)
 
+	@Pyro4.expose
 	def set_log_server(self, server):
 		self.log_forwarder.set_server(server)
 		log.debug('Successfully set the log forwarding server')
 
+	@Pyro4.expose
 	def ping(self):
 		return 'pong'
 
+	@Pyro4.expose
 	def stop(self):
 		self.stop_serving = True
diff --git a/tox.ini b/tox.ini
index 0b5ef44..ff693dc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,7 +18,7 @@ commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz -
 [testenv:integration]
 deps =
     nose
-    Pyro4 >= 4.26
+    Pyro4 >= 4.30
     pyvbox >= 0.2.0
 commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
 

From 209651ef3809846328ee236321400c656a3a1a85 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 14:09:21 +0100
Subject: [PATCH 143/345] Dry run working in tests

---
 bootstrapvz/base/remote/remote.py        |  2 --
 tests/integration/tools/__init__.py      |  5 +++++
 tests/integration/tools/build_servers.py |  2 +-
 tests/integration/virtualbox_tests.py    | 12 +++++++-----
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/base/remote/remote.py
index aeffa31..6dc9a37 100644
--- a/bootstrapvz/base/remote/remote.py
+++ b/bootstrapvz/base/remote/remote.py
@@ -4,8 +4,6 @@
 
 def main():
 	"""Main function for invoking the bootstrap process remotely
-
-
 	"""
 	# Get the commandline arguments
 	opts = get_opts()
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index dbe0446..2ca8b8c 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -2,6 +2,11 @@ from bootstrapvz.common.tools import load_data
 from build_servers import LocalBuildServer
 from build_servers import RemoteBuildServer
 
+# Register deserialization handlers for objects
+# that will pass between server and client
+from bootstrapvz.base.remote import register_deserialization_handlers
+register_deserialization_handlers()
+
 
 # Snatched from here: http://stackoverflow.com/a/7205107
 def merge_dicts(*args):
diff --git a/tests/integration/tools/build_servers.py b/tests/integration/tools/build_servers.py
index c103624..95daf41 100644
--- a/tests/integration/tools/build_servers.py
+++ b/tests/integration/tools/build_servers.py
@@ -5,7 +5,7 @@ class BuildServer(object):
 
 	def __init__(self, settings):
 		self.settings = settings
-		self.build_settings = settings.get('build_settings', None)
+		self.build_settings = settings.get('build_settings', {})
 		self.can_bootstrap = settings['can_bootstrap']
 		self.release = settings.get('release', None)
 
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 21bc847..ea7eaed 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,10 +1,11 @@
 import tools
 from manifests import partials
+from bootstrapvz.base.manifest import Manifest
 
 
 def test_virtualbox_unpartitioned_extlinux():
 	import yaml
-	specific_settings = yaml.load("""
+	manifest_data = yaml.load("""
 provider:
   name: virtualbox
 system:
@@ -14,11 +15,12 @@ volume:
   partitions:
     type: msdos
 """)
-	manifest = tools.merge_dicts(partials['base'], partials['stable64'],
-	                             partials['unpartitioned'], specific_settings)
+	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
+	                                  partials['unpartitioned'], manifest_data)
 
-	build_server = tools.pick_build_server(manifest)
-	manifest['provider']['guest_additions'] = build_server.build_settings['guest_additions']
+	build_server = tools.pick_build_server(manifest_data)
+	manifest_data['provider']['guest_additions'] = build_server.build_settings['guest_additions']
+	manifest = Manifest(data=manifest_data)
 
 	bootstrap_info = tools.bootstrap(manifest, build_server)
 

From 909e5cee469ca8f9b45ce52ab4db98641293fc96 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 14:13:43 +0100
Subject: [PATCH 144/345] Put remote module into pkg root

---
 bootstrap-vz-remote                                   | 2 +-
 bootstrap-vz-server                                   | 2 +-
 bootstrapvz/{base => }/remote/__init__.py             | 0
 bootstrapvz/{base => }/remote/callback.py             | 0
 bootstrapvz/{base/remote/remote.py => remote/main.py} | 2 ++
 bootstrapvz/{base => }/remote/server.py               | 0
 bootstrapvz/{base => }/remote/ssh_rpc_manager.py      | 0
 tests/integration/tools/__init__.py                   | 4 ++--
 8 files changed, 6 insertions(+), 4 deletions(-)
 rename bootstrapvz/{base => }/remote/__init__.py (100%)
 rename bootstrapvz/{base => }/remote/callback.py (100%)
 rename bootstrapvz/{base/remote/remote.py => remote/main.py} (97%)
 rename bootstrapvz/{base => }/remote/server.py (100%)
 rename bootstrapvz/{base => }/remote/ssh_rpc_manager.py (100%)

diff --git a/bootstrap-vz-remote b/bootstrap-vz-remote
index b5050b8..d7b7254 100755
--- a/bootstrap-vz-remote
+++ b/bootstrap-vz-remote
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 
 if __name__ == '__main__':
-	from bootstrapvz.base.remote.remote import main
+	from bootstrapvz.remote.main import main
 	main()
diff --git a/bootstrap-vz-server b/bootstrap-vz-server
index a79f005..bf941a0 100755
--- a/bootstrap-vz-server
+++ b/bootstrap-vz-server
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
 
 if __name__ == '__main__':
-	from bootstrapvz.base.remote.server import main
+	from bootstrapvz.remote.server import main
 	main()
diff --git a/bootstrapvz/base/remote/__init__.py b/bootstrapvz/remote/__init__.py
similarity index 100%
rename from bootstrapvz/base/remote/__init__.py
rename to bootstrapvz/remote/__init__.py
diff --git a/bootstrapvz/base/remote/callback.py b/bootstrapvz/remote/callback.py
similarity index 100%
rename from bootstrapvz/base/remote/callback.py
rename to bootstrapvz/remote/callback.py
diff --git a/bootstrapvz/base/remote/remote.py b/bootstrapvz/remote/main.py
similarity index 97%
rename from bootstrapvz/base/remote/remote.py
rename to bootstrapvz/remote/main.py
index 6dc9a37..aa0d719 100644
--- a/bootstrapvz/base/remote/remote.py
+++ b/bootstrapvz/remote/main.py
@@ -46,6 +46,8 @@ Options:
                      If <path> is `-' file logging will be disabled.
   --pause-on-error   Pause on error, before rollback
   --dry-run          Don't actually run the tasks
+  --color=auto|always|never
+                     Colorize the console output [default: auto]
   --debug            Print debugging information
   -h, --help         show this help
 	"""
diff --git a/bootstrapvz/base/remote/server.py b/bootstrapvz/remote/server.py
similarity index 100%
rename from bootstrapvz/base/remote/server.py
rename to bootstrapvz/remote/server.py
diff --git a/bootstrapvz/base/remote/ssh_rpc_manager.py b/bootstrapvz/remote/ssh_rpc_manager.py
similarity index 100%
rename from bootstrapvz/base/remote/ssh_rpc_manager.py
rename to bootstrapvz/remote/ssh_rpc_manager.py
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 2ca8b8c..11efa2d 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -4,7 +4,7 @@ from build_servers import RemoteBuildServer
 
 # Register deserialization handlers for objects
 # that will pass between server and client
-from bootstrapvz.base.remote import register_deserialization_handlers
+from bootstrapvz.remote import register_deserialization_handlers
 register_deserialization_handlers()
 
 
@@ -47,7 +47,7 @@ def bootstrap(manifest, build_server):
 		from bootstrapvz.base.main import run
 		bootstrap_info = run(manifest)
 	else:
-		from bootstrapvz.base.remote.remote import run
+		from bootstrapvz.remote.main import run
 		bootstrap_info = run(manifest, build_server.settings)
 	return bootstrap_info
 

From 3d38ce04ef005bd1cc0182267892c5f422e28be9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 14:17:06 +0100
Subject: [PATCH 145/345] Move build_servers module into bootstrapvz

---
 .../integration/tools => bootstrapvz/remote}/build_servers.py | 0
 tests/integration/tools/__init__.py                           | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename {tests/integration/tools => bootstrapvz/remote}/build_servers.py (100%)

diff --git a/tests/integration/tools/build_servers.py b/bootstrapvz/remote/build_servers.py
similarity index 100%
rename from tests/integration/tools/build_servers.py
rename to bootstrapvz/remote/build_servers.py
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 11efa2d..7ccf1cc 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,6 +1,6 @@
 from bootstrapvz.common.tools import load_data
-from build_servers import LocalBuildServer
-from build_servers import RemoteBuildServer
+from bootstrapvz.remote.build_servers import LocalBuildServer
+from bootstrapvz.remote.build_servers import RemoteBuildServer
 
 # Register deserialization handlers for objects
 # that will pass between server and client

From cee05e3fd06713e057e9e2a62df6148506db61b0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 15:42:38 +0100
Subject: [PATCH 146/345] Refactor...

---
 .gitignore                                  |   2 +-
 bootstrapvz/remote/build-servers-schema.yml |  49 ++++++++++
 bootstrapvz/remote/build_servers.py         | 103 +++++++++++++++++++-
 bootstrapvz/remote/main.py                  |  58 ++++++-----
 tests/integration/tools/__init__.py         |  19 +---
 tests/integration/virtualbox_tests.py       |   3 +-
 6 files changed, 187 insertions(+), 47 deletions(-)
 create mode 100644 bootstrapvz/remote/build-servers-schema.yml

diff --git a/.gitignore b/.gitignore
index d2b3547..3422ad6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,4 +12,4 @@
 # Testing
 /.coverage
 /.tox/
-/build_servers.yml
+/build-servers.yml
diff --git a/bootstrapvz/remote/build-servers-schema.yml b/bootstrapvz/remote/build-servers-schema.yml
new file mode 100644
index 0000000..3990704
--- /dev/null
+++ b/bootstrapvz/remote/build-servers-schema.yml
@@ -0,0 +1,49 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Build server settings list
+type: object
+properties:
+  local:
+    type: object
+    properties:
+      type: {enum: [local]}
+      can_bootstrap: {$ref: '#/definitions/can_bootstrap'}
+      release: {type: string}
+      build_settings: {$ref: '#/definitions/build_settings'}
+    required: [type, can_bootstrap, release]
+patternProperties:
+  ^(?!local).*$: {$ref: '#/definitions/ssh'}
+
+definitions:
+  absolute_path:
+    type: string
+    pattern: ^/[^\0]+$
+
+  can_bootstrap:
+    type: array
+    items:
+      enum:
+      - virtualbox
+      - ec2-ebs
+      - ec2-s3
+
+  build_settings:
+    type: object
+    properties:
+      guest_additions: {$ref: '#/definitions/absolute_path'}
+
+  ssh:
+    type: object
+    properties:
+      type: {enum: [ssh]}
+      can_bootstrap: {$ref: '#/definitions/can_bootstrap'}
+      build_settings: {$ref: '#/definitions/build_settings'}
+      release: {type: string}
+      address: {type: string}
+      port: {type: integer}
+      username: {type: string}
+      password: {type: string}
+      root_password: {type: string}
+      keyfile: {$ref: '#/definitions/absolute_path'}
+      server_bin: {$ref: '#/definitions/absolute_path'}
+    required: [type, can_bootstrap, release]
diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index 95daf41..64ce968 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -1,4 +1,38 @@
 from bootstrapvz.common.tools import log_check_call
+import logging
+log = logging.getLogger(__name__)
+
+
+def pick_build_server(build_servers, preferences, manifest):
+	# Validate the build servers list
+	from bootstrapvz.common.tools import load_data
+	import os.path
+	schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml')))
+	import jsonschema
+	jsonschema.validate(build_servers, schema)
+
+	if manifest.provider['name'] == 'ec2':
+		must_bootstrap = 'ec2-' + manifest.volume['backing']
+	else:
+		must_bootstrap = manifest.provider['name']
+
+	def matches(name, settings):
+		if preferences.get('name', name) != name:
+			return False
+		if preferences.get('release', settings['release']) != settings['release']:
+			return False
+		if must_bootstrap not in settings['can_bootstrap']:
+			return False
+		return True
+
+	for name, settings in build_servers.iteritems():
+		if not matches(name, settings):
+			continue
+		if settings['type'] == 'local':
+			return LocalBuildServer(settings)
+		else:
+			return RemoteBuildServer(settings)
+	raise Exception('Unable to find a build server that matches your preferences.')
 
 
 class BuildServer(object):
@@ -17,6 +51,7 @@ class LocalBuildServer(BuildServer):
 class RemoteBuildServer(BuildServer):
 
 	def __init__(self, settings):
+		super(RemoteBuildServer, self).__init__(settings)
 		self.address = settings['address']
 		self.port = settings['port']
 		self.username = settings['username']
@@ -24,7 +59,61 @@ class RemoteBuildServer(BuildServer):
 		self.root_password = settings['root_password']
 		self.keyfile = settings['keyfile']
 		self.server_bin = settings['server_bin']
-		super(RemoteBuildServer, self).__init__(settings)
+
+		# We can't use :0 for the forwarding ports because
+		# A: It's quite hard to retrieve the port on the remote after the daemon has started
+		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
+		[self.local_server_port, self.local_callback_port] = getNPorts(2)
+		[self.remote_server_port, self.remote_callback_port] = getNPorts(2)
+
+	def connect(self):
+		log.debug('Opening SSH connection')
+		import subprocess
+
+		server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)]
+
+		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
+		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
+		                  '-p', str(self.settings['port']),
+		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
+		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
+		                  addr_arg]
+		full_cmd = ssh_cmd + ['--'] + server_cmd
+		import sys
+		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr)
+
+		# Check that we can connect to the server
+		try:
+			import Pyro4
+			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
+			self.connection = Pyro4.Proxy(server_uri)
+
+			log.debug('Connecting to the RPC daemon')
+			remaining_retries = 5
+			while True:
+				try:
+					self.connection.ping()
+					break
+				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
+					if remaining_retries > 0:
+						remaining_retries -= 1
+						from time import sleep
+						sleep(2)
+					else:
+						raise e
+		except (Exception, KeyboardInterrupt) as e:
+			self.ssh_process.terminate()
+			raise e
+		return self.connection
+
+	def disconnect(self):
+		if hasattr(self, 'connection'):
+			log.debug('Stopping the RPC daemon')
+			self.connection.stop()
+			self.connection._pyroRelease()
+		if hasattr(self, 'ssh_process'):
+			log.debug('Waiting for the SSH connection to terminate')
+			self.ssh_process.wait()
 
 	def download(self, src, dst):
 		src_arg = '{user}@{host}:{path}'.format(self.username, self.address, src)
@@ -38,3 +127,15 @@ class RemoteBuildServer(BuildServer):
 		                  '--',
 		                  'sudo', 'rm', path]
 		log_check_call(ssh_cmd)
+
+
+def getNPorts(n, port_range=(1024, 65535)):
+	import random
+	ports = []
+	for i in range(0, n):
+		while True:
+			port = random.randrange(*port_range)
+			if port not in ports:
+				ports.append(port)
+				break
+	return ports
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index aa0d719..88019cd 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -12,9 +12,17 @@ def main():
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(path=opts['MANIFEST'])
 
-	# Load the build servers
+	# load the build servers file
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data(opts['--servers'])
+	# Pick a build server
+	from build_servers import pick_build_server
+	preferences = {}
+	if opts['--name'] is not None:
+		preferences['name'] = opts['--name']
+	if opts['--release'] is not None:
+		preferences['release'] = opts['--release']
+	build_server = pick_build_server(build_servers, preferences, manifest)
 
 	# Set up logging
 	from bootstrapvz.base.main import setup_loggers
@@ -27,7 +35,7 @@ def main():
 
 	# Everything has been set up, connect to the server and begin the bootstrapping process
 	run(manifest,
-	    build_servers[opts['SERVER']],
+	    build_server,
 	    debug=opts['--debug'],
 	    dry_run=opts['--dry-run'])
 
@@ -38,56 +46,54 @@ def get_opts():
 	from docopt import docopt
 	usage = """bootstrap-vz-remote
 
-Usage: bootstrap-vz-remote [options] --servers=<path> SERVER MANIFEST
+Usage: bootstrap-vz-remote [options] --servers=<path> MANIFEST
 
 Options:
-  --servers <path>   Path to list of build servers
-  --log <path>       Log to given directory [default: /var/log/bootstrap-vz]
-                     If <path> is `-' file logging will be disabled.
-  --pause-on-error   Pause on error, before rollback
-  --dry-run          Don't actually run the tasks
+  --servers <path>    Path to list of build servers
+  --name <name>       Selects specific server from the build servers list
+  --release <release> Require the build server OS to be a specific release
+  --log <path>        Log to given directory [default: /var/log/bootstrap-vz]
+                      If <path> is `-' file logging will be disabled.
+  --pause-on-error    Pause on error, before rollback
+  --dry-run           Don't actually run the tasks
   --color=auto|always|never
                      Colorize the console output [default: auto]
-  --debug            Print debugging information
-  -h, --help         show this help
+  --debug             Print debugging information
+  -h, --help          show this help
 	"""
 	return docopt(usage)
 
 
-def run(manifest, server, debug=False, dry_run=False):
+def run(manifest, build_server, debug=False, dry_run=False):
 	"""Connects to the remote build server, starts an RPC daemin
 	on the other side and initiates a remote bootstrapping procedure
 	"""
 	bootstrap_info = None
-
-	from ssh_rpc_manager import SSHRPCManager
-	manager = SSHRPCManager(server)
 	try:
-		# Connect to the build server and start the RPC daemon
-		manager.start()
-		server = manager.rpc_server
+		# Connect to the build server
+		connection = build_server.connect()
 		# Start a callback server on this side, so that we may receive log entries
 		from callback import CallbackServer
-		callback_server = CallbackServer(listen_port=manager.local_callback_port,
-		                                 remote_port=manager.remote_callback_port)
+		callback_server = CallbackServer(listen_port=build_server.local_callback_port,
+		                                 remote_port=build_server.remote_callback_port)
 		from bootstrapvz.base.log import LogServer
 		log_server = LogServer()
 		try:
 			# Start the callback server (in a background thread)
 			callback_server.start(log_server)
 			# Tell the RPC daemon about the callback server
-			server.set_log_server(log_server)
+			connection.set_log_server(log_server)
 
 			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = server.run(manifest,
-			                            debug=debug,
-			                            # We can't pause the bootstrapping process remotely, yet...
-			                            pause_on_error=False,
-			                            dry_run=dry_run)
+			bootstrap_info = connection.run(manifest,
+			                                debug=debug,
+			                                # We can't pause the bootstrapping process remotely, yet...
+			                                pause_on_error=False,
+			                                dry_run=dry_run)
 		finally:
 			# Stop the callback server
 			callback_server.stop()
 	finally:
 		# Stop the RPC daemon and close the SSH connection
-		manager.stop()
+		build_server.disconnect()
 	return bootstrap_info
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 7ccf1cc..e2009c0 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,6 +1,4 @@
-from bootstrapvz.common.tools import load_data
 from bootstrapvz.remote.build_servers import LocalBuildServer
-from bootstrapvz.remote.build_servers import RemoteBuildServer
 
 # Register deserialization handlers for objects
 # that will pass between server and client
@@ -27,28 +25,13 @@ def merge_dicts(*args):
 	return reduce(merge, args, {})
 
 
-def pick_build_server(manifest):
-	if manifest['provider']['name'] == 'ec2':
-		img_type = 'ec2-' + manifest['volume']['backing']
-	else:
-		img_type = manifest['provider']['name']
-
-	# tox makes sure that the cwd is the project root
-	build_servers = load_data('build_servers.yml')
-	settings = next((server for name, server in build_servers.iteritems() if img_type in server['can_bootstrap']), None)
-	if settings['type'] == 'local':
-		return LocalBuildServer(settings)
-	else:
-		return RemoteBuildServer(settings)
-
-
 def bootstrap(manifest, build_server):
 	if isinstance(build_server, LocalBuildServer):
 		from bootstrapvz.base.main import run
 		bootstrap_info = run(manifest)
 	else:
 		from bootstrapvz.remote.main import run
-		bootstrap_info = run(manifest, build_server.settings)
+		bootstrap_info = run(manifest, build_server)
 	return bootstrap_info
 
 
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index ea7eaed..cd357da 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -18,7 +18,8 @@ volume:
 	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
 	                                  partials['unpartitioned'], manifest_data)
 
-	build_server = tools.pick_build_server(manifest_data)
+	manifest = Manifest(data=manifest_data)
+	build_server = tools.pick_build_server(manifest)
 	manifest_data['provider']['guest_additions'] = build_server.build_settings['guest_additions']
 	manifest = Manifest(data=manifest_data)
 

From ece717a79ff5dde22e5b28c0b691d0917af86556 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 15:54:31 +0100
Subject: [PATCH 147/345] Fix bugs when picking the build server

---
 bootstrapvz/remote/build_servers.py   | 13 +++++++++----
 bootstrapvz/remote/main.py            | 16 +++++++++++-----
 tests/integration/__init__.py         |  4 ++++
 tests/integration/tools/__init__.py   |  3 +--
 tests/integration/virtualbox_tests.py |  7 ++++---
 5 files changed, 29 insertions(+), 14 deletions(-)

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index 64ce968..e9e9b6b 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -3,7 +3,7 @@ import logging
 log = logging.getLogger(__name__)
 
 
-def pick_build_server(build_servers, preferences, manifest):
+def pick_build_server(build_servers, manifest, preferences={}):
 	# Validate the build servers list
 	from bootstrapvz.common.tools import load_data
 	import os.path
@@ -11,10 +11,10 @@ def pick_build_server(build_servers, preferences, manifest):
 	import jsonschema
 	jsonschema.validate(build_servers, schema)
 
-	if manifest.provider['name'] == 'ec2':
-		must_bootstrap = 'ec2-' + manifest.volume['backing']
+	if manifest['provider']['name'] == 'ec2':
+		must_bootstrap = 'ec2-' + manifest['volume']['backing']
 	else:
-		must_bootstrap = manifest.provider['name']
+		must_bootstrap = manifest['provider']['name']
 
 	def matches(name, settings):
 		if preferences.get('name', name) != name:
@@ -43,6 +43,11 @@ class BuildServer(object):
 		self.can_bootstrap = settings['can_bootstrap']
 		self.release = settings.get('release', None)
 
+	def apply_build_settings(self, manifest_data):
+		if manifest_data['provider']['name'] == 'virtualbox' and 'guest_additions' in manifest_data['provider']:
+			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
+		return manifest_data
+
 
 class LocalBuildServer(BuildServer):
 	pass
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index 88019cd..2f52803 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -8,12 +8,11 @@ def main():
 	# Get the commandline arguments
 	opts = get_opts()
 
-	# Load the manifest
-	from bootstrapvz.base.manifest import Manifest
-	manifest = Manifest(path=opts['MANIFEST'])
+	from bootstrapvz.common.tools import load_data
+	# load the manifest data, we might want to modify it later on
+	manifest_data = load_data(opts['MANIFEST'])
 
 	# load the build servers file
-	from bootstrapvz.common.tools import load_data
 	build_servers = load_data(opts['--servers'])
 	# Pick a build server
 	from build_servers import pick_build_server
@@ -22,7 +21,14 @@ def main():
 		preferences['name'] = opts['--name']
 	if opts['--release'] is not None:
 		preferences['release'] = opts['--release']
-	build_server = pick_build_server(build_servers, preferences, manifest)
+	build_server = pick_build_server(build_servers, manifest_data, preferences)
+
+	# Apply the build server settings to the manifest (e.g. the virtualbox guest additions path)
+	manifest_data = build_server.apply_build_settings(manifest_data)
+
+	# Load the manifest
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(path=opts['MANIFEST'], data=manifest_data)
 
 	# Set up logging
 	from bootstrapvz.base.main import setup_loggers
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index e69de29..5e6a71c 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -0,0 +1,4 @@
+from bootstrapvz.common.tools import load_data
+
+# tox ensures that the cwd is the project root
+build_servers = load_data('build-servers.yml')
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index e2009c0..32f1d50 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,5 +1,3 @@
-from bootstrapvz.remote.build_servers import LocalBuildServer
-
 # Register deserialization handlers for objects
 # that will pass between server and client
 from bootstrapvz.remote import register_deserialization_handlers
@@ -26,6 +24,7 @@ def merge_dicts(*args):
 
 
 def bootstrap(manifest, build_server):
+	from bootstrapvz.remote.build_servers import LocalBuildServer
 	if isinstance(build_server, LocalBuildServer):
 		from bootstrapvz.base.main import run
 		bootstrap_info = run(manifest)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index cd357da..adcc190 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,6 +1,8 @@
 import tools
 from manifests import partials
 from bootstrapvz.base.manifest import Manifest
+from bootstrapvz.remote.build_servers import pick_build_server
+from . import build_servers
 
 
 def test_virtualbox_unpartitioned_extlinux():
@@ -18,9 +20,8 @@ volume:
 	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
 	                                  partials['unpartitioned'], manifest_data)
 
-	manifest = Manifest(data=manifest_data)
-	build_server = tools.pick_build_server(manifest)
-	manifest_data['provider']['guest_additions'] = build_server.build_settings['guest_additions']
+	build_server = pick_build_server(build_servers, manifest_data)
+	manifest_data = build_server.apply_build_settings(manifest_data)
 	manifest = Manifest(data=manifest_data)
 
 	bootstrap_info = tools.bootstrap(manifest, build_server)

From bc8967529dfa3f8e96aebf64a4fa2822265630c7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 15:58:05 +0100
Subject: [PATCH 148/345] Move remote log stuff back into remote module

---
 bootstrapvz/base/log.py      | 31 -------------------------------
 bootstrapvz/remote/log.py    | 32 ++++++++++++++++++++++++++++++++
 bootstrapvz/remote/main.py   |  2 +-
 bootstrapvz/remote/server.py |  2 +-
 4 files changed, 34 insertions(+), 33 deletions(-)
 create mode 100644 bootstrapvz/remote/log.py

diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py
index 0083eb5..8b01b04 100644
--- a/bootstrapvz/base/log.py
+++ b/bootstrapvz/base/log.py
@@ -102,34 +102,3 @@ class FileFormatter(SourceFormatter):
 	"""
 	def format(self, record):
 		return super(FileFormatter, self).format(record)
-
-
-class LogForwarder(logging.Handler):
-
-	def __init__(self, level=logging.NOTSET):
-		self.server = None
-		super(LogForwarder, self).__init__(level)
-
-	def set_server(self, server):
-		self.server = server
-
-	def emit(self, record):
-		if self.server is not None:
-			if record.exc_info is not None:
-				import traceback
-				exc_type, exc_value, exc_traceback = record.exc_info
-				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
-			# TODO: Use serpent instead
-			import pickle
-			self.server.handle(pickle.dumps(record))
-
-
-class LogServer(object):
-
-	def handle(self, pickled_record):
-		import pickle
-		record = pickle.loads(pickled_record)
-		log = logging.getLogger()
-		record.extra = getattr(record, 'extra', {})
-		record.extra['source'] = 'remote'
-		log.handle(record)
diff --git a/bootstrapvz/remote/log.py b/bootstrapvz/remote/log.py
new file mode 100644
index 0000000..793ffea
--- /dev/null
+++ b/bootstrapvz/remote/log.py
@@ -0,0 +1,32 @@
+import logging
+
+
+class LogForwarder(logging.Handler):
+
+	def __init__(self, level=logging.NOTSET):
+		self.server = None
+		super(LogForwarder, self).__init__(level)
+
+	def set_server(self, server):
+		self.server = server
+
+	def emit(self, record):
+		if self.server is not None:
+			if record.exc_info is not None:
+				import traceback
+				exc_type, exc_value, exc_traceback = record.exc_info
+				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
+			# TODO: Use serpent instead
+			import pickle
+			self.server.handle(pickle.dumps(record))
+
+
+class LogServer(object):
+
+	def handle(self, pickled_record):
+		import pickle
+		record = pickle.loads(pickled_record)
+		log = logging.getLogger()
+		record.extra = getattr(record, 'extra', {})
+		record.extra['source'] = 'remote'
+		log.handle(record)
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index 2f52803..0289b15 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -82,7 +82,7 @@ def run(manifest, build_server, debug=False, dry_run=False):
 		from callback import CallbackServer
 		callback_server = CallbackServer(listen_port=build_server.local_callback_port,
 		                                 remote_port=build_server.remote_callback_port)
-		from bootstrapvz.base.log import LogServer
+		from log import LogServer
 		log_server = LogServer()
 		try:
 			# Start the callback server (in a background thread)
diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index 045ecac..2d43f44 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -15,7 +15,7 @@ def main():
 
 
 def setup_logging():
-	from bootstrapvz.base.log import LogForwarder
+	from log import LogForwarder
 	log_forwarder = LogForwarder()
 	root = logging.getLogger()
 	root.addHandler(log_forwarder)

From 056d279b6586687454efa2759a68e54f44477583 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 15:59:52 +0100
Subject: [PATCH 149/345] Read import in base.__init__

---
 bootstrapvz/base/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index 305d2e3..be224ce 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -1,5 +1,6 @@
 from phase import Phase
 from task import Task
+from main import main
 
 __all__ = ['Phase', 'Task', 'main']
 

From 2b6fefd789bf5c5f1b2a90a988d0b1b094fb1c68 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 19:16:27 +0100
Subject: [PATCH 150/345] Fix serialization of exceptions

---
 bootstrapvz/base/manifest.py     |  7 +++++++
 bootstrapvz/common/exceptions.py |  2 ++
 bootstrapvz/remote/__init__.py   | 21 ++++++++++++++++++++-
 bootstrapvz/remote/callback.py   | 25 +++++++++++++++++--------
 bootstrapvz/remote/log.py        | 17 ++++-------------
 bootstrapvz/remote/main.py       |  6 ++----
 bootstrapvz/remote/server.py     |  7 ++++---
 7 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 2854068..27e7686 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -136,3 +136,10 @@ class Manifest(object):
 		return {'__class__': 'bootstrapvz.base.manifest.Manifest',
 		        'path': self.path,
 		        'data': self.data}
+
+	def __setstate__(self, state):
+		self.path = state['path']
+		self.load(state['data'])
+		self.initialize()
+		self.validate()
+		self.parse()
diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index a349c07..dee0d53 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -2,6 +2,7 @@
 
 class ManifestError(Exception):
 	def __init__(self, message, manifest_path, data_path=None):
+		super(ManifestError, self).__init__(message)
 		self.message = message
 		self.manifest_path = manifest_path
 		self.data_path = data_path
@@ -16,6 +17,7 @@ class ManifestError(Exception):
 
 class TaskListError(Exception):
 	def __init__(self, message):
+		super(TaskListError, self).__init__(message)
 		self.message = message
 
 	def __str__(self):
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 3c61646..a7ec9da 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -1,22 +1,31 @@
 """Remote module containing methods to bootstrap remotely
 """
+import bootstrapvz.common.exceptions
 
 
 def register_deserialization_handlers():
 	from Pyro4.util import SerializerBase
 	SerializerBase.register_dict_to_class('bootstrapvz.base.manifest.Manifest', deserialize_manifest)
 	SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo)
+	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.ManifestError', deserialize_exception)
+	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.TaskListError', deserialize_exception)
+	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.TaskError', deserialize_exception)
 
 
 def unregister_deserialization_handlers():
 	from Pyro4.util import SerializerBase
 	SerializerBase.unregister_dict_to_class('bootstrapvz.base.manifest.Manifest')
 	SerializerBase.unregister_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation')
+	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.ManifestError')
+	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.TaskListError')
+	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.TaskError')
 
 
 def deserialize_manifest(classname, state):
 	from bootstrapvz.base.manifest import Manifest
-	return Manifest(path=state['path'], data=state['data'])
+	manifest = Manifest.__new__(Manifest)
+	manifest.__setstate__(state)
+	return manifest
 
 
 def deserialize_bootstrapinfo(classname, state):
@@ -24,3 +33,13 @@ def deserialize_bootstrapinfo(classname, state):
 	bootstrap_info = BootstrapInformation.__new__(BootstrapInformation)
 	bootstrap_info.__setstate__(state)
 	return bootstrap_info
+
+deserialize_map = {'bootstrapvz.common.exceptions.ManifestError': bootstrapvz.common.exceptions.ManifestError,
+                   'bootstrapvz.common.exceptions.TaskListError': bootstrapvz.common.exceptions.TaskListError,
+                   'bootstrapvz.common.exceptions.TaskError': bootstrapvz.common.exceptions.TaskError,
+                   }
+
+
+def deserialize_exception(classname, data):
+	from Pyro4.util import SerializerBase
+	return SerializerBase.make_exception(deserialize_map[classname], data)
diff --git a/bootstrapvz/remote/callback.py b/bootstrapvz/remote/callback.py
index 25b197b..f6f1f6e 100644
--- a/bootstrapvz/remote/callback.py
+++ b/bootstrapvz/remote/callback.py
@@ -1,20 +1,19 @@
+import Pyro4
 import logging
+
+Pyro4.config.REQUIRE_EXPOSE = True
 log = logging.getLogger(__name__)
 
 
 class CallbackServer(object):
 
 	def __init__(self, listen_port, remote_port):
-		self.listen_port = listen_port
-		self.remote_port = remote_port
-
-	def start(self, log_server):
-		import Pyro4
-		self.daemon = Pyro4.Daemon(host='localhost', port=self.listen_port,
-		                           nathost='localhost', natport=self.remote_port,
+		self.daemon = Pyro4.Daemon(host='localhost', port=listen_port,
+		                           nathost='localhost', natport=remote_port,
 		                           unixsocket=None)
-		self.daemon.register(log_server)
+		self.daemon.register(self)
 
+	def start(self):
 		def serve():
 			self.daemon.requestLoop()
 		from threading import Thread
@@ -27,3 +26,13 @@ class CallbackServer(object):
 			self.daemon.shutdown()
 		if hasattr(self, 'thread'):
 			self.thread.join()
+
+	@Pyro4.expose
+	def handle_log(self, pickled_record):
+		import pickle
+		record = pickle.loads(pickled_record)
+		log = logging.getLogger()
+		record.extra = getattr(record, 'extra', {})
+		record.extra['source'] = 'remote'
+		log.handle(record)
+
diff --git a/bootstrapvz/remote/log.py b/bootstrapvz/remote/log.py
index 793ffea..fc7d66d 100644
--- a/bootstrapvz/remote/log.py
+++ b/bootstrapvz/remote/log.py
@@ -15,18 +15,9 @@ class LogForwarder(logging.Handler):
 			if record.exc_info is not None:
 				import traceback
 				exc_type, exc_value, exc_traceback = record.exc_info
-				record.exc_info = traceback.print_exception(exc_type, exc_value, exc_traceback)
+				record.extra = getattr(record, 'extra', {})
+				record.extra['traceback'] = traceback.format_exception(exc_type, exc_value, exc_traceback)
+				record.exc_info = None
 			# TODO: Use serpent instead
 			import pickle
-			self.server.handle(pickle.dumps(record))
-
-
-class LogServer(object):
-
-	def handle(self, pickled_record):
-		import pickle
-		record = pickle.loads(pickled_record)
-		log = logging.getLogger()
-		record.extra = getattr(record, 'extra', {})
-		record.extra['source'] = 'remote'
-		log.handle(record)
+			self.server.handle_log(pickle.dumps(record))
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index 0289b15..b038dd3 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -82,13 +82,11 @@ def run(manifest, build_server, debug=False, dry_run=False):
 		from callback import CallbackServer
 		callback_server = CallbackServer(listen_port=build_server.local_callback_port,
 		                                 remote_port=build_server.remote_callback_port)
-		from log import LogServer
-		log_server = LogServer()
 		try:
 			# Start the callback server (in a background thread)
-			callback_server.start(log_server)
+			callback_server.start()
 			# Tell the RPC daemon about the callback server
-			connection.set_log_server(log_server)
+			connection.set_callback_server(callback_server)
 
 			# Everything has been set up, begin the bootstrapping process
 			bootstrap_info = connection.run(manifest,
diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index 2d43f44..fdd11c6 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -56,9 +56,10 @@ class Server(object):
 		return run(*args, **kwargs)
 
 	@Pyro4.expose
-	def set_log_server(self, server):
-		self.log_forwarder.set_server(server)
-		log.debug('Successfully set the log forwarding server')
+	def set_callback_server(self, server):
+		self.callback_server = server
+		self.log_forwarder.set_server(self.callback_server)
+		log.debug('Forwarding logs to the callback server now')
 
 	@Pyro4.expose
 	def ping(self):

From ab18516f7920f8cf7cb40429a7613ab163e3d76b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 30 Nov 2014 19:19:14 +0100
Subject: [PATCH 151/345] Support SIGINT over the network

---
 bootstrapvz/base/main.py            |  4 ++--
 bootstrapvz/base/tasklist.py        |  5 ++++-
 bootstrapvz/remote/build_servers.py | 10 +++++++++-
 bootstrapvz/remote/callback.py      |  7 +++++++
 bootstrapvz/remote/main.py          | 11 +++++++++++
 bootstrapvz/remote/server.py        |  4 ++++
 6 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index d949bcd..fe38e7d 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -83,7 +83,7 @@ def setup_loggers(opts):
 	root.addHandler(console_handler)
 
 
-def run(manifest, debug=False, pause_on_error=False, dry_run=False):
+def run(manifest, debug=False, pause_on_error=False, dry_run=False, check_continue=None):
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
@@ -106,7 +106,7 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	log = logging.getLogger(__name__)
 	try:
 		# Run all the tasks the tasklist has gathered
-		tasklist.run(info=bootstrap_info, dry_run=dry_run)
+		tasklist.run(info=bootstrap_info, dry_run=dry_run, check_continue=check_continue)
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
 	except (Exception, KeyboardInterrupt) as e:
diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 8e5dbb1..342896e 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -15,7 +15,7 @@ class TaskList(object):
 		self.tasks = tasks
 		self.tasks_completed = []
 
-	def run(self, info, dry_run=False):
+	def run(self, info, dry_run=False, check_continue=None):
 		"""Converts the taskgraph into a list and runs all tasks in that list
 
 		:param dict info: The bootstrap information object
@@ -27,6 +27,9 @@ class TaskList(object):
 		log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list))))
 
 		for task in task_list:
+			# Check if we should abort the run (used for asynchronous run abortion through remote building)
+			if callable(check_continue) and not check_continue():
+				raise TaskListError('Run was aborted.')
 			# Tasks are not required to have a description
 			if hasattr(task, 'description'):
 				log.info(task.description)
diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index e9e9b6b..fcf8d79 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -77,6 +77,13 @@ class RemoteBuildServer(BuildServer):
 
 		server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)]
 
+		def set_process_group():
+			# Changes the process group of a command so that any SIGINT
+			# for the main thread will not be propagated to it.
+			# We'd like to handle SIGINT ourselves (i.e. propagate the shutdown to the serverside)
+			import os
+			os.setpgrp()
+
 		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
 		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
 		                  '-p', str(self.settings['port']),
@@ -85,7 +92,8 @@ class RemoteBuildServer(BuildServer):
 		                  addr_arg]
 		full_cmd = ssh_cmd + ['--'] + server_cmd
 		import sys
-		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr)
+		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr,
+		                                    preexec_fn=set_process_group)
 
 		# Check that we can connect to the server
 		try:
diff --git a/bootstrapvz/remote/callback.py b/bootstrapvz/remote/callback.py
index f6f1f6e..2c385cc 100644
--- a/bootstrapvz/remote/callback.py
+++ b/bootstrapvz/remote/callback.py
@@ -12,6 +12,7 @@ class CallbackServer(object):
 		                           nathost='localhost', natport=remote_port,
 		                           unixsocket=None)
 		self.daemon.register(self)
+		self.abort = False
 
 	def start(self):
 		def serve():
@@ -36,3 +37,9 @@ class CallbackServer(object):
 		record.extra['source'] = 'remote'
 		log.handle(record)
 
+	@Pyro4.expose
+	def get_abort_run(self):
+		return self.abort
+
+	def abort_run(self):
+		self.abort = True
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index b038dd3..eb247f3 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -88,12 +88,23 @@ def run(manifest, build_server, debug=False, dry_run=False):
 			# Tell the RPC daemon about the callback server
 			connection.set_callback_server(callback_server)
 
+			# Replace the standard SIGINT handler with a remote call to the server
+			# so that it may abort the run.
+			def abort(signum, frame):
+				import logging
+				logging.getLogger(__name__).warn('SIGINT received, asking remote to abort.')
+				callback_server.abort_run()
+			import signal
+			orig_sigint = signal.signal(signal.SIGINT, abort)
+
 			# Everything has been set up, begin the bootstrapping process
 			bootstrap_info = connection.run(manifest,
 			                                debug=debug,
 			                                # We can't pause the bootstrapping process remotely, yet...
 			                                pause_on_error=False,
 			                                dry_run=dry_run)
+			# Restore the old SIGINT handler
+			signal.signal(signal.SIGINT, orig_sigint)
 		finally:
 			# Stop the callback server
 			callback_server.stop()
diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index fdd11c6..a802af2 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -52,7 +52,11 @@ class Server(object):
 
 	@Pyro4.expose
 	def run(self, *args, **kwargs):
+
+		def abort_run():
+			return not self.callback_server.get_abort_run()
 		from bootstrapvz.base.main import run
+		kwargs['check_continue'] = abort_run
 		return run(*args, **kwargs)
 
 	@Pyro4.expose

From 09fee291a891f48c1534f732b576fcccc9b77a78 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 1 Dec 2014 00:09:13 +0100
Subject: [PATCH 152/345] Fix bugs in remote build server

---
 bootstrapvz/remote/build_servers.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index fcf8d79..013c769 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -46,6 +46,8 @@ class BuildServer(object):
 	def apply_build_settings(self, manifest_data):
 		if manifest_data['provider']['name'] == 'virtualbox' and 'guest_additions' in manifest_data['provider']:
 			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
+		if 'apt_proxy' in self.build_settings:
+			manifest_data.get('plugins', {})['apt_proxy'] = self.build_settings['apt_proxy']
 		return manifest_data
 
 
@@ -75,7 +77,7 @@ class RemoteBuildServer(BuildServer):
 		log.debug('Opening SSH connection')
 		import subprocess
 
-		server_cmd = ['sudo', self.settings['server_bin'], '--listen', str(self.remote_server_port)]
+		server_cmd = ['sudo', self.server_bin, '--listen', str(self.remote_server_port)]
 
 		def set_process_group():
 			# Changes the process group of a command so that any SIGINT
@@ -85,8 +87,8 @@ class RemoteBuildServer(BuildServer):
 			os.setpgrp()
 
 		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
-		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
-		                  '-p', str(self.settings['port']),
+		ssh_cmd = ['ssh', '-i', self.keyfile,
+		                  '-p', str(self.port),
 		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
 		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
 		                  addr_arg]
@@ -129,13 +131,13 @@ class RemoteBuildServer(BuildServer):
 			self.ssh_process.wait()
 
 	def download(self, src, dst):
-		src_arg = '{user}@{host}:{path}'.format(self.username, self.address, src)
-		log_check_call(['scp', '-i', self.keyfile, '-P', self.port,
+		src_arg = '{user}@{host}:{path}'.format(user=self.username, host=self.address, path=src)
+		log_check_call(['scp', '-i', self.keyfile, '-P', str(self.port),
 		                src_arg, dst])
 
 	def delete(self, path):
-		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
-		                  '-p', str(self.settings['port']),
+		ssh_cmd = ['ssh', '-i', self.keyfile,
+		                  '-p', str(self.port),
 		                  self.username + '@' + self.address,
 		                  '--',
 		                  'sudo', 'rm', path]

From 288c5f4c57d97ba5dbc6e52a0846353e9641a6e0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 1 Dec 2014 00:09:46 +0100
Subject: [PATCH 153/345] Always use tarball, minor fixes to test

---
 tests/integration/manifests/base.yml  | 1 +
 tests/integration/virtualbox_tests.py | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/integration/manifests/base.yml b/tests/integration/manifests/base.yml
index 849a31a..3480441 100644
--- a/tests/integration/manifests/base.yml
+++ b/tests/integration/manifests/base.yml
@@ -2,6 +2,7 @@
 provider: {}
 bootstrapper:
   workspace: /target
+  tarball: true
 image:
   name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
   description: Debian {system.release} {system.architecture}
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index adcc190..84f990b 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -26,12 +26,14 @@ volume:
 
 	bootstrap_info = tools.bootstrap(manifest, build_server)
 
-	if isinstance(build_server, tools.build_servers.LocalBuildServer):
+	from bootstrapvz.remote.build_servers import LocalBuildServer
+	if isinstance(build_server, LocalBuildServer):
 		image_path = bootstrap_info.volume.image_path
 	else:
 		import tempfile
 		handle, image_path = tempfile.mkstemp()
-		handle.close()
+		import os
+		os.close(handle)
 		build_server.download(bootstrap_info.volume.image_path, image_path)
 		build_server.delete(bootstrap_info.volume.image_path)
 

From a8e5c2d6e40a59532cff4b2ae0a4c5dbce173e67 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 14 Dec 2014 15:38:44 +0100
Subject: [PATCH 154/345] Fix problem with read rights on remote build machine

---
 bootstrapvz/remote/build_servers.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index 013c769..cdcbdf0 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -131,16 +131,20 @@ class RemoteBuildServer(BuildServer):
 			self.ssh_process.wait()
 
 	def download(self, src, dst):
+		# Make sure we can read the file as {user}
+		self._remote_command(['sudo', 'chown', self.username, src])
 		src_arg = '{user}@{host}:{path}'.format(user=self.username, host=self.address, path=src)
 		log_check_call(['scp', '-i', self.keyfile, '-P', str(self.port),
 		                src_arg, dst])
 
 	def delete(self, path):
+		self._remote_command(['sudo', 'rm', path])
+
+	def _remote_command(self, command):
 		ssh_cmd = ['ssh', '-i', self.keyfile,
 		                  '-p', str(self.port),
 		                  self.username + '@' + self.address,
-		                  '--',
-		                  'sudo', 'rm', path]
+		                  '--'] + command
 		log_check_call(ssh_cmd)
 
 

From ea3eeae0641a78b3a96308010adfe4e6394833da Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 14 Dec 2014 15:41:04 +0100
Subject: [PATCH 155/345] vbox instance creation working

---
 tests/integration/tools/images.py     | 14 ++++-----
 tests/integration/tools/instances.py  | 41 +++++++++++++--------------
 tests/integration/virtualbox_tests.py | 18 ++++++++----
 3 files changed, 39 insertions(+), 34 deletions(-)

diff --git a/tests/integration/tools/images.py b/tests/integration/tools/images.py
index 928338b..2bda849 100644
--- a/tests/integration/tools/images.py
+++ b/tests/integration/tools/images.py
@@ -1,3 +1,4 @@
+import virtualbox
 
 
 class Image(object):
@@ -14,12 +15,11 @@ class VirtualBoxImage(Image):
 	def __init__(self, manifest, image_path):
 		super(VirtualBoxImage, self).__init__(manifest)
 		self.image_path = image_path
-		self.medium = self.vbox.open_medium(location=self.image.image_path,
-		                                    decive_type=self.vbox.library.DeviceType.HardDisk,
-		                                    access_mode=self.vbox.library.AccessMode.read_only,
-		                                    force_new_uuid=False)
+		self.vbox = virtualbox.VirtualBox()
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    virtualbox.library.DeviceType.hard_disk,  # decive_type
+		                                    virtualbox.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
 
 	def destroy(self):
-		self.medium.delete_storage()
-		import os
-		os.remove(self.image_path)
+		self.medium.close()
diff --git a/tests/integration/tools/instances.py b/tests/integration/tools/instances.py
index 075fc20..d9dc228 100644
--- a/tests/integration/tools/instances.py
+++ b/tests/integration/tools/instances.py
@@ -1,4 +1,5 @@
 from bootstrapvz.common.tools import log_check_call
+import virtualbox
 
 
 class Instance(object):
@@ -24,11 +25,12 @@ class VirtualBoxInstance(Instance):
 
 	def __init__(self, name, image):
 		super(VirtualBoxInstance, self).__init__(name, image)
-		import virtualbox
 		self.vbox = virtualbox.VirtualBox()
+		manager = virtualbox.Manager()
+		self.session = manager.get_session()
 
 	def create(self):
-		if self.image.manifest['system']['architecture'] == 'x86':
+		if self.image.manifest.system['architecture'] == 'x86':
 			os_type = 'Debian'
 		else:
 			os_type = 'Debian_64'
@@ -37,27 +39,19 @@ class VirtualBoxInstance(Instance):
 		self.machine.save_settings()
 		self.machine.cpu_count = self.cpus
 		self.machine.memory_size = self.memory
-		self.machine.attach_device(name='root', controller_port=0, device=0,
-		                           type_p=self.vbox.library.DeviceType.HardDisk,
-		                           medium=self.image.medium)
 		self.vbox.register_machine(self.machine)
-		# [self.uuid] = log_check_call(['VBoxManage', 'createvm'
-		#                               '--name', self.name])
-		# log_check_call(['VBoxManage', 'modifyvm', self.uuid,
-		#                 '--cpus', self.cpus,
-		#                 '--memory', self.memory])
-		# log_check_call(['VBoxManage', 'storageattach', self.uuid,
-		#                 '--storagectl', '"SATA Controller"',
-		#                 '--device', '0',
-		#                 '--port', '0',
-		#                 '--type', 'hdd',
-		#                 '--medium', self.image.image_path])
+		self.machine.lock_machine(self.session, virtualbox.library.LockType.write)
+		strg_ctrl = self.session.machine.add_storage_controller('SATA Controller',
+		                                                        virtualbox.library.StorageBus.sata)
+		strg_ctrl.port_count = 1
+		self.session.machine.attach_device(name='SATA Controller', controller_port=0, device=0,
+		                                   type_p=virtualbox.library.DeviceType.hard_disk,
+		                                   medium=self.image.medium)
+		self.session.machine.save_settings()
+		self.session.unlock_machine()
 
 	def boot(self):
-		self.session = self.vbox.Session()
 		self.machine.launch_vm_process(self.session, 'headless')
-		# log_check_call(['VBoxManage', 'startvm', self.uuid,
-		#                 '--type', 'headless'])
 
 	def shutdown(self):
 		self.session.console.power_down()
@@ -65,5 +59,10 @@ class VirtualBoxInstance(Instance):
 		                '--type', 'headless'])
 
 	def destroy(self):
-		self.machine.unregister(self.vbox.CleanupMode.full)
-		self.machine.remove(delete=True)
+		if hasattr(self, 'machine'):
+			self.machine.lock_machine(self.session, virtualbox.library.LockType.write)
+			self.session.machine.detach_device(name='SATA Controller', controller_port=0, device=0)
+			self.session.machine.save_settings()
+			self.session.unlock_machine()
+			self.machine.unregister(virtualbox.library.CleanupMode.unregister_only)
+			self.machine.remove(delete=True)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 84f990b..2f00928 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,4 +1,6 @@
 import tools
+import tools.images
+import tools.instances
 from manifests import partials
 from bootstrapvz.base.manifest import Manifest
 from bootstrapvz.remote.build_servers import pick_build_server
@@ -39,14 +41,18 @@ volume:
 
 	try:
 		image = tools.images.VirtualBoxImage(manifest, image_path)
+		try:
+			instance = tools.instances.VirtualBoxInstance('unpartitioned_extlinux', image)
+			instance.create()
+			# instance.boot()
 
-		instance = tools.instances.VirtualBoxInstance(image)
-		instance.create()
-		instance.boot()
+			# tools.test(instance)
 
-		tools.test(instance)
+		finally:
+			if 'instance' in locals():
+				instance.destroy()
 	finally:
-		if 'instance' in locals():
-			instance.destroy()
 		if 'image' in locals():
 			image.destroy()
+		import os
+		os.remove(image_path)

From e2cddbca4c4b2ee432a801381c4d4907d8d73d5a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 18 Dec 2014 21:47:51 +0100
Subject: [PATCH 156/345] Can now create and boot vbox image

---
 tests/integration/tools/__init__.py   | 10 ++++++++++
 tests/integration/tools/instances.py  | 23 +++++++++++++----------
 tests/integration/virtualbox_tests.py |  5 ++++-
 3 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 32f1d50..c858563 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -23,6 +23,16 @@ def merge_dicts(*args):
 	return reduce(merge, args, {})
 
 
+def waituntil(predicate, timeout=5, interval=0.05):
+	import time
+	threshhold = time.time() + timeout
+	while time.time() < threshhold:
+		if predicate():
+			return True
+		time.sleep(interval)
+	return False
+
+
 def bootstrap(manifest, build_server):
 	from bootstrapvz.remote.build_servers import LocalBuildServer
 	if isinstance(build_server, LocalBuildServer):
diff --git a/tests/integration/tools/instances.py b/tests/integration/tools/instances.py
index d9dc228..87591e5 100644
--- a/tests/integration/tools/instances.py
+++ b/tests/integration/tools/instances.py
@@ -1,4 +1,3 @@
-from bootstrapvz.common.tools import log_check_call
 import virtualbox
 
 
@@ -36,9 +35,9 @@ class VirtualBoxInstance(Instance):
 			os_type = 'Debian_64'
 		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
 		                                        groups=[], os_type_id=os_type, flags='')
-		self.machine.save_settings()
 		self.machine.cpu_count = self.cpus
 		self.machine.memory_size = self.memory
+		self.machine.save_settings()  # save settings, so that we can register it
 		self.vbox.register_machine(self.machine)
 		self.machine.lock_machine(self.session, virtualbox.library.LockType.write)
 		strg_ctrl = self.session.machine.add_storage_controller('SATA Controller',
@@ -47,22 +46,26 @@ class VirtualBoxInstance(Instance):
 		self.session.machine.attach_device(name='SATA Controller', controller_port=0, device=0,
 		                                   type_p=virtualbox.library.DeviceType.hard_disk,
 		                                   medium=self.image.medium)
-		self.session.machine.save_settings()
-		self.session.unlock_machine()
+		self.session.machine.save_settings()  # save changes to the controller
+		self._unlock_machine()
 
 	def boot(self):
-		self.machine.launch_vm_process(self.session, 'headless')
+		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
 
 	def shutdown(self):
-		self.session.console.power_down()
-		log_check_call(['VBoxManage', 'stopvm', self.uuid,
-		                '--type', 'headless'])
+		self.session.console.power_down().wait_for_completion(-1)
+		self._unlock_machine()
 
 	def destroy(self):
 		if hasattr(self, 'machine'):
-			self.machine.lock_machine(self.session, virtualbox.library.LockType.write)
+			self.machine.lock_machine(self.session, virtualbox.library.LockType.shared)
 			self.session.machine.detach_device(name='SATA Controller', controller_port=0, device=0)
 			self.session.machine.save_settings()
-			self.session.unlock_machine()
+			self._unlock_machine()
 			self.machine.unregister(virtualbox.library.CleanupMode.unregister_only)
 			self.machine.remove(delete=True)
+
+	def _unlock_machine(self):
+		from . import waituntil
+		self.session.unlock_machine()
+		waituntil(lambda: self.machine.session_state != virtualbox.library.SessionState.locked)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 2f00928..08b5a24 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -44,7 +44,10 @@ volume:
 		try:
 			instance = tools.instances.VirtualBoxInstance('unpartitioned_extlinux', image)
 			instance.create()
-			# instance.boot()
+			try:
+				instance.boot()
+			finally:
+				instance.shutdown()
 
 			# tools.test(instance)
 

From e9137ac1722320361854b18089dec99c2c251035 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 19 Dec 2014 01:25:38 +0100
Subject: [PATCH 157/345] Make serialization a lot more easy to handle

---
 bootstrapvz/base/bootstrapinfo.py         |   4 +-
 bootstrapvz/base/fs/partitionmaps/none.py |   9 ++
 bootstrapvz/base/manifest.py              |   2 +-
 bootstrapvz/common/bytes.py               |   8 ++
 bootstrapvz/common/fs/qemuvolume.py       |   4 +
 bootstrapvz/common/fsm_proxy.py           |  13 +++
 bootstrapvz/remote/__init__.py            | 102 +++++++++++++++-------
 tests/integration/virtualbox_tests.py     |   5 +-
 8 files changed, 109 insertions(+), 38 deletions(-)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 70dfa4d..0039253 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -130,10 +130,10 @@ class BootstrapInformation(object):
 
 	def __getstate__(self):
 		state = self.__dict__.copy()
-		exclude_keys = ['volume', 'source_lists', 'preference_lists', 'packages']
+		exclude_keys = ['source_lists', 'preference_lists', 'packages']
 		for key in exclude_keys:
 			del state[key]
-		state['__class__'] = 'bootstrapvz.base.bootstrapinfo.BootstrapInformation'
+		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
 		return state
 
 	def __setstate__(self, state):
diff --git a/bootstrapvz/base/fs/partitionmaps/none.py b/bootstrapvz/base/fs/partitionmaps/none.py
index d9b122a..439fe48 100644
--- a/bootstrapvz/base/fs/partitionmaps/none.py
+++ b/bootstrapvz/base/fs/partitionmaps/none.py
@@ -32,3 +32,12 @@ class NoPartitions(object):
 		:rtype: Bytes
 		"""
 		return self.root.get_end()
+
+	def __getstate__(self):
+		state = self.__dict__.copy()
+		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
+		return state
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 27e7686..02ddae8 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -133,7 +133,7 @@ class Manifest(object):
 		raise ManifestError(message, self.path, data_path)
 
 	def __getstate__(self):
-		return {'__class__': 'bootstrapvz.base.manifest.Manifest',
+		return {'__class__': self.__module__ + '.' + self.__class__.__name__,
 		        'path': self.path,
 		        'data': self.data}
 
diff --git a/bootstrapvz/common/bytes.py b/bootstrapvz/common/bytes.py
index cd6d9c4..a64e0ec 100644
--- a/bootstrapvz/common/bytes.py
+++ b/bootstrapvz/common/bytes.py
@@ -126,6 +126,14 @@ class Bytes(object):
 			self.qty %= other
 		return self
 
+	def __getstate__(self):
+		return {'__class__': self.__module__ + '.' + self.__class__.__name__,
+		        'qty': self.qty,
+		        }
+
+	def __setstate__(self, state):
+		self.qty = state['qty']
+
 
 class UnitError(Exception):
 	pass
diff --git a/bootstrapvz/common/fs/qemuvolume.py b/bootstrapvz/common/fs/qemuvolume.py
index 2f9adfe..26d3300 100644
--- a/bootstrapvz/common/fs/qemuvolume.py
+++ b/bootstrapvz/common/fs/qemuvolume.py
@@ -77,3 +77,7 @@ class QEMUVolume(LoopbackVolume):
 			if not self._is_nbd_used(device_name):
 				return os.path.join('/dev', device_name)
 		raise VolumeError('Unable to find free nbd device.')
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
diff --git a/bootstrapvz/common/fsm_proxy.py b/bootstrapvz/common/fsm_proxy.py
index bf02c73..a968b92 100644
--- a/bootstrapvz/common/fsm_proxy.py
+++ b/bootstrapvz/common/fsm_proxy.py
@@ -43,6 +43,19 @@ class FSMProxy(object):
 			if not hasattr(self, event):
 				setattr(self, event, make_proxy(fsm, event))
 
+	def __getstate__(self):
+		state = {}
+		for key, value in self.__dict__.iteritems():
+			if callable(value) or key == 'fsm':
+				continue
+			state[key] = value
+		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
+		return state
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
+
 
 class FSMProxyError(Exception):
 	pass
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index a7ec9da..4aae205 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -1,45 +1,83 @@
 """Remote module containing methods to bootstrap remotely
 """
-import bootstrapvz.common.exceptions
+from Pyro4.util import SerializerBase
+import logging
+log = logging.getLogger(__name__)
+
+supported_classes = ['bootstrapvz.base.manifest.Manifest',
+                     'bootstrapvz.base.bootstrapinfo.BootstrapInformation',
+                     'bootstrapvz.common.fs.loopbackvolume.LoopbackVolume',
+                     'bootstrapvz.common.fs.qemuvolume.QEMUVolume',
+                     'bootstrapvz.common.fs.virtualdiskimage.VirtualDiskImage',
+                     'bootstrapvz.common.fs.virtualmachinedisk.VirtualMachineDisk',
+                     'bootstrapvz.base.fs.partitionmaps.gpt.GPTPartitionMap',
+                     'bootstrapvz.base.fs.partitionmaps.msdos.MSDOSPartitionMap',
+                     'bootstrapvz.base.fs.partitionmaps.none.NoPartitions',
+                     'bootstrapvz.base.fs.partitions.gpt.GPTPartition',
+                     'bootstrapvz.base.fs.partitions.gpt_swap.GPTSwapPartition',
+                     'bootstrapvz.base.fs.partitions.msdos.MSDOSPartition',
+                     'bootstrapvz.base.fs.partitions.msdos_swap.MSDOSSwapPartition',
+                     'bootstrapvz.base.fs.partitions.single.SinglePartition',
+                     'bootstrapvz.base.fs.partitions.unformatted.UnformattedPartition',
+                     'bootstrapvz.common.bytes.Bytes',
+                     ]
+
+supported_exceptions = ['bootstrapvz.common.exceptions.ManifestError',
+                        'bootstrapvz.common.exceptions.TaskListError',
+                        'bootstrapvz.common.exceptions.TaskError',
+                        'bootstrapvz.base.fs.exceptions.VolumeError',
+                        'bootstrapvz.base.fs.exceptions.PartitionError',
+                        'bootstrapvz.base.pkg.exceptions.PackageError',
+                        'bootstrapvz.base.pkg.exceptions.SourceError',
+                        'bootstrapvz.common.bytes.UnitError',
+                        'bootstrapvz.common.fsm_proxy.FSMProxyError',
+                        ]
 
 
 def register_deserialization_handlers():
-	from Pyro4.util import SerializerBase
-	SerializerBase.register_dict_to_class('bootstrapvz.base.manifest.Manifest', deserialize_manifest)
-	SerializerBase.register_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation', deserialize_bootstrapinfo)
-	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.ManifestError', deserialize_exception)
-	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.TaskListError', deserialize_exception)
-	SerializerBase.register_dict_to_class('bootstrapvz.common.exceptions.TaskError', deserialize_exception)
+	for supported_class in supported_classes:
+		SerializerBase.register_dict_to_class(supported_class, deserialize)
+	for supported_exc in supported_exceptions:
+		SerializerBase.register_dict_to_class(supported_exc, deserialize_exception)
 
 
 def unregister_deserialization_handlers():
+	for supported_class in supported_classes:
+		SerializerBase.unregister_dict_to_class(supported_class, deserialize)
+	for supported_exc in supported_exceptions:
+		SerializerBase.unregister_dict_to_class(supported_exc, deserialize_exception)
+
+
+def deserialize_exception(fq_classname, data):
+	class_object = get_class_object(fq_classname)
 	from Pyro4.util import SerializerBase
-	SerializerBase.unregister_dict_to_class('bootstrapvz.base.manifest.Manifest')
-	SerializerBase.unregister_dict_to_class('bootstrapvz.base.bootstrapinfo.BootstrapInformation')
-	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.ManifestError')
-	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.TaskListError')
-	SerializerBase.unregister_dict_to_class('bootstrapvz.common.exceptions.TaskError')
+	return SerializerBase.make_exception(class_object, data)
 
 
-def deserialize_manifest(classname, state):
-	from bootstrapvz.base.manifest import Manifest
-	manifest = Manifest.__new__(Manifest)
-	manifest.__setstate__(state)
-	return manifest
+def deserialize(fq_classname, data):
+	class_object = get_class_object(fq_classname)
+	from Pyro4.util import SerpentSerializer
+	from Pyro4.errors import SecurityError
+	ser = SerpentSerializer()
+	state = {}
+	for key, value in data.items():
+		try:
+			state[key] = ser.recreate_classes(value)
+		except SecurityError as e:
+			msg = 'Unable to deserialize key `{key}\' on {class_name}'.format(key=key, class_name=fq_classname)
+			import pprint
+			msg += pprint.pformat(data)
+			raise Exception(msg, e)
+
+	instance = class_object.__new__(class_object)
+	instance.__setstate__(state)
+	return instance
 
 
-def deserialize_bootstrapinfo(classname, state):
-	from bootstrapvz.base.bootstrapinfo import BootstrapInformation
-	bootstrap_info = BootstrapInformation.__new__(BootstrapInformation)
-	bootstrap_info.__setstate__(state)
-	return bootstrap_info
-
-deserialize_map = {'bootstrapvz.common.exceptions.ManifestError': bootstrapvz.common.exceptions.ManifestError,
-                   'bootstrapvz.common.exceptions.TaskListError': bootstrapvz.common.exceptions.TaskListError,
-                   'bootstrapvz.common.exceptions.TaskError': bootstrapvz.common.exceptions.TaskError,
-                   }
-
-
-def deserialize_exception(classname, data):
-	from Pyro4.util import SerializerBase
-	return SerializerBase.make_exception(deserialize_map[classname], data)
+def get_class_object(fq_classname):
+	parts = fq_classname.split('.')
+	module_name = '.'.join(parts[:-1])
+	class_name = parts[-1]
+	import importlib
+	imported_module = importlib.import_module(module_name)
+	return getattr(imported_module, class_name)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 08b5a24..caf39e9 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -38,6 +38,7 @@ volume:
 		os.close(handle)
 		build_server.download(bootstrap_info.volume.image_path, image_path)
 		build_server.delete(bootstrap_info.volume.image_path)
+		# image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
 
 	try:
 		image = tools.images.VirtualBoxImage(manifest, image_path)
@@ -46,11 +47,9 @@ volume:
 			instance.create()
 			try:
 				instance.boot()
+				# tools.reachable_with_ssh(instance)
 			finally:
 				instance.shutdown()
-
-			# tools.test(instance)
-
 		finally:
 			if 'instance' in locals():
 				instance.destroy()

From 71e0d943fcb08c2671ca51f53db501a7f72fe947 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 19 Dec 2014 01:26:33 +0100
Subject: [PATCH 158/345] Make json and yaml files conflict instead of override

---
 tests/integration/manifests/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
index 7d84774..dd6bfa3 100644
--- a/tests/integration/manifests/__init__.py
+++ b/tests/integration/manifests/__init__.py
@@ -5,11 +5,11 @@ from bootstrapvz.common.tools import load_data
 partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml'))
 partial_yaml = glob.glob(os.path.join(os.path.dirname(__file__), '*.json'))
 
-def dictkey(path):
-	return 
-# yaml overrides json
 partials = {}
 for path in partial_json + partial_yaml:
 	key = os.path.splitext(os.path.basename(path))[0]
+	if key in partials:
+		msg = 'Error when loading partial manifests: The partial {key} exists twice'.format(key=key)
+		raise Exception(msg)
 	partials[key] = load_data(path)
 	

From ad52df37bd50e433c92c269266f98c842c674194 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 19 Dec 2014 01:27:16 +0100
Subject: [PATCH 159/345] Add partial for config of root password

---
 tests/integration/manifests/__init__.py       | 7 ++++++-
 tests/integration/manifests/root_password.yml | 4 ++++
 tests/integration/virtualbox_tests.py         | 3 ++-
 3 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 tests/integration/manifests/root_password.yml

diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
index dd6bfa3..a535f30 100644
--- a/tests/integration/manifests/__init__.py
+++ b/tests/integration/manifests/__init__.py
@@ -12,4 +12,9 @@ for path in partial_json + partial_yaml:
 		msg = 'Error when loading partial manifests: The partial {key} exists twice'.format(key=key)
 		raise Exception(msg)
 	partials[key] = load_data(path)
-	
+
+import random
+import string
+pool = string.ascii_uppercase + string.ascii_lowercase + string.digits
+random_password = ''.join(random.choice(pool) for _ in range(16))
+partials['root_password']['plugins']['root_password']['password'] = random_password
diff --git a/tests/integration/manifests/root_password.yml b/tests/integration/manifests/root_password.yml
new file mode 100644
index 0000000..fdc826a
--- /dev/null
+++ b/tests/integration/manifests/root_password.yml
@@ -0,0 +1,4 @@
+---
+plugins:
+  root_password:
+    password: random password set by the partial manifest loader
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index caf39e9..2cd3f02 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -20,7 +20,8 @@ volume:
     type: msdos
 """)
 	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
-	                                  partials['unpartitioned'], manifest_data)
+	                                  partials['unpartitioned'], partials['root_password'],
+	                                  manifest_data)
 
 	build_server = pick_build_server(build_servers, manifest_data)
 	manifest_data = build_server.apply_build_settings(manifest_data)

From e8d6e7f6023293459a1fba572bb3ec542879cd9f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 19 Dec 2014 01:43:10 +0100
Subject: [PATCH 160/345] factor AbstractPartition.Mount out into separate
 module and make it serializable

---
 bootstrapvz/base/fs/partitions/abstract.py | 40 +-----------------
 bootstrapvz/base/fs/partitions/mount.py    | 49 ++++++++++++++++++++++
 bootstrapvz/remote/__init__.py             |  1 +
 tests/integration/virtualbox_tests.py      |  1 -
 4 files changed, 52 insertions(+), 39 deletions(-)
 create mode 100644 bootstrapvz/base/fs/partitions/mount.py

diff --git a/bootstrapvz/base/fs/partitions/abstract.py b/bootstrapvz/base/fs/partitions/abstract.py
index 6d3cf48..038e393 100644
--- a/bootstrapvz/base/fs/partitions/abstract.py
+++ b/bootstrapvz/base/fs/partitions/abstract.py
@@ -1,6 +1,5 @@
 from abc import ABCMeta
 from abc import abstractmethod
-import os.path
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.fsm_proxy import FSMProxy
 
@@ -19,42 +18,6 @@ class AbstractPartition(FSMProxy):
 	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
 	          ]
 
-	class Mount(object):
-		"""Represents a mount into the partition
-		"""
-		def __init__(self, source, destination, opts):
-			"""
-			:param str,AbstractPartition source: The path from where we mount or a partition
-			:param str destination: The path of the mountpoint
-			:param list opts: List of options to pass to the mount command
-			"""
-			self.source      = source
-			self.destination = destination
-			self.opts        = opts
-
-		def mount(self, prefix):
-			"""Performs the mount operation or forwards it to another partition
-
-			:param str prefix: Path prefix of the mountpoint
-			"""
-			mount_dir = os.path.join(prefix, self.destination)
-			# If the source is another partition, we tell that partition to mount itself
-			if isinstance(self.source, AbstractPartition):
-				self.source.mount(destination=mount_dir)
-			else:
-				log_check_call(['mount'] + self.opts + [self.source, mount_dir])
-			self.mount_dir = mount_dir
-
-		def unmount(self):
-			"""Performs the unmount operation or asks the partition to unmount itself
-			"""
-			# If its a partition, it can unmount itself
-			if isinstance(self.source, AbstractPartition):
-				self.source.unmount()
-			else:
-				log_check_call(['umount', self.mount_dir])
-			del self.mount_dir
-
 	def __init__(self, size, filesystem, format_command):
 		"""
 		:param Bytes size: Size of the partition
@@ -143,7 +106,8 @@ class AbstractPartition(FSMProxy):
 		:param list opts: Any options that should be passed to the mount command
 		"""
 		# Create a new mount object, mount it if the partition is mounted and put it in the mounts dict
-		mount = self.Mount(source, destination, opts)
+		from mount import Mount
+		mount = Mount(source, destination, opts)
 		if self.fsm.current == 'mounted':
 			mount.mount(self.mount_dir)
 		self.mounts[destination] = mount
diff --git a/bootstrapvz/base/fs/partitions/mount.py b/bootstrapvz/base/fs/partitions/mount.py
new file mode 100644
index 0000000..7ac7e4b
--- /dev/null
+++ b/bootstrapvz/base/fs/partitions/mount.py
@@ -0,0 +1,49 @@
+from abstract import AbstractPartition
+import os.path
+from bootstrapvz.common.tools import log_check_call
+
+
+class Mount(object):
+	"""Represents a mount into the partition
+	"""
+	def __init__(self, source, destination, opts):
+		"""
+		:param str,AbstractPartition source: The path from where we mount or a partition
+		:param str destination: The path of the mountpoint
+		:param list opts: List of options to pass to the mount command
+		"""
+		self.source      = source
+		self.destination = destination
+		self.opts        = opts
+
+	def mount(self, prefix):
+		"""Performs the mount operation or forwards it to another partition
+
+		:param str prefix: Path prefix of the mountpoint
+		"""
+		mount_dir = os.path.join(prefix, self.destination)
+		# If the source is another partition, we tell that partition to mount itself
+		if isinstance(self.source, AbstractPartition):
+			self.source.mount(destination=mount_dir)
+		else:
+			log_check_call(['mount'] + self.opts + [self.source, mount_dir])
+		self.mount_dir = mount_dir
+
+	def unmount(self):
+		"""Performs the unmount operation or asks the partition to unmount itself
+		"""
+		# If its a partition, it can unmount itself
+		if isinstance(self.source, AbstractPartition):
+			self.source.unmount()
+		else:
+			log_check_call(['umount', self.mount_dir])
+		del self.mount_dir
+
+	def __getstate__(self):
+		state = self.__dict__.copy()
+		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
+		return state
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 4aae205..0990417 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -13,6 +13,7 @@ supported_classes = ['bootstrapvz.base.manifest.Manifest',
                      'bootstrapvz.base.fs.partitionmaps.gpt.GPTPartitionMap',
                      'bootstrapvz.base.fs.partitionmaps.msdos.MSDOSPartitionMap',
                      'bootstrapvz.base.fs.partitionmaps.none.NoPartitions',
+                     'bootstrapvz.base.fs.partitions.mount.Mount',
                      'bootstrapvz.base.fs.partitions.gpt.GPTPartition',
                      'bootstrapvz.base.fs.partitions.gpt_swap.GPTSwapPartition',
                      'bootstrapvz.base.fs.partitions.msdos.MSDOSPartition',
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 2cd3f02..1537c02 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -39,7 +39,6 @@ volume:
 		os.close(handle)
 		build_server.download(bootstrap_info.volume.image_path, image_path)
 		build_server.delete(bootstrap_info.volume.image_path)
-		# image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
 
 	try:
 		image = tools.images.VirtualBoxImage(manifest, image_path)

From 8090d3c5bc5ea0ab9bf960c76262a30c48dc2760 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 19 Dec 2014 21:55:32 +0100
Subject: [PATCH 161/345] Log to file on the remote

---
 bootstrapvz/remote/server.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index a802af2..28e9e44 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -15,11 +15,22 @@ def main():
 
 
 def setup_logging():
+	root = logging.getLogger()
+	root.setLevel(logging.NOTSET)
+
 	from log import LogForwarder
 	log_forwarder = LogForwarder()
-	root = logging.getLogger()
 	root.addHandler(log_forwarder)
-	root.setLevel(logging.NOTSET)
+
+	from datetime import datetime
+	import os.path
+	from bootstrapvz.base.log import get_file_handler
+	timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
+	filename = '{timestamp}_remote.log'.format(timestamp=timestamp)
+	logfile_path = os.path.join('/var/log/bootstrap-vz', filename)
+	file_handler = get_file_handler(logfile_path, True)
+	root.addHandler(file_handler)
+
 	return log_forwarder
 
 

From 6f23bcaafc3c9e0d3f0f2a394eb591009787193a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 01:17:56 +0100
Subject: [PATCH 162/345] Add apt_proxy partial

---
 tests/integration/manifests/apt_proxy.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 tests/integration/manifests/apt_proxy.yml

diff --git a/tests/integration/manifests/apt_proxy.yml b/tests/integration/manifests/apt_proxy.yml
new file mode 100644
index 0000000..36e0026
--- /dev/null
+++ b/tests/integration/manifests/apt_proxy.yml
@@ -0,0 +1,5 @@
+---
+plugins:
+  apt_proxy:
+    address: 127.0.0.1
+    port: 3142

From 6a0bef147ad4b746ddd85c00e40f6fd452845043 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 01:21:00 +0100
Subject: [PATCH 163/345] Refactor. Locking a vbox is now quite a bit easier

---
 tests/integration/images/__init__.py      | 10 +++
 tests/integration/images/virtualbox.py    | 18 +++++
 tests/integration/instances/__init__.py   | 16 ++++
 tests/integration/instances/virtualbox.py | 95 +++++++++++++++++++++++
 tests/integration/tools/images.py         | 25 ------
 tests/integration/tools/instances.py      | 71 -----------------
 tests/integration/virtualbox_tests.py     | 45 ++++++-----
 7 files changed, 165 insertions(+), 115 deletions(-)
 create mode 100644 tests/integration/images/__init__.py
 create mode 100644 tests/integration/images/virtualbox.py
 create mode 100644 tests/integration/instances/__init__.py
 create mode 100644 tests/integration/instances/virtualbox.py
 delete mode 100644 tests/integration/tools/images.py
 delete mode 100644 tests/integration/tools/instances.py

diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py
new file mode 100644
index 0000000..9945ad8
--- /dev/null
+++ b/tests/integration/images/__init__.py
@@ -0,0 +1,10 @@
+
+
+class Image(object):
+
+	def __init__(self, manifest):
+		self.manifest = manifest
+
+	def destroy(self):
+		pass
+
diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
new file mode 100644
index 0000000..1584140
--- /dev/null
+++ b/tests/integration/images/virtualbox.py
@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+from . import Image
+import virtualbox as vboxapi
+
+
+class VirtualBoxImage(Image):
+
+	def __init__(self, manifest, image_path):
+		super(VirtualBoxImage, self).__init__(manifest)
+		self.image_path = image_path
+		self.vbox = vboxapi.VirtualBox()
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    vboxapi.library.DeviceType.hard_disk,  # decive_type
+		                                    vboxapi.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
+
+	def destroy(self):
+		self.medium.close()
diff --git a/tests/integration/instances/__init__.py b/tests/integration/instances/__init__.py
new file mode 100644
index 0000000..049fc0f
--- /dev/null
+++ b/tests/integration/instances/__init__.py
@@ -0,0 +1,16 @@
+
+
+class Instance(object):
+
+	def __init__(self, name, image):
+		self.name = name
+		self.image = image
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass
diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
new file mode 100644
index 0000000..69fed19
--- /dev/null
+++ b/tests/integration/instances/virtualbox.py
@@ -0,0 +1,95 @@
+from __future__ import absolute_import
+from . import Instance
+import virtualbox as vboxapi
+
+
+class VirtualBoxInstance(Instance):
+
+	cpus = 1
+	memory = 256
+
+	def __init__(self, name, image):
+		super(VirtualBoxInstance, self).__init__(name, image)
+		self.vbox = vboxapi.VirtualBox()
+		manager = vboxapi.Manager()
+		self.session = manager.get_session()
+
+	def create(self):
+		# create machine
+		os_type = {'x86': 'Debian',
+		           'amd64': 'Debian_64'}.get(self.image.manifest.system['architecture'])
+		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
+		                                        groups=[], os_type_id=os_type, flags='')
+		self.machine.cpu_count = self.cpus
+		self.machine.memory_size = self.memory
+		self.machine.save_settings()  # save settings, so that we can register it
+		self.vbox.register_machine(self.machine)
+
+		# attach image
+		with self.Lock(self.machine, self.session) as machine:
+			strg_ctrl = machine.add_storage_controller('SATA Controller',
+			                                           vboxapi.library.StorageBus.sata)
+			strg_ctrl.port_count = 1
+			machine.attach_device(name='SATA Controller', controller_port=0, device=0,
+			                      type_p=vboxapi.library.DeviceType.hard_disk,
+			                      medium=self.image.medium)
+			machine.save_settings()
+
+		# redirect serial port
+		with self.Lock(self.machine, self.session) as machine:
+			serial_port = machine.get_serial_port(0)
+			serial_port.enabled = True
+			import tempfile
+			handle, self.serial_port_path = tempfile.mkstemp()
+			import os
+			os.close(handle)
+			serial_port.path = self.serial_port_path
+			serial_port.host_mode = vboxapi.library.PortMode.host_pipe
+			# serial_port.server = True
+			machine.save_settings()
+
+	def boot(self):
+		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
+
+	def shutdown(self):
+		self.session.console.power_down().wait_for_completion(-1)
+		self.Lock(self.machine, self.session).unlock()
+
+	def destroy(self):
+		if hasattr(self, 'machine'):
+			try:
+				with self.Lock(self.machine, self.session) as machine:
+					machine.detach_device(name='SATA Controller', controller_port=0, device=0)
+					machine.save_settings()
+			except vboxapi.library.VBoxErrorObjectNotFound:
+				pass
+			self.machine.unregister(vboxapi.library.CleanupMode.unregister_only)
+			self.machine.remove(delete=True)
+
+	class Lock(object):
+		def __init__(self, machine, session):
+			self.machine = machine
+			self.session = session
+
+		def __enter__(self):
+			return self.lock()
+
+		def __exit__(self, type, value, traceback):
+			return self.unlock()
+
+		def lock(self):
+			self.unlock()
+			self.machine.lock_machine(self.session, vboxapi.library.LockType.write)
+			return self.session.machine
+
+		def unlock(self):
+			from ..tools import waituntil
+			if self.machine.session_state == vboxapi.library.SessionState.unlocked:
+				return
+			if self.machine.session_state == vboxapi.library.SessionState.unlocking:
+				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
+				return
+			if self.machine.session_state == vboxapi.library.SessionState.spawning:
+				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.locked)
+			self.session.unlock_machine()
+			waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
diff --git a/tests/integration/tools/images.py b/tests/integration/tools/images.py
deleted file mode 100644
index 2bda849..0000000
--- a/tests/integration/tools/images.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import virtualbox
-
-
-class Image(object):
-
-	def __init__(self, manifest):
-		self.manifest = manifest
-
-	def destroy(self):
-		pass
-
-
-class VirtualBoxImage(Image):
-
-	def __init__(self, manifest, image_path):
-		super(VirtualBoxImage, self).__init__(manifest)
-		self.image_path = image_path
-		self.vbox = virtualbox.VirtualBox()
-		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    virtualbox.library.DeviceType.hard_disk,  # decive_type
-		                                    virtualbox.library.AccessMode.read_only,  # access_mode
-		                                    False)  # force_new_uuid
-
-	def destroy(self):
-		self.medium.close()
diff --git a/tests/integration/tools/instances.py b/tests/integration/tools/instances.py
deleted file mode 100644
index 87591e5..0000000
--- a/tests/integration/tools/instances.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import virtualbox
-
-
-class Instance(object):
-
-	def __init__(self, name, image):
-		self.name = name
-		self.image = image
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass
-
-
-class VirtualBoxInstance(Instance):
-
-	cpus = 1
-	memory = 256
-
-	def __init__(self, name, image):
-		super(VirtualBoxInstance, self).__init__(name, image)
-		self.vbox = virtualbox.VirtualBox()
-		manager = virtualbox.Manager()
-		self.session = manager.get_session()
-
-	def create(self):
-		if self.image.manifest.system['architecture'] == 'x86':
-			os_type = 'Debian'
-		else:
-			os_type = 'Debian_64'
-		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
-		                                        groups=[], os_type_id=os_type, flags='')
-		self.machine.cpu_count = self.cpus
-		self.machine.memory_size = self.memory
-		self.machine.save_settings()  # save settings, so that we can register it
-		self.vbox.register_machine(self.machine)
-		self.machine.lock_machine(self.session, virtualbox.library.LockType.write)
-		strg_ctrl = self.session.machine.add_storage_controller('SATA Controller',
-		                                                        virtualbox.library.StorageBus.sata)
-		strg_ctrl.port_count = 1
-		self.session.machine.attach_device(name='SATA Controller', controller_port=0, device=0,
-		                                   type_p=virtualbox.library.DeviceType.hard_disk,
-		                                   medium=self.image.medium)
-		self.session.machine.save_settings()  # save changes to the controller
-		self._unlock_machine()
-
-	def boot(self):
-		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
-
-	def shutdown(self):
-		self.session.console.power_down().wait_for_completion(-1)
-		self._unlock_machine()
-
-	def destroy(self):
-		if hasattr(self, 'machine'):
-			self.machine.lock_machine(self.session, virtualbox.library.LockType.shared)
-			self.session.machine.detach_device(name='SATA Controller', controller_port=0, device=0)
-			self.session.machine.save_settings()
-			self._unlock_machine()
-			self.machine.unregister(virtualbox.library.CleanupMode.unregister_only)
-			self.machine.remove(delete=True)
-
-	def _unlock_machine(self):
-		from . import waituntil
-		self.session.unlock_machine()
-		waituntil(lambda: self.machine.session_state != virtualbox.library.SessionState.locked)
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 1537c02..0fd95f0 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,10 +1,10 @@
 import tools
-import tools.images
-import tools.instances
 from manifests import partials
 from bootstrapvz.base.manifest import Manifest
 from bootstrapvz.remote.build_servers import pick_build_server
 from . import build_servers
+from images.virtualbox import VirtualBoxImage
+from instances.virtualbox import VirtualBoxInstance
 
 
 def test_virtualbox_unpartitioned_extlinux():
@@ -19,42 +19,49 @@ volume:
   partitions:
     type: msdos
 """)
-	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'],
-	                                  partials['unpartitioned'], partials['root_password'],
+	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'], partials['unpartitioned'],
+	                                  partials['root_password'], partials['apt_proxy'],
 	                                  manifest_data)
 
 	build_server = pick_build_server(build_servers, manifest_data)
 	manifest_data = build_server.apply_build_settings(manifest_data)
 	manifest = Manifest(data=manifest_data)
 
-	bootstrap_info = tools.bootstrap(manifest, build_server)
+	# bootstrap_info = tools.bootstrap(manifest, build_server)
+	from bootstrapvz.base.bootstrapinfo import BootstrapInformation
+	bootstrap_info = BootstrapInformation(manifest)
+	bootstrap_info.volume.image_path = '/target/debian-wheezy-amd64-141218.vdi'
 
 	from bootstrapvz.remote.build_servers import LocalBuildServer
 	if isinstance(build_server, LocalBuildServer):
 		image_path = bootstrap_info.volume.image_path
 	else:
-		import tempfile
-		handle, image_path = tempfile.mkstemp()
-		import os
-		os.close(handle)
-		build_server.download(bootstrap_info.volume.image_path, image_path)
-		build_server.delete(bootstrap_info.volume.image_path)
+		# import tempfile
+		# handle, image_path = tempfile.mkstemp()
+		# import os
+		# os.close(handle)
+		# build_server.download(bootstrap_info.volume.image_path, image_path)
+		# build_server.delete(bootstrap_info.volume.image_path)
+		image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
 
+	image = VirtualBoxImage(manifest, image_path)
+	lines = []
 	try:
-		image = tools.images.VirtualBoxImage(manifest, image_path)
+		instance = VirtualBoxInstance('unpartitioned_extlinux', image)
 		try:
-			instance = tools.instances.VirtualBoxInstance('unpartitioned_extlinux', image)
 			instance.create()
 			try:
 				instance.boot()
+				lines = instance.get_console_output()
 				# tools.reachable_with_ssh(instance)
 			finally:
 				instance.shutdown()
 		finally:
-			if 'instance' in locals():
-				instance.destroy()
+			instance.destroy()
 	finally:
-		if 'image' in locals():
-			image.destroy()
-		import os
-		os.remove(image_path)
+		# pass
+		if len(lines) > 0:
+			raise Exception('\n'.join(lines))
+		# image.destroy()
+		# import os
+		# os.remove(image_path)

From ed98ab30fdc7b8abaac7c9597ccc77d67827f66f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 02:25:47 +0100
Subject: [PATCH 164/345] Able to get console output from the machine now! It's
 blocking though, so maybe there should be a check for "Entering runlevel: 2"
 or some shit...

---
 tests/integration/instances/virtualbox.py | 27 ++++++++++++++++++++++-
 tests/integration/virtualbox_tests.py     | 13 +++++------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 69fed19..fa3b7cd 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -45,12 +45,37 @@ class VirtualBoxInstance(Instance):
 			os.close(handle)
 			serial_port.path = self.serial_port_path
 			serial_port.host_mode = vboxapi.library.PortMode.host_pipe
-			# serial_port.server = True
+			serial_port.server = True  # Create the socket on startup
 			machine.save_settings()
 
 	def boot(self):
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
 
+	def get_console_output(self):
+		import socket
+		import select
+		import errno
+		import sys
+		console = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+		console.connect(self.serial_port_path)
+
+		console.setblocking(0)
+		output = ''
+		continue_select = True
+		while continue_select:
+			read_ready, _, _ = select.select([console], [], [])
+			if console in read_ready:
+				while True:
+					try:
+						sys.stdout.write(console.recv(1024))
+						break
+					except socket.error, e:
+						if e.errno != errno.EWOULDBLOCK:
+							raise Exception(e)
+						continue_select = False
+		console.close()
+		return output
+
 	def shutdown(self):
 		self.session.console.power_down().wait_for_completion(-1)
 		self.Lock(self.machine, self.session).unlock()
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 0fd95f0..99b4459 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -45,23 +45,20 @@ volume:
 		image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
 
 	image = VirtualBoxImage(manifest, image_path)
-	lines = []
+	output = None
 	try:
 		instance = VirtualBoxInstance('unpartitioned_extlinux', image)
 		try:
 			instance.create()
 			try:
 				instance.boot()
-				lines = instance.get_console_output()
+				output = instance.get_console_output()
 				# tools.reachable_with_ssh(instance)
 			finally:
 				instance.shutdown()
 		finally:
 			instance.destroy()
 	finally:
-		# pass
-		if len(lines) > 0:
-			raise Exception('\n'.join(lines))
-		# image.destroy()
-		# import os
-		# os.remove(image_path)
+		image.destroy()
+		import os
+		os.remove(image_path)

From 27950af66e207bd8f78bd23ed154c065bfbe10d6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 02:43:20 +0100
Subject: [PATCH 165/345] Check runlevel in order to terminate reading from the
 console

---
 tests/integration/instances/virtualbox.py | 15 ++++++++++++---
 tests/integration/virtualbox_tests.py     |  2 --
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index fa3b7cd..9766fe8 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -50,24 +50,33 @@ class VirtualBoxInstance(Instance):
 
 	def boot(self):
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
+		self.console_output = self._read_console_output()
 
-	def get_console_output(self):
+	def _read_console_output(self):
 		import socket
 		import select
 		import errno
 		import sys
 		console = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 		console.connect(self.serial_port_path)
-
 		console.setblocking(0)
+
+		runlvl_check = 'INIT: Entering runlevel: 2'
 		output = ''
+		ptr = 0
 		continue_select = True
 		while continue_select:
 			read_ready, _, _ = select.select([console], [], [])
 			if console in read_ready:
 				while True:
 					try:
-						sys.stdout.write(console.recv(1024))
+						read = console.recv(1024)
+						output += read
+						sys.stdout.write(read)
+						if runlvl_check in output[ptr:]:
+							continue_select = False
+						else:
+							ptr = len(output) - len(runlvl_check)
 						break
 					except socket.error, e:
 						if e.errno != errno.EWOULDBLOCK:
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 99b4459..36fb8ad 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -45,14 +45,12 @@ volume:
 		image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
 
 	image = VirtualBoxImage(manifest, image_path)
-	output = None
 	try:
 		instance = VirtualBoxInstance('unpartitioned_extlinux', image)
 		try:
 			instance.create()
 			try:
 				instance.boot()
-				output = instance.get_console_output()
 				# tools.reachable_with_ssh(instance)
 			finally:
 				instance.shutdown()

From 200c5086e5fc45988a659691bdd3a8f54f1a33b9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 15:40:48 +0100
Subject: [PATCH 166/345] Extend sed_i to raise Exceptions when the expected
 amount of replacements is not met

---
 bootstrapvz/common/exceptions.py |  8 ++++++++
 bootstrapvz/common/tools.py      | 23 +++++++++++++++++++++--
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index dee0d53..2e1930e 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -26,3 +26,11 @@ class TaskListError(Exception):
 
 class TaskError(Exception):
 	pass
+
+
+class NoMatchesError(Exception):
+	pass
+
+
+class TooManyMatchesError(Exception):
+	pass
diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 79d5202..1715c66 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -56,11 +56,30 @@ def log_call(command, stdin=None, env=None, shell=False, cwd=None):
 	return process.returncode, stdout, stderr
 
 
-def sed_i(file_path, pattern, subst):
+def sed_i(file_path, pattern, subst, expected_replacements=1):
+	replacement_count = inline_replace(file_path, pattern, subst)
+	if replacement_count < expected_replacements:
+		from exceptions import NoMatchesError
+		msg = ('There were no matches for the expression `{exp}\' in the file `{path}\''
+		       .format(exp=pattern, path=file_path))
+		raise NoMatchesError(msg)
+	if replacement_count > expected_replacements:
+		from exceptions import TooManyMatchesError
+		msg = ('There were too many matches for the expression `{exp}\' in the file `{path}\''
+		       .format(exp=pattern, path=file_path))
+		raise TooManyMatchesError(msg)
+
+
+def inline_replace(file_path, pattern, subst):
 	import fileinput
 	import re
+	replacement_count = 0
 	for line in fileinput.input(files=file_path, inplace=True):
-		print re.sub(pattern, subst, line),
+		replacement = re.sub(pattern, subst, line)
+		if replacement != line:
+			replacement_count += 1
+		print replacement,
+	return replacement_count
 
 
 def load_json(path):

From 58d66fea68069d7098608c614e9f387b1a5bb3ae Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 15:42:14 +0100
Subject: [PATCH 167/345] Make extlinux output boot messages to the serial
 console (fixes #136)

---
 bootstrapvz/common/task_groups.py | 2 +-
 bootstrapvz/common/tasks/boot.py  | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index 2ae5e50..b35fbb3 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -138,7 +138,7 @@ def get_bootloader_group(manifest):
 			group.append(boot.InstallGrub_2)
 	if manifest.system['bootloader'] == 'extlinux':
 		group.extend([boot.AddExtlinuxPackage,
-		              boot.ConfigureExtLinux,
+		              boot.ConfigureExtlinux,
 		              boot.InstallExtLinux])
 	return group
 
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 71c6452..81c8e02 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -148,7 +148,7 @@ class AddExtlinuxPackage(Task):
 			info.packages.add('syslinux-common')
 
 
-class ConfigureExtLinux(Task):
+class ConfigureExtlinux(Task):
 	description = 'Configuring extlinux'
 	phase = phases.system_modification
 	predecessors = [filesystem.FStab]
@@ -157,14 +157,14 @@ class ConfigureExtLinux(Task):
 	def run(cls, info):
 		from bootstrapvz.common.tools import sed_i
 		extlinux_def = os.path.join(info.root, 'etc/default/extlinux')
-		sed_i(extlinux_def, '^EXTLINUX_PARAMETERS="ro quiet"',
-		                    'EXTLINUX_PARAMETERS="ro console=ttyS0"')
+		sed_i(extlinux_def, r'^EXTLINUX_PARAMETERS="([^"]+)"$',
+		                    r'EXTLINUX_PARAMETERS="\1 console=ttyS0"')
 
 
 class InstallExtLinux(Task):
 	description = 'Installing extlinux'
 	phase = phases.system_modification
-	predecessors = [filesystem.FStab]
+	predecessors = [filesystem.FStab, ConfigureExtlinux]
 
 	@classmethod
 	def run(cls, info):

From c71a8230fe626a90557fea182d1ed889f56cd28e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 15:43:28 +0100
Subject: [PATCH 168/345] Make tests/ a module, so that test scripts can import
 and run the tests manually

---
 tests/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 tests/__init__.py

diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29

From a11e46661105666319f67d984c2a48eaafa44830 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 16:07:00 +0100
Subject: [PATCH 169/345] Generalize reading from socket

---
 tests/integration/instances/virtualbox.py | 36 ++------------------
 tests/integration/tools/__init__.py       | 40 +++++++++++++++++++++++
 tests/integration/tools/exceptions.py     |  4 +++
 3 files changed, 46 insertions(+), 34 deletions(-)
 create mode 100644 tests/integration/tools/exceptions.py

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 9766fe8..953e72b 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -50,40 +50,8 @@ class VirtualBoxInstance(Instance):
 
 	def boot(self):
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
-		self.console_output = self._read_console_output()
-
-	def _read_console_output(self):
-		import socket
-		import select
-		import errno
-		import sys
-		console = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-		console.connect(self.serial_port_path)
-		console.setblocking(0)
-
-		runlvl_check = 'INIT: Entering runlevel: 2'
-		output = ''
-		ptr = 0
-		continue_select = True
-		while continue_select:
-			read_ready, _, _ = select.select([console], [], [])
-			if console in read_ready:
-				while True:
-					try:
-						read = console.recv(1024)
-						output += read
-						sys.stdout.write(read)
-						if runlvl_check in output[ptr:]:
-							continue_select = False
-						else:
-							ptr = len(output) - len(runlvl_check)
-						break
-					except socket.error, e:
-						if e.errno != errno.EWOULDBLOCK:
-							raise Exception(e)
-						continue_select = False
-		console.close()
-		return output
+		from ..tools import read_from_socket
+		self.console_output = read_from_socket(self.serial_port_path, 'INIT: Entering runlevel: 2', 20)
 
 	def shutdown(self):
 		self.session.console.power_down().wait_for_completion(-1)
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index c858563..7760a1d 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -33,6 +33,46 @@ def waituntil(predicate, timeout=5, interval=0.05):
 	return False
 
 
+def read_from_socket(socket_path, termination_string, timeout):
+		import socket
+		import select
+		import errno
+		console = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+		console.connect(socket_path)
+		console.setblocking(0)
+
+		output = ''
+		ptr = 0
+		continue_select = True
+		nooutput_for = 0
+		select_timeout = .5
+		while continue_select:
+			read_ready, _, _ = select.select([console], [], [], select_timeout)
+			if console in read_ready:
+				nooutput_for = 0
+				while True:
+					try:
+						output += console.recv(1024)
+						if termination_string in output[ptr:]:
+							continue_select = False
+						else:
+							ptr = len(output) - len(termination_string)
+						break
+					except socket.error, e:
+						if e.errno != errno.EWOULDBLOCK:
+							raise Exception(e)
+						continue_select = False
+			else:
+				nooutput_for += select_timeout
+			if nooutput_for > timeout:
+				from exceptions import SocketReadTimeout
+				msg = ('Reading from socket `{path}\' timed out after {seconds} seconds.'
+				       .format(path=socket_path, timeout=nooutput_for))
+				raise SocketReadTimeout(msg)
+		console.close()
+		return output
+
+
 def bootstrap(manifest, build_server):
 	from bootstrapvz.remote.build_servers import LocalBuildServer
 	if isinstance(build_server, LocalBuildServer):
diff --git a/tests/integration/tools/exceptions.py b/tests/integration/tools/exceptions.py
new file mode 100644
index 0000000..830c625
--- /dev/null
+++ b/tests/integration/tools/exceptions.py
@@ -0,0 +1,4 @@
+
+
+class SocketReadTimeout(Exception):
+	pass

From 9c6af89e7837761f7442d4bb187b042d096cc238 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 16:52:31 +0100
Subject: [PATCH 170/345] Abstract bootstrapping, instance creation, booting
 etc.. This all now runs using a simple with: statement

---
 tests/integration/images/__init__.py          |  4 -
 tests/integration/images/virtualbox.py        | 18 ++++-
 tests/integration/instances/virtualbox.py     | 23 ++++++
 tests/integration/manifests/__init__.py       | 26 +++++++
 tests/integration/manifests/extlinux.yml      |  3 +
 tests/integration/manifests/gpt.yml           |  4 +
 tests/integration/manifests/grub.yml          |  3 +
 tests/integration/manifests/msdos.yml         |  4 +
 .../manifests/single_partition.yml            |  6 ++
 tests/integration/manifests/unpartitioned.yml |  2 +-
 tests/integration/tools/__init__.py           | 34 ---------
 tests/integration/tools/bootable_manifest.py  | 57 ++++++++++++++
 tests/integration/virtualbox_tests.py         | 74 ++++---------------
 13 files changed, 158 insertions(+), 100 deletions(-)
 create mode 100644 tests/integration/manifests/extlinux.yml
 create mode 100644 tests/integration/manifests/gpt.yml
 create mode 100644 tests/integration/manifests/grub.yml
 create mode 100644 tests/integration/manifests/msdos.yml
 create mode 100644 tests/integration/manifests/single_partition.yml
 create mode 100644 tests/integration/tools/bootable_manifest.py

diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py
index 9945ad8..5260afa 100644
--- a/tests/integration/images/__init__.py
+++ b/tests/integration/images/__init__.py
@@ -4,7 +4,3 @@ class Image(object):
 
 	def __init__(self, manifest):
 		self.manifest = manifest
-
-	def destroy(self):
-		pass
-
diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
index 1584140..acb9274 100644
--- a/tests/integration/images/virtualbox.py
+++ b/tests/integration/images/virtualbox.py
@@ -9,10 +9,26 @@ class VirtualBoxImage(Image):
 		super(VirtualBoxImage, self).__init__(manifest)
 		self.image_path = image_path
 		self.vbox = vboxapi.VirtualBox()
+
+	def open(self):
 		self.medium = self.vbox.open_medium(self.image_path,  # location
 		                                    vboxapi.library.DeviceType.hard_disk,  # decive_type
 		                                    vboxapi.library.AccessMode.read_only,  # access_mode
 		                                    False)  # force_new_uuid
 
-	def destroy(self):
+	def close(self):
 		self.medium.close()
+
+	def get_instance(self):
+		import hashlib
+		from ..instances.virtualbox import VirtualBoxInstance
+		image_hash = hashlib.sha1(self.image_path).hexdigest()
+		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
+		return VirtualBoxInstance(name, self)
+
+	def __enter__(self):
+		self.open()
+		return self.get_instance()
+
+	def __exit__(self, type, value, traceback):
+		self.close()
diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 953e72b..11e6788 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -68,6 +68,29 @@ class VirtualBoxInstance(Instance):
 			self.machine.unregister(vboxapi.library.CleanupMode.unregister_only)
 			self.machine.remove(delete=True)
 
+	def up(self):
+		try:
+			self.create()
+			try:
+				self.boot()
+			except Exception as e:
+				self.shutdown()
+				raise e
+		except Exception as e:
+			self.destroy()
+			raise e
+
+	def down(self):
+		self.shutdown()
+		self.destroy()
+
+	def __enter__(self):
+		self.up()
+		return self
+
+	def __exit__(self, type, value, traceback):
+		self.down()
+
 	class Lock(object):
 		def __init__(self, machine, session):
 			self.machine = machine
diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
index a535f30..4eb6dfa 100644
--- a/tests/integration/manifests/__init__.py
+++ b/tests/integration/manifests/__init__.py
@@ -18,3 +18,29 @@ import string
 pool = string.ascii_uppercase + string.ascii_lowercase + string.digits
 random_password = ''.join(random.choice(pool) for _ in range(16))
 partials['root_password']['plugins']['root_password']['password'] = random_password
+
+
+def merge_manifest_data(standard_partials=[], custom=[]):
+	import yaml
+	manifest_data = [partials[name] for name in standard_partials]
+	manifest_data.extend(yaml.load(data) for data in custom)
+	return merge_dicts(*manifest_data)
+
+
+# Snatched from here: http://stackoverflow.com/a/7205107
+def merge_dicts(*args):
+	def merge(a, b, path=None):
+		if path is None:
+			path = []
+		for key in b:
+			if key in a:
+				if isinstance(a[key], dict) and isinstance(b[key], dict):
+					merge(a[key], b[key], path + [str(key)])
+				elif a[key] == b[key]:
+					pass
+				else:
+					raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
+			else:
+						a[key] = b[key]
+		return a
+	return reduce(merge, args, {})
diff --git a/tests/integration/manifests/extlinux.yml b/tests/integration/manifests/extlinux.yml
new file mode 100644
index 0000000..13a8a43
--- /dev/null
+++ b/tests/integration/manifests/extlinux.yml
@@ -0,0 +1,3 @@
+---
+system:
+  bootloader: extlinux
diff --git a/tests/integration/manifests/gpt.yml b/tests/integration/manifests/gpt.yml
new file mode 100644
index 0000000..1ece4c2
--- /dev/null
+++ b/tests/integration/manifests/gpt.yml
@@ -0,0 +1,4 @@
+---
+volume:
+  partitions:
+    type: gpt
diff --git a/tests/integration/manifests/grub.yml b/tests/integration/manifests/grub.yml
new file mode 100644
index 0000000..4dfa67f
--- /dev/null
+++ b/tests/integration/manifests/grub.yml
@@ -0,0 +1,3 @@
+---
+system:
+  bootloader: grub
diff --git a/tests/integration/manifests/msdos.yml b/tests/integration/manifests/msdos.yml
new file mode 100644
index 0000000..5795f95
--- /dev/null
+++ b/tests/integration/manifests/msdos.yml
@@ -0,0 +1,4 @@
+---
+volume:
+  partitions:
+    type: msdos
diff --git a/tests/integration/manifests/single_partition.yml b/tests/integration/manifests/single_partition.yml
new file mode 100644
index 0000000..ce4c247
--- /dev/null
+++ b/tests/integration/manifests/single_partition.yml
@@ -0,0 +1,6 @@
+---
+volume:
+  partitions:
+    root:
+      filesystem: ext4
+      size: 1GiB
diff --git a/tests/integration/manifests/unpartitioned.yml b/tests/integration/manifests/unpartitioned.yml
index ef7f97d..4f881ba 100644
--- a/tests/integration/manifests/unpartitioned.yml
+++ b/tests/integration/manifests/unpartitioned.yml
@@ -1,7 +1,7 @@
 ---
 volume:
-  type: none
   partitions:
+    type: none
     root:
       filesystem: ext4
       size: 1GiB
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 7760a1d..66910b6 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -4,25 +4,6 @@ from bootstrapvz.remote import register_deserialization_handlers
 register_deserialization_handlers()
 
 
-# Snatched from here: http://stackoverflow.com/a/7205107
-def merge_dicts(*args):
-	def merge(a, b, path=None):
-		if path is None:
-			path = []
-		for key in b:
-			if key in a:
-				if isinstance(a[key], dict) and isinstance(b[key], dict):
-					merge(a[key], b[key], path + [str(key)])
-				elif a[key] == b[key]:
-					pass
-				else:
-					raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
-			else:
-						a[key] = b[key]
-		return a
-	return reduce(merge, args, {})
-
-
 def waituntil(predicate, timeout=5, interval=0.05):
 	import time
 	threshhold = time.time() + timeout
@@ -71,18 +52,3 @@ def read_from_socket(socket_path, termination_string, timeout):
 				raise SocketReadTimeout(msg)
 		console.close()
 		return output
-
-
-def bootstrap(manifest, build_server):
-	from bootstrapvz.remote.build_servers import LocalBuildServer
-	if isinstance(build_server, LocalBuildServer):
-		from bootstrapvz.base.main import run
-		bootstrap_info = run(manifest)
-	else:
-		from bootstrapvz.remote.main import run
-		bootstrap_info = run(manifest, build_server)
-	return bootstrap_info
-
-
-def test(instance):
-	pass
diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py
new file mode 100644
index 0000000..5d86245
--- /dev/null
+++ b/tests/integration/tools/bootable_manifest.py
@@ -0,0 +1,57 @@
+from bootstrapvz.remote.build_servers import LocalBuildServer
+from ..images.virtualbox import VirtualBoxImage
+
+
+class BootableManifest(object):
+
+	def __init__(self, manifest_data):
+		self.manifest_data = manifest_data
+
+	def pick_build_server(self, path='build-servers.yml'):
+		from bootstrapvz.common.tools import load_data
+		build_servers = load_data(path)
+		from bootstrapvz.remote.build_servers import pick_build_server
+		return pick_build_server(build_servers, self.manifest_data)
+
+	def get_manifest(self, build_server):
+		manifest_data = build_server.apply_build_settings(self.manifest_data)
+		from bootstrapvz.base.manifest import Manifest
+		return Manifest(data=manifest_data)
+
+	def bootstrap(self, manifest, build_server):
+		if isinstance(build_server, LocalBuildServer):
+			from bootstrapvz.base.main import run
+			bootstrap_info = run(manifest)
+		else:
+			from bootstrapvz.remote.main import run
+			bootstrap_info = run(manifest, build_server)
+		return bootstrap_info
+
+	def get_image(self, build_server, bootstrap_info, manifest):
+		if isinstance(build_server, LocalBuildServer):
+			image_path = bootstrap_info.volume.image_path
+		else:
+			import tempfile
+			handle, image_path = tempfile.mkstemp()
+			import os
+			os.close(handle)
+			build_server.download(bootstrap_info.volume.image_path, image_path)
+			build_server.delete(bootstrap_info.volume.image_path)
+		image_type = {'virtualbox': VirtualBoxImage}
+		return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path)
+
+	def __enter__(self):
+		self.build_server = self.pick_build_server()
+		self.manifest = self.get_manifest(self.build_server)
+		self.bootstrap_info = self.bootstrap(self.manifest, self.build_server)
+		self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest)
+		self.image.open()
+		self.instance = self.image.get_instance()
+		self.instance.up()
+		return self.instance
+
+	def __exit__(self, type, value, traceback):
+		if hasattr(self, 'instance'):
+			self.instance.down()
+		if hasattr(self, 'image'):
+			self.image.close()
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 36fb8ad..6e2e122 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,62 +1,16 @@
-import tools
-from manifests import partials
-from bootstrapvz.base.manifest import Manifest
-from bootstrapvz.remote.build_servers import pick_build_server
-from . import build_servers
-from images.virtualbox import VirtualBoxImage
-from instances.virtualbox import VirtualBoxInstance
+from manifests import merge_manifest_data
+from tools.bootable_manifest import BootableManifest
+
+partials = {'vbox': 'provider: {name: virtualbox}',
+            'vdi': 'volume: {backing: vdi}',
+            'vmdk': 'volume: {backing: vmdk}',
+            }
 
 
-def test_virtualbox_unpartitioned_extlinux():
-	import yaml
-	manifest_data = yaml.load("""
-provider:
-  name: virtualbox
-system:
-  bootloader: extlinux
-volume:
-  backing: vdi
-  partitions:
-    type: msdos
-""")
-	manifest_data = tools.merge_dicts(partials['base'], partials['stable64'], partials['unpartitioned'],
-	                                  partials['root_password'], partials['apt_proxy'],
-	                                  manifest_data)
-
-	build_server = pick_build_server(build_servers, manifest_data)
-	manifest_data = build_server.apply_build_settings(manifest_data)
-	manifest = Manifest(data=manifest_data)
-
-	# bootstrap_info = tools.bootstrap(manifest, build_server)
-	from bootstrapvz.base.bootstrapinfo import BootstrapInformation
-	bootstrap_info = BootstrapInformation(manifest)
-	bootstrap_info.volume.image_path = '/target/debian-wheezy-amd64-141218.vdi'
-
-	from bootstrapvz.remote.build_servers import LocalBuildServer
-	if isinstance(build_server, LocalBuildServer):
-		image_path = bootstrap_info.volume.image_path
-	else:
-		# import tempfile
-		# handle, image_path = tempfile.mkstemp()
-		# import os
-		# os.close(handle)
-		# build_server.download(bootstrap_info.volume.image_path, image_path)
-		# build_server.delete(bootstrap_info.volume.image_path)
-		image_path = '/Users/anders/Workspace/cloud/images/debian-wheezy-amd64-141130.vmdk'
-
-	image = VirtualBoxImage(manifest, image_path)
-	try:
-		instance = VirtualBoxInstance('unpartitioned_extlinux', image)
-		try:
-			instance.create()
-			try:
-				instance.boot()
-				# tools.reachable_with_ssh(instance)
-			finally:
-				instance.shutdown()
-		finally:
-			instance.destroy()
-	finally:
-		image.destroy()
-		import os
-		os.remove(image_path)
+def test_virtualbox_partitioned_extlinux():
+	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vdi']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)

From 70c282e80464971c7e91773b2e469f61460ba956 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 17:59:39 +0100
Subject: [PATCH 171/345] Add log statements and remove unused SSHRPCManager

---
 bootstrapvz/remote/build_servers.py       | 21 ++++---
 bootstrapvz/remote/ssh_rpc_manager.py     | 72 -----------------------
 tests/integration/images/virtualbox.py    |  4 ++
 tests/integration/instances/virtualbox.py | 12 ++++
 4 files changed, 28 insertions(+), 81 deletions(-)
 delete mode 100644 bootstrapvz/remote/ssh_rpc_manager.py

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index cdcbdf0..bdaa8a2 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -29,15 +29,16 @@ def pick_build_server(build_servers, manifest, preferences={}):
 		if not matches(name, settings):
 			continue
 		if settings['type'] == 'local':
-			return LocalBuildServer(settings)
+			return LocalBuildServer(name, settings)
 		else:
-			return RemoteBuildServer(settings)
+			return RemoteBuildServer(name, settings)
 	raise Exception('Unable to find a build server that matches your preferences.')
 
 
 class BuildServer(object):
 
-	def __init__(self, settings):
+	def __init__(self, name, settings):
+		self.name = name
 		self.settings = settings
 		self.build_settings = settings.get('build_settings', {})
 		self.can_bootstrap = settings['can_bootstrap']
@@ -57,8 +58,8 @@ class LocalBuildServer(BuildServer):
 
 class RemoteBuildServer(BuildServer):
 
-	def __init__(self, settings):
-		super(RemoteBuildServer, self).__init__(settings)
+	def __init__(self, name, settings):
+		super(RemoteBuildServer, self).__init__(name, settings)
 		self.address = settings['address']
 		self.port = settings['port']
 		self.username = settings['username']
@@ -74,7 +75,7 @@ class RemoteBuildServer(BuildServer):
 		[self.remote_server_port, self.remote_callback_port] = getNPorts(2)
 
 	def connect(self):
-		log.debug('Opening SSH connection')
+		log.debug('Opening SSH connection to build server `{name}\''.format(name=self.name))
 		import subprocess
 
 		server_cmd = ['sudo', self.server_bin, '--listen', str(self.remote_server_port)]
@@ -103,7 +104,7 @@ class RemoteBuildServer(BuildServer):
 			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
 			self.connection = Pyro4.Proxy(server_uri)
 
-			log.debug('Connecting to the RPC daemon')
+			log.debug('Connecting to the RPC daemon on build server `{name}\''.format(name=self.name))
 			remaining_retries = 5
 			while True:
 				try:
@@ -123,14 +124,15 @@ class RemoteBuildServer(BuildServer):
 
 	def disconnect(self):
 		if hasattr(self, 'connection'):
-			log.debug('Stopping the RPC daemon')
+			log.debug('Stopping the RPC daemon on build server `{name}\''.format(name=self.name))
 			self.connection.stop()
 			self.connection._pyroRelease()
 		if hasattr(self, 'ssh_process'):
-			log.debug('Waiting for the SSH connection to terminate')
+			log.debug('Waiting for SSH connection to build server `{name}\' to terminate'.format(name=self.name))
 			self.ssh_process.wait()
 
 	def download(self, src, dst):
+		log.debug('Downloading file `{path}\' from build server `{name}\''.format(path=src, name=self.name))
 		# Make sure we can read the file as {user}
 		self._remote_command(['sudo', 'chown', self.username, src])
 		src_arg = '{user}@{host}:{path}'.format(user=self.username, host=self.address, path=src)
@@ -138,6 +140,7 @@ class RemoteBuildServer(BuildServer):
 		                src_arg, dst])
 
 	def delete(self, path):
+		log.debug('Deleting file `{path}\' on build server `{name}\''.format(path=path, name=self.name))
 		self._remote_command(['sudo', 'rm', path])
 
 	def _remote_command(self, command):
diff --git a/bootstrapvz/remote/ssh_rpc_manager.py b/bootstrapvz/remote/ssh_rpc_manager.py
deleted file mode 100644
index 39fd037..0000000
--- a/bootstrapvz/remote/ssh_rpc_manager.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import logging
-log = logging.getLogger(__name__)
-
-
-class SSHRPCManager(object):
-
-	def __init__(self, settings):
-		self.settings = settings
-
-		# We can't use :0 because
-		# A: It's quite hard to retrieve the port on the remote after the daemon has started
-		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
-		[self.local_server_port, self.local_callback_port] = self.getNPorts(2)
-		[self.remote_server_port, self.remote_callback_port] = self.getNPorts(2)
-
-	def getNPorts(self, n, port_range=(1024, 65535)):
-		import random
-		ports = []
-		for i in range(0, n):
-			while True:
-				port = random.randrange(*port_range)
-				if port not in ports:
-					ports.append(port)
-					break
-		return ports
-
-	def start(self):
-		log.debug('Opening SSH connection')
-		import subprocess
-
-		ssh_cmd = ['ssh', '-i', self.settings['keyfile'],
-		                  '-p', str(self.settings['port']),
-		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
-		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
-		                  self.settings['username'] + '@' + self.settings['address'],
-		                  '--',
-		                  'sudo', self.settings['server_bin'],
-		                  '--listen', str(self.remote_server_port)]
-		import sys
-		self.ssh_process = subprocess.Popen(args=ssh_cmd, stdout=sys.stderr, stderr=sys.stderr)
-
-		# Check that we can connect to the server
-		try:
-			import Pyro4
-			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
-			self.rpc_server = Pyro4.Proxy(server_uri)
-
-			log.debug('Connecting to the RPC daemon')
-			remaining_retries = 5
-			while True:
-				try:
-					self.rpc_server.ping()
-					break
-				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
-					if remaining_retries > 0:
-						remaining_retries -= 1
-						from time import sleep
-						sleep(2)
-					else:
-						raise e
-		except (Exception, KeyboardInterrupt) as e:
-			self.ssh_process.terminate()
-			raise e
-
-	def stop(self):
-		if hasattr(self, 'rpc_server'):
-			log.debug('Stopping the RPC daemon')
-			self.rpc_server.stop()
-			self.rpc_server._pyroRelease()
-		if hasattr(self, 'ssh_process'):
-			log.debug('Waiting for the SSH connection to terminate')
-			self.ssh_process.wait()
diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
index acb9274..9d09692 100644
--- a/tests/integration/images/virtualbox.py
+++ b/tests/integration/images/virtualbox.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 from . import Image
 import virtualbox as vboxapi
+import logging
+log = logging.getLogger(__name__)
 
 
 class VirtualBoxImage(Image):
@@ -11,12 +13,14 @@ class VirtualBoxImage(Image):
 		self.vbox = vboxapi.VirtualBox()
 
 	def open(self):
+		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
 		self.medium = self.vbox.open_medium(self.image_path,  # location
 		                                    vboxapi.library.DeviceType.hard_disk,  # decive_type
 		                                    vboxapi.library.AccessMode.read_only,  # access_mode
 		                                    False)  # force_new_uuid
 
 	def close(self):
+		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
 		self.medium.close()
 
 	def get_instance(self):
diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 11e6788..06d9da8 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -1,6 +1,8 @@
 from __future__ import absolute_import
 from . import Instance
 import virtualbox as vboxapi
+import logging
+log = logging.getLogger(__name__)
 
 
 class VirtualBoxInstance(Instance):
@@ -15,6 +17,7 @@ class VirtualBoxInstance(Instance):
 		self.session = manager.get_session()
 
 	def create(self):
+		log.debug('Creating vbox machine `{name}\''.format(name=self.name))
 		# create machine
 		os_type = {'x86': 'Debian',
 		           'amd64': 'Debian_64'}.get(self.image.manifest.system['architecture'])
@@ -26,6 +29,7 @@ class VirtualBoxInstance(Instance):
 		self.vbox.register_machine(self.machine)
 
 		# attach image
+		log.debug('Attaching SATA storage controller to vbox machine `{name}\''.format(name=self.name))
 		with self.Lock(self.machine, self.session) as machine:
 			strg_ctrl = machine.add_storage_controller('SATA Controller',
 			                                           vboxapi.library.StorageBus.sata)
@@ -36,6 +40,7 @@ class VirtualBoxInstance(Instance):
 			machine.save_settings()
 
 		# redirect serial port
+		log.debug('Enabling serial port on vbox machine `{name}\''.format(name=self.name))
 		with self.Lock(self.machine, self.session) as machine:
 			serial_port = machine.get_serial_port(0)
 			serial_port.enabled = True
@@ -49,24 +54,31 @@ class VirtualBoxInstance(Instance):
 			machine.save_settings()
 
 	def boot(self):
+		log.debug('Booting vbox machine `{name}\''.format(name=self.name))
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
 		from ..tools import read_from_socket
 		self.console_output = read_from_socket(self.serial_port_path, 'INIT: Entering runlevel: 2', 20)
 
 	def shutdown(self):
+		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))
 		self.session.console.power_down().wait_for_completion(-1)
 		self.Lock(self.machine, self.session).unlock()
 
 	def destroy(self):
+		log.debug('Destroying vbox machine `{name}\''.format(name=self.name))
 		if hasattr(self, 'machine'):
 			try:
+				log.debug('Detaching SATA storage controller from vbox machine `{name}\''.format(name=self.name))
 				with self.Lock(self.machine, self.session) as machine:
 					machine.detach_device(name='SATA Controller', controller_port=0, device=0)
 					machine.save_settings()
 			except vboxapi.library.VBoxErrorObjectNotFound:
 				pass
+			log.debug('Unregistering and removing vbox machine `{name}\''.format(name=self.name))
 			self.machine.unregister(vboxapi.library.CleanupMode.unregister_only)
 			self.machine.remove(delete=True)
+		else:
+			log.debug('vbox machine `{name}\' was not created, skipping destruction'.format(name=self.name))
 
 	def up(self):
 		try:

From 8bb34c604b7796d04999e5cb979c9dbb83775d43 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 20 Dec 2014 23:18:27 +0100
Subject: [PATCH 172/345] Fix serious bug in merge_dicts where the original
 dict would be modified

---
 tests/integration/manifests/__init__.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
index 4eb6dfa..615e3e5 100644
--- a/tests/integration/manifests/__init__.py
+++ b/tests/integration/manifests/__init__.py
@@ -29,9 +29,21 @@ def merge_manifest_data(standard_partials=[], custom=[]):
 
 # Snatched from here: http://stackoverflow.com/a/7205107
 def merge_dicts(*args):
-	def merge(a, b, path=None):
-		if path is None:
-			path = []
+	def clone(obj):
+		copy = obj
+		if isinstance(obj, dict):
+			copy = {}
+			for key, value in obj.iteritems():
+				copy[key] = clone(value)
+		if isinstance(obj, list):
+			copy = []
+			copy.extend(obj)
+		if isinstance(obj, set):
+			copy = set()
+			copy.update(obj)
+		return copy
+
+	def merge(a, b, path=[]):
 		for key in b:
 			if key in a:
 				if isinstance(a[key], dict) and isinstance(b[key], dict):
@@ -39,8 +51,8 @@ def merge_dicts(*args):
 				elif a[key] == b[key]:
 					pass
 				else:
-					raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
+					raise Exception('Conflict at `{path}\''.format(path='.'.join(path + [str(key)])))
 			else:
-						a[key] = b[key]
+				a[key] = clone(b[key])
 		return a
 	return reduce(merge, args, {})

From 50fabe65ecab75b2b938fd1fa588c645ab6723d0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 21 Dec 2014 17:28:11 +0100
Subject: [PATCH 173/345] remove duplicated code

---
 tests/integration/__init__.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
index 5e6a71c..e69de29 100644
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -1,4 +0,0 @@
-from bootstrapvz.common.tools import load_data
-
-# tox ensures that the cwd is the project root
-build_servers = load_data('build-servers.yml')

From bbec32a9876cb34110b10c1fb4200b1e5f5a8a65 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 21 Dec 2014 17:28:52 +0100
Subject: [PATCH 174/345] Use stable/unstable, rather than wheezy/jessie

---
 tests/integration/manifests/stable64.yml   | 2 +-
 tests/integration/manifests/stable86.yml   | 4 ++++
 tests/integration/manifests/unstable64.yml | 4 ++++
 3 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/manifests/stable86.yml
 create mode 100644 tests/integration/manifests/unstable64.yml

diff --git a/tests/integration/manifests/stable64.yml b/tests/integration/manifests/stable64.yml
index 4489269..8f3cb82 100644
--- a/tests/integration/manifests/stable64.yml
+++ b/tests/integration/manifests/stable64.yml
@@ -1,4 +1,4 @@
 ---
 system:
-  release: wheezy
+  release: stable
   architecture: amd64
diff --git a/tests/integration/manifests/stable86.yml b/tests/integration/manifests/stable86.yml
new file mode 100644
index 0000000..1e6def2
--- /dev/null
+++ b/tests/integration/manifests/stable86.yml
@@ -0,0 +1,4 @@
+---
+system:
+  release: stable
+  architecture: x86
diff --git a/tests/integration/manifests/unstable64.yml b/tests/integration/manifests/unstable64.yml
new file mode 100644
index 0000000..4cdf9eb
--- /dev/null
+++ b/tests/integration/manifests/unstable64.yml
@@ -0,0 +1,4 @@
+---
+system:
+  release: unstable
+  architecture: amd64

From 943aab111f57f63de91a507de4ad15b9e23c6baa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 21 Dec 2014 18:55:44 +0100
Subject: [PATCH 175/345] Add some more tests

---
 tests/integration/manifests/oldstable64.yml |  4 +
 tests/integration/virtualbox_tests.py       | 81 ++++++++++++++++++++-
 2 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 tests/integration/manifests/oldstable64.yml

diff --git a/tests/integration/manifests/oldstable64.yml b/tests/integration/manifests/oldstable64.yml
new file mode 100644
index 0000000..3281ca7
--- /dev/null
+++ b/tests/integration/manifests/oldstable64.yml
@@ -0,0 +1,4 @@
+---
+system:
+  release: oldstable
+  architecture: amd64
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 6e2e122..0d5c20a 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,5 +1,6 @@
 from manifests import merge_manifest_data
 from tools.bootable_manifest import BootableManifest
+from nose.plugins.skip import Skip
 
 partials = {'vbox': 'provider: {name: virtualbox}',
             'vdi': 'volume: {backing: vdi}',
@@ -7,10 +8,84 @@ partials = {'vbox': 'provider: {name: virtualbox}',
             }
 
 
-def test_virtualbox_partitioned_extlinux():
-	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition',
+def test_unpartitioned_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned',
 	                'root_password', 'apt_proxy']
-	custom_partials = [partials['vbox'], partials['vdi']]
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_grub_oldstable():
+	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_unpartitioned_extlinux():
+	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_extlinux():
+	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_grub():
+	std_partials = ['base', 'stable64', 'grub', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_unpartitioned_extlinux_unstable():
+	raise Skip('Jessie not yet working with extlinux')
+	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'unpartitioned',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_extlinux_unstable():
+	raise Skip('Jessie not yet working with extlinux')
+	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_partitioned_grub_unstable():
+	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'single_partition',
+	                'root_password', 'apt_proxy']
+	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
 		print(instance.console_output)

From 81060b2439d920a3c746585f8bf1466be8af46c7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 21 Dec 2014 19:42:43 +0100
Subject: [PATCH 176/345] 5ffbdc4 made no sense, apt_proxy was already a build
 setting

---
 bootstrapvz/remote/build-servers-schema.yml |  7 ++++++
 tests/integration/manifests/apt_proxy.yml   |  5 ----
 tests/integration/virtualbox_tests.py       | 27 +++++++--------------
 3 files changed, 16 insertions(+), 23 deletions(-)
 delete mode 100644 tests/integration/manifests/apt_proxy.yml

diff --git a/bootstrapvz/remote/build-servers-schema.yml b/bootstrapvz/remote/build-servers-schema.yml
index 3990704..ceba3eb 100644
--- a/bootstrapvz/remote/build-servers-schema.yml
+++ b/bootstrapvz/remote/build-servers-schema.yml
@@ -31,6 +31,13 @@ definitions:
     type: object
     properties:
       guest_additions: {$ref: '#/definitions/absolute_path'}
+      apt_proxy:
+        type: object
+        properties:
+          address: {type: string}
+          port: {type: integer}
+          persistent: {type: boolean}
+        required: [address, port]
 
   ssh:
     type: object
diff --git a/tests/integration/manifests/apt_proxy.yml b/tests/integration/manifests/apt_proxy.yml
deleted file mode 100644
index 36e0026..0000000
--- a/tests/integration/manifests/apt_proxy.yml
+++ /dev/null
@@ -1,5 +0,0 @@
----
-plugins:
-  apt_proxy:
-    address: 127.0.0.1
-    port: 3142
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 0d5c20a..26fa4c3 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -9,8 +9,7 @@ partials = {'vbox': 'provider: {name: virtualbox}',
 
 
 def test_unpartitioned_extlinux_oldstable():
-	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -18,8 +17,7 @@ def test_unpartitioned_extlinux_oldstable():
 
 
 def test_partitioned_extlinux_oldstable():
-	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -27,8 +25,7 @@ def test_partitioned_extlinux_oldstable():
 
 
 def test_partitioned_grub_oldstable():
-	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -36,8 +33,7 @@ def test_partitioned_grub_oldstable():
 
 
 def test_unpartitioned_extlinux():
-	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -45,8 +41,7 @@ def test_unpartitioned_extlinux():
 
 
 def test_partitioned_extlinux():
-	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -54,8 +49,7 @@ def test_partitioned_extlinux():
 
 
 def test_partitioned_grub():
-	std_partials = ['base', 'stable64', 'grub', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'stable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -64,8 +58,7 @@ def test_partitioned_grub():
 
 def test_unpartitioned_extlinux_unstable():
 	raise Skip('Jessie not yet working with extlinux')
-	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'unpartitioned',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -74,8 +67,7 @@ def test_unpartitioned_extlinux_unstable():
 
 def test_partitioned_extlinux_unstable():
 	raise Skip('Jessie not yet working with extlinux')
-	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -83,8 +75,7 @@ def test_partitioned_extlinux_unstable():
 
 
 def test_partitioned_grub_unstable():
-	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'single_partition',
-	                'root_password', 'apt_proxy']
+	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:

From 2aebc870a4d3358d7db196ecf53ccbf41bc54d33 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 13:39:40 +0100
Subject: [PATCH 177/345] Fix extlinux config for squeeze

---
 bootstrapvz/common/tasks/boot.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 81c8e02..dbc419d 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -155,6 +155,10 @@ class ConfigureExtlinux(Task):
 
 	@classmethod
 	def run(cls, info):
+		if info.release_codename == 'squeeze':
+			# On squeeze /etc/default/extlinux is generated when running extlinux-update
+			log_check_call(['chroot', info.root,
+			                'extlinux-update'])
 		from bootstrapvz.common.tools import sed_i
 		extlinux_def = os.path.join(info.root, 'etc/default/extlinux')
 		sed_i(extlinux_def, r'^EXTLINUX_PARAMETERS="([^"]+)"$',

From b6976eb6e944ed02cc50eefab254d39682310f43 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 13:39:54 +0100
Subject: [PATCH 178/345] Lowercase L for InstallExlinux taskname

---
 bootstrapvz/common/task_groups.py | 2 +-
 bootstrapvz/common/tasks/boot.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index b35fbb3..330c598 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -139,7 +139,7 @@ def get_bootloader_group(manifest):
 	if manifest.system['bootloader'] == 'extlinux':
 		group.extend([boot.AddExtlinuxPackage,
 		              boot.ConfigureExtlinux,
-		              boot.InstallExtLinux])
+		              boot.InstallExtlinux])
 	return group
 
 
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index dbc419d..9610fd5 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -165,7 +165,7 @@ class ConfigureExtlinux(Task):
 		                    r'EXTLINUX_PARAMETERS="\1 console=ttyS0"')
 
 
-class InstallExtLinux(Task):
+class InstallExtlinux(Task):
 	description = 'Installing extlinux'
 	phase = phases.system_modification
 	predecessors = [filesystem.FStab, ConfigureExtlinux]

From a241842ef941ebf9e921d661ecf46e06c32b7b2a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 13:40:16 +0100
Subject: [PATCH 179/345] Better dict merging (less side-effectful)

---
 tests/integration/manifests/__init__.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py
index 615e3e5..d6afbd6 100644
--- a/tests/integration/manifests/__init__.py
+++ b/tests/integration/manifests/__init__.py
@@ -32,15 +32,11 @@ def merge_dicts(*args):
 	def clone(obj):
 		copy = obj
 		if isinstance(obj, dict):
-			copy = {}
-			for key, value in obj.iteritems():
-				copy[key] = clone(value)
+			copy = {key: clone(value) for key, value in obj.iteritems()}
 		if isinstance(obj, list):
-			copy = []
-			copy.extend(obj)
+			copy = [clone(value) for value in obj]
 		if isinstance(obj, set):
-			copy = set()
-			copy.update(obj)
+			copy = set([clone(value) for value in obj])
 		return copy
 
 	def merge(a, b, path=[]):

From ce8dd02cbf2524dcc4df1e7a6716396afe8128f3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 13:40:33 +0100
Subject: [PATCH 180/345] Fix dict merge conflict in
 test_unpartitioned_extlinux_unstable

---
 tests/integration/virtualbox_tests.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 26fa4c3..36c8b67 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -58,7 +58,7 @@ def test_partitioned_grub():
 
 def test_unpartitioned_extlinux_unstable():
 	raise Skip('Jessie not yet working with extlinux')
-	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'unpartitioned', 'root_password']
+	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:

From 5823c9119bfb892a913a843bbb2050826538a3fb Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 14:14:43 +0100
Subject: [PATCH 181/345] Split grub and extlinux installs into separate
 modules

---
 bootstrapvz/common/task_groups.py          |  17 +--
 bootstrapvz/common/tasks/boot.py           | 144 ---------------------
 bootstrapvz/common/tasks/extlinux.py       |  58 +++++++++
 bootstrapvz/common/tasks/grub.py           |  96 ++++++++++++++
 bootstrapvz/plugins/docker_daemon/tasks.py |   6 +-
 bootstrapvz/providers/azure/tasks/boot.py  |   4 +-
 bootstrapvz/providers/ec2/__init__.py      |   3 +-
 bootstrapvz/providers/gce/tasks/boot.py    |   4 +-
 8 files changed, 172 insertions(+), 160 deletions(-)
 create mode 100644 bootstrapvz/common/tasks/extlinux.py
 create mode 100644 bootstrapvz/common/tasks/grub.py

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index 330c598..d6370d0 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -1,7 +1,8 @@
 from tasks import workspace
 from tasks import packages
 from tasks import host
-from tasks import boot
+from tasks import grub
+from tasks import extlinux
 from tasks import bootstrap
 from tasks import volume
 from tasks import loopback
@@ -129,17 +130,17 @@ locale_group = [locale.LocaleBootstrapPackage,
 def get_bootloader_group(manifest):
 	group = []
 	if manifest.system['bootloader'] == 'grub':
-		group.extend([boot.AddGrubPackage,
-		              boot.ConfigureGrub])
+		group.extend([grub.AddGrubPackage,
+		              grub.ConfigureGrub])
 		from bootstrapvz.common.tools import get_codename
 		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
-			group.append(boot.InstallGrub_1_99)
+			group.append(grub.InstallGrub_1_99)
 		else:
-			group.append(boot.InstallGrub_2)
+			group.append(grub.InstallGrub_2)
 	if manifest.system['bootloader'] == 'extlinux':
-		group.extend([boot.AddExtlinuxPackage,
-		              boot.ConfigureExtlinux,
-		              boot.InstallExtlinux])
+		group.extend([extlinux.AddExtlinuxPackage,
+		              extlinux.ConfigureExtlinux,
+		              extlinux.InstallExtlinux])
 	return group
 
 
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 9610fd5..3d7d509 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -1,9 +1,5 @@
 from bootstrapvz.base import Task
 from .. import phases
-from ..tools import log_check_call
-import apt
-import filesystem
-from bootstrapvz.base.fs import partitionmaps
 import os.path
 
 
@@ -45,143 +41,3 @@ class DisableGetTTYs(Task):
 		for i in range(2, 7):
 			i = str(i)
 			sed_i(inittab_path, '^' + i + ttyx + i, '#' + i + ttyx + i)
-
-
-class AddGrubPackage(Task):
-	description = 'Adding grub package'
-	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
-
-	@classmethod
-	def run(cls, info):
-		info.packages.add('grub-pc')
-
-
-class ConfigureGrub(Task):
-	description = 'Configuring grub'
-	phase = phases.system_modification
-	predecessors = [filesystem.FStab]
-
-	@classmethod
-	def run(cls, info):
-		from bootstrapvz.common.tools import sed_i
-		grub_def = os.path.join(info.root, 'etc/default/grub')
-		sed_i(grub_def, '^#GRUB_TERMINAL=console', 'GRUB_TERMINAL=console')
-		sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT="quiet"',
-		                'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0"')
-
-
-class InstallGrub_1_99(Task):
-	description = 'Installing grub 1.99'
-	phase = phases.system_modification
-	predecessors = [filesystem.FStab]
-
-	@classmethod
-	def run(cls, info):
-
-		from ..fs import remount
-		p_map = info.volume.partition_map
-
-		def link_fn():
-			info.volume.link_dm_node()
-			if isinstance(p_map, partitionmaps.none.NoPartitions):
-				p_map.root.device_path = info.volume.device_path
-
-		def unlink_fn():
-			info.volume.unlink_dm_node()
-			if isinstance(p_map, partitionmaps.none.NoPartitions):
-				p_map.root.device_path = info.volume.device_path
-
-		# GRUB cannot deal with installing to loopback devices
-		# so we fake a real harddisk with dmsetup.
-		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
-		from ..fs.loopbackvolume import LoopbackVolume
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, link_fn)
-		try:
-			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
-			device_map_path = os.path.join(info.root, 'boot/grub/device.map')
-			partition_prefix = 'msdos'
-			if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
-				partition_prefix = 'gpt'
-			with open(device_map_path, 'w') as device_map:
-				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
-				if not isinstance(p_map, partitionmaps.none.NoPartitions):
-					for idx, partition in enumerate(info.volume.partition_map.partitions):
-						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
-						                 .format(device_path=partition.device_path,
-						                         prefix=partition_prefix,
-						                         idx=idx + 1))
-
-			# Install grub
-			log_check_call(['chroot', info.root, 'grub-install', device_path])
-			log_check_call(['chroot', info.root, 'update-grub'])
-		except Exception as e:
-			if isinstance(info.volume, LoopbackVolume):
-				remount(info.volume, unlink_fn)
-			raise e
-
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, unlink_fn)
-
-
-class InstallGrub_2(Task):
-	description = 'Installing grub 2'
-	phase = phases.system_modification
-	predecessors = [filesystem.FStab]
-
-	@classmethod
-	def run(cls, info):
-		log_check_call(['chroot', info.root, 'grub-install', info.volume.device_path])
-		log_check_call(['chroot', info.root, 'update-grub'])
-
-
-class AddExtlinuxPackage(Task):
-	description = 'Adding extlinux package'
-	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
-
-	@classmethod
-	def run(cls, info):
-		info.packages.add('extlinux')
-		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
-			info.packages.add('syslinux-common')
-
-
-class ConfigureExtlinux(Task):
-	description = 'Configuring extlinux'
-	phase = phases.system_modification
-	predecessors = [filesystem.FStab]
-
-	@classmethod
-	def run(cls, info):
-		if info.release_codename == 'squeeze':
-			# On squeeze /etc/default/extlinux is generated when running extlinux-update
-			log_check_call(['chroot', info.root,
-			                'extlinux-update'])
-		from bootstrapvz.common.tools import sed_i
-		extlinux_def = os.path.join(info.root, 'etc/default/extlinux')
-		sed_i(extlinux_def, r'^EXTLINUX_PARAMETERS="([^"]+)"$',
-		                    r'EXTLINUX_PARAMETERS="\1 console=ttyS0"')
-
-
-class InstallExtlinux(Task):
-	description = 'Installing extlinux'
-	phase = phases.system_modification
-	predecessors = [filesystem.FStab, ConfigureExtlinux]
-
-	@classmethod
-	def run(cls, info):
-		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
-			bootloader = '/usr/lib/syslinux/gptmbr.bin'
-		else:
-			bootloader = '/usr/lib/extlinux/mbr.bin'
-		log_check_call(['chroot', info.root,
-		                'dd', 'bs=440', 'count=1',
-		                'if=' + bootloader,
-		                'of=' + info.volume.device_path])
-		log_check_call(['chroot', info.root,
-		                'extlinux',
-		                '--install', '/boot/extlinux'])
-		log_check_call(['chroot', info.root,
-		                'extlinux-update'])
diff --git a/bootstrapvz/common/tasks/extlinux.py b/bootstrapvz/common/tasks/extlinux.py
new file mode 100644
index 0000000..a050700
--- /dev/null
+++ b/bootstrapvz/common/tasks/extlinux.py
@@ -0,0 +1,58 @@
+from bootstrapvz.base import Task
+from .. import phases
+from ..tools import log_check_call
+import apt
+import filesystem
+from bootstrapvz.base.fs import partitionmaps
+import os.path
+
+
+class AddExtlinuxPackage(Task):
+	description = 'Adding extlinux package'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('extlinux')
+		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
+			info.packages.add('syslinux-common')
+
+
+class ConfigureExtlinux(Task):
+	description = 'Configuring extlinux'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+		if info.release_codename == 'squeeze':
+			# On squeeze /etc/default/extlinux is generated when running extlinux-update
+			log_check_call(['chroot', info.root,
+			                'extlinux-update'])
+		from bootstrapvz.common.tools import sed_i
+		extlinux_def = os.path.join(info.root, 'etc/default/extlinux')
+		sed_i(extlinux_def, r'^EXTLINUX_PARAMETERS="([^"]+)"$',
+		                    r'EXTLINUX_PARAMETERS="\1 console=ttyS0"')
+
+
+class InstallExtlinux(Task):
+	description = 'Installing extlinux'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab, ConfigureExtlinux]
+
+	@classmethod
+	def run(cls, info):
+		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
+			bootloader = '/usr/lib/syslinux/gptmbr.bin'
+		else:
+			bootloader = '/usr/lib/extlinux/mbr.bin'
+		log_check_call(['chroot', info.root,
+		                'dd', 'bs=440', 'count=1',
+		                'if=' + bootloader,
+		                'of=' + info.volume.device_path])
+		log_check_call(['chroot', info.root,
+		                'extlinux',
+		                '--install', '/boot/extlinux'])
+		log_check_call(['chroot', info.root,
+		                'extlinux-update'])
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
new file mode 100644
index 0000000..ed781a1
--- /dev/null
+++ b/bootstrapvz/common/tasks/grub.py
@@ -0,0 +1,96 @@
+from bootstrapvz.base import Task
+from .. import phases
+from ..tools import log_check_call
+import apt
+import filesystem
+from bootstrapvz.base.fs import partitionmaps
+import os.path
+
+
+class AddGrubPackage(Task):
+	description = 'Adding grub package'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('grub-pc')
+
+
+class ConfigureGrub(Task):
+	description = 'Configuring grub'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import sed_i
+		grub_def = os.path.join(info.root, 'etc/default/grub')
+		sed_i(grub_def, '^#GRUB_TERMINAL=console', 'GRUB_TERMINAL=console')
+		sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT="quiet"',
+		                'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0"')
+
+
+class InstallGrub_1_99(Task):
+	description = 'Installing grub 1.99'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+
+		from ..fs import remount
+		p_map = info.volume.partition_map
+
+		def link_fn():
+			info.volume.link_dm_node()
+			if isinstance(p_map, partitionmaps.none.NoPartitions):
+				p_map.root.device_path = info.volume.device_path
+
+		def unlink_fn():
+			info.volume.unlink_dm_node()
+			if isinstance(p_map, partitionmaps.none.NoPartitions):
+				p_map.root.device_path = info.volume.device_path
+
+		# GRUB cannot deal with installing to loopback devices
+		# so we fake a real harddisk with dmsetup.
+		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
+		from ..fs.loopbackvolume import LoopbackVolume
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, link_fn)
+		try:
+			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
+			device_map_path = os.path.join(info.root, 'boot/grub/device.map')
+			partition_prefix = 'msdos'
+			if isinstance(p_map, partitionmaps.gpt.GPTPartitionMap):
+				partition_prefix = 'gpt'
+			with open(device_map_path, 'w') as device_map:
+				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
+				if not isinstance(p_map, partitionmaps.none.NoPartitions):
+					for idx, partition in enumerate(info.volume.partition_map.partitions):
+						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
+						                 .format(device_path=partition.device_path,
+						                         prefix=partition_prefix,
+						                         idx=idx + 1))
+
+			# Install grub
+			log_check_call(['chroot', info.root, 'grub-install', device_path])
+			log_check_call(['chroot', info.root, 'update-grub'])
+		except Exception as e:
+			if isinstance(info.volume, LoopbackVolume):
+				remount(info.volume, unlink_fn)
+			raise e
+
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, unlink_fn)
+
+
+class InstallGrub_2(Task):
+	description = 'Installing grub 2'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab]
+
+	@classmethod
+	def run(cls, info):
+		log_check_call(['chroot', info.root, 'grub-install', info.volume.device_path])
+		log_check_call(['chroot', info.root, 'update-grub'])
diff --git a/bootstrapvz/plugins/docker_daemon/tasks.py b/bootstrapvz/plugins/docker_daemon/tasks.py
index 93fcb16..59bef18 100644
--- a/bootstrapvz/plugins/docker_daemon/tasks.py
+++ b/bootstrapvz/plugins/docker_daemon/tasks.py
@@ -1,6 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import boot
+from bootstrapvz.common.tasks import grub
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.tools import sed_i
@@ -63,8 +63,8 @@ class AddDockerInit(Task):
 class EnableMemoryCgroup(Task):
 	description = 'Change grub configuration to enable the memory cgroup'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
-	predecessors = [boot.ConfigureGrub, gceboot.ConfigureGrub]
+	successors = [grub.InstallGrub_1_99, grub.InstallGrub_2]
+	predecessors = [grub.ConfigureGrub, gceboot.ConfigureGrub]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/azure/tasks/boot.py b/bootstrapvz/providers/azure/tasks/boot.py
index 7f95276..8c61fff 100644
--- a/bootstrapvz/providers/azure/tasks/boot.py
+++ b/bootstrapvz/providers/azure/tasks/boot.py
@@ -1,12 +1,12 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import boot
+from bootstrapvz.common.tasks import grub
 
 
 class ConfigureGrub(Task):
 	description = 'Change grub configuration to allow for ttyS0 output'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
+	successors = [grub.InstallGrub_1_99, grub.InstallGrub_2]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index d58ff04..3399831 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -11,6 +11,7 @@ import tasks.initd
 from bootstrapvz.common.tasks import volume
 from bootstrapvz.common.tasks import filesystem
 from bootstrapvz.common.tasks import boot
+from bootstrapvz.common.tasks import grub
 from bootstrapvz.common.tasks import initd
 from bootstrapvz.common.tasks import loopback
 from bootstrapvz.common.tasks import kernel
@@ -90,7 +91,7 @@ def resolve_tasks(taskset, manifest):
 		taskset.add(initd.AdjustExpandRootScript)
 
 	if manifest.system['bootloader'] == 'pvgrub':
-		taskset.add(boot.AddGrubPackage)
+		taskset.add(grub.AddGrubPackage)
 		taskset.add(tasks.boot.ConfigurePVGrub)
 
 	if manifest.volume['backing'].lower() == 'ebs':
diff --git a/bootstrapvz/providers/gce/tasks/boot.py b/bootstrapvz/providers/gce/tasks/boot.py
index 6d03025..224c7fc 100644
--- a/bootstrapvz/providers/gce/tasks/boot.py
+++ b/bootstrapvz/providers/gce/tasks/boot.py
@@ -1,13 +1,13 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import boot
+from bootstrapvz.common.tasks import grub
 import os.path
 
 
 class ConfigureGrub(Task):
 	description = 'Change grub configuration to allow for ttyS0 output'
 	phase = phases.system_modification
-	successors = [boot.InstallGrub_1_99, boot.InstallGrub_2]
+	successors = [grub.InstallGrub_1_99, grub.InstallGrub_2]
 
 	@classmethod
 	def run(cls, info):

From 7b77b484f2f4415eddd439da5e5a349a1ec70f6c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 15:24:30 +0100
Subject: [PATCH 182/345] Fix #179

---
 bootstrapvz/common/tasks/boot.py | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 3d7d509..9f028c1 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -33,11 +33,18 @@ class DisableGetTTYs(Task):
 
 	@classmethod
 	def run(cls, info):
-		from ..tools import sed_i
-		inittab_path = os.path.join(info.root, 'etc/inittab')
-		tty1 = '1:2345:respawn:/sbin/getty 38400 tty1'
-		sed_i(inittab_path, '^' + tty1, '#' + tty1)
-		ttyx = ':23:respawn:/sbin/getty 38400 tty'
-		for i in range(2, 7):
-			i = str(i)
-			sed_i(inittab_path, '^' + i + ttyx + i, '#' + i + ttyx + i)
+		# Forward compatible check for jessie,
+		# we should probably get some version numbers up in dis bitch
+		if info.release_codename in ['squeeze', 'wheezy']:
+			from ..tools import sed_i
+			inittab_path = os.path.join(info.root, 'etc/inittab')
+			tty1 = '1:2345:respawn:/sbin/getty 38400 tty1'
+			sed_i(inittab_path, '^' + tty1, '#' + tty1)
+			ttyx = ':23:respawn:/sbin/getty 38400 tty'
+			for i in range(2, 7):
+				i = str(i)
+				sed_i(inittab_path, '^' + i + ttyx + i, '#' + i + ttyx + i)
+		else:
+			tty_path = 'etc/systemd/system/getty.target.wants/getty@tty{num}.service'
+			for i in range(2, 7):
+				os.remove(os.path.join(info.root, tty_path.format(num=i)))

From 6ade2a90c21e6d886bf35c5400a1caf5d8b1fd76 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 15:24:50 +0100
Subject: [PATCH 183/345] More verbose image name

---
 tests/integration/manifests/base.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/manifests/base.yml b/tests/integration/manifests/base.yml
index 3480441..3dec767 100644
--- a/tests/integration/manifests/base.yml
+++ b/tests/integration/manifests/base.yml
@@ -4,7 +4,7 @@ bootstrapper:
   workspace: /target
   tarball: true
 image:
-  name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
+  name: deb-{system.release}-{system.architecture}-{system.bootloader}-{volume.partitions.type}-{%y}{%m}{%d}
   description: Debian {system.release} {system.architecture}
 system:
   charmap: UTF-8

From 456a68ea25b77406daf6550f4c13b46110f0dfaa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 31 Dec 2014 15:41:40 +0100
Subject: [PATCH 184/345] Remove unneccessary import

---
 bootstrapvz/remote/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 0990417..d381830 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -51,7 +51,6 @@ def unregister_deserialization_handlers():
 
 def deserialize_exception(fq_classname, data):
 	class_object = get_class_object(fq_classname)
-	from Pyro4.util import SerializerBase
 	return SerializerBase.make_exception(class_object, data)
 
 

From 9a6975ce7dbd9f89cbb21a98dcbaa81b260268f2 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 1 Jan 2015 18:45:39 +0100
Subject: [PATCH 185/345] Add new Sectors unit, enhance Bytes unit, add unit
 tests for both

---
 bootstrapvz/common/bytes.py   |  69 ++++++++++-----
 bootstrapvz/common/sectors.py | 154 ++++++++++++++++++++++++++++++++++
 tests/unit/__init__.py        |   0
 tests/unit/bytes.py           | 108 ++++++++++++++++++++++++
 tests/unit/sectors.py         | 128 ++++++++++++++++++++++++++++
 5 files changed, 436 insertions(+), 23 deletions(-)
 create mode 100644 bootstrapvz/common/sectors.py
 create mode 100644 tests/unit/__init__.py
 create mode 100644 tests/unit/bytes.py
 create mode 100644 tests/unit/sectors.py

diff --git a/bootstrapvz/common/bytes.py b/bootstrapvz/common/bytes.py
index a64e0ec..d558aa6 100644
--- a/bootstrapvz/common/bytes.py
+++ b/bootstrapvz/common/bytes.py
@@ -1,3 +1,14 @@
+from exceptions import UnitError
+
+
+def onlybytes(msg):
+	def decorator(func):
+		def check_other(self, other):
+			if not isinstance(other, Bytes):
+				raise UnitError(msg)
+			return func(self, other)
+		return check_other
+	return decorator
 
 
 class Bytes(object):
@@ -61,25 +72,48 @@ class Bytes(object):
 	def __long__(self):
 		return self.qty
 
+	def __abs__(self):
+		return self.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __lt__(self, other):
+		return self.qty < other.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __le__(self, other):
+		return self.qty <= other.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __eq__(self, other):
+		return self.qty == other.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __ne__(self, other):
+		return self.qty != other.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __ge__(self, other):
+		return self.qty >= other.qty
+
+	@onlybytes('Can only compare Bytes to Bytes')
+	def __gt__(self, other):
+		return self.qty > other.qty
+
+	@onlybytes('Can only add Bytes to Bytes')
 	def __add__(self, other):
-		if not isinstance(other, Bytes):
-			raise UnitError('Can only add Bytes to Bytes')
 		return Bytes(self.qty + other.qty)
 
+	@onlybytes('Can only add Bytes to Bytes')
 	def __iadd__(self, other):
-		if not isinstance(other, Bytes):
-			raise UnitError('Can only add Bytes to Bytes')
 		self.qty += other.qty
 		return self
 
+	@onlybytes('Can only subtract Bytes from Bytes')
 	def __sub__(self, other):
-		if not isinstance(other, Bytes):
-			raise UnitError('Can only subtract Bytes from Bytes')
 		return Bytes(self.qty - other.qty)
 
+	@onlybytes('Can only subtract Bytes from Bytes')
 	def __isub__(self, other):
-		if not isinstance(other, Bytes):
-			raise UnitError('Can only subtract Bytes from Bytes')
 		self.qty -= other.qty
 		return self
 
@@ -110,20 +144,13 @@ class Bytes(object):
 			self.qty /= other
 		return self
 
+	@onlybytes('Can only take modulus of Bytes with Bytes')
 	def __mod__(self, other):
-		if isinstance(other, Bytes):
-			return self.qty % other.qty
-		if not isinstance(other, (int, long)):
-			raise UnitError('Can only take modulus of Bytes with integers or Bytes')
-		return Bytes(self.qty % other)
+		return Bytes(self.qty % other.qty)
 
+	@onlybytes('Can only take modulus of Bytes with Bytes')
 	def __imod__(self, other):
-		if isinstance(other, Bytes):
-			self.qty %= other.qty
-		else:
-			if not isinstance(other, (int, long)):
-				raise UnitError('Can only divide Bytes with integers or Bytes')
-			self.qty %= other
+		self.qty %= other.qty
 		return self
 
 	def __getstate__(self):
@@ -133,7 +160,3 @@ class Bytes(object):
 
 	def __setstate__(self, state):
 		self.qty = state['qty']
-
-
-class UnitError(Exception):
-	pass
diff --git a/bootstrapvz/common/sectors.py b/bootstrapvz/common/sectors.py
new file mode 100644
index 0000000..ecdbc49
--- /dev/null
+++ b/bootstrapvz/common/sectors.py
@@ -0,0 +1,154 @@
+from exceptions import UnitError
+from bytes import Bytes
+
+
+def onlysectors(msg):
+	def decorator(func):
+		def check_other(self, other):
+			if not isinstance(other, Sectors):
+				raise UnitError(msg)
+			return func(self, other)
+		return check_other
+	return decorator
+
+
+class Sectors(object):
+
+	def __init__(self, quantity, sector_size):
+		if isinstance(sector_size, Bytes):
+			self.sector_size = sector_size
+		else:
+			self.sector_size = Bytes(sector_size)
+
+		if isinstance(quantity, Bytes):
+			self.bytes = quantity
+		else:
+			if isinstance(quantity, (int, long)):
+				self.bytes = self.sector_size * quantity
+			else:
+				self.bytes = Bytes(quantity)
+
+	def get_sectors(self):
+		return self.bytes / self.sector_size
+
+	def __repr__(self):
+		return str(self.get_sectors()) + 's'
+
+	def __str__(self):
+		return self.__repr__()
+
+	def __int__(self):
+		return self.get_sectors()
+
+	def __long__(self):
+		return self.get_sectors()
+
+	def __abs__(self):
+		return self.get_sectors()
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __lt__(self, other):
+		return self.bytes < other.bytes
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __le__(self, other):
+		return self.bytes <= other.bytes
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __eq__(self, other):
+		return self.bytes == other.bytes
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __ne__(self, other):
+		return self.bytes != other.bytes
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __ge__(self, other):
+		return self.bytes >= other.bytes
+
+	@onlysectors('Can only compare sectors with sectors')
+	def __gt__(self, other):
+		return self.bytes > other.bytes
+
+	def __add__(self, other):
+		if not isinstance(other, Sectors):
+			raise UnitError('Can only add sectors to sectors')
+		if self.sector_size != other.sector_size:
+			raise UnitError('Cannot sum sectors with different sector sizes')
+		return Sectors(self.bytes + other.bytes, self.sector_size)
+
+	def __iadd__(self, other):
+		if not isinstance(other, (Bytes, Sectors)):
+			raise UnitError('Can only add Bytes or sectors to sectors')
+		if isinstance(other, Bytes):
+			self.bytes += other
+		if isinstance(other, Sectors):
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot sum sectors with different sector sizes')
+			self.bytes += other.bytes
+		return self
+
+	def __sub__(self, other):
+		if not isinstance(other, (Sectors, int, long)):
+			raise UnitError('Can only subtract sectors or integers from sectors')
+		if isinstance(other, int):
+			return Sectors(self.bytes - self.sector_size * other, self.sector_size)
+		else:
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot subtract sectors with different sector sizes')
+			return Sectors(self.bytes - other.bytes, self.sector_size)
+
+	def __isub__(self, other):
+		if not isinstance(other, (Sectors, int, long)):
+			raise UnitError('Can only subtract sectors or integers from sectors')
+		if isinstance(other, int):
+			self.bytes -= self.sector_size * other
+		else:
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot subtract sectors with different sector sizes')
+			self.bytes -= other.bytes
+		return self
+
+	def __mul__(self, other):
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only multiply sectors with integers')
+		return Sectors(self.bytes * other, self.sector_size)
+
+	def __imul__(self, other):
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only multiply sectors with integers')
+		self.bytes *= other
+		return self
+
+	def __div__(self, other):
+		if isinstance(other, Sectors):
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot divide sectors with different sector sizes')
+			return self.bytes / other.bytes
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only divide sectors with integers or sectors')
+		return Sectors(self.bytes / other, self.sector_size)
+
+	def __idiv__(self, other):
+		if isinstance(other, Sectors):
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot divide sectors with different sector sizes')
+			self.bytes /= other.bytes
+		else:
+			if not isinstance(other, (int, long)):
+				raise UnitError('Can only divide sectors with integers or sectors')
+			self.bytes /= other
+		return self
+
+	@onlysectors('Can only take modulus of sectors with sectors')
+	def __mod__(self, other):
+		if self.sector_size != other.sector_size:
+			raise UnitError('Cannot take modulus of sectors with different sector sizes')
+		return Sectors(self.bytes % other.bytes, self.sector_size)
+
+	@onlysectors('Can only take modulus of sectors with sectors')
+	def __imod__(self, other):
+		if self.sector_size != other.sector_size:
+			raise UnitError('Cannot take modulus of sectors with different sector sizes')
+		self.bytes %= other.bytes
+		return self
diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/unit/bytes.py b/tests/unit/bytes.py
new file mode 100644
index 0000000..19e8b08
--- /dev/null
+++ b/tests/unit/bytes.py
@@ -0,0 +1,108 @@
+from nose.tools import eq_
+from nose.tools import raises
+from bootstrapvz.common.bytes import Bytes
+from bootstrapvz.common.exceptions import UnitError
+
+
+def test_lt():
+	assert Bytes('1MiB') < Bytes('2MiB')
+
+
+def test_le():
+	assert Bytes('1MiB') <= Bytes('2MiB')
+	assert Bytes('1MiB') <= Bytes('1MiB')
+
+
+def test_eq():
+	eq_(Bytes('1MiB'), Bytes('1MiB'))
+
+
+def test_neq():
+	assert Bytes('15MiB') != Bytes('1MiB')
+
+
+def test_gt():
+	assert Bytes('2MiB') > Bytes('1MiB')
+
+
+def test_ge():
+	assert Bytes('2MiB') >= Bytes('1MiB')
+	assert Bytes('2MiB') >= Bytes('2MiB')
+
+
+def test_eq_unit():
+	eq_(Bytes('1024MiB'), Bytes('1GiB'))
+
+
+def test_add():
+	eq_(Bytes('2GiB'), Bytes('1GiB') + Bytes('1GiB'))
+
+
+def test_iadd():
+	b = Bytes('1GiB')
+	b += Bytes('1GiB')
+	eq_(Bytes('2GiB'), b)
+
+
+def test_sub():
+	eq_(Bytes('1GiB'), Bytes('2GiB') - Bytes('1GiB'))
+
+
+def test_isub():
+	b = Bytes('2GiB')
+	b -= Bytes('1GiB')
+	eq_(Bytes('1GiB'), b)
+
+
+def test_mul():
+	eq_(Bytes('2GiB'), Bytes('1GiB') * 2)
+
+
+@raises(UnitError)
+def test_mul_bytes():
+	Bytes('1GiB') * Bytes('1GiB')
+
+
+def test_imul():
+	b = Bytes('1GiB')
+	b *= 2
+	eq_(Bytes('2GiB'), b)
+
+
+def test_div():
+	eq_(Bytes('1GiB'), Bytes('2GiB') / 2)
+
+
+def test_div_bytes():
+	eq_(2, Bytes('2GiB') / Bytes('1GiB'))
+
+
+def test_idiv():
+	b = Bytes('2GiB')
+	b /= 2
+	eq_(Bytes('1GiB'), b)
+
+
+def test_mod():
+	eq_(Bytes('256MiB'), Bytes('1GiB') % Bytes('768MiB'))
+
+
+@raises(UnitError)
+def test_mod_int():
+	Bytes('1GiB') % 768
+
+
+def test_imod():
+	b = Bytes('1GiB')
+	b %= Bytes('768MiB')
+	eq_(Bytes('256MiB'), b)
+
+
+@raises(UnitError)
+def test_imod_int():
+	b = Bytes('1GiB')
+	b %= 5
+
+
+def test_abs():
+	eq_(pow(1024, 3), abs(Bytes('1GiB')))
diff --git a/tests/unit/sectors.py b/tests/unit/sectors.py
new file mode 100644
index 0000000..b15bac7
--- /dev/null
+++ b/tests/unit/sectors.py
@@ -0,0 +1,128 @@
+from nose.tools import eq_
+from nose.tools import raises
+from bootstrapvz.common.sectors import Sectors
+from bootstrapvz.common.bytes import Bytes
+from bootstrapvz.common.exceptions import UnitError
+
+std_secsz = Bytes(512)
+
+
+def test_init_with_int():
+	eq_(4, abs(Sectors(4, std_secsz)))
+	secsize = 4096
+	eq_(Sectors('1MiB', secsize), Sectors(256, secsize))
+
+
+def test_lt():
+	assert Sectors('1MiB', std_secsz) < Sectors('2MiB', std_secsz)
+
+
+def test_le():
+	assert Sectors('1MiB', std_secsz) <= Sectors('2MiB', std_secsz)
+	assert Sectors('1MiB', std_secsz) <= Sectors('1MiB', std_secsz)
+
+
+def test_eq():
+	eq_(Sectors('1MiB', std_secsz), Sectors('1MiB', std_secsz))
+
+
+def test_neq():
+	assert Sectors('15MiB', std_secsz) != Sectors('1MiB', std_secsz)
+
+
+def test_gt():
+	assert Sectors('2MiB', std_secsz) > Sectors('1MiB', std_secsz)
+
+
+def test_ge():
+	assert Sectors('2MiB', std_secsz) >= Sectors('1MiB', std_secsz)
+	assert Sectors('2MiB', std_secsz) >= Sectors('2MiB', std_secsz)
+
+
+def test_eq_unit():
+	eq_(Sectors('1024MiB', std_secsz), Sectors('1GiB', std_secsz))
+
+
+def test_add():
+	eq_(Sectors('2GiB', std_secsz), Sectors('1GiB', std_secsz) + Sectors('1GiB', std_secsz))
+
+
+@raises(UnitError)
+def test_add_with_diff_secsize():
+	Sectors('1GiB', Bytes(512)) + Sectors('1GiB', Bytes(4096))
+
+
+def test_iadd():
+	s = Sectors('1GiB', std_secsz)
+	s += Sectors('1GiB', std_secsz)
+	eq_(Sectors('2GiB', std_secsz), s)
+
+
+def test_sub():
+	eq_(Sectors('1GiB', std_secsz), Sectors('2GiB', std_secsz) - Sectors('1GiB', std_secsz))
+
+
+def test_sub_int():
+	secsize = Bytes('4KiB')
+	eq_(Sectors('1MiB', secsize), Sectors('1028KiB', secsize) - 1)
+
+
+def test_isub():
+	s = Sectors('2GiB', std_secsz)
+	s -= Sectors('1GiB', std_secsz)
+	eq_(Sectors('1GiB', std_secsz), s)
+
+
+def test_mul():
+	eq_(Sectors('2GiB', std_secsz), Sectors('1GiB', std_secsz) * 2)
+
+
+@raises(UnitError)
+def test_mul_bytes():
+	Sectors('1GiB', std_secsz) * Sectors('1GiB', std_secsz)
+
+
+def test_imul():
+	s = Sectors('1GiB', std_secsz)
+	s *= 2
+	eq_(Sectors('2GiB', std_secsz), s)
+
+
+def test_div():
+	eq_(Sectors('1GiB', std_secsz), Sectors('2GiB', std_secsz) / 2)
+
+
+def test_div_bytes():
+	eq_(2, Sectors('2GiB', std_secsz) / Sectors('1GiB', std_secsz))
+
+
+def test_idiv():
+	s = Sectors('2GiB', std_secsz)
+	s /= 2
+	eq_(Sectors('1GiB', std_secsz), s)
+
+
+def test_mod():
+	eq_(Sectors('256MiB', std_secsz), Sectors('1GiB', std_secsz) % Sectors('768MiB', std_secsz))
+
+
+@raises(UnitError)
+def test_mod_int():
+	Sectors('1GiB', std_secsz) % 768
+
+
+def test_imod():
+	s = Sectors('1GiB', std_secsz)
+	s %= Sectors('768MiB', std_secsz)
+	eq_(Sectors('256MiB', std_secsz), s)
+
+
+@raises(UnitError)
+def test_imod_int():
+	s = Sectors('1GiB', std_secsz)
+	s %= 5
+
+
+def test_abs():
+	secsize = 512
+	eq_(pow(1024, 3) / secsize, abs(Sectors('1GiB', secsize)))

From 3ff1c57980ad89f9576299c50791aaedcb04daff Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 1 Jan 2015 21:00:59 +0100
Subject: [PATCH 186/345] Don't require qemu for raw volumes, use `truncate`
 instead

---
 bootstrapvz/common/fs/loopbackvolume.py | 4 ++--
 bootstrapvz/common/tasks/loopback.py    | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/common/fs/loopbackvolume.py b/bootstrapvz/common/fs/loopbackvolume.py
index 8f6f7cd..49f905f 100644
--- a/bootstrapvz/common/fs/loopbackvolume.py
+++ b/bootstrapvz/common/fs/loopbackvolume.py
@@ -11,8 +11,8 @@ class LoopbackVolume(Volume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		vol_size = str(self.size.get_qty_in('MiB')) + 'M'
-		log_check_call(['qemu-img', 'create', '-f', 'raw', self.image_path, vol_size])
+		size_opt = '--size={mib}M'.format(mib=self.size.get_qty_in('MiB'))
+		log_check_call(['truncate', 'create', size_opt, self.image_path])
 
 	def _before_attach(self, e):
 		[self.loop_device_path] = log_check_call(['losetup', '--show', '--find', self.image_path])
diff --git a/bootstrapvz/common/tasks/loopback.py b/bootstrapvz/common/tasks/loopback.py
index 77f562a..d52b3de 100644
--- a/bootstrapvz/common/tasks/loopback.py
+++ b/bootstrapvz/common/tasks/loopback.py
@@ -12,12 +12,12 @@ class AddRequiredCommands(Task):
 	@classmethod
 	def run(cls, info):
 		from ..fs.loopbackvolume import LoopbackVolume
-		if isinstance(info.volume, LoopbackVolume):
-			info.host_dependencies['qemu-img'] = 'qemu-utils'
-			info.host_dependencies['losetup'] = 'mount'
 		from ..fs.qemuvolume import QEMUVolume
-		if isinstance(info.volume, QEMUVolume):
+		if type(info.volume) is LoopbackVolume:
 			info.host_dependencies['losetup'] = 'mount'
+			info.host_dependencies['truncate'] = 'coreutils'
+		if isinstance(info.volume, QEMUVolume):
+			info.host_dependencies['qemu-img'] = 'qemu-utils'
 
 
 class Create(Task):

From a476248ed67d98dcdec57ed71d96838f0c9d0ff5 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 1 Jan 2015 21:09:16 +0100
Subject: [PATCH 187/345] Partition volumes by sectors instead of bytes This
 allows for finer grained control over the partition sizes and gaps

---
 bootstrapvz/base/fs/__init__.py               | 36 +++++++++++--------
 bootstrapvz/base/fs/partitionmaps/abstract.py |  7 ++++
 bootstrapvz/base/fs/partitionmaps/gpt.py      | 23 ++++++++----
 bootstrapvz/base/fs/partitionmaps/msdos.py    | 23 ++++++++----
 bootstrapvz/base/fs/partitionmaps/none.py     | 10 +++---
 bootstrapvz/base/fs/partitions/base.py        |  6 ++--
 bootstrapvz/base/fs/partitions/gap.py         | 17 +++++++++
 bootstrapvz/common/exceptions.py              |  4 +++
 bootstrapvz/common/fs/loopbackvolume.py       |  2 +-
 bootstrapvz/common/fs/qemuvolume.py           |  2 +-
 bootstrapvz/common/tasks/filesystem.py        | 14 +++++---
 bootstrapvz/plugins/prebootstrapped/tasks.py  |  2 +-
 bootstrapvz/plugins/vagrant/tasks.py          |  2 +-
 bootstrapvz/providers/ec2/ebsvolume.py        |  2 +-
 bootstrapvz/providers/ec2/tasks/ami.py        |  2 +-
 15 files changed, 105 insertions(+), 47 deletions(-)
 create mode 100644 bootstrapvz/base/fs/partitions/gap.py

diff --git a/bootstrapvz/base/fs/__init__.py b/bootstrapvz/base/fs/__init__.py
index 6ffe21f..9f7742b 100644
--- a/bootstrapvz/base/fs/__init__.py
+++ b/bootstrapvz/base/fs/__init__.py
@@ -9,27 +9,33 @@ def load_volume(data, bootloader):
 	:return: The volume that represents all information pertaining to the volume we bootstrap on.
 	:rtype: Volume
 	"""
-	# Create a mapping between valid partition maps in the manifest and their corresponding classes
+	# Map valid partition maps in the manifest and their corresponding classes
 	from partitionmaps.gpt import GPTPartitionMap
 	from partitionmaps.msdos import MSDOSPartitionMap
 	from partitionmaps.none import NoPartitions
-	partition_maps = {'none': NoPartitions,
-	                  'gpt': GPTPartitionMap,
-	                  'msdos': MSDOSPartitionMap,
-	                  }
-	# Instantiate the partition map
-	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader)
+	partition_map = {'none': NoPartitions,
+	                 'gpt': GPTPartitionMap,
+	                 'msdos': MSDOSPartitionMap,
+	                 }.get(data['partitions']['type'])
 
-	# Create a mapping between valid volume backings in the manifest and their corresponding classes
+	# Map valid volume backings in the manifest and their corresponding classes
 	from bootstrapvz.common.fs.loopbackvolume import LoopbackVolume
 	from bootstrapvz.providers.ec2.ebsvolume import EBSVolume
 	from bootstrapvz.common.fs.virtualdiskimage import VirtualDiskImage
 	from bootstrapvz.common.fs.virtualmachinedisk import VirtualMachineDisk
-	volume_backings = {'raw': LoopbackVolume,
-	                   's3':  LoopbackVolume,
-	                   'vdi': VirtualDiskImage,
-	                   'vmdk': VirtualMachineDisk,
-	                   'ebs': EBSVolume
-	                   }
+	volume_backing = {'raw': LoopbackVolume,
+	                  's3':  LoopbackVolume,
+	                  'vdi': VirtualDiskImage,
+	                  'vmdk': VirtualMachineDisk,
+	                  'ebs': EBSVolume
+	                  }.get(data['backing'])
+
+	# Instantiate the partition map
+	from bootstrapvz.common.bytes import Bytes
+	# Only operate with a physical sector size of 512 bytes for now,
+	# not sure if we can change that for some of the virtual disks
+	sector_size = Bytes('512B')
+	partition_map = partition_map(data['partitions'], sector_size, bootloader)
+
 	# Create the volume with the partition map as an argument
-	return volume_backings.get(data['backing'])(partition_map)
+	return volume_backing(partition_map)
diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py
index 43f9e79..d1e08ec 100644
--- a/bootstrapvz/base/fs/partitionmaps/abstract.py
+++ b/bootstrapvz/base/fs/partitionmaps/abstract.py
@@ -1,5 +1,6 @@
 from abc import ABCMeta
 from abc import abstractmethod
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.fsm_proxy import FSMProxy
 from ..exceptions import PartitionError
@@ -86,6 +87,8 @@ class AbstractPartitionMap(FSMProxy):
 
 			# Check if any partition was not mapped
 			for idx, partition in enumerate(self.partitions):
+				if isinstance(partition, PartitionGap):
+					continue
 				if partition.fsm.current not in ['mapped', 'formatted']:
 					raise PartitionError('kpartx did not map partition #' + str(idx + 1))
 
@@ -111,6 +114,8 @@ class AbstractPartitionMap(FSMProxy):
 		volume = event.volume
 		# Run through all partitions before unmapping and make sure they can all be unmapped
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			if partition.fsm.cannot('unmap'):
 				msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
 				raise PartitionError(msg)
@@ -118,4 +123,6 @@ class AbstractPartitionMap(FSMProxy):
 		log_check_call(['kpartx', '-ds', volume.device_path])
 		# Call unmap on all partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.unmap()
diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index 8472338..be54c32 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -1,6 +1,7 @@
 from abstract import AbstractPartitionMap
 from ..partitions.gpt import GPTPartition
 from ..partitions.gpt_swap import GPTSwapPartition
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -8,12 +9,14 @@ class GPTPartitionMap(AbstractPartitionMap):
 	"""Represents a GPT partition map
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# List of partitions
 		self.partitions = []
 
@@ -26,27 +29,27 @@ class GPTPartitionMap(AbstractPartitionMap):
 		# next partition.
 		if bootloader == 'grub':
 			from ..partitions.unformatted import UnformattedPartition
-			self.grub_boot = UnformattedPartition(Bytes('1007KiB'), last_partition())
+			self.grub_boot = UnformattedPartition(Sectors('1007KiB', sector_size), last_partition())
 			# Mark the partition as a bios_grub partition
 			self.grub_boot.flags.append('bios_grub')
 			self.partitions.append(self.grub_boot)
 
 		# The boot and swap partitions are optional
 		if 'boot' in data:
-			self.boot = GPTPartition(Bytes(data['boot']['size']),
+			self.boot = GPTPartition(Sectors(data['boot']['size'], sector_size),
 			                         data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                         'boot', last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition())
+			self.swap = GPTSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
 			self.partitions.append(self.swap)
-		self.root = GPTPartition(Bytes(data['root']['size']),
+		self.root = GPTPartition(Sectors(data['root']['size'], sector_size),
 		                         data['root']['filesystem'], data['root'].get('format_command', None),
 		                         'root', last_partition())
 		self.partitions.append(self.root)
 
 		# We need to move the first partition to make space for the gpt offset
-		gpt_offset = Bytes('17KiB')
+		gpt_offset = Sectors('17KiB', sector_size)
 		self.partitions[0].offset += gpt_offset
 
 		if hasattr(self, 'grub_boot'):
@@ -58,6 +61,10 @@ class GPTPartitionMap(AbstractPartitionMap):
 			# Avoid increasing the volume size because of gpt_offset
 			self.partitions[0].size -= gpt_offset
 
+		# Leave the last sector unformatted
+		self.partitions[-1].size -= 1
+		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
+
 		super(GPTPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
@@ -70,4 +77,6 @@ class GPTPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'gpt'])
 		# Create the partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.create(volume)
diff --git a/bootstrapvz/base/fs/partitionmaps/msdos.py b/bootstrapvz/base/fs/partitionmaps/msdos.py
index 1b726a5..d959e37 100644
--- a/bootstrapvz/base/fs/partitionmaps/msdos.py
+++ b/bootstrapvz/base/fs/partitionmaps/msdos.py
@@ -1,6 +1,7 @@
 from abstract import AbstractPartitionMap
 from ..partitions.msdos import MSDOSPartition
 from ..partitions.msdos_swap import MSDOSSwapPartition
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -9,12 +10,14 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 	Sometimes also called MBR (but that confuses the hell out of me, so ms-dos it is)
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# List of partitions
 		self.partitions = []
 
@@ -24,14 +27,14 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 
 		# The boot and swap partitions are optional
 		if 'boot' in data:
-			self.boot = MSDOSPartition(Bytes(data['boot']['size']),
+			self.boot = MSDOSPartition(Sectors(data['boot']['size'], sector_size),
 			                           data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                           last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition())
+			self.swap = MSDOSSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
 			self.partitions.append(self.swap)
-		self.root = MSDOSPartition(Bytes(data['root']['size']),
+		self.root = MSDOSPartition(Sectors(data['root']['size'], sector_size),
 		                           data['root']['filesystem'], data['root'].get('format_command', None),
 		                           last_partition())
 		self.partitions.append(self.root)
@@ -44,13 +47,17 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		# The MBR offset is included in the grub offset, so if we don't use grub
 		# we should reduce the size of the first partition and move it by only 512 bytes.
 		if bootloader == 'grub':
-			offset = Bytes('2MiB')
+			offset = Sectors('2MiB', sector_size)
 		else:
-			offset = Bytes('512B')
+			offset = Sectors('512B', sector_size)
 
 		self.partitions[0].offset += offset
 		self.partitions[0].size -= offset
 
+		# Leave the last sector unformatted
+		self.partitions[-1].size -= 1
+		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
+
 		super(MSDOSPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
@@ -61,4 +68,6 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'msdos'])
 		# Create the partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.create(volume)
diff --git a/bootstrapvz/base/fs/partitionmaps/none.py b/bootstrapvz/base/fs/partitionmaps/none.py
index 439fe48..944f8a5 100644
--- a/bootstrapvz/base/fs/partitionmaps/none.py
+++ b/bootstrapvz/base/fs/partitionmaps/none.py
@@ -7,14 +7,16 @@ class NoPartitions(object):
 	simply always deal with partition maps and then let the base abstract that away.
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# In the NoPartitions partitions map we only have a single 'partition'
-		self.root = SinglePartition(Bytes(data['root']['size']),
+		self.root = SinglePartition(Sectors(data['root']['size'], sector_size),
 		                            data['root']['filesystem'], data['root'].get('format_command', None))
 		self.partitions = [self.root]
 
@@ -29,7 +31,7 @@ class NoPartitions(object):
 		"""Returns the total size the partitions occupy
 
 		:return: The size of all the partitions
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
 		return self.root.get_end()
 
diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index e67e733..019218c 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -28,9 +28,9 @@ class BasePartition(AbstractPartition):
 		# By saving the previous partition we have
 		# a linked list that partitions can go backwards in to find the first partition.
 		self.previous = previous
-		from bootstrapvz.common.bytes import Bytes
-		# Initialize the offset to 0 bytes, may be changed later
-		self.offset = Bytes(0)
+		from bootstrapvz.common.sectors import Sectors
+		# Initialize the offset to 0 sectors, may be changed later
+		self.offset = Sectors(0, size.sector_size)
 		# List of flags that parted should put on the partition
 		self.flags = []
 		super(BasePartition, self).__init__(size, filesystem, format_command)
diff --git a/bootstrapvz/base/fs/partitions/gap.py b/bootstrapvz/base/fs/partitions/gap.py
new file mode 100644
index 0000000..9aadb97
--- /dev/null
+++ b/bootstrapvz/base/fs/partitions/gap.py
@@ -0,0 +1,17 @@
+from base import BasePartition
+
+
+class PartitionGap(BasePartition):
+	"""Represents a non-existent partition
+	A gap in the partitionmap
+	"""
+
+	# The states for our state machine. It can neither be create nor mapped.
+	events = []
+
+	def __init__(self, size, previous):
+		"""
+		:param Bytes size: Size of the partition
+		:param BasePartition previous: The partition that preceeds this one
+		"""
+		super(PartitionGap, self).__init__(size, None, None, previous)
diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index 2e1930e..909d7d3 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -34,3 +34,7 @@ class NoMatchesError(Exception):
 
 class TooManyMatchesError(Exception):
 	pass
+
+
+class UnitError(Exception):
+	pass
diff --git a/bootstrapvz/common/fs/loopbackvolume.py b/bootstrapvz/common/fs/loopbackvolume.py
index 49f905f..2db9edb 100644
--- a/bootstrapvz/common/fs/loopbackvolume.py
+++ b/bootstrapvz/common/fs/loopbackvolume.py
@@ -11,7 +11,7 @@ class LoopbackVolume(Volume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		size_opt = '--size={mib}M'.format(mib=self.size.get_qty_in('MiB'))
+		size_opt = '--size={mib}M'.format(mib=self.size.bytes.get_qty_in('MiB'))
 		log_check_call(['truncate', 'create', size_opt, self.image_path])
 
 	def _before_attach(self, e):
diff --git a/bootstrapvz/common/fs/qemuvolume.py b/bootstrapvz/common/fs/qemuvolume.py
index 26d3300..0e8c2f0 100644
--- a/bootstrapvz/common/fs/qemuvolume.py
+++ b/bootstrapvz/common/fs/qemuvolume.py
@@ -8,7 +8,7 @@ class QEMUVolume(LoopbackVolume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		vol_size = str(self.size.get_qty_in('MiB')) + 'M'
+		vol_size = str(self.size.bytes.get_qty_in('MiB')) + 'M'
 		log_check_call(['qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size])
 
 	def _check_nbd_module(self):
diff --git a/bootstrapvz/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py
index 8c4ab8a..7fb7511 100644
--- a/bootstrapvz/common/tasks/filesystem.py
+++ b/bootstrapvz/common/tasks/filesystem.py
@@ -25,9 +25,11 @@ class Format(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
+		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		for partition in info.volume.partition_map.partitions:
-			if not isinstance(partition, UnformattedPartition):
-				partition.format()
+			if isinstance(partition, (UnformattedPartition, PartitionGap)):
+				continue
+			partition.format()
 
 
 class TuneVolumeFS(Task):
@@ -38,12 +40,14 @@ class TuneVolumeFS(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
+		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		import re
 		# Disable the time based filesystem check
 		for partition in info.volume.partition_map.partitions:
-			if not isinstance(partition, UnformattedPartition):
-				if re.match('^ext[2-4]$', partition.filesystem) is not None:
-					log_check_call(['tune2fs', '-i', '0', partition.device_path])
+			if isinstance(partition, (UnformattedPartition, PartitionGap)):
+				continue
+			if re.match('^ext[2-4]$', partition.filesystem) is not None:
+				log_check_call(['tune2fs', '-i', '0', partition.device_path])
 
 
 class AddXFSProgs(Task):
diff --git a/bootstrapvz/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py
index d56c666..7937b30 100644
--- a/bootstrapvz/plugins/prebootstrapped/tasks.py
+++ b/bootstrapvz/plugins/prebootstrapped/tasks.py
@@ -34,7 +34,7 @@ class CreateFromSnapshot(Task):
 	@classmethod
 	def run(cls, info):
 		snapshot = info.manifest.plugins['prebootstrapped']['snapshot']
-		ebs_volume = info._ec2['connection'].create_volume(info.volume.size.get_qty_in('GiB'),
+		ebs_volume = info._ec2['connection'].create_volume(info.volume.size.bytes.get_qty_in('GiB'),
 		                                                   info._ec2['host']['availabilityZone'],
 		                                                   snapshot=snapshot)
 		while ebs_volume.volume_state() != 'available':
diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py
index 1810146..9f673cf 100644
--- a/bootstrapvz/plugins/vagrant/tasks.py
+++ b/bootstrapvz/plugins/vagrant/tasks.py
@@ -184,7 +184,7 @@ class PackageBox(Task):
 		# VHDURI = "http://go.microsoft.com/fwlink/?LinkId=137171"
 		volume_uuid = info.volume.get_uuid()
 		[disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces)
-		attr(disk, 'ovf:capacity', info.volume.size.get_qty_in('B'))
+		attr(disk, 'ovf:capacity', info.volume.size.bytes.get_qty_in('B'))
 		attr(disk, 'ovf:format', info.volume.ovf_uri)
 		attr(disk, 'ovf:uuid', volume_uuid)
 
diff --git a/bootstrapvz/providers/ec2/ebsvolume.py b/bootstrapvz/providers/ec2/ebsvolume.py
index 2cd5549..270bc75 100644
--- a/bootstrapvz/providers/ec2/ebsvolume.py
+++ b/bootstrapvz/providers/ec2/ebsvolume.py
@@ -11,7 +11,7 @@ class EBSVolume(Volume):
 	def _before_create(self, e):
 		conn = e.connection
 		zone = e.zone
-		size = self.size.get_qty_in('GiB')
+		size = self.size.bytes.get_qty_in('GiB')
 		self.volume = conn.create_volume(size, zone)
 		while self.volume.volume_state() != 'available':
 			time.sleep(5)
diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 7a39f2e..4e2da26 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -109,7 +109,7 @@ class RegisterAMI(Task):
 			from boto.ec2.blockdevicemapping import BlockDeviceType
 			from boto.ec2.blockdevicemapping import BlockDeviceMapping
 			block_device = BlockDeviceType(snapshot_id=info._ec2['snapshot'].id, delete_on_termination=True,
-			                               size=info.volume.size.get_qty_in('GiB'))
+			                               size=info.volume.size.bytes.get_qty_in('GiB'))
 			registration_params['block_device_map'] = BlockDeviceMapping()
 			registration_params['block_device_map'][root_dev_name] = block_device
 

From 1cca37985d41c20fde30779846550ae0ab8d6be2 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 2 Jan 2015 14:40:24 +0100
Subject: [PATCH 188/345] Mov tox main config to top of tox file

---
 tox.ini | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index ff693dc..6dcc230 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,9 +1,10 @@
+[tox]
+envlist = flake8, unit
+
 [flake8]
 ignore = E101,E221,E241,E501,W191
 max-line-length = 110
 
-[tox]
-envlist = flake8, unit
 
 [testenv:flake8]
 deps = flake8

From 4be7e8966c29e9b86b08d439054d82706f736324 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 2 Jan 2015 14:40:52 +0100
Subject: [PATCH 189/345] Add {posargs} to nosetest commands

---
 tox.ini | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 6dcc230..2257010 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,20 +8,20 @@ max-line-length = 110
 
 [testenv:flake8]
 deps = flake8
-commands = flake8 bootstrapvz/ --exclude=minify_json.py
+commands = flake8 bootstrapvz/ --exclude=minify_json.py {posargs}
 
 [testenv:unit]
 deps =
     nose
     nose-cov
-commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive
+commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive {posargs}
 
 [testenv:integration]
 deps =
     nose
     Pyro4 >= 4.30
     pyvbox >= 0.2.0
-commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive
+commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive {posargs}
 
 [testenv:docs]
 changedir = docs

From 7310129f4e7aa842c88c95a408ef8c3e5ff416fa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 2 Jan 2015 14:50:05 +0100
Subject: [PATCH 190/345] Disable getty processes on jessie through logind
 config

---
 bootstrapvz/common/assets/systemd/logind.conf | 5 +++++
 bootstrapvz/common/tasks/boot.py              | 8 +++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 bootstrapvz/common/assets/systemd/logind.conf

diff --git a/bootstrapvz/common/assets/systemd/logind.conf b/bootstrapvz/common/assets/systemd/logind.conf
new file mode 100644
index 0000000..b6c4d12
--- /dev/null
+++ b/bootstrapvz/common/assets/systemd/logind.conf
@@ -0,0 +1,5 @@
+
+[Login]
+# Disable all TTY getters
+NAutoVTs=0
+ReserveVT=0
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 9f028c1..4cc121c 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from .. import phases
 import os.path
+from . import assets
 
 
 class UpdateInitramfs(Task):
@@ -45,6 +46,7 @@ class DisableGetTTYs(Task):
 				i = str(i)
 				sed_i(inittab_path, '^' + i + ttyx + i, '#' + i + ttyx + i)
 		else:
-			tty_path = 'etc/systemd/system/getty.target.wants/getty@tty{num}.service'
-			for i in range(2, 7):
-				os.remove(os.path.join(info.root, tty_path.format(num=i)))
+			from shutil import copy
+			logind_asset_path = os.path.join(assets, 'systemd/logind.conf')
+			logind_destination = os.path.join(info.root, 'etc/systemd/logind.conf')
+			copy(logind_asset_path, logind_destination)

From 7f84e405f0ce0150fcd68f1b570fea1ef8d78c22 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 2 Jan 2015 15:11:36 +0100
Subject: [PATCH 191/345] Cast to int before comparing in ec2
 validate_manifest()

---
 bootstrapvz/providers/ec2/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index 3399831..2a5dd65 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -33,7 +33,7 @@ def validate_manifest(data, validator, error):
 		for key, partition in data['volume']['partitions'].iteritems():
 			if key != 'type':
 				volume_size += Bytes(partition['size'])
-		if volume_size % Bytes('1GiB') != 0:
+		if int(volume_size % Bytes('1GiB')) != 0:
 			msg = ('The volume size must be a multiple of 1GiB when using EBS backing')
 			error(msg, ['volume', 'partitions'])
 	else:

From b537f2d987088273788ee5f5eb9d551720f319a1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 2 Jan 2015 15:12:48 +0100
Subject: [PATCH 192/345] Remove __abs__ from bytes and sectors, one should
 convert to int instead

---
 bootstrapvz/common/bytes.py   | 3 ---
 bootstrapvz/common/sectors.py | 3 ---
 tests/unit/bytes.py           | 4 ++--
 tests/unit/sectors.py         | 4 ++--
 4 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/bootstrapvz/common/bytes.py b/bootstrapvz/common/bytes.py
index d558aa6..f1e48ba 100644
--- a/bootstrapvz/common/bytes.py
+++ b/bootstrapvz/common/bytes.py
@@ -72,9 +72,6 @@ class Bytes(object):
 	def __long__(self):
 		return self.qty
 
-	def __abs__(self):
-		return self.qty
-
 	@onlybytes('Can only compare Bytes to Bytes')
 	def __lt__(self, other):
 		return self.qty < other.qty
diff --git a/bootstrapvz/common/sectors.py b/bootstrapvz/common/sectors.py
index ecdbc49..23679e7 100644
--- a/bootstrapvz/common/sectors.py
+++ b/bootstrapvz/common/sectors.py
@@ -43,9 +43,6 @@ class Sectors(object):
 	def __long__(self):
 		return self.get_sectors()
 
-	def __abs__(self):
-		return self.get_sectors()
-
 	@onlysectors('Can only compare sectors with sectors')
 	def __lt__(self, other):
 		return self.bytes < other.bytes
diff --git a/tests/unit/bytes.py b/tests/unit/bytes.py
index 19e8b08..7c95797 100644
--- a/tests/unit/bytes.py
+++ b/tests/unit/bytes.py
@@ -104,5 +104,5 @@ def test_imod_int():
 	b %= 5
 
 
-def test_abs():
-	eq_(pow(1024, 3), abs(Bytes('1GiB')))
+def test_convert_int():
+	eq_(pow(1024, 3), int(Bytes('1GiB')))
diff --git a/tests/unit/sectors.py b/tests/unit/sectors.py
index b15bac7..7a86ae9 100644
--- a/tests/unit/sectors.py
+++ b/tests/unit/sectors.py
@@ -123,6 +123,6 @@ def test_imod_int():
 	s %= 5
 
 
-def test_abs():
+def test_convert_int():
 	secsize = 512
-	eq_(pow(1024, 3) / secsize, abs(Sectors('1GiB', secsize)))
+	eq_(pow(1024, 3) / secsize, int(Sectors('1GiB', secsize)))

From f43e2480a5581128bd25167e51d1a81d3d5f4069 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 3 Jan 2015 12:44:32 +0100
Subject: [PATCH 193/345] Allow serialization of Sectors class

---
 bootstrapvz/common/sectors.py  | 10 ++++++++++
 bootstrapvz/remote/__init__.py |  1 +
 2 files changed, 11 insertions(+)

diff --git a/bootstrapvz/common/sectors.py b/bootstrapvz/common/sectors.py
index 23679e7..ee7f37f 100644
--- a/bootstrapvz/common/sectors.py
+++ b/bootstrapvz/common/sectors.py
@@ -149,3 +149,13 @@ class Sectors(object):
 			raise UnitError('Cannot take modulus of sectors with different sector sizes')
 		self.bytes %= other.bytes
 		return self
+
+	def __getstate__(self):
+		return {'__class__': self.__module__ + '.' + self.__class__.__name__,
+		        'sector_size': self.sector_size,
+		        'bytes': self.bytes,
+		        }
+
+	def __setstate__(self, state):
+		self.sector_size = state['sector_size']
+		self.bytes = state['bytes']
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index d381830..bf4fd24 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -21,6 +21,7 @@ supported_classes = ['bootstrapvz.base.manifest.Manifest',
                      'bootstrapvz.base.fs.partitions.single.SinglePartition',
                      'bootstrapvz.base.fs.partitions.unformatted.UnformattedPartition',
                      'bootstrapvz.common.bytes.Bytes',
+                     'bootstrapvz.common.sectors.Sectors',
                      ]
 
 supported_exceptions = ['bootstrapvz.common.exceptions.ManifestError',

From 16eaade09a4df99bbd4c87ba19a411d73207200d Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 3 Jan 2015 12:44:57 +0100
Subject: [PATCH 194/345] Fix sectors/bytes type problem with single partition

---
 bootstrapvz/base/fs/partitionmaps/abstract.py | 2 +-
 bootstrapvz/base/fs/partitions/abstract.py    | 2 +-
 bootstrapvz/base/fs/partitions/base.py        | 2 +-
 bootstrapvz/base/fs/partitions/single.py      | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py
index d1e08ec..15cfb91 100644
--- a/bootstrapvz/base/fs/partitionmaps/abstract.py
+++ b/bootstrapvz/base/fs/partitionmaps/abstract.py
@@ -38,7 +38,7 @@ class AbstractPartitionMap(FSMProxy):
 		"""Returns the total size the partitions occupy
 
 		:return: The size of all partitions
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
 		# We just need the endpoint of the last partition
 		return self.partitions[-1].get_end()
diff --git a/bootstrapvz/base/fs/partitions/abstract.py b/bootstrapvz/base/fs/partitions/abstract.py
index 038e393..10e66e4 100644
--- a/bootstrapvz/base/fs/partitions/abstract.py
+++ b/bootstrapvz/base/fs/partitions/abstract.py
@@ -53,7 +53,7 @@ class AbstractPartition(FSMProxy):
 		"""Gets the end of the partition
 
 		:return: The end of the partition
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
 		return self.get_start() + self.size
 
diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 019218c..23af1b8 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -59,7 +59,7 @@ class BasePartition(AbstractPartition):
 		"""Gets the starting byte of this partition
 
 		:return: The starting byte of this partition
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
 		if self.previous is None:
 			# If there is no previous partition, this partition begins at the offset
diff --git a/bootstrapvz/base/fs/partitions/single.py b/bootstrapvz/base/fs/partitions/single.py
index 80683bc..e890493 100644
--- a/bootstrapvz/base/fs/partitions/single.py
+++ b/bootstrapvz/base/fs/partitions/single.py
@@ -9,8 +9,8 @@ class SinglePartition(AbstractPartition):
 		"""Gets the starting byte of this partition
 
 		:return: The starting byte of this partition
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
 		# On an unpartitioned volume there is no offset and no previous partition
-		return Bytes(0)
+		return Sectors(0, self.size.sector_size)

From 5383811eccfa52b57f0fd55472154790299ba19b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 3 Jan 2015 13:26:14 +0100
Subject: [PATCH 195/345] Allow serialization of PartitionGap class

---
 bootstrapvz/remote/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index bf4fd24..9f6b4eb 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -20,6 +20,7 @@ supported_classes = ['bootstrapvz.base.manifest.Manifest',
                      'bootstrapvz.base.fs.partitions.msdos_swap.MSDOSSwapPartition',
                      'bootstrapvz.base.fs.partitions.single.SinglePartition',
                      'bootstrapvz.base.fs.partitions.unformatted.UnformattedPartition',
+                     'bootstrapvz.base.fs.partitions.gap.PartitionGap',
                      'bootstrapvz.common.bytes.Bytes',
                      'bootstrapvz.common.sectors.Sectors',
                      ]

From eaf8f8ea0f5c9c09658fc525d6f0ed6f977fb39f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 3 Jan 2015 14:40:02 +0100
Subject: [PATCH 196/345] Remove leftover pprint

---
 bootstrapvz/remote/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 9f6b4eb..cd4102b 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -67,8 +67,6 @@ def deserialize(fq_classname, data):
 			state[key] = ser.recreate_classes(value)
 		except SecurityError as e:
 			msg = 'Unable to deserialize key `{key}\' on {class_name}'.format(key=key, class_name=fq_classname)
-			import pprint
-			msg += pprint.pformat(data)
 			raise Exception(msg, e)
 
 	instance = class_object.__new__(class_object)

From e1946fcb685986c9482066cfea321740e72ff023 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 15 Jan 2015 22:44:55 +0100
Subject: [PATCH 197/345] Simpler check for whether to create image or use
 image in prebootstrapped plugin

---
 bootstrapvz/plugins/prebootstrapped/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/plugins/prebootstrapped/__init__.py b/bootstrapvz/plugins/prebootstrapped/__init__.py
index bcdc4fc..614e049 100644
--- a/bootstrapvz/plugins/prebootstrapped/__init__.py
+++ b/bootstrapvz/plugins/prebootstrapped/__init__.py
@@ -37,13 +37,13 @@ def resolve_tasks(taskset, manifest):
 	              guest_additions.InstallGuestAdditions,
 	              ]
 	if manifest.volume['backing'] == 'ebs':
-		if 'snapshot' in settings and settings['snapshot'] is not None:
+		if settings.get('snapshot', None) is not None:
 			taskset.add(CreateFromSnapshot)
 			[taskset.discard(task) for task in skip_tasks]
 		else:
 			taskset.add(Snapshot)
 	else:
-		if 'image' in settings and settings['image'] is not None:
+		if settings.get('image', None) is not None:
 			taskset.add(CreateFromImage)
 			[taskset.discard(task) for task in skip_tasks]
 		else:

From bd4cf250a25e1c135638d28a062d8cdddeb418e3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:50:03 +0100
Subject: [PATCH 198/345] Fix spelling

---
 tests/integration/images/virtualbox.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
index 9d09692..97a599f 100644
--- a/tests/integration/images/virtualbox.py
+++ b/tests/integration/images/virtualbox.py
@@ -15,7 +15,7 @@ class VirtualBoxImage(Image):
 	def open(self):
 		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
 		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    vboxapi.library.DeviceType.hard_disk,  # decive_type
+		                                    vboxapi.library.DeviceType.hard_disk,  # device_type
 		                                    vboxapi.library.AccessMode.read_only,  # access_mode
 		                                    False)  # force_new_uuid
 

From 87f2d889b75bfc9c66d4cca1c047dfe8176dc9b3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:50:27 +0100
Subject: [PATCH 199/345] Delete image after use, apparently the vbox version
 of delete is to do nothing

---
 tests/integration/images/virtualbox.py       | 6 ++++++
 tests/integration/tools/bootable_manifest.py | 1 +
 2 files changed, 7 insertions(+)

diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
index 97a599f..a09fde3 100644
--- a/tests/integration/images/virtualbox.py
+++ b/tests/integration/images/virtualbox.py
@@ -23,6 +23,12 @@ class VirtualBoxImage(Image):
 		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
 		self.medium.close()
 
+	def destroy(self):
+		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
+		import os
+		os.remove(self.image_path)
+		del self.image_path
+
 	def get_instance(self):
 		import hashlib
 		from ..instances.virtualbox import VirtualBoxInstance
diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py
index 5d86245..4aa14b9 100644
--- a/tests/integration/tools/bootable_manifest.py
+++ b/tests/integration/tools/bootable_manifest.py
@@ -55,3 +55,4 @@ class BootableManifest(object):
 			self.instance.down()
 		if hasattr(self, 'image'):
 			self.image.close()
+			self.image.destroy()

From 0fe3c9e9845973f8bbfe34f6788efbcc95ec3578 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:51:58 +0100
Subject: [PATCH 200/345] Cleanup adfter keyboard interrupts (also do a better
 job of cleaning up)

---
 tests/integration/instances/virtualbox.py    |  4 +--
 tests/integration/tools/bootable_manifest.py | 29 ++++++++++++++------
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 06d9da8..8e37e17 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -85,10 +85,10 @@ class VirtualBoxInstance(Instance):
 			self.create()
 			try:
 				self.boot()
-			except Exception as e:
+			except (Exception, KeyboardInterrupt) as e:
 				self.shutdown()
 				raise e
-		except Exception as e:
+		except (Exception, KeyboardInterrupt) as e:
 			self.destroy()
 			raise e
 
diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py
index 4aa14b9..7a449ac 100644
--- a/tests/integration/tools/bootable_manifest.py
+++ b/tests/integration/tools/bootable_manifest.py
@@ -35,19 +35,30 @@ class BootableManifest(object):
 			handle, image_path = tempfile.mkstemp()
 			import os
 			os.close(handle)
-			build_server.download(bootstrap_info.volume.image_path, image_path)
-			build_server.delete(bootstrap_info.volume.image_path)
+			try:
+				build_server.download(bootstrap_info.volume.image_path, image_path)
+			except (Exception, KeyboardInterrupt) as e:
+				os.remove(image_path)
+				raise e
+			finally:
+				build_server.delete(bootstrap_info.volume.image_path)
 		image_type = {'virtualbox': VirtualBoxImage}
 		return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path)
 
 	def __enter__(self):
-		self.build_server = self.pick_build_server()
-		self.manifest = self.get_manifest(self.build_server)
-		self.bootstrap_info = self.bootstrap(self.manifest, self.build_server)
-		self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest)
-		self.image.open()
-		self.instance = self.image.get_instance()
-		self.instance.up()
+		try:
+			self.build_server = self.pick_build_server()
+			self.manifest = self.get_manifest(self.build_server)
+			self.bootstrap_info = self.bootstrap(self.manifest, self.build_server)
+			self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest)
+			self.image.open()
+			self.instance = self.image.get_instance()
+			self.instance.up()
+		except (Exception, KeyboardInterrupt) as e:
+			if hasattr(self, 'image'):
+				self.image.close()
+				self.image.destroy()
+			raise e
 		return self.instance
 
 	def __exit__(self, type, value, traceback):

From 7ef88d284dcdf3ab3de28e2ad41603c045638806 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:52:16 +0100
Subject: [PATCH 201/345] Don't try unlocking the machine before locking it If
 it's already locked, that should be an error.

---
 tests/integration/instances/virtualbox.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 8e37e17..07c2a1e 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -115,7 +115,6 @@ class VirtualBoxInstance(Instance):
 			return self.unlock()
 
 		def lock(self):
-			self.unlock()
 			self.machine.lock_machine(self.session, vboxapi.library.LockType.write)
 			return self.session.machine
 

From 767b32d20ebe0bb6d6f1f2ea3669b81a3b3ac694 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:52:42 +0100
Subject: [PATCH 202/345] Improve read_from_socket, a lot...

---
 tests/integration/tools/__init__.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 66910b6..625070b 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -14,7 +14,7 @@ def waituntil(predicate, timeout=5, interval=0.05):
 	return False
 
 
-def read_from_socket(socket_path, termination_string, timeout):
+def read_from_socket(socket_path, termination_string, timeout, read_timeout=0.5):
 		import socket
 		import select
 		import errno
@@ -22,15 +22,15 @@ def read_from_socket(socket_path, termination_string, timeout):
 		console.connect(socket_path)
 		console.setblocking(0)
 
+		from timeit import default_timer
+		start = default_timer()
+
 		output = ''
 		ptr = 0
 		continue_select = True
-		nooutput_for = 0
-		select_timeout = .5
 		while continue_select:
-			read_ready, _, _ = select.select([console], [], [], select_timeout)
+			read_ready, _, _ = select.select([console], [], [], read_timeout)
 			if console in read_ready:
-				nooutput_for = 0
 				while True:
 					try:
 						output += console.recv(1024)
@@ -43,12 +43,11 @@ def read_from_socket(socket_path, termination_string, timeout):
 						if e.errno != errno.EWOULDBLOCK:
 							raise Exception(e)
 						continue_select = False
-			else:
-				nooutput_for += select_timeout
-			if nooutput_for > timeout:
+			if default_timer() - start > timeout:
 				from exceptions import SocketReadTimeout
-				msg = ('Reading from socket `{path}\' timed out after {seconds} seconds.'
-				       .format(path=socket_path, timeout=nooutput_for))
+				msg = ('Reading from socket `{path}\' timed out after {seconds} seconds.\n'
+				       'Here is the output so far:\n{output}'
+				       .format(path=socket_path, seconds=timeout, output=output))
 				raise SocketReadTimeout(msg)
 		console.close()
 		return output

From 149db4c989aac279c3a666f23c1c38642728bee5 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:53:35 +0100
Subject: [PATCH 203/345] Add new task: DeterminKernelVersion, this can
 potentially fix a lot of small problems

---
 bootstrapvz/common/task_groups.py  |  5 +++++
 bootstrapvz/common/tasks/kernel.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index d6370d0..f5b3592 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -15,6 +15,7 @@ from tasks import locale
 from tasks import network
 from tasks import initd
 from tasks import ssh
+from tasks import kernel
 
 
 def get_standard_groups(manifest):
@@ -26,6 +27,7 @@ def get_standard_groups(manifest):
 	if 'boot' in manifest.volume['partitions']:
 		group.extend(boot_partition_group)
 	group.extend(mounting_group)
+	group.extend(kernel_group)
 	group.extend(get_fs_specific_group(manifest))
 	group.extend(get_network_group(manifest))
 	group.extend(get_apt_group(manifest))
@@ -76,6 +78,9 @@ mounting_group = [filesystem.CreateMountDir,
                   filesystem.DeleteMountDir,
                   ]
 
+kernel_group = [kernel.DetermineKernelVersion,
+                ]
+
 ssh_group = [ssh.AddOpenSSHPackage,
              ssh.DisableSSHPasswordAuthentication,
              ssh.DisableSSHDNSLookup,
diff --git a/bootstrapvz/common/tasks/kernel.py b/bootstrapvz/common/tasks/kernel.py
index be978f7..5acecf6 100644
--- a/bootstrapvz/common/tasks/kernel.py
+++ b/bootstrapvz/common/tasks/kernel.py
@@ -1,6 +1,7 @@
 from bootstrapvz.base import Task
 from .. import phases
 from ..tasks import packages
+import logging
 
 
 class AddDKMSPackages(Task):
@@ -22,5 +23,30 @@ class UpdateInitramfs(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.common.tools import log_check_call
-	        # Update initramfs (-u) for all currently installed kernel versions (-k all)
+		# Update initramfs (-u) for all currently installed kernel versions (-k all)
 		log_check_call(['chroot', info.root, 'update-initramfs', '-u', '-k', 'all'])
+
+
+class DetermineKernelVersion(Task):
+	description = 'Determining kernel version'
+	phase = phases.package_installation
+	predecessors = [packages.InstallPackages]
+
+	@classmethod
+	def run(cls, info):
+		# Snatched from `extlinux-update' in wheezy
+		# list the files in boot/ that match vmlinuz-*
+		# sort what the * matches, the first entry is the kernel version
+		import os.path
+		import re
+		regexp = re.compile('^vmlinuz-(?P<version>.+)$')
+
+		def get_kernel_version(vmlinuz_path):
+			vmlinux_basename = os.path.basename(vmlinuz_path)
+			return regexp.match(vmlinux_basename).group('version')
+		from glob import glob
+		boot = os.path.join(info.root, 'boot')
+		vmlinuz_paths = glob('{boot}/vmlinuz-*'.format(boot=boot))
+		kernels = map(get_kernel_version, vmlinuz_paths)
+		info.kernel_version = sorted(kernels, reverse=True)[0]
+		logging.getLogger(__name__).debug('Kernel version is {version}'.format(version=info.kernel_version))

From b67b174eb5b99af23b8108930008b0d6db23df3b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 01:54:05 +0100
Subject: [PATCH 204/345] Extend build server logging when downloading an image

---
 bootstrapvz/remote/build_servers.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index bdaa8a2..be117dc 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -132,7 +132,9 @@ class RemoteBuildServer(BuildServer):
 			self.ssh_process.wait()
 
 	def download(self, src, dst):
-		log.debug('Downloading file `{path}\' from build server `{name}\''.format(path=src, name=self.name))
+		log.debug('Downloading file `{src}\' from '
+		          'build server `{name}\' to `{dst}\''
+		          .format(src=src, dst=dst, name=self.name))
 		# Make sure we can read the file as {user}
 		self._remote_command(['sudo', 'chown', self.username, src])
 		src_arg = '{user}@{host}:{path}'.format(user=self.username, host=self.address, path=src)

From ac7e32d35e4d45b093b5d47a1de1386143884249 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 16 Jan 2015 23:49:08 +0100
Subject: [PATCH 205/345] Preserve stacktraces when reraising exceptions

---
 bootstrapvz/base/fs/partitionmaps/abstract.py | 4 ++--
 bootstrapvz/base/main.py                      | 2 +-
 bootstrapvz/common/tasks/apt.py               | 2 +-
 bootstrapvz/common/tasks/grub.py              | 4 ++--
 bootstrapvz/common/tasks/packages.py          | 2 +-
 bootstrapvz/remote/build_servers.py           | 8 ++++----
 tests/integration/tools/bootable_manifest.py  | 4 ++--
 7 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py
index 15cfb91..b4475a4 100644
--- a/bootstrapvz/base/fs/partitionmaps/abstract.py
+++ b/bootstrapvz/base/fs/partitionmaps/abstract.py
@@ -92,13 +92,13 @@ class AbstractPartitionMap(FSMProxy):
 				if partition.fsm.current not in ['mapped', 'formatted']:
 					raise PartitionError('kpartx did not map partition #' + str(idx + 1))
 
-		except PartitionError as e:
+		except PartitionError:
 			# Revert any mapping and reraise the error
 			for partition in self.partitions:
 				if not partition.fsm.can('unmap'):
 					partition.unmap()
 			log_check_call(['kpartx', '-ds', volume.device_path])
-			raise e
+			raise
 
 	def unmap(self, volume):
 		"""Unmaps the partition
diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index fe38e7d..55c3ec3 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -138,5 +138,5 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False, check_contin
 		# Run the rollback tasklist
 		rollback_tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		log.info('Successfully completed rollback')
-		raise e
+		raise
 	return bootstrap_info
diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index f009469..6572674 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -164,7 +164,7 @@ class AptUpgrade(Task):
 				       'This can sometimes occur when package retrieval times out or a package extraction failed. '
 				       'apt might succeed if you try bootstrapping again.')
 				logging.getLogger(__name__).warn(msg)
-			raise e
+			raise
 
 
 class PurgeUnusedPackages(Task):
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index ed781a1..022c4f2 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -76,10 +76,10 @@ class InstallGrub_1_99(Task):
 			# Install grub
 			log_check_call(['chroot', info.root, 'grub-install', device_path])
 			log_check_call(['chroot', info.root, 'update-grub'])
-		except Exception as e:
+		except Exception:
 			if isinstance(info.volume, LoopbackVolume):
 				remount(info.volume, unlink_fn)
-			raise e
+			raise
 
 		if isinstance(info.volume, LoopbackVolume):
 			remount(info.volume, unlink_fn)
diff --git a/bootstrapvz/common/tasks/packages.py b/bootstrapvz/common/tasks/packages.py
index a5b41b0..f6e41af 100644
--- a/bootstrapvz/common/tasks/packages.py
+++ b/bootstrapvz/common/tasks/packages.py
@@ -69,7 +69,7 @@ class InstallPackages(Task):
 					       'This can sometimes occur when package retrieval times out or a package extraction failed. '
 					       'apt might succeed if you try bootstrapping again.')
 					logging.getLogger(__name__).warn(msg)
-			raise e
+			raise
 
 	@classmethod
 	def install_local(cls, info, local_packages):
diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index be117dc..e1981e6 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -110,16 +110,16 @@ class RemoteBuildServer(BuildServer):
 				try:
 					self.connection.ping()
 					break
-				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError) as e:
+				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError):
 					if remaining_retries > 0:
 						remaining_retries -= 1
 						from time import sleep
 						sleep(2)
 					else:
-						raise e
-		except (Exception, KeyboardInterrupt) as e:
+						raise
+		except (Exception, KeyboardInterrupt):
 			self.ssh_process.terminate()
-			raise e
+			raise
 		return self.connection
 
 	def disconnect(self):
diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py
index 7a449ac..d537996 100644
--- a/tests/integration/tools/bootable_manifest.py
+++ b/tests/integration/tools/bootable_manifest.py
@@ -54,11 +54,11 @@ class BootableManifest(object):
 			self.image.open()
 			self.instance = self.image.get_instance()
 			self.instance.up()
-		except (Exception, KeyboardInterrupt) as e:
+		except (Exception, KeyboardInterrupt):
 			if hasattr(self, 'image'):
 				self.image.close()
 				self.image.destroy()
-			raise e
+			raise
 		return self.instance
 
 	def __exit__(self, type, value, traceback):

From f6e4903a8e028c85147e5c67b673176d95cc2127 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 17 Jan 2015 12:27:29 +0100
Subject: [PATCH 206/345] Update the kernel image after package installation

---
 bootstrapvz/common/task_groups.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index f5b3592..2efb409 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -79,6 +79,7 @@ mounting_group = [filesystem.CreateMountDir,
                   ]
 
 kernel_group = [kernel.DetermineKernelVersion,
+                kernel.UpdateInitramfs,
                 ]
 
 ssh_group = [ssh.AddOpenSSHPackage,

From 6b6b636f3be54c384aa6a6a54711cae0a39a2cd9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 17 Jan 2015 12:29:33 +0100
Subject: [PATCH 207/345] extlinux is now running on jessie

---
 bootstrapvz/common/assets/extlinux/boot.txt   |  1 +
 .../common/assets/extlinux/extlinux.conf      | 17 ++++++
 bootstrapvz/common/task_groups.py             | 12 +++--
 bootstrapvz/common/tasks/extlinux.py          | 52 ++++++++++++++++++-
 4 files changed, 77 insertions(+), 5 deletions(-)
 create mode 100644 bootstrapvz/common/assets/extlinux/boot.txt
 create mode 100644 bootstrapvz/common/assets/extlinux/extlinux.conf

diff --git a/bootstrapvz/common/assets/extlinux/boot.txt b/bootstrapvz/common/assets/extlinux/boot.txt
new file mode 100644
index 0000000..ddef765
--- /dev/null
+++ b/bootstrapvz/common/assets/extlinux/boot.txt
@@ -0,0 +1 @@
+Wait 5 seconds or press ENTER to 
diff --git a/bootstrapvz/common/assets/extlinux/extlinux.conf b/bootstrapvz/common/assets/extlinux/extlinux.conf
new file mode 100644
index 0000000..ca26700
--- /dev/null
+++ b/bootstrapvz/common/assets/extlinux/extlinux.conf
@@ -0,0 +1,17 @@
+default l0
+prompt 1
+timeout 50
+
+
+label l0
+	menu label Debian GNU/Linux, kernel {kernel_version}
+	linux /boot/vmlinuz-{kernel_version}
+	append initrd=/boot/initrd.img-{kernel_version} root=UUID={UUID} ro quiet console=ttyS0
+
+label l0r
+	menu label Debian GNU/Linux, kernel {kernel_version} (recovery mode)
+	linux /boot/vmlinuz-{kernel_version}
+	append initrd=/boot/initrd.img-{kernel_version} root=UUID={UUID} ro console=ttyS0 single
+	text help
+   This option boots the system into recovery mode (single-user)
+	endtext
diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index 2efb409..d4417e9 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -134,19 +134,23 @@ locale_group = [locale.LocaleBootstrapPackage,
 
 
 def get_bootloader_group(manifest):
+	from bootstrapvz.common.tools import get_codename
 	group = []
 	if manifest.system['bootloader'] == 'grub':
 		group.extend([grub.AddGrubPackage,
 		              grub.ConfigureGrub])
-		from bootstrapvz.common.tools import get_codename
 		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
 			group.append(grub.InstallGrub_1_99)
 		else:
 			group.append(grub.InstallGrub_2)
 	if manifest.system['bootloader'] == 'extlinux':
-		group.extend([extlinux.AddExtlinuxPackage,
-		              extlinux.ConfigureExtlinux,
-		              extlinux.InstallExtlinux])
+		group.append(extlinux.AddExtlinuxPackage)
+		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+			group.extend([extlinux.ConfigureExtlinux,
+			              extlinux.InstallExtlinux])
+		else:
+			group.extend([extlinux.ConfigureExtlinuxJessie,
+			              extlinux.InstallExtlinuxJessie])
 	return group
 
 
diff --git a/bootstrapvz/common/tasks/extlinux.py b/bootstrapvz/common/tasks/extlinux.py
index a050700..8c502d5 100644
--- a/bootstrapvz/common/tasks/extlinux.py
+++ b/bootstrapvz/common/tasks/extlinux.py
@@ -3,8 +3,9 @@ from .. import phases
 from ..tools import log_check_call
 import apt
 import filesystem
+import kernel
 from bootstrapvz.base.fs import partitionmaps
-import os.path
+import os
 
 
 class AddExtlinuxPackage(Task):
@@ -56,3 +57,52 @@ class InstallExtlinux(Task):
 		                '--install', '/boot/extlinux'])
 		log_check_call(['chroot', info.root,
 		                'extlinux-update'])
+
+
+class ConfigureExtlinuxJessie(Task):
+	description = 'Configuring extlinux'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		extlinux_path = os.path.join(info.root, 'boot/extlinux')
+		os.mkdir(extlinux_path)
+
+		from . import assets
+		extlinux_assets_path = os.path.join(assets, 'extlinux')
+		extlinux_tpl_path = os.path.join(extlinux_assets_path, 'extlinux.conf')
+		with open(extlinux_tpl_path) as extlinux_tpl:
+			extlinux_config = extlinux_tpl.read()
+
+		root_uuid = info.volume.partition_map.root.get_uuid()
+		extlinux_config = extlinux_config.format(kernel_version=info.kernel_version, UUID=root_uuid)
+		extlinux_config_path = os.path.join(extlinux_path, 'extlinux.conf')
+		with open(extlinux_config_path, 'w') as extlinux_conf_handle:
+			extlinux_conf_handle.write(extlinux_config)
+		from shutil import copy
+		# Copy the boot message
+		boot_txt_path = os.path.join(extlinux_assets_path, 'boot.txt')
+		copy(boot_txt_path, os.path.join(extlinux_path, 'boot.txt'))
+
+
+class InstallExtlinuxJessie(Task):
+	description = 'Installing extlinux'
+	phase = phases.system_modification
+	predecessors = [filesystem.FStab, ConfigureExtlinuxJessie]
+	# Make sure the kernel image is updated after we have installed the bootloader
+	successors = [kernel.UpdateInitramfs]
+
+	@classmethod
+	def run(cls, info):
+		if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap):
+			# Yeah, somebody saw it fit to uppercase that folder in jessie. Why? BECAUSE
+			bootloader = '/usr/lib/EXTLINUX/gptmbr.bin'
+		else:
+			bootloader = '/usr/lib/EXTLINUX/mbr.bin'
+		log_check_call(['chroot', info.root,
+		                'dd', 'bs=440', 'count=1',
+		                'if=' + bootloader,
+		                'of=' + info.volume.device_path])
+		log_check_call(['chroot', info.root,
+		                'extlinux',
+		                '--install', '/boot/extlinux'])

From 8d8a8230925c7c1416d9882af980eca832744860 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 17 Jan 2015 12:29:47 +0100
Subject: [PATCH 208/345] Properly detect successful boot on jessie

---
 tests/integration/instances/virtualbox.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 07c2a1e..deaddcd 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -57,7 +57,14 @@ class VirtualBoxInstance(Instance):
 		log.debug('Booting vbox machine `{name}\''.format(name=self.name))
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
 		from ..tools import read_from_socket
-		self.console_output = read_from_socket(self.serial_port_path, 'INIT: Entering runlevel: 2', 20)
+		# Gotta figure out a more reliable way to check when the system is done booting.
+		# Maybe bootstrapped unit test images should have a startup script that issues
+		# a callback to the host.
+		if self.image.manifest.system['release'] in ['squeeze', 'wheezy']:
+			termination_string = 'INIT: Entering runlevel: 2'
+		else:
+			termination_string = 'Debian GNU/Linux'
+		self.console_output = read_from_socket(self.serial_port_path, termination_string, 20)
 
 	def shutdown(self):
 		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))

From 744001a9ab81b1dff07484c5a1573306c99e1a77 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 17 Jan 2015 12:42:05 +0100
Subject: [PATCH 209/345] grub now works on jessie

---
 bootstrapvz/common/tasks/grub.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index 022c4f2..b528712 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -3,6 +3,7 @@ from .. import phases
 from ..tools import log_check_call
 import apt
 import filesystem
+import kernel
 from bootstrapvz.base.fs import partitionmaps
 import os.path
 
@@ -89,6 +90,8 @@ class InstallGrub_2(Task):
 	description = 'Installing grub 2'
 	phase = phases.system_modification
 	predecessors = [filesystem.FStab]
+	# Make sure the kernel image is updated after we have installed the bootloader
+	successors = [kernel.UpdateInitramfs]
 
 	@classmethod
 	def run(cls, info):

From 44dcdec0ec89c6d3c5c6162a4c6a9a7c840ab84c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 17 Jan 2015 12:42:47 +0100
Subject: [PATCH 210/345] No need to skip jessie/extlinux tests any longer

---
 tests/integration/virtualbox_tests.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 36c8b67..7411975 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -57,7 +57,6 @@ def test_partitioned_grub():
 
 
 def test_unpartitioned_extlinux_unstable():
-	raise Skip('Jessie not yet working with extlinux')
 	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -66,7 +65,6 @@ def test_unpartitioned_extlinux_unstable():
 
 
 def test_partitioned_extlinux_unstable():
-	raise Skip('Jessie not yet working with extlinux')
 	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)

From 31b9cb5caaeb6572c759c073bdcfb170757dcf6b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 18 Jan 2015 23:01:15 +0100
Subject: [PATCH 211/345] The dmsetup function still divided self.size by 512,
 which is not necessary any longer since we are calculating in actual sectors
 now

---
 bootstrapvz/base/fs/volume.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/base/fs/volume.py b/bootstrapvz/base/fs/volume.py
index 043094a..38d4991 100644
--- a/bootstrapvz/base/fs/volume.py
+++ b/bootstrapvz/base/fs/volume.py
@@ -65,7 +65,7 @@ class Volume(FSMProxy):
 
 	def _before_link_dm_node(self, e):
 		"""Links the volume using the device mapper
-		This allows us to create a 'window' into the volume that acts like a volum in itself.
+		This allows us to create a 'window' into the volume that acts like a volume in itself.
 		Mainly it is used to fool grub into thinking that it is working with a real volume,
 		rather than a loopback device or a network block device.
 
@@ -95,9 +95,9 @@ class Volume(FSMProxy):
 		start_sector = getattr(e, 'start_sector', 0)
 
 		# The number of sectors that should be mapped
-		sectors = getattr(e, 'sectors', int(self.size / 512) - start_sector)
+		sectors = getattr(e, 'sectors', int(self.size) - start_sector)
 
-		# This is the table  we send to dmsetup, so that it may create a decie mapping for us.
+		# This is the table we send to dmsetup, so that it may create a device mapping for us.
 		table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}'
 		         .format(log_start_sec=logical_start_sector,
 		                 sectors=sectors,

From 016fef606fdf29960d1cdb8a1b89cb880153d782 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:20:29 +0100
Subject: [PATCH 212/345] Account for PartitioGap in a few more places

---
 bootstrapvz/base/fs/partitionmaps/abstract.py |  9 ++++++---
 bootstrapvz/base/fs/partitions/gap.py         | 17 +++++++++++++++++
 bootstrapvz/common/tasks/grub.py              |  3 +++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py
index b4475a4..ed25770 100644
--- a/bootstrapvz/base/fs/partitionmaps/abstract.py
+++ b/bootstrapvz/base/fs/partitionmaps/abstract.py
@@ -75,6 +75,9 @@ class AbstractPartitionMap(FSMProxy):
 			                    '{device_path} (?P<blk_offset>\d+)$'
 			                    .format(device_path=volume.device_path))
 			log_check_call(['kpartx', '-as', volume.device_path])
+
+			mappable_partitions = filter(lambda p: not isinstance(p, PartitionGap), self.partitions)
+
 			import os.path
 			# Run through the kpartx output and map the paths to the partitions
 			for mapping in mappings:
@@ -83,19 +86,19 @@ class AbstractPartitionMap(FSMProxy):
 					raise PartitionError('Unable to parse kpartx output: ' + mapping)
 				partition_path = os.path.join('/dev/mapper', match.group('name'))
 				p_idx = int(match.group('p_idx')) - 1
-				self.partitions[p_idx].map(partition_path)
+				mappable_partitions[p_idx].map(partition_path)
 
 			# Check if any partition was not mapped
 			for idx, partition in enumerate(self.partitions):
 				if isinstance(partition, PartitionGap):
 					continue
 				if partition.fsm.current not in ['mapped', 'formatted']:
-					raise PartitionError('kpartx did not map partition #' + str(idx + 1))
+					raise PartitionError('kpartx did not map partition #' + str(partition.get_index()))
 
 		except PartitionError:
 			# Revert any mapping and reraise the error
 			for partition in self.partitions:
-				if not partition.fsm.can('unmap'):
+				if partition.fsm.can('unmap'):
 					partition.unmap()
 			log_check_call(['kpartx', '-ds', volume.device_path])
 			raise
diff --git a/bootstrapvz/base/fs/partitions/gap.py b/bootstrapvz/base/fs/partitions/gap.py
index 9aadb97..e7b197b 100644
--- a/bootstrapvz/base/fs/partitions/gap.py
+++ b/bootstrapvz/base/fs/partitions/gap.py
@@ -15,3 +15,20 @@ class PartitionGap(BasePartition):
 		:param BasePartition previous: The partition that preceeds this one
 		"""
 		super(PartitionGap, self).__init__(size, None, None, previous)
+
+	def get_index(self):
+		"""Gets the index of this partition in the partition map
+		Note that PartitionGap.get_index() simply returns the index of the
+		previous partition, since a gap does not count towards
+		the number of partitions.
+		If there is no previous partition 0 will be returned
+		(although partitions really are 1 indexed)
+
+		:return: The index of the partition in the partition map
+		:rtype: int
+		"""
+		if self.previous is None:
+			return 0
+		else:
+			# Recursive call to the previous partition, walking up the chain...
+			return self.previous.get_index()
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index b528712..ab5bec2 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -68,7 +68,10 @@ class InstallGrub_1_99(Task):
 			with open(device_map_path, 'w') as device_map:
 				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
 				if not isinstance(p_map, partitionmaps.none.NoPartitions):
+					from bootstrapvz.base.fs.partitions.gap import PartitionGap
 					for idx, partition in enumerate(info.volume.partition_map.partitions):
+						if isinstance(partition, PartitionGap):
+							continue
 						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
 						                 .format(device_path=partition.device_path,
 						                         prefix=partition_prefix,

From 1569797e5b69f471bbcadd58929e064772e5709e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:20:43 +0100
Subject: [PATCH 213/345] Properly set the states of different partitions in
 prebootstrapped plugin

---
 bootstrapvz/plugins/prebootstrapped/tasks.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py
index 7937b30..b1074c2 100644
--- a/bootstrapvz/plugins/prebootstrapped/tasks.py
+++ b/bootstrapvz/plugins/prebootstrapped/tasks.py
@@ -80,12 +80,20 @@ def set_fs_states(volume):
 		volume.fsm.current = 'detached'
 
 		p_map = volume.partition_map
-		partitions_state = 'attached'
 		from bootstrapvz.base.fs.partitionmaps.none import NoPartitions
-		if isinstance(p_map, NoPartitions):
-			partitions_state = 'formatted'
-		else:
+		if not isinstance(p_map, NoPartitions):
 			p_map.fsm.current = 'unmapped'
-			partitions_state = 'unmapped_fmt'
+
+		from bootstrapvz.base.fs.partitions.gap import PartitionGap
+		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
+		from bootstrapvz.base.fs.partitions.single import SinglePartition
 		for partition in p_map.partitions:
-			partition.fsm.current = partitions_state
+			if isinstance(partition, PartitionGap):
+				continue
+			if isinstance(partition, UnformattedPartition):
+				partition.fsm.current = 'unmapped'
+				continue
+			if isinstance(partition, SinglePartition):
+				partition.fsm.current = 'formatted'
+				continue
+			partition.fsm.current = 'unmapped_fmt'

From b4cd90597784287ccf8ff65a6900d665952580e4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:21:16 +0100
Subject: [PATCH 214/345] Fix serialization of UnitError

---
 bootstrapvz/remote/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index cd4102b..1b61c35 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -32,7 +32,7 @@ supported_exceptions = ['bootstrapvz.common.exceptions.ManifestError',
                         'bootstrapvz.base.fs.exceptions.PartitionError',
                         'bootstrapvz.base.pkg.exceptions.PackageError',
                         'bootstrapvz.base.pkg.exceptions.SourceError',
-                        'bootstrapvz.common.bytes.UnitError',
+                        'bootstrapvz.common.exceptions.UnitError',
                         'bootstrapvz.common.fsm_proxy.FSMProxyError',
                         ]
 

From f0402d6a9bfa7adfbd4ca73b0137b7856eb79f93 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:21:30 +0100
Subject: [PATCH 215/345] Preserve stacktrace when reraising in vbox instance
 up()

---
 tests/integration/instances/virtualbox.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index deaddcd..9de6fb2 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -92,12 +92,12 @@ class VirtualBoxInstance(Instance):
 			self.create()
 			try:
 				self.boot()
-			except (Exception, KeyboardInterrupt) as e:
+			except (Exception, KeyboardInterrupt):
 				self.shutdown()
-				raise e
-		except (Exception, KeyboardInterrupt) as e:
+				raise
+		except (Exception, KeyboardInterrupt):
 			self.destroy()
-			raise e
+			raise
 
 	def down(self):
 		self.shutdown()

From 4d74c72d99e6646cc6a210e6ea50a2e60e333d2c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:22:12 +0100
Subject: [PATCH 216/345] Simplify GPT a little by explicitly taking care of
 the GPT primary & secondary Do the math for the GPT offset a little
 differently

---
 bootstrapvz/base/fs/partitionmaps/gpt.py | 34 +++++++++++-------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index be54c32..a9bf06e 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -24,12 +24,18 @@ class GPTPartitionMap(AbstractPartitionMap):
 		def last_partition():
 			return self.partitions[-1] if len(self.partitions) > 0 else None
 
-		# If we are using the grub bootloader we need to create an unformatted partition
-		# at the beginning of the map. Its size is 1007kb, which we will steal from the
-		# next partition.
+		# The first 34 sectors are reserved for the primary GPT
+		primary_gpt = PartitionGap(Sectors(34, sector_size), last_partition())
+		self.partitions.append(primary_gpt)
+
 		if bootloader == 'grub':
+			# If we are using the grub bootloader we need to create an unformatted partition
+			# at the beginning of the map. Its size is 1007kb, which seems to be chosen so that
+			# gpt_primary + grub = 1024KiB
+			# So lets just specify grub size as 1MiB - 34 sectors
 			from ..partitions.unformatted import UnformattedPartition
-			self.grub_boot = UnformattedPartition(Sectors('1007KiB', sector_size), last_partition())
+			grub_size = Sectors('1MiB', sector_size) - primary_gpt.size
+			self.grub_boot = UnformattedPartition(grub_size, last_partition())
 			# Mark the partition as a bios_grub partition
 			self.grub_boot.flags.append('bios_grub')
 			self.partitions.append(self.grub_boot)
@@ -48,22 +54,14 @@ class GPTPartitionMap(AbstractPartitionMap):
 		                         'root', last_partition())
 		self.partitions.append(self.root)
 
-		# We need to move the first partition to make space for the gpt offset
-		gpt_offset = Sectors('17KiB', sector_size)
-		self.partitions[0].offset += gpt_offset
+		# The last 34 sectors are reserved for the secondary GPT
+		secondary_gpt = PartitionGap(Sectors(34, sector_size), last_partition())
+		self.partitions.append(secondary_gpt)
 
+		# reduce the size of the root partition so that the overall volume size is not exceeded
+		self.root.size -= primary_gpt.size + secondary_gpt.size
 		if hasattr(self, 'grub_boot'):
-			# grub_boot should not increase the size of the volume,
-			# so we reduce the size of the succeeding partition.
-			# gpt_offset is included here, because of the offset we added above (grub_boot is partition[0])
-			self.partitions[1].size -= self.grub_boot.get_end()
-		else:
-			# Avoid increasing the volume size because of gpt_offset
-			self.partitions[0].size -= gpt_offset
-
-		# Leave the last sector unformatted
-		self.partitions[-1].size -= 1
-		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
+			self.root.size -= self.grub_boot.size
 
 		super(GPTPartitionMap, self).__init__(bootloader)
 

From 9a50cea204eab69de3265102978aea1281a42a5c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:38:16 +0100
Subject: [PATCH 217/345] Add 1 sector gap between partitions on GPT

---
 bootstrapvz/base/fs/partitionmaps/gpt.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index a9bf06e..0a402ad 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -45,13 +45,21 @@ class GPTPartitionMap(AbstractPartitionMap):
 			self.boot = GPTPartition(Sectors(data['boot']['size'], sector_size),
 			                         data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                         'boot', last_partition())
+			# Offset all partitions by 1 sector.
+			# parted in jessie has changed and no longer allows partitions to be right next to each other.
+			self.boot.offset = Sectors(1, sector_size)
+			self.boot.size -= self.boot.offset
 			self.partitions.append(self.boot)
 		if 'swap' in data:
 			self.swap = GPTSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
+			self.swap.offset = Sectors(1, sector_size)
+			self.swap.size -= self.swap.offset
 			self.partitions.append(self.swap)
 		self.root = GPTPartition(Sectors(data['root']['size'], sector_size),
 		                         data['root']['filesystem'], data['root'].get('format_command', None),
 		                         'root', last_partition())
+		self.root.offset = Sectors(1, sector_size)
+		self.root.size -= self.root.offset
 		self.partitions.append(self.root)
 
 		# The last 34 sectors are reserved for the secondary GPT

From 0f82fbd6af55ba5402e6e492b8c0f1b3ea261934 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 01:39:48 +0100
Subject: [PATCH 218/345] Add some GPT tests

---
 tests/integration/virtualbox_tests.py | 62 ++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 6 deletions(-)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 7411975..8d096fa 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -16,7 +16,7 @@ def test_unpartitioned_extlinux_oldstable():
 		print(instance.console_output)
 
 
-def test_partitioned_extlinux_oldstable():
+def test_msdos_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -24,7 +24,16 @@ def test_partitioned_extlinux_oldstable():
 		print(instance.console_output)
 
 
-def test_partitioned_grub_oldstable():
+def test_gpt_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_msdos_grub_oldstable():
+	raise Skip('grub install on squeeze is broken')
 	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -32,6 +41,15 @@ def test_partitioned_grub_oldstable():
 		print(instance.console_output)
 
 
+def test_gpt_grub_oldstable():
+	raise Skip('grub install on squeeze is broken')
+	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
 def test_unpartitioned_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
@@ -40,7 +58,7 @@ def test_unpartitioned_extlinux():
 		print(instance.console_output)
 
 
-def test_partitioned_extlinux():
+def test_msdos_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -48,7 +66,15 @@ def test_partitioned_extlinux():
 		print(instance.console_output)
 
 
-def test_partitioned_grub():
+def test_gpt_extlinux():
+	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_msdos_grub():
 	std_partials = ['base', 'stable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -56,6 +82,14 @@ def test_partitioned_grub():
 		print(instance.console_output)
 
 
+def test_gpt_grub():
+	std_partials = ['base', 'stable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
 def test_unpartitioned_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
@@ -64,7 +98,7 @@ def test_unpartitioned_extlinux_unstable():
 		print(instance.console_output)
 
 
-def test_partitioned_extlinux_unstable():
+def test_msdos_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -72,9 +106,25 @@ def test_partitioned_extlinux_unstable():
 		print(instance.console_output)
 
 
-def test_partitioned_grub_unstable():
+def test_gpt_extlinux_unstable():
+	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)
+
+
+def test_msdos_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
 		print(instance.console_output)
+
+
+def test_gpt_grub_unstable():
+	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['vbox'], partials['vmdk']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	with BootableManifest(manifest_data) as instance:
+		print(instance.console_output)

From 0e19b4c1ed4da969504542cc2965ea3e440599ca Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 16:37:31 +0100
Subject: [PATCH 219/345] Fix boot detection and extend boot timeout

---
 tests/integration/instances/virtualbox.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/virtualbox.py
index 9de6fb2..ad908ef 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/virtualbox.py
@@ -60,11 +60,12 @@ class VirtualBoxInstance(Instance):
 		# Gotta figure out a more reliable way to check when the system is done booting.
 		# Maybe bootstrapped unit test images should have a startup script that issues
 		# a callback to the host.
-		if self.image.manifest.system['release'] in ['squeeze', 'wheezy']:
+		from bootstrapvz.common.tools import get_codename
+		if get_codename(self.image.manifest.system['release']) in ['squeeze', 'wheezy']:
 			termination_string = 'INIT: Entering runlevel: 2'
 		else:
 			termination_string = 'Debian GNU/Linux'
-		self.console_output = read_from_socket(self.serial_port_path, termination_string, 20)
+		self.console_output = read_from_socket(self.serial_port_path, termination_string, 120)
 
 	def shutdown(self):
 		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))

From e5dd68acc74ffcd62819a6b8d14e44d5d6d1deb0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 17:19:27 +0100
Subject: [PATCH 220/345] Extlinux booting on gpt now working

---
 bootstrapvz/base/fs/partitionmaps/gpt.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index 0a402ad..afae06a 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -36,8 +36,6 @@ class GPTPartitionMap(AbstractPartitionMap):
 			from ..partitions.unformatted import UnformattedPartition
 			grub_size = Sectors('1MiB', sector_size) - primary_gpt.size
 			self.grub_boot = UnformattedPartition(grub_size, last_partition())
-			# Mark the partition as a bios_grub partition
-			self.grub_boot.flags.append('bios_grub')
 			self.partitions.append(self.grub_boot)
 
 		# The boot and swap partitions are optional
@@ -71,6 +69,14 @@ class GPTPartitionMap(AbstractPartitionMap):
 		if hasattr(self, 'grub_boot'):
 			self.root.size -= self.grub_boot.size
 
+		# Set the boot flag on the right partition
+		if hasattr(self, 'grub_boot'):
+			# Mark the partition as a bios_grub partition
+			self.grub_boot.flags.append('bios_grub')
+		else:
+			# Mark the boot partition, or root, if boot does not exist
+			getattr(self, 'boot', self.root).flags.append('legacy_boot')
+
 		super(GPTPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):

From d105d10c76a53b175cc43161f8215559341d5047 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 18:40:46 +0100
Subject: [PATCH 221/345] raise proper Skip exception rather than the plugin
 class

---
 tests/integration/virtualbox_tests.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 8d096fa..a20aef5 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,6 +1,6 @@
 from manifests import merge_manifest_data
 from tools.bootable_manifest import BootableManifest
-from nose.plugins.skip import Skip
+from unittest.case import SkipTest
 
 partials = {'vbox': 'provider: {name: virtualbox}',
             'vdi': 'volume: {backing: vdi}',
@@ -33,7 +33,7 @@ def test_gpt_extlinux_oldstable():
 
 
 def test_msdos_grub_oldstable():
-	raise Skip('grub install on squeeze is broken')
+	raise SkipTest('grub install on squeeze is broken')
 	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -42,7 +42,7 @@ def test_msdos_grub_oldstable():
 
 
 def test_gpt_grub_oldstable():
-	raise Skip('grub install on squeeze is broken')
+	raise SkipTest('grub install on squeeze is broken')
 	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'single_partition', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)

From 05006f2d83630a30a65f8457d178715dad893d96 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 23:11:00 +0100
Subject: [PATCH 222/345] Allow using integers for more operations with Sectors

---
 bootstrapvz/common/sectors.py | 95 +++++++++++++++++++++--------------
 1 file changed, 56 insertions(+), 39 deletions(-)

diff --git a/bootstrapvz/common/sectors.py b/bootstrapvz/common/sectors.py
index ee7f37f..92db8f7 100644
--- a/bootstrapvz/common/sectors.py
+++ b/bootstrapvz/common/sectors.py
@@ -68,87 +68,104 @@ class Sectors(object):
 		return self.bytes > other.bytes
 
 	def __add__(self, other):
-		if not isinstance(other, Sectors):
-			raise UnitError('Can only add sectors to sectors')
-		if self.sector_size != other.sector_size:
-			raise UnitError('Cannot sum sectors with different sector sizes')
-		return Sectors(self.bytes + other.bytes, self.sector_size)
+		if isinstance(other, (int, long)):
+			return Sectors(self.bytes + self.sector_size * other, self.sector_size)
+		if isinstance(other, Bytes):
+			return Sectors(self.bytes + other, self.sector_size)
+		if isinstance(other, Sectors):
+			if self.sector_size != other.sector_size:
+				raise UnitError('Cannot sum sectors with different sector sizes')
+			return Sectors(self.bytes + other.bytes, self.sector_size)
+		raise UnitError('Can only add sectors, bytes or integers to sectors')
 
 	def __iadd__(self, other):
-		if not isinstance(other, (Bytes, Sectors)):
-			raise UnitError('Can only add Bytes or sectors to sectors')
+		if isinstance(other, (int, long)):
+			self.bytes += self.sector_size * other
+			return self
 		if isinstance(other, Bytes):
 			self.bytes += other
+			return self
 		if isinstance(other, Sectors):
 			if self.sector_size != other.sector_size:
 				raise UnitError('Cannot sum sectors with different sector sizes')
 			self.bytes += other.bytes
-		return self
+			return self
+		raise UnitError('Can only add sectors, bytes or integers to sectors')
 
 	def __sub__(self, other):
-		if not isinstance(other, (Sectors, int, long)):
-			raise UnitError('Can only subtract sectors or integers from sectors')
-		if isinstance(other, int):
+		if isinstance(other, (int, long)):
 			return Sectors(self.bytes - self.sector_size * other, self.sector_size)
-		else:
+		if isinstance(other, Bytes):
+			return Sectors(self.bytes - other, self.sector_size)
+		if isinstance(other, Sectors):
 			if self.sector_size != other.sector_size:
 				raise UnitError('Cannot subtract sectors with different sector sizes')
 			return Sectors(self.bytes - other.bytes, self.sector_size)
+		raise UnitError('Can only subtract sectors, bytes or integers from sectors')
 
 	def __isub__(self, other):
-		if not isinstance(other, (Sectors, int, long)):
-			raise UnitError('Can only subtract sectors or integers from sectors')
-		if isinstance(other, int):
+		if isinstance(other, (int, long)):
 			self.bytes -= self.sector_size * other
-		else:
+			return self
+		if isinstance(other, Bytes):
+			self.bytes -= other
+			return self
+		if isinstance(other, Sectors):
 			if self.sector_size != other.sector_size:
 				raise UnitError('Cannot subtract sectors with different sector sizes')
 			self.bytes -= other.bytes
-		return self
+			return self
+		raise UnitError('Can only subtract sectors, bytes or integers from sectors')
 
 	def __mul__(self, other):
-		if not isinstance(other, (int, long)):
+		if isinstance(other, (int, long)):
+			return Sectors(self.bytes * other, self.sector_size)
+		else:
 			raise UnitError('Can only multiply sectors with integers')
-		return Sectors(self.bytes * other, self.sector_size)
 
 	def __imul__(self, other):
-		if not isinstance(other, (int, long)):
+		if isinstance(other, (int, long)):
+			self.bytes *= other
+			return self
+		else:
 			raise UnitError('Can only multiply sectors with integers')
-		self.bytes *= other
-		return self
 
 	def __div__(self, other):
+		if isinstance(other, (int, long)):
+			return Sectors(self.bytes / other, self.sector_size)
 		if isinstance(other, Sectors):
 			if self.sector_size != other.sector_size:
+				return self.bytes / other.bytes
+			else:
 				raise UnitError('Cannot divide sectors with different sector sizes')
-			return self.bytes / other.bytes
-		if not isinstance(other, (int, long)):
-			raise UnitError('Can only divide sectors with integers or sectors')
-		return Sectors(self.bytes / other, self.sector_size)
+		raise UnitError('Can only divide sectors with integers or sectors')
 
 	def __idiv__(self, other):
-		if isinstance(other, Sectors):
-			if self.sector_size != other.sector_size:
-				raise UnitError('Cannot divide sectors with different sector sizes')
-			self.bytes /= other.bytes
-		else:
-			if not isinstance(other, (int, long)):
-				raise UnitError('Can only divide sectors with integers or sectors')
+		if isinstance(other, (int, long)):
 			self.bytes /= other
-		return self
+			return self
+		if isinstance(other, Sectors):
+			if self.sector_size == other.sector_size:
+				self.bytes /= other.bytes
+				return self
+			else:
+				raise UnitError('Cannot divide sectors with different sector sizes')
+		raise UnitError('Can only divide sectors with integers or sectors')
 
 	@onlysectors('Can only take modulus of sectors with sectors')
 	def __mod__(self, other):
-		if self.sector_size != other.sector_size:
+		if self.sector_size == other.sector_size:
+			return Sectors(self.bytes % other.bytes, self.sector_size)
+		else:
 			raise UnitError('Cannot take modulus of sectors with different sector sizes')
-		return Sectors(self.bytes % other.bytes, self.sector_size)
 
 	@onlysectors('Can only take modulus of sectors with sectors')
 	def __imod__(self, other):
-		if self.sector_size != other.sector_size:
+		if self.sector_size == other.sector_size:
+			self.bytes %= other.bytes
+			return self
+		else:
 			raise UnitError('Cannot take modulus of sectors with different sector sizes')
-		self.bytes %= other.bytes
-		return self
 
 	def __getstate__(self):
 		return {'__class__': self.__module__ + '.' + self.__class__.__name__,

From a62ce1705ae30f0c37fec9d0233ab03864ad4f5f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 23:11:19 +0100
Subject: [PATCH 223/345] Remove the PartitionGap class, it's way too much of a
 hassle to work with Instead pad_start and pad_end have been introduced

---
 bootstrapvz/base/fs/partitionmaps/abstract.py | 11 +---
 bootstrapvz/base/fs/partitionmaps/gpt.py      | 58 +++++++++----------
 bootstrapvz/base/fs/partitionmaps/msdos.py    | 17 +++---
 bootstrapvz/base/fs/partitions/abstract.py    |  6 +-
 bootstrapvz/base/fs/partitions/base.py        | 18 +++---
 bootstrapvz/base/fs/partitions/gap.py         | 34 -----------
 bootstrapvz/base/fs/partitions/single.py      |  1 -
 bootstrapvz/common/tasks/filesystem.py        |  6 +-
 bootstrapvz/common/tasks/grub.py              |  3 -
 bootstrapvz/plugins/prebootstrapped/tasks.py  |  3 -
 bootstrapvz/remote/__init__.py                |  1 -
 11 files changed, 53 insertions(+), 105 deletions(-)
 delete mode 100644 bootstrapvz/base/fs/partitions/gap.py

diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py
index ed25770..cde7c2f 100644
--- a/bootstrapvz/base/fs/partitionmaps/abstract.py
+++ b/bootstrapvz/base/fs/partitionmaps/abstract.py
@@ -1,6 +1,5 @@
 from abc import ABCMeta
 from abc import abstractmethod
-from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.fsm_proxy import FSMProxy
 from ..exceptions import PartitionError
@@ -76,8 +75,6 @@ class AbstractPartitionMap(FSMProxy):
 			                    .format(device_path=volume.device_path))
 			log_check_call(['kpartx', '-as', volume.device_path])
 
-			mappable_partitions = filter(lambda p: not isinstance(p, PartitionGap), self.partitions)
-
 			import os.path
 			# Run through the kpartx output and map the paths to the partitions
 			for mapping in mappings:
@@ -86,12 +83,10 @@ class AbstractPartitionMap(FSMProxy):
 					raise PartitionError('Unable to parse kpartx output: ' + mapping)
 				partition_path = os.path.join('/dev/mapper', match.group('name'))
 				p_idx = int(match.group('p_idx')) - 1
-				mappable_partitions[p_idx].map(partition_path)
+				self.partitions[p_idx].map(partition_path)
 
 			# Check if any partition was not mapped
 			for idx, partition in enumerate(self.partitions):
-				if isinstance(partition, PartitionGap):
-					continue
 				if partition.fsm.current not in ['mapped', 'formatted']:
 					raise PartitionError('kpartx did not map partition #' + str(partition.get_index()))
 
@@ -117,8 +112,6 @@ class AbstractPartitionMap(FSMProxy):
 		volume = event.volume
 		# Run through all partitions before unmapping and make sure they can all be unmapped
 		for partition in self.partitions:
-			if isinstance(partition, PartitionGap):
-				continue
 			if partition.fsm.cannot('unmap'):
 				msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
 				raise PartitionError(msg)
@@ -126,6 +119,4 @@ class AbstractPartitionMap(FSMProxy):
 		log_check_call(['kpartx', '-ds', volume.device_path])
 		# Call unmap on all partitions
 		for partition in self.partitions:
-			if isinstance(partition, PartitionGap):
-				continue
 			partition.unmap()
diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index afae06a..b8918d3 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -1,7 +1,6 @@
 from abstract import AbstractPartitionMap
 from ..partitions.gpt import GPTPartition
 from ..partitions.gpt_swap import GPTSwapPartition
-from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -24,57 +23,60 @@ class GPTPartitionMap(AbstractPartitionMap):
 		def last_partition():
 			return self.partitions[-1] if len(self.partitions) > 0 else None
 
-		# The first 34 sectors are reserved for the primary GPT
-		primary_gpt = PartitionGap(Sectors(34, sector_size), last_partition())
-		self.partitions.append(primary_gpt)
-
 		if bootloader == 'grub':
 			# If we are using the grub bootloader we need to create an unformatted partition
 			# at the beginning of the map. Its size is 1007kb, which seems to be chosen so that
-			# gpt_primary + grub = 1024KiB
-			# So lets just specify grub size as 1MiB - 34 sectors
+			# primary gpt + grub = 1024KiB
+			# The 34 sectors for the primary gpt will be subtracted later on
 			from ..partitions.unformatted import UnformattedPartition
-			grub_size = Sectors('1MiB', sector_size) - primary_gpt.size
-			self.grub_boot = UnformattedPartition(grub_size, last_partition())
+			self.grub_boot = UnformattedPartition(Sectors('1MiB', sector_size), last_partition())
 			self.partitions.append(self.grub_boot)
 
+		# Offset all partitions by 1 sector.
+		# parted in jessie has changed and no longer allows
+		# gpt partitions to be right next to each other.
+		partition_gap = Sectors(1, sector_size)
+
 		# The boot and swap partitions are optional
 		if 'boot' in data:
 			self.boot = GPTPartition(Sectors(data['boot']['size'], sector_size),
 			                         data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                         'boot', last_partition())
-			# Offset all partitions by 1 sector.
-			# parted in jessie has changed and no longer allows partitions to be right next to each other.
-			self.boot.offset = Sectors(1, sector_size)
-			self.boot.size -= self.boot.offset
+			if self.boot.previous is not None:
+				# No need to pad if this is the first partition
+				self.boot.pad_start += partition_gap
+				self.boot.size -= partition_gap
 			self.partitions.append(self.boot)
+
 		if 'swap' in data:
 			self.swap = GPTSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
-			self.swap.offset = Sectors(1, sector_size)
-			self.swap.size -= self.swap.offset
+			if self.swap.previous is not None:
+				self.swap.pad_start += partition_gap
+				self.swap.size -= partition_gap
 			self.partitions.append(self.swap)
+
 		self.root = GPTPartition(Sectors(data['root']['size'], sector_size),
 		                         data['root']['filesystem'], data['root'].get('format_command', None),
 		                         'root', last_partition())
-		self.root.offset = Sectors(1, sector_size)
-		self.root.size -= self.root.offset
+		if self.root.previous is not None:
+			self.root.pad_start += partition_gap
+			self.root.size -= partition_gap
 		self.partitions.append(self.root)
 
-		# The last 34 sectors are reserved for the secondary GPT
-		secondary_gpt = PartitionGap(Sectors(34, sector_size), last_partition())
-		self.partitions.append(secondary_gpt)
+		# The first and last 34 sectors are reserved for the primary/secondary GPT
+		primary_gpt_size = Sectors(34, sector_size)
+		self.partitions[0].pad_start += primary_gpt_size
+		self.partitions[0].size -= primary_gpt_size
 
-		# reduce the size of the root partition so that the overall volume size is not exceeded
-		self.root.size -= primary_gpt.size + secondary_gpt.size
-		if hasattr(self, 'grub_boot'):
-			self.root.size -= self.grub_boot.size
+		secondary_gpt_size = Sectors(34, sector_size)
+		self.partitions[-1].pad_end += secondary_gpt_size
+		self.partitions[-1].size -= secondary_gpt_size
 
-		# Set the boot flag on the right partition
 		if hasattr(self, 'grub_boot'):
-			# Mark the partition as a bios_grub partition
+			# Mark the grub partition as a bios_grub partition
 			self.grub_boot.flags.append('bios_grub')
 		else:
-			# Mark the boot partition, or root, if boot does not exist
+			# Not using grub, mark the boot partition or root as bootable
 			getattr(self, 'boot', self.root).flags.append('legacy_boot')
 
 		super(GPTPartitionMap, self).__init__(bootloader)
@@ -89,6 +91,4 @@ class GPTPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'gpt'])
 		# Create the partitions
 		for partition in self.partitions:
-			if isinstance(partition, PartitionGap):
-				continue
 			partition.create(volume)
diff --git a/bootstrapvz/base/fs/partitionmaps/msdos.py b/bootstrapvz/base/fs/partitionmaps/msdos.py
index d959e37..89934c5 100644
--- a/bootstrapvz/base/fs/partitionmaps/msdos.py
+++ b/bootstrapvz/base/fs/partitionmaps/msdos.py
@@ -1,7 +1,6 @@
 from abstract import AbstractPartitionMap
 from ..partitions.msdos import MSDOSPartition
 from ..partitions.msdos_swap import MSDOSSwapPartition
-from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -31,9 +30,11 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 			                           data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                           last_partition())
 			self.partitions.append(self.boot)
+
 		if 'swap' in data:
 			self.swap = MSDOSSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
 			self.partitions.append(self.swap)
+
 		self.root = MSDOSPartition(Sectors(data['root']['size'], sector_size),
 		                           data['root']['filesystem'], data['root'].get('format_command', None),
 		                           last_partition())
@@ -47,16 +48,18 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		# The MBR offset is included in the grub offset, so if we don't use grub
 		# we should reduce the size of the first partition and move it by only 512 bytes.
 		if bootloader == 'grub':
-			offset = Sectors('2MiB', sector_size)
+			mbr_offset = Sectors('2MiB', sector_size)
 		else:
-			offset = Sectors('512B', sector_size)
+			mbr_offset = Sectors('512B', sector_size)
 
-		self.partitions[0].offset += offset
-		self.partitions[0].size -= offset
+		self.partitions[0].pad_start += mbr_offset
+		self.partitions[0].size -= mbr_offset
 
 		# Leave the last sector unformatted
+		# parted in jessie thinks that a partition 10 sectors in size
+		# goes from sector 0 to sector 9 (instead of 0 to 10)
+		self.partitions[-1].pad_end += 1
 		self.partitions[-1].size -= 1
-		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
 
 		super(MSDOSPartitionMap, self).__init__(bootloader)
 
@@ -68,6 +71,4 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'msdos'])
 		# Create the partitions
 		for partition in self.partitions:
-			if isinstance(partition, PartitionGap):
-				continue
 			partition.create(volume)
diff --git a/bootstrapvz/base/fs/partitions/abstract.py b/bootstrapvz/base/fs/partitions/abstract.py
index 10e66e4..9d481d7 100644
--- a/bootstrapvz/base/fs/partitions/abstract.py
+++ b/bootstrapvz/base/fs/partitions/abstract.py
@@ -1,5 +1,6 @@
 from abc import ABCMeta
 from abc import abstractmethod
+from bootstrapvz.common.sectors import Sectors
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.fsm_proxy import FSMProxy
 
@@ -27,6 +28,9 @@ class AbstractPartition(FSMProxy):
 		self.size           = size
 		self.filesystem     = filesystem
 		self.format_command = format_command
+		# Initialize the start & end padding to 0 sectors, may be changed later
+		self.pad_start = Sectors(0, size.sector_size)
+		self.pad_end = Sectors(0, size.sector_size)
 		# Path to the partition
 		self.device_path    = None
 		# Dictionary with mount points as keys and Mount objects as values
@@ -55,7 +59,7 @@ class AbstractPartition(FSMProxy):
 		:return: The end of the partition
 		:rtype: Sectors
 		"""
-		return self.get_start() + self.size
+		return self.get_start() + self.pad_start + self.size + self.pad_end
 
 	def _before_format(self, e):
 		"""Formats the partition
diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 23af1b8..3e5cbab 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -1,4 +1,5 @@
 from abstract import AbstractPartition
+from bootstrapvz.common.sectors import Sectors
 
 
 class BasePartition(AbstractPartition):
@@ -25,12 +26,9 @@ class BasePartition(AbstractPartition):
 		:param list format_command: Optional format command, valid variables are fs, device_path and size
 		:param BasePartition previous: The partition that preceeds this one
 		"""
-		# By saving the previous partition we have
-		# a linked list that partitions can go backwards in to find the first partition.
+		# By saving the previous partition we have a linked list
+		# that partitions can go backwards in to find the first partition.
 		self.previous = previous
-		from bootstrapvz.common.sectors import Sectors
-		# Initialize the offset to 0 sectors, may be changed later
-		self.offset = Sectors(0, size.sector_size)
 		# List of flags that parted should put on the partition
 		self.flags = []
 		super(BasePartition, self).__init__(size, filesystem, format_command)
@@ -62,11 +60,9 @@ class BasePartition(AbstractPartition):
 		:rtype: Sectors
 		"""
 		if self.previous is None:
-			# If there is no previous partition, this partition begins at the offset
-			return self.offset
+			return Sectors(0, self.size.sector_size)
 		else:
-			# Get the end of the previous partition and add the offset of this partition
-			return self.previous.get_end() + self.offset
+			return self.previous.get_end()
 
 	def map(self, device_path):
 		"""Maps the partition to a device_path
@@ -81,8 +77,8 @@ class BasePartition(AbstractPartition):
 		from bootstrapvz.common.tools import log_check_call
 		# The create command is failry simple, start and end are just Bytes objects coerced into strings
 		create_command = ('mkpart primary {start} {end}'
-		                  .format(start=str(self.get_start()),
-		                          end=str(self.get_end())))
+		                  .format(start=str(self.get_start() + self.pad_start),
+		                          end=str(self.get_end() - self.pad_end)))
 		# Create the partition
 		log_check_call(['parted', '--script', '--align', 'none', e.volume.device_path,
 		                '--', create_command])
diff --git a/bootstrapvz/base/fs/partitions/gap.py b/bootstrapvz/base/fs/partitions/gap.py
deleted file mode 100644
index e7b197b..0000000
--- a/bootstrapvz/base/fs/partitions/gap.py
+++ /dev/null
@@ -1,34 +0,0 @@
-from base import BasePartition
-
-
-class PartitionGap(BasePartition):
-	"""Represents a non-existent partition
-	A gap in the partitionmap
-	"""
-
-	# The states for our state machine. It can neither be create nor mapped.
-	events = []
-
-	def __init__(self, size, previous):
-		"""
-		:param Bytes size: Size of the partition
-		:param BasePartition previous: The partition that preceeds this one
-		"""
-		super(PartitionGap, self).__init__(size, None, None, previous)
-
-	def get_index(self):
-		"""Gets the index of this partition in the partition map
-		Note that PartitionGap.get_index() simply returns the index of the
-		previous partition, since a gap does not count towards
-		the number of partitions.
-		If there is no previous partition 0 will be returned
-		(although partitions really are 1 indexed)
-
-		:return: The index of the partition in the partition map
-		:rtype: int
-		"""
-		if self.previous is None:
-			return 0
-		else:
-			# Recursive call to the previous partition, walking up the chain...
-			return self.previous.get_index()
diff --git a/bootstrapvz/base/fs/partitions/single.py b/bootstrapvz/base/fs/partitions/single.py
index e890493..e10b74c 100644
--- a/bootstrapvz/base/fs/partitions/single.py
+++ b/bootstrapvz/base/fs/partitions/single.py
@@ -12,5 +12,4 @@ class SinglePartition(AbstractPartition):
 		:rtype: Sectors
 		"""
 		from bootstrapvz.common.sectors import Sectors
-		# On an unpartitioned volume there is no offset and no previous partition
 		return Sectors(0, self.size.sector_size)
diff --git a/bootstrapvz/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py
index 7fb7511..a6cb782 100644
--- a/bootstrapvz/common/tasks/filesystem.py
+++ b/bootstrapvz/common/tasks/filesystem.py
@@ -25,9 +25,8 @@ class Format(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
-		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		for partition in info.volume.partition_map.partitions:
-			if isinstance(partition, (UnformattedPartition, PartitionGap)):
+			if isinstance(partition, UnformattedPartition):
 				continue
 			partition.format()
 
@@ -40,11 +39,10 @@ class TuneVolumeFS(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
-		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		import re
 		# Disable the time based filesystem check
 		for partition in info.volume.partition_map.partitions:
-			if isinstance(partition, (UnformattedPartition, PartitionGap)):
+			if isinstance(partition, UnformattedPartition):
 				continue
 			if re.match('^ext[2-4]$', partition.filesystem) is not None:
 				log_check_call(['tune2fs', '-i', '0', partition.device_path])
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index ab5bec2..b528712 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -68,10 +68,7 @@ class InstallGrub_1_99(Task):
 			with open(device_map_path, 'w') as device_map:
 				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
 				if not isinstance(p_map, partitionmaps.none.NoPartitions):
-					from bootstrapvz.base.fs.partitions.gap import PartitionGap
 					for idx, partition in enumerate(info.volume.partition_map.partitions):
-						if isinstance(partition, PartitionGap):
-							continue
 						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
 						                 .format(device_path=partition.device_path,
 						                         prefix=partition_prefix,
diff --git a/bootstrapvz/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py
index b1074c2..4711e46 100644
--- a/bootstrapvz/plugins/prebootstrapped/tasks.py
+++ b/bootstrapvz/plugins/prebootstrapped/tasks.py
@@ -84,12 +84,9 @@ def set_fs_states(volume):
 		if not isinstance(p_map, NoPartitions):
 			p_map.fsm.current = 'unmapped'
 
-		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
 		from bootstrapvz.base.fs.partitions.single import SinglePartition
 		for partition in p_map.partitions:
-			if isinstance(partition, PartitionGap):
-				continue
 			if isinstance(partition, UnformattedPartition):
 				partition.fsm.current = 'unmapped'
 				continue
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 1b61c35..7397a5a 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -20,7 +20,6 @@ supported_classes = ['bootstrapvz.base.manifest.Manifest',
                      'bootstrapvz.base.fs.partitions.msdos_swap.MSDOSSwapPartition',
                      'bootstrapvz.base.fs.partitions.single.SinglePartition',
                      'bootstrapvz.base.fs.partitions.unformatted.UnformattedPartition',
-                     'bootstrapvz.base.fs.partitions.gap.PartitionGap',
                      'bootstrapvz.common.bytes.Bytes',
                      'bootstrapvz.common.sectors.Sectors',
                      ]

From 4cbc8e4f8e36d11c1280d7c1f3cef513486df73c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 19 Jan 2015 23:53:21 +0100
Subject: [PATCH 224/345] Related to 26bb907, both gpt and msdos partitions
 need the 1 sector gap now

---
 bootstrapvz/base/fs/partitionmaps/gpt.py   |  2 +-
 bootstrapvz/base/fs/partitionmaps/msdos.py | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index b8918d3..3113ceb 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -34,7 +34,7 @@ class GPTPartitionMap(AbstractPartitionMap):
 
 		# Offset all partitions by 1 sector.
 		# parted in jessie has changed and no longer allows
-		# gpt partitions to be right next to each other.
+		# partitions to be right next to each other.
 		partition_gap = Sectors(1, sector_size)
 
 		# The boot and swap partitions are optional
diff --git a/bootstrapvz/base/fs/partitionmaps/msdos.py b/bootstrapvz/base/fs/partitionmaps/msdos.py
index 89934c5..6c0d25d 100644
--- a/bootstrapvz/base/fs/partitionmaps/msdos.py
+++ b/bootstrapvz/base/fs/partitionmaps/msdos.py
@@ -31,13 +31,25 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 			                           last_partition())
 			self.partitions.append(self.boot)
 
+		# Offset all partitions by 1 sector.
+		# parted in jessie has changed and no longer allows
+		# partitions to be right next to each other.
+		partition_gap = Sectors(1, sector_size)
+
 		if 'swap' in data:
 			self.swap = MSDOSSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
+			if self.swap.previous is not None:
+				# No need to pad if this is the first partition
+				self.swap.pad_start += partition_gap
+				self.swap.size -= partition_gap
 			self.partitions.append(self.swap)
 
 		self.root = MSDOSPartition(Sectors(data['root']['size'], sector_size),
 		                           data['root']['filesystem'], data['root'].get('format_command', None),
 		                           last_partition())
+		if self.root.previous is not None:
+			self.root.pad_start += partition_gap
+			self.root.size -= partition_gap
 		self.partitions.append(self.root)
 
 		# Mark boot as the boot partition, or root, if boot does not exist

From b70e24a848c3bc9f5f2cb6d7999b4eaf32c03f8e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 20 Jan 2015 08:31:12 +0100
Subject: [PATCH 225/345] Switch to using 3 partitions when testing gpt+msdos
 Make boot partition a little bigger

---
 tests/integration/manifests/partitioned.yml |  4 ++--
 tests/integration/virtualbox_tests.py       | 24 ++++++++++-----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/integration/manifests/partitioned.yml b/tests/integration/manifests/partitioned.yml
index e6686c4..c44eab1 100644
--- a/tests/integration/manifests/partitioned.yml
+++ b/tests/integration/manifests/partitioned.yml
@@ -3,9 +3,9 @@ volume:
   partitions:
     boot:
       filesystem: ext2
-      size: 32MiB
+      size: 64MiB
     root:
       filesystem: ext4
-      size: 864MiB
+      size: 832MiB
     swap:
       size: 128MiB
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index a20aef5..ef98abc 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -17,7 +17,7 @@ def test_unpartitioned_extlinux_oldstable():
 
 
 def test_msdos_extlinux_oldstable():
-	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -25,7 +25,7 @@ def test_msdos_extlinux_oldstable():
 
 
 def test_gpt_extlinux_oldstable():
-	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -34,7 +34,7 @@ def test_gpt_extlinux_oldstable():
 
 def test_msdos_grub_oldstable():
 	raise SkipTest('grub install on squeeze is broken')
-	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -43,7 +43,7 @@ def test_msdos_grub_oldstable():
 
 def test_gpt_grub_oldstable():
 	raise SkipTest('grub install on squeeze is broken')
-	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -59,7 +59,7 @@ def test_unpartitioned_extlinux():
 
 
 def test_msdos_extlinux():
-	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -67,7 +67,7 @@ def test_msdos_extlinux():
 
 
 def test_gpt_extlinux():
-	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -75,7 +75,7 @@ def test_gpt_extlinux():
 
 
 def test_msdos_grub():
-	std_partials = ['base', 'stable64', 'grub', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -83,7 +83,7 @@ def test_msdos_grub():
 
 
 def test_gpt_grub():
-	std_partials = ['base', 'stable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -99,7 +99,7 @@ def test_unpartitioned_extlinux_unstable():
 
 
 def test_msdos_extlinux_unstable():
-	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -107,7 +107,7 @@ def test_msdos_extlinux_unstable():
 
 
 def test_gpt_extlinux_unstable():
-	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -115,7 +115,7 @@ def test_gpt_extlinux_unstable():
 
 
 def test_msdos_grub_unstable():
-	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'single_partition', 'root_password']
+	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:
@@ -123,7 +123,7 @@ def test_msdos_grub_unstable():
 
 
 def test_gpt_grub_unstable():
-	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'single_partition', 'root_password']
+	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with BootableManifest(manifest_data) as instance:

From 75e70c96f78c3014b12aee5cd58bdcab7db1e766 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 21 Jan 2015 22:32:04 +0100
Subject: [PATCH 226/345] Fix extlinux booting on jessie when /boot is on
 another partition

---
 .../common/assets/extlinux/extlinux.conf      |  8 +++---
 bootstrapvz/common/tasks/extlinux.py          | 27 ++++++++++++-------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/bootstrapvz/common/assets/extlinux/extlinux.conf b/bootstrapvz/common/assets/extlinux/extlinux.conf
index ca26700..68ce106 100644
--- a/bootstrapvz/common/assets/extlinux/extlinux.conf
+++ b/bootstrapvz/common/assets/extlinux/extlinux.conf
@@ -5,13 +5,13 @@ timeout 50
 
 label l0
 	menu label Debian GNU/Linux, kernel {kernel_version}
-	linux /boot/vmlinuz-{kernel_version}
-	append initrd=/boot/initrd.img-{kernel_version} root=UUID={UUID} ro quiet console=ttyS0
+	linux {boot_prefix}/vmlinuz-{kernel_version}
+	append initrd={boot_prefix}/initrd.img-{kernel_version} root=UUID={root_uuid} ro quiet console=ttyS0
 
 label l0r
 	menu label Debian GNU/Linux, kernel {kernel_version} (recovery mode)
-	linux /boot/vmlinuz-{kernel_version}
-	append initrd=/boot/initrd.img-{kernel_version} root=UUID={UUID} ro console=ttyS0 single
+	linux {boot_prefix}/vmlinuz-{kernel_version}
+	append initrd={boot_prefix}/initrd.img-{kernel_version} root=UUID={root_uuid} ro console=ttyS0 single
 	text help
    This option boots the system into recovery mode (single-user)
 	endtext
diff --git a/bootstrapvz/common/tasks/extlinux.py b/bootstrapvz/common/tasks/extlinux.py
index 8c502d5..1a3b8b5 100644
--- a/bootstrapvz/common/tasks/extlinux.py
+++ b/bootstrapvz/common/tasks/extlinux.py
@@ -69,19 +69,26 @@ class ConfigureExtlinuxJessie(Task):
 		os.mkdir(extlinux_path)
 
 		from . import assets
-		extlinux_assets_path = os.path.join(assets, 'extlinux')
-		extlinux_tpl_path = os.path.join(extlinux_assets_path, 'extlinux.conf')
-		with open(extlinux_tpl_path) as extlinux_tpl:
-			extlinux_config = extlinux_tpl.read()
+		with open(os.path.join(assets, 'extlinux/extlinux.conf')) as template:
+			extlinux_config_tpl = template.read()
 
-		root_uuid = info.volume.partition_map.root.get_uuid()
-		extlinux_config = extlinux_config.format(kernel_version=info.kernel_version, UUID=root_uuid)
-		extlinux_config_path = os.path.join(extlinux_path, 'extlinux.conf')
-		with open(extlinux_config_path, 'w') as extlinux_conf_handle:
+		config_vars = {'root_uuid': info.volume.partition_map.root.get_uuid(),
+		               'kernel_version': info.kernel_version}
+		# Check if / and /boot are on the same partition
+		# If not, /boot will actually be / when booting
+		if hasattr(info.volume.partition_map, 'boot'):
+			config_vars['boot_prefix'] = ''
+		else:
+			config_vars['boot_prefix'] = '/boot'
+
+		extlinux_config = extlinux_config_tpl.format(**config_vars)
+
+		with open(os.path.join(extlinux_path, 'extlinux.conf'), 'w') as extlinux_conf_handle:
 			extlinux_conf_handle.write(extlinux_config)
-		from shutil import copy
+
 		# Copy the boot message
-		boot_txt_path = os.path.join(extlinux_assets_path, 'boot.txt')
+		from shutil import copy
+		boot_txt_path = os.path.join(assets, 'extlinux/boot.txt')
 		copy(boot_txt_path, os.path.join(extlinux_path, 'boot.txt'))
 
 

From f1bfee24a0fef4e75f5b6722322494f4368b3e63 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 11:35:30 +0100
Subject: [PATCH 227/345] Prepare integration testing framework to work with
 providers other than virtualbox Also, convert with context handlers in
 classes to functions with generators This makes the code a lot more readable

---
 bootstrapvz/remote/build_servers.py           |   9 +-
 tests/integration/images/__init__.py          |   8 +-
 tests/integration/images/image.py             |   6 +
 tests/integration/images/vbox.py              |  64 +++++++++++
 tests/integration/images/virtualbox.py        |  44 --------
 tests/integration/instances/__init__.py       |  16 ---
 tests/integration/instances/instance.py       |  16 +++
 .../instances/{virtualbox.py => vbox.py}      | 104 +++++++-----------
 tests/integration/tools/__init__.py           |  26 ++++-
 tests/integration/tools/bootable_manifest.py  |  69 ------------
 tests/integration/virtualbox_tests.py         |  32 +++---
 11 files changed, 181 insertions(+), 213 deletions(-)
 create mode 100644 tests/integration/images/image.py
 create mode 100644 tests/integration/images/vbox.py
 delete mode 100644 tests/integration/images/virtualbox.py
 create mode 100644 tests/integration/instances/instance.py
 rename tests/integration/instances/{virtualbox.py => vbox.py} (60%)
 delete mode 100644 tests/integration/tools/bootable_manifest.py

diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index e1981e6..46b3758 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -53,7 +53,10 @@ class BuildServer(object):
 
 
 class LocalBuildServer(BuildServer):
-	pass
+
+	def run(self, manifest):
+		from bootstrapvz.base.main import run
+		return run(manifest)
 
 
 class RemoteBuildServer(BuildServer):
@@ -152,6 +155,10 @@ class RemoteBuildServer(BuildServer):
 		                  '--'] + command
 		log_check_call(ssh_cmd)
 
+	def run(self, manifest):
+		from bootstrapvz.remote.main import run
+		return run(manifest, self)
+
 
 def getNPorts(n, port_range=(1024, 65535)):
 	import random
diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py
index 5260afa..041717f 100644
--- a/tests/integration/images/__init__.py
+++ b/tests/integration/images/__init__.py
@@ -1,6 +1,6 @@
 
 
-class Image(object):
-
-	def __init__(self, manifest):
-		self.manifest = manifest
+def initialize_image(manifest, build_server, bootstrap_info):
+	if manifest.provider['name'] == 'virtualbox':
+		import vbox
+		return vbox.initialize_image(manifest, build_server, bootstrap_info)
diff --git a/tests/integration/images/image.py b/tests/integration/images/image.py
new file mode 100644
index 0000000..5260afa
--- /dev/null
+++ b/tests/integration/images/image.py
@@ -0,0 +1,6 @@
+
+
+class Image(object):
+
+	def __init__(self, manifest):
+		self.manifest = manifest
diff --git a/tests/integration/images/vbox.py b/tests/integration/images/vbox.py
new file mode 100644
index 0000000..300ae45
--- /dev/null
+++ b/tests/integration/images/vbox.py
@@ -0,0 +1,64 @@
+from image import Image
+import virtualbox
+import logging
+from contextlib import contextmanager
+log = logging.getLogger(__name__)
+
+
+def initialize_image(manifest, build_server, bootstrap_info):
+	from bootstrapvz.remote.build_servers import LocalBuildServer
+	if isinstance(build_server, LocalBuildServer):
+		image_path = bootstrap_info.volume.image_path
+	else:
+		import tempfile
+		handle, image_path = tempfile.mkstemp()
+		import os
+		os.close(handle)
+		try:
+			build_server.download(bootstrap_info.volume.image_path, image_path)
+		except (Exception, KeyboardInterrupt):
+			os.remove(image_path)
+			raise
+		finally:
+			build_server.delete(bootstrap_info.volume.image_path)
+	image = VirtualBoxImage(manifest, image_path)
+	return image
+
+
+class VirtualBoxImage(Image):
+
+	def __init__(self, manifest, image_path):
+		super(VirtualBoxImage, self).__init__(manifest)
+		self.image_path = image_path
+		self.vbox = virtualbox.VirtualBox()
+
+	def open(self):
+		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
+		                                    virtualbox.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
+
+	def close(self):
+		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
+		self.medium.close()
+
+	def destroy(self):
+		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
+		import os
+		os.remove(self.image_path)
+		del self.image_path
+
+	@contextmanager
+	def get_instance(self):
+		import hashlib
+		image_hash = hashlib.sha1(self.image_path).hexdigest()
+		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
+
+		self.open()
+		try:
+			from ..instances.vbox import boot_image
+			with boot_image(name, self) as instance:
+				yield instance
+		finally:
+			self.close()
diff --git a/tests/integration/images/virtualbox.py b/tests/integration/images/virtualbox.py
deleted file mode 100644
index a09fde3..0000000
--- a/tests/integration/images/virtualbox.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from __future__ import absolute_import
-from . import Image
-import virtualbox as vboxapi
-import logging
-log = logging.getLogger(__name__)
-
-
-class VirtualBoxImage(Image):
-
-	def __init__(self, manifest, image_path):
-		super(VirtualBoxImage, self).__init__(manifest)
-		self.image_path = image_path
-		self.vbox = vboxapi.VirtualBox()
-
-	def open(self):
-		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
-		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    vboxapi.library.DeviceType.hard_disk,  # device_type
-		                                    vboxapi.library.AccessMode.read_only,  # access_mode
-		                                    False)  # force_new_uuid
-
-	def close(self):
-		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
-		self.medium.close()
-
-	def destroy(self):
-		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
-		import os
-		os.remove(self.image_path)
-		del self.image_path
-
-	def get_instance(self):
-		import hashlib
-		from ..instances.virtualbox import VirtualBoxInstance
-		image_hash = hashlib.sha1(self.image_path).hexdigest()
-		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
-		return VirtualBoxInstance(name, self)
-
-	def __enter__(self):
-		self.open()
-		return self.get_instance()
-
-	def __exit__(self, type, value, traceback):
-		self.close()
diff --git a/tests/integration/instances/__init__.py b/tests/integration/instances/__init__.py
index 049fc0f..e69de29 100644
--- a/tests/integration/instances/__init__.py
+++ b/tests/integration/instances/__init__.py
@@ -1,16 +0,0 @@
-
-
-class Instance(object):
-
-	def __init__(self, name, image):
-		self.name = name
-		self.image = image
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass
diff --git a/tests/integration/instances/instance.py b/tests/integration/instances/instance.py
new file mode 100644
index 0000000..049fc0f
--- /dev/null
+++ b/tests/integration/instances/instance.py
@@ -0,0 +1,16 @@
+
+
+class Instance(object):
+
+	def __init__(self, name, image):
+		self.name = name
+		self.image = image
+
+	def boot(self):
+		pass
+
+	def shutdown(self):
+		pass
+
+	def destroy(self):
+		pass
diff --git a/tests/integration/instances/virtualbox.py b/tests/integration/instances/vbox.py
similarity index 60%
rename from tests/integration/instances/virtualbox.py
rename to tests/integration/instances/vbox.py
index ad908ef..1adc8d4 100644
--- a/tests/integration/instances/virtualbox.py
+++ b/tests/integration/instances/vbox.py
@@ -1,10 +1,24 @@
-from __future__ import absolute_import
-from . import Instance
-import virtualbox as vboxapi
+from instance import Instance
+import virtualbox
+from contextlib import contextmanager
 import logging
 log = logging.getLogger(__name__)
 
 
+@contextmanager
+def boot_image(name, image):
+	instance = VirtualBoxInstance(name, image)
+	try:
+		instance.create()
+		try:
+			instance.boot()
+			yield instance
+		finally:
+			instance.shutdown()
+	finally:
+		instance.destroy()
+
+
 class VirtualBoxInstance(Instance):
 
 	cpus = 1
@@ -12,8 +26,8 @@ class VirtualBoxInstance(Instance):
 
 	def __init__(self, name, image):
 		super(VirtualBoxInstance, self).__init__(name, image)
-		self.vbox = vboxapi.VirtualBox()
-		manager = vboxapi.Manager()
+		self.vbox = virtualbox.VirtualBox()
+		manager = virtualbox.Manager()
 		self.session = manager.get_session()
 
 	def create(self):
@@ -30,18 +44,18 @@ class VirtualBoxInstance(Instance):
 
 		# attach image
 		log.debug('Attaching SATA storage controller to vbox machine `{name}\''.format(name=self.name))
-		with self.Lock(self.machine, self.session) as machine:
+		with lock(self.machine, self.session) as machine:
 			strg_ctrl = machine.add_storage_controller('SATA Controller',
-			                                           vboxapi.library.StorageBus.sata)
+			                                           virtualbox.library.StorageBus.sata)
 			strg_ctrl.port_count = 1
 			machine.attach_device(name='SATA Controller', controller_port=0, device=0,
-			                      type_p=vboxapi.library.DeviceType.hard_disk,
+			                      type_p=virtualbox.library.DeviceType.hard_disk,
 			                      medium=self.image.medium)
 			machine.save_settings()
 
 		# redirect serial port
 		log.debug('Enabling serial port on vbox machine `{name}\''.format(name=self.name))
-		with self.Lock(self.machine, self.session) as machine:
+		with lock(self.machine, self.session) as machine:
 			serial_port = machine.get_serial_port(0)
 			serial_port.enabled = True
 			import tempfile
@@ -49,7 +63,7 @@ class VirtualBoxInstance(Instance):
 			import os
 			os.close(handle)
 			serial_port.path = self.serial_port_path
-			serial_port.host_mode = vboxapi.library.PortMode.host_pipe
+			serial_port.host_mode = virtualbox.library.PortMode.host_pipe
 			serial_port.server = True  # Create the socket on startup
 			machine.save_settings()
 
@@ -70,70 +84,36 @@ class VirtualBoxInstance(Instance):
 	def shutdown(self):
 		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))
 		self.session.console.power_down().wait_for_completion(-1)
-		self.Lock(self.machine, self.session).unlock()
+		lock(self.machine, self.session).unlock()
 
 	def destroy(self):
 		log.debug('Destroying vbox machine `{name}\''.format(name=self.name))
 		if hasattr(self, 'machine'):
 			try:
 				log.debug('Detaching SATA storage controller from vbox machine `{name}\''.format(name=self.name))
-				with self.Lock(self.machine, self.session) as machine:
+				with lock(self.machine, self.session) as machine:
 					machine.detach_device(name='SATA Controller', controller_port=0, device=0)
 					machine.save_settings()
-			except vboxapi.library.VBoxErrorObjectNotFound:
+			except virtualbox.library.VBoxErrorObjectNotFound:
 				pass
 			log.debug('Unregistering and removing vbox machine `{name}\''.format(name=self.name))
-			self.machine.unregister(vboxapi.library.CleanupMode.unregister_only)
+			self.machine.unregister(virtualbox.library.CleanupMode.unregister_only)
 			self.machine.remove(delete=True)
 		else:
 			log.debug('vbox machine `{name}\' was not created, skipping destruction'.format(name=self.name))
 
-	def up(self):
-		try:
-			self.create()
-			try:
-				self.boot()
-			except (Exception, KeyboardInterrupt):
-				self.shutdown()
-				raise
-		except (Exception, KeyboardInterrupt):
-			self.destroy()
-			raise
 
-	def down(self):
-		self.shutdown()
-		self.destroy()
-
-	def __enter__(self):
-		self.up()
-		return self
-
-	def __exit__(self, type, value, traceback):
-		self.down()
-
-	class Lock(object):
-		def __init__(self, machine, session):
-			self.machine = machine
-			self.session = session
-
-		def __enter__(self):
-			return self.lock()
-
-		def __exit__(self, type, value, traceback):
-			return self.unlock()
-
-		def lock(self):
-			self.machine.lock_machine(self.session, vboxapi.library.LockType.write)
-			return self.session.machine
-
-		def unlock(self):
-			from ..tools import waituntil
-			if self.machine.session_state == vboxapi.library.SessionState.unlocked:
-				return
-			if self.machine.session_state == vboxapi.library.SessionState.unlocking:
-				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
-				return
-			if self.machine.session_state == vboxapi.library.SessionState.spawning:
-				waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.locked)
-			self.session.unlock_machine()
-			waituntil(lambda: self.machine.session_state == vboxapi.library.SessionState.unlocked)
+@contextmanager
+def lock(machine, session):
+	machine.lock_machine(session, virtualbox.library.LockType.write)
+	yield session.machine
+	from ..tools import waituntil
+	if machine.session_state == virtualbox.library.SessionState.unlocked:
+		return
+	if machine.session_state == virtualbox.library.SessionState.unlocking:
+		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)
+		return
+	if machine.session_state == virtualbox.library.SessionState.spawning:
+		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.locked)
+	session.unlock_machine()
+	waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 625070b..42bc46a 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,9 +1,33 @@
+from contextlib import contextmanager
+from bootstrapvz.remote import register_deserialization_handlers
+
 # Register deserialization handlers for objects
 # that will pass between server and client
-from bootstrapvz.remote import register_deserialization_handlers
 register_deserialization_handlers()
 
 
+@contextmanager
+def boot_manifest(manifest_data):
+	from bootstrapvz.common.tools import load_data
+	build_servers = load_data('build-servers.yml')
+	from bootstrapvz.remote.build_servers import pick_build_server
+	build_server = pick_build_server(build_servers, manifest_data)
+
+	manifest_data = build_server.apply_build_settings(manifest_data)
+	from bootstrapvz.base.manifest import Manifest
+	manifest = Manifest(data=manifest_data)
+
+	bootstrap_info = build_server.run(manifest)
+
+	from ..images import initialize_image
+	image = initialize_image(manifest, build_server, bootstrap_info)
+	try:
+		with image.get_instance() as instance:
+			yield instance
+	finally:
+		image.destroy()
+
+
 def waituntil(predicate, timeout=5, interval=0.05):
 	import time
 	threshhold = time.time() + timeout
diff --git a/tests/integration/tools/bootable_manifest.py b/tests/integration/tools/bootable_manifest.py
deleted file mode 100644
index d537996..0000000
--- a/tests/integration/tools/bootable_manifest.py
+++ /dev/null
@@ -1,69 +0,0 @@
-from bootstrapvz.remote.build_servers import LocalBuildServer
-from ..images.virtualbox import VirtualBoxImage
-
-
-class BootableManifest(object):
-
-	def __init__(self, manifest_data):
-		self.manifest_data = manifest_data
-
-	def pick_build_server(self, path='build-servers.yml'):
-		from bootstrapvz.common.tools import load_data
-		build_servers = load_data(path)
-		from bootstrapvz.remote.build_servers import pick_build_server
-		return pick_build_server(build_servers, self.manifest_data)
-
-	def get_manifest(self, build_server):
-		manifest_data = build_server.apply_build_settings(self.manifest_data)
-		from bootstrapvz.base.manifest import Manifest
-		return Manifest(data=manifest_data)
-
-	def bootstrap(self, manifest, build_server):
-		if isinstance(build_server, LocalBuildServer):
-			from bootstrapvz.base.main import run
-			bootstrap_info = run(manifest)
-		else:
-			from bootstrapvz.remote.main import run
-			bootstrap_info = run(manifest, build_server)
-		return bootstrap_info
-
-	def get_image(self, build_server, bootstrap_info, manifest):
-		if isinstance(build_server, LocalBuildServer):
-			image_path = bootstrap_info.volume.image_path
-		else:
-			import tempfile
-			handle, image_path = tempfile.mkstemp()
-			import os
-			os.close(handle)
-			try:
-				build_server.download(bootstrap_info.volume.image_path, image_path)
-			except (Exception, KeyboardInterrupt) as e:
-				os.remove(image_path)
-				raise e
-			finally:
-				build_server.delete(bootstrap_info.volume.image_path)
-		image_type = {'virtualbox': VirtualBoxImage}
-		return image_type.get(self.manifest_data['provider']['name'])(manifest, image_path)
-
-	def __enter__(self):
-		try:
-			self.build_server = self.pick_build_server()
-			self.manifest = self.get_manifest(self.build_server)
-			self.bootstrap_info = self.bootstrap(self.manifest, self.build_server)
-			self.image = self.get_image(self.build_server, self.bootstrap_info, self.manifest)
-			self.image.open()
-			self.instance = self.image.get_instance()
-			self.instance.up()
-		except (Exception, KeyboardInterrupt):
-			if hasattr(self, 'image'):
-				self.image.close()
-				self.image.destroy()
-			raise
-		return self.instance
-
-	def __exit__(self, type, value, traceback):
-		if hasattr(self, 'instance'):
-			self.instance.down()
-		if hasattr(self, 'image'):
-			self.image.close()
-			self.image.destroy()
diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index ef98abc..a5930aa 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -1,5 +1,5 @@
 from manifests import merge_manifest_data
-from tools.bootable_manifest import BootableManifest
+from tools import boot_manifest
 from unittest.case import SkipTest
 
 partials = {'vbox': 'provider: {name: virtualbox}',
@@ -12,7 +12,7 @@ def test_unpartitioned_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -20,7 +20,7 @@ def test_msdos_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -28,7 +28,7 @@ def test_gpt_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -37,7 +37,7 @@ def test_msdos_grub_oldstable():
 	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -46,7 +46,7 @@ def test_gpt_grub_oldstable():
 	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -54,7 +54,7 @@ def test_unpartitioned_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -62,7 +62,7 @@ def test_msdos_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -70,7 +70,7 @@ def test_gpt_extlinux():
 	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -78,7 +78,7 @@ def test_msdos_grub():
 	std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -86,7 +86,7 @@ def test_gpt_grub():
 	std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -94,7 +94,7 @@ def test_unpartitioned_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -102,7 +102,7 @@ def test_msdos_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -110,7 +110,7 @@ def test_gpt_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -118,7 +118,7 @@ def test_msdos_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
 
 
@@ -126,5 +126,5 @@ def test_gpt_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with BootableManifest(manifest_data) as instance:
+	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)

From 36728cf6481b43d068c1c94a42d65a4b1ed43fea Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 11:37:26 +0100
Subject: [PATCH 228/345] Since bootstrap-vz uses `sudo' to elevate privileges,
 the root password will never be needed

---
 bootstrapvz/remote/build-servers-schema.yml | 1 -
 bootstrapvz/remote/build_servers.py         | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/bootstrapvz/remote/build-servers-schema.yml b/bootstrapvz/remote/build-servers-schema.yml
index ceba3eb..20f0b85 100644
--- a/bootstrapvz/remote/build-servers-schema.yml
+++ b/bootstrapvz/remote/build-servers-schema.yml
@@ -50,7 +50,6 @@ definitions:
       port: {type: integer}
       username: {type: string}
       password: {type: string}
-      root_password: {type: string}
       keyfile: {$ref: '#/definitions/absolute_path'}
       server_bin: {$ref: '#/definitions/absolute_path'}
     required: [type, can_bootstrap, release]
diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers.py
index 46b3758..bd8e8ab 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers.py
@@ -66,8 +66,7 @@ class RemoteBuildServer(BuildServer):
 		self.address = settings['address']
 		self.port = settings['port']
 		self.username = settings['username']
-		self.password = settings['password']
-		self.root_password = settings['root_password']
+		self.password = settings.get('password', None)
 		self.keyfile = settings['keyfile']
 		self.server_bin = settings['server_bin']
 

From ef37cd257f63084e23fb30da26a7e64c6facf745 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 11:55:15 +0100
Subject: [PATCH 229/345] Move build servers into separate module

---
 bootstrapvz/remote/build_servers/__init__.py  | 46 ++++++++++++
 .../build-servers-schema.yml                  |  0
 .../remote/build_servers/build_server.py      | 17 +++++
 bootstrapvz/remote/build_servers/local.py     |  8 +++
 .../remote.py}                                | 70 +------------------
 tests/integration/images/vbox.py              |  2 +-
 6 files changed, 74 insertions(+), 69 deletions(-)
 create mode 100644 bootstrapvz/remote/build_servers/__init__.py
 rename bootstrapvz/remote/{ => build_servers}/build-servers-schema.yml (100%)
 create mode 100644 bootstrapvz/remote/build_servers/build_server.py
 create mode 100644 bootstrapvz/remote/build_servers/local.py
 rename bootstrapvz/remote/{build_servers.py => build_servers/remote.py} (65%)

diff --git a/bootstrapvz/remote/build_servers/__init__.py b/bootstrapvz/remote/build_servers/__init__.py
new file mode 100644
index 0000000..f6da332
--- /dev/null
+++ b/bootstrapvz/remote/build_servers/__init__.py
@@ -0,0 +1,46 @@
+
+
+def pick_build_server(build_servers, manifest, preferences={}):
+	# Validate the build servers list
+	from bootstrapvz.common.tools import load_data
+	import os.path
+	schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml')))
+	import jsonschema
+	jsonschema.validate(build_servers, schema)
+
+	if manifest['provider']['name'] == 'ec2':
+		must_bootstrap = 'ec2-' + manifest['volume']['backing']
+	else:
+		must_bootstrap = manifest['provider']['name']
+
+	def matches(name, settings):
+		if preferences.get('name', name) != name:
+			return False
+		if preferences.get('release', settings['release']) != settings['release']:
+			return False
+		if must_bootstrap not in settings['can_bootstrap']:
+			return False
+		return True
+
+	for name, settings in build_servers.iteritems():
+		if not matches(name, settings):
+			continue
+		if settings['type'] == 'local':
+			from local import LocalBuildServer
+			return LocalBuildServer(name, settings)
+		else:
+			from remote import RemoteBuildServer
+			return RemoteBuildServer(name, settings)
+	raise Exception('Unable to find a build server that matches your preferences.')
+
+
+def getNPorts(n, port_range=(1024, 65535)):
+	import random
+	ports = []
+	for i in range(0, n):
+		while True:
+			port = random.randrange(*port_range)
+			if port not in ports:
+				ports.append(port)
+				break
+	return ports
diff --git a/bootstrapvz/remote/build-servers-schema.yml b/bootstrapvz/remote/build_servers/build-servers-schema.yml
similarity index 100%
rename from bootstrapvz/remote/build-servers-schema.yml
rename to bootstrapvz/remote/build_servers/build-servers-schema.yml
diff --git a/bootstrapvz/remote/build_servers/build_server.py b/bootstrapvz/remote/build_servers/build_server.py
new file mode 100644
index 0000000..bb57c49
--- /dev/null
+++ b/bootstrapvz/remote/build_servers/build_server.py
@@ -0,0 +1,17 @@
+
+
+class BuildServer(object):
+
+	def __init__(self, name, settings):
+		self.name = name
+		self.settings = settings
+		self.build_settings = settings.get('build_settings', {})
+		self.can_bootstrap = settings['can_bootstrap']
+		self.release = settings.get('release', None)
+
+	def apply_build_settings(self, manifest_data):
+		if manifest_data['provider']['name'] == 'virtualbox' and 'guest_additions' in manifest_data['provider']:
+			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
+		if 'apt_proxy' in self.build_settings:
+			manifest_data.get('plugins', {})['apt_proxy'] = self.build_settings['apt_proxy']
+		return manifest_data
diff --git a/bootstrapvz/remote/build_servers/local.py b/bootstrapvz/remote/build_servers/local.py
new file mode 100644
index 0000000..8332995
--- /dev/null
+++ b/bootstrapvz/remote/build_servers/local.py
@@ -0,0 +1,8 @@
+from build_server import BuildServer
+
+
+class LocalBuildServer(BuildServer):
+
+	def run(self, manifest):
+		from bootstrapvz.base.main import run
+		return run(manifest)
diff --git a/bootstrapvz/remote/build_servers.py b/bootstrapvz/remote/build_servers/remote.py
similarity index 65%
rename from bootstrapvz/remote/build_servers.py
rename to bootstrapvz/remote/build_servers/remote.py
index bd8e8ab..c14c23f 100644
--- a/bootstrapvz/remote/build_servers.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -1,64 +1,9 @@
+from build_server import BuildServer
 from bootstrapvz.common.tools import log_check_call
 import logging
 log = logging.getLogger(__name__)
 
 
-def pick_build_server(build_servers, manifest, preferences={}):
-	# Validate the build servers list
-	from bootstrapvz.common.tools import load_data
-	import os.path
-	schema = load_data(os.path.normpath(os.path.join(os.path.dirname(__file__), 'build-servers-schema.yml')))
-	import jsonschema
-	jsonschema.validate(build_servers, schema)
-
-	if manifest['provider']['name'] == 'ec2':
-		must_bootstrap = 'ec2-' + manifest['volume']['backing']
-	else:
-		must_bootstrap = manifest['provider']['name']
-
-	def matches(name, settings):
-		if preferences.get('name', name) != name:
-			return False
-		if preferences.get('release', settings['release']) != settings['release']:
-			return False
-		if must_bootstrap not in settings['can_bootstrap']:
-			return False
-		return True
-
-	for name, settings in build_servers.iteritems():
-		if not matches(name, settings):
-			continue
-		if settings['type'] == 'local':
-			return LocalBuildServer(name, settings)
-		else:
-			return RemoteBuildServer(name, settings)
-	raise Exception('Unable to find a build server that matches your preferences.')
-
-
-class BuildServer(object):
-
-	def __init__(self, name, settings):
-		self.name = name
-		self.settings = settings
-		self.build_settings = settings.get('build_settings', {})
-		self.can_bootstrap = settings['can_bootstrap']
-		self.release = settings.get('release', None)
-
-	def apply_build_settings(self, manifest_data):
-		if manifest_data['provider']['name'] == 'virtualbox' and 'guest_additions' in manifest_data['provider']:
-			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
-		if 'apt_proxy' in self.build_settings:
-			manifest_data.get('plugins', {})['apt_proxy'] = self.build_settings['apt_proxy']
-		return manifest_data
-
-
-class LocalBuildServer(BuildServer):
-
-	def run(self, manifest):
-		from bootstrapvz.base.main import run
-		return run(manifest)
-
-
 class RemoteBuildServer(BuildServer):
 
 	def __init__(self, name, settings):
@@ -70,6 +15,7 @@ class RemoteBuildServer(BuildServer):
 		self.keyfile = settings['keyfile']
 		self.server_bin = settings['server_bin']
 
+		from . import getNPorts
 		# We can't use :0 for the forwarding ports because
 		# A: It's quite hard to retrieve the port on the remote after the daemon has started
 		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
@@ -157,15 +103,3 @@ class RemoteBuildServer(BuildServer):
 	def run(self, manifest):
 		from bootstrapvz.remote.main import run
 		return run(manifest, self)
-
-
-def getNPorts(n, port_range=(1024, 65535)):
-	import random
-	ports = []
-	for i in range(0, n):
-		while True:
-			port = random.randrange(*port_range)
-			if port not in ports:
-				ports.append(port)
-				break
-	return ports
diff --git a/tests/integration/images/vbox.py b/tests/integration/images/vbox.py
index 300ae45..f8d7803 100644
--- a/tests/integration/images/vbox.py
+++ b/tests/integration/images/vbox.py
@@ -6,7 +6,7 @@ log = logging.getLogger(__name__)
 
 
 def initialize_image(manifest, build_server, bootstrap_info):
-	from bootstrapvz.remote.build_servers import LocalBuildServer
+	from bootstrapvz.remote.build_servers.local import LocalBuildServer
 	if isinstance(build_server, LocalBuildServer):
 		image_path = bootstrap_info.volume.image_path
 	else:

From 5b48ce58c52fd66b9d4ffe92afbf59734b6edbec Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 13:30:32 +0100
Subject: [PATCH 230/345] Fix lock handling for virtualbox tests

---
 tests/integration/instances/vbox.py | 34 ++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/tests/integration/instances/vbox.py b/tests/integration/instances/vbox.py
index 1adc8d4..1ab9927 100644
--- a/tests/integration/instances/vbox.py
+++ b/tests/integration/instances/vbox.py
@@ -1,6 +1,7 @@
 from instance import Instance
 import virtualbox
 from contextlib import contextmanager
+from ..tools import waituntil
 import logging
 log = logging.getLogger(__name__)
 
@@ -84,7 +85,8 @@ class VirtualBoxInstance(Instance):
 	def shutdown(self):
 		log.debug('Shutting down vbox machine `{name}\''.format(name=self.name))
 		self.session.console.power_down().wait_for_completion(-1)
-		lock(self.machine, self.session).unlock()
+		if not waituntil(lambda: self.machine.session_state == virtualbox.library.SessionState.unlocked):
+			raise LockingException('Timeout while waiting for the machine to become unlocked')
 
 	def destroy(self):
 		log.debug('Destroying vbox machine `{name}\''.format(name=self.name))
@@ -105,15 +107,27 @@ class VirtualBoxInstance(Instance):
 
 @contextmanager
 def lock(machine, session):
+	if machine.session_state != virtualbox.library.SessionState.unlocked:
+		msg = ('Acquiring lock on machine failed, state was `{state}\' '
+		       'instead of `Unlocked\'.'.format(state=str(machine.session_state)))
+		raise LockingException(msg)
+
 	machine.lock_machine(session, virtualbox.library.LockType.write)
 	yield session.machine
-	from ..tools import waituntil
-	if machine.session_state == virtualbox.library.SessionState.unlocked:
-		return
-	if machine.session_state == virtualbox.library.SessionState.unlocking:
-		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)
-		return
-	if machine.session_state == virtualbox.library.SessionState.spawning:
-		waituntil(lambda: machine.session_state == virtualbox.library.SessionState.locked)
+
+	if machine.session_state != virtualbox.library.SessionState.locked:
+		if not waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked):
+			msg = ('Error before trying to release lock on machine, state was `{state}\' '
+			       'instead of `Locked\'.'.format(state=str(machine.session_state)))
+			raise LockingException(msg)
+
 	session.unlock_machine()
-	waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked)
+
+	if not waituntil(lambda: machine.session_state == virtualbox.library.SessionState.unlocked):
+		msg = ('Timeout while trying to release lock on machine, '
+		       'last state was `{state}\''.format(state=str(machine.session_state)))
+		raise LockingException(msg)
+
+
+class LockingException(Exception):
+	pass

From 17a4511ee14f8c480cbb50ade420c76992e7c3c2 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 13:31:34 +0100
Subject: [PATCH 231/345] Convert remote building state management to work in
 with statements exception and state handling is a lot easier now, less class
 vars etc.

---
 .../remote/{ => build_servers}/callback.py    |  14 +--
 bootstrapvz/remote/build_servers/remote.py    | 102 +++++++++++-------
 bootstrapvz/remote/main.py                    |  52 +++------
 3 files changed, 85 insertions(+), 83 deletions(-)
 rename bootstrapvz/remote/{ => build_servers}/callback.py (80%)

diff --git a/bootstrapvz/remote/callback.py b/bootstrapvz/remote/build_servers/callback.py
similarity index 80%
rename from bootstrapvz/remote/callback.py
rename to bootstrapvz/remote/build_servers/callback.py
index 2c385cc..79cf41b 100644
--- a/bootstrapvz/remote/callback.py
+++ b/bootstrapvz/remote/build_servers/callback.py
@@ -14,19 +14,19 @@ class CallbackServer(object):
 		self.daemon.register(self)
 		self.abort = False
 
-	def start(self):
+	def __enter__(self):
 		def serve():
 			self.daemon.requestLoop()
 		from threading import Thread
 		self.thread = Thread(target=serve)
-		log.debug('Starting the callback server')
+		log.debug('Starting callback server')
 		self.thread.start()
+		return self
 
-	def stop(self):
-		if hasattr(self, 'daemon'):
-			self.daemon.shutdown()
-		if hasattr(self, 'thread'):
-			self.thread.join()
+	def __exit__(self, type, value, traceback):
+		log.debug('Shutting down callback server')
+		self.daemon.shutdown()
+		self.thread.join()
 
 	@Pyro4.expose
 	def handle_log(self, pickled_record):
diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py
index c14c23f..6e13b26 100644
--- a/bootstrapvz/remote/build_servers/remote.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -1,5 +1,6 @@
 from build_server import BuildServer
 from bootstrapvz.common.tools import log_check_call
+from contextlib import contextmanager
 import logging
 log = logging.getLogger(__name__)
 
@@ -15,18 +16,27 @@ class RemoteBuildServer(BuildServer):
 		self.keyfile = settings['keyfile']
 		self.server_bin = settings['server_bin']
 
+	@contextmanager
+	def connect(self):
+		with self.spawn_server() as forwards:
+			with connect_pyro('localhost', forwards['local_server_port']) as connection:
+				from callback import CallbackServer
+				args = {'listen_port': forwards['local_callback_port'],
+				        'remote_port': forwards['remote_callback_port']}
+				with CallbackServer(**args) as callback_server:
+					connection.set_callback_server(callback_server)
+					yield (connection, callback_server)
+
+	@contextmanager
+	def spawn_server(self):
 		from . import getNPorts
 		# We can't use :0 for the forwarding ports because
 		# A: It's quite hard to retrieve the port on the remote after the daemon has started
 		# B: SSH doesn't accept 0:localhost:0 as a port forwarding option
-		[self.local_server_port, self.local_callback_port] = getNPorts(2)
-		[self.remote_server_port, self.remote_callback_port] = getNPorts(2)
+		[local_server_port, local_callback_port] = getNPorts(2)
+		[remote_server_port, remote_callback_port] = getNPorts(2)
 
-	def connect(self):
-		log.debug('Opening SSH connection to build server `{name}\''.format(name=self.name))
-		import subprocess
-
-		server_cmd = ['sudo', self.server_bin, '--listen', str(self.remote_server_port)]
+		server_cmd = ['sudo', self.server_bin, '--listen', str(remote_server_port)]
 
 		def set_process_group():
 			# Changes the process group of a command so that any SIGINT
@@ -38,46 +48,28 @@ class RemoteBuildServer(BuildServer):
 		addr_arg = '{user}@{host}'.format(user=self.username, host=self.address)
 		ssh_cmd = ['ssh', '-i', self.keyfile,
 		                  '-p', str(self.port),
-		                  '-L' + str(self.local_server_port) + ':localhost:' + str(self.remote_server_port),
-		                  '-R' + str(self.remote_callback_port) + ':localhost:' + str(self.local_callback_port),
+		                  '-L' + str(local_server_port) + ':localhost:' + str(remote_server_port),
+		                  '-R' + str(remote_callback_port) + ':localhost:' + str(local_callback_port),
 		                  addr_arg]
 		full_cmd = ssh_cmd + ['--'] + server_cmd
+
+		log.debug('Opening SSH connection to build server `{name}\''.format(name=self.name))
 		import sys
-		self.ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr,
-		                                    preexec_fn=set_process_group)
-
-		# Check that we can connect to the server
+		import subprocess
+		ssh_process = subprocess.Popen(args=full_cmd, stdout=sys.stderr, stderr=sys.stderr,
+		                               preexec_fn=set_process_group)
 		try:
-			import Pyro4
-			server_uri = 'PYRO:server@localhost:{server_port}'.format(server_port=self.local_server_port)
-			self.connection = Pyro4.Proxy(server_uri)
-
-			log.debug('Connecting to the RPC daemon on build server `{name}\''.format(name=self.name))
-			remaining_retries = 5
-			while True:
-				try:
-					self.connection.ping()
-					break
-				except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError):
-					if remaining_retries > 0:
-						remaining_retries -= 1
-						from time import sleep
-						sleep(2)
-					else:
-						raise
+			yield {'local_server_port': local_server_port,
+			       'local_callback_port': local_callback_port,
+			       'remote_server_port': remote_server_port,
+			       'remote_callback_port': remote_callback_port}
 		except (Exception, KeyboardInterrupt):
-			self.ssh_process.terminate()
+			log.debug('Forcefully terminating SSH connection to the build server')
+			ssh_process.terminate()
 			raise
-		return self.connection
-
-	def disconnect(self):
-		if hasattr(self, 'connection'):
-			log.debug('Stopping the RPC daemon on build server `{name}\''.format(name=self.name))
-			self.connection.stop()
-			self.connection._pyroRelease()
-		if hasattr(self, 'ssh_process'):
-			log.debug('Waiting for SSH connection to build server `{name}\' to terminate'.format(name=self.name))
-			self.ssh_process.wait()
+		else:
+			log.debug('Waiting for SSH connection to the build server to close')
+			ssh_process.wait()
 
 	def download(self, src, dst):
 		log.debug('Downloading file `{src}\' from '
@@ -103,3 +95,31 @@ class RemoteBuildServer(BuildServer):
 	def run(self, manifest):
 		from bootstrapvz.remote.main import run
 		return run(manifest, self)
+
+
+@contextmanager
+def connect_pyro(host, port):
+	import Pyro4
+	server_uri = 'PYRO:server@{host}:{port}'.format(host=host, port=port)
+	connection = Pyro4.Proxy(server_uri)
+
+	log.debug('Connecting to the RPC daemon')
+	remaining_retries = 5
+	while True:
+		try:
+			connection.ping()
+			break
+		except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError):
+			if remaining_retries > 0:
+				remaining_retries -= 1
+				from time import sleep
+				sleep(2)
+			else:
+				raise
+
+	try:
+		yield connection
+	finally:
+		log.debug('Stopping the RPC daemon')
+		connection.stop()
+		connection._pyroRelease()
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index eb247f3..6508136 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -75,40 +75,22 @@ def run(manifest, build_server, debug=False, dry_run=False):
 	on the other side and initiates a remote bootstrapping procedure
 	"""
 	bootstrap_info = None
-	try:
-		# Connect to the build server
-		connection = build_server.connect()
-		# Start a callback server on this side, so that we may receive log entries
-		from callback import CallbackServer
-		callback_server = CallbackServer(listen_port=build_server.local_callback_port,
-		                                 remote_port=build_server.remote_callback_port)
-		try:
-			# Start the callback server (in a background thread)
-			callback_server.start()
-			# Tell the RPC daemon about the callback server
-			connection.set_callback_server(callback_server)
+	with build_server.connect() as (connection, callback_server):
+		# Replace the standard SIGINT handler with a remote call to the server
+		# so that it may abort the run.
+		def abort(signum, frame):
+			import logging
+			logging.getLogger(__name__).warn('SIGINT received, asking remote to abort.')
+			callback_server.abort_run()
+		import signal
+		orig_sigint = signal.signal(signal.SIGINT, abort)
 
-			# Replace the standard SIGINT handler with a remote call to the server
-			# so that it may abort the run.
-			def abort(signum, frame):
-				import logging
-				logging.getLogger(__name__).warn('SIGINT received, asking remote to abort.')
-				callback_server.abort_run()
-			import signal
-			orig_sigint = signal.signal(signal.SIGINT, abort)
-
-			# Everything has been set up, begin the bootstrapping process
-			bootstrap_info = connection.run(manifest,
-			                                debug=debug,
-			                                # We can't pause the bootstrapping process remotely, yet...
-			                                pause_on_error=False,
-			                                dry_run=dry_run)
-			# Restore the old SIGINT handler
-			signal.signal(signal.SIGINT, orig_sigint)
-		finally:
-			# Stop the callback server
-			callback_server.stop()
-	finally:
-		# Stop the RPC daemon and close the SSH connection
-		build_server.disconnect()
+		# Everything has been set up, begin the bootstrapping process
+		bootstrap_info = connection.run(manifest,
+		                                debug=debug,
+		                                # We can't pause the bootstrapping process remotely, yet...
+		                                pause_on_error=False,
+		                                dry_run=dry_run)
+		# Restore the old SIGINT handler
+		signal.signal(signal.SIGINT, orig_sigint)
 	return bootstrap_info

From e9a3845281d95b7ce6b8dbbd013d14d92c563d6c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 17:31:46 +0100
Subject: [PATCH 232/345] Fix serialization of CalledProcessError

---
 bootstrapvz/common/tools.py    |  9 +++++++--
 bootstrapvz/remote/__init__.py | 25 +++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 1715c66..2829c51 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -3,9 +3,14 @@ import os
 
 def log_check_call(command, stdin=None, env=None, shell=False, cwd=None):
 	status, stdout, stderr = log_call(command, stdin, env, shell, cwd)
+	from subprocess import CalledProcessError
 	if status != 0:
-		from subprocess import CalledProcessError
-		raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr))
+		e = CalledProcessError(status, ' '.join(command), '\n'.join(stderr))
+		# Fix Pyro4's fixIronPythonExceptionForPickle() by setting the args property,
+		# even though we use our own serialization (at least I think that's the problem).
+		# See bootstrapvz.remote.serialize_called_process_error for more info.
+		setattr(e, 'args', (status, ' '.join(command), '\n'.join(stderr)))
+		raise e
 	return stdout
 
 
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index 7397a5a..ad0049d 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -33,6 +33,7 @@ supported_exceptions = ['bootstrapvz.common.exceptions.ManifestError',
                         'bootstrapvz.base.pkg.exceptions.SourceError',
                         'bootstrapvz.common.exceptions.UnitError',
                         'bootstrapvz.common.fsm_proxy.FSMProxyError',
+                        'subprocess.CalledProcessError',
                         ]
 
 
@@ -41,6 +42,8 @@ def register_deserialization_handlers():
 		SerializerBase.register_dict_to_class(supported_class, deserialize)
 	for supported_exc in supported_exceptions:
 		SerializerBase.register_dict_to_class(supported_exc, deserialize_exception)
+	import subprocess
+	SerializerBase.register_class_to_dict(subprocess.CalledProcessError, serialize_called_process_error)
 
 
 def unregister_deserialization_handlers():
@@ -73,6 +76,28 @@ def deserialize(fq_classname, data):
 	return instance
 
 
+def serialize_called_process_error(obj):
+	# This is by far the weirdest exception serialization.
+	# There is a bug in both Pyro4 and the Python subprocess module.
+	# CalledProcessError does not populate its args property,
+	# although according to https://docs.python.org/2/library/exceptions.html#exceptions.BaseException.args
+	# it should...
+	# So we populate that property during serialization instead
+	# (the code is grabbed directly from Pyro4's class_to_dict())
+	# However, Pyro4 still cannot figure out to call the deserializer
+	# unless we also use setattr() on the exception to set the args below
+	# (before throwing it).
+	# Mind you, the error "__init__() takes at least 3 arguments (2 given)"
+	# is thrown *on the server* if we don't use setattr().
+	# It's all very confusing to me and I'm not entirely
+	# sure what the exact problem is. Regardless - it works, so there.
+	return {'__class__': obj.__class__.__module__ + '.' + obj.__class__.__name__,
+	        '__exception__': True,
+	        'args': (obj.returncode, obj.cmd, obj.output),
+	        'attributes': vars(obj)  # add custom exception attributes
+	        }
+
+
 def get_class_object(fq_classname):
 	parts = fq_classname.split('.')
 	module_name = '.'.join(parts[:-1])

From a0e3ba218fb739c80cb268c24a25f6c3aa0e3f52 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 17:32:23 +0100
Subject: [PATCH 233/345] Always attempt to close SSH gracefully

---
 bootstrapvz/remote/build_servers/remote.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py
index 6e13b26..165fc92 100644
--- a/bootstrapvz/remote/build_servers/remote.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -63,13 +63,17 @@ class RemoteBuildServer(BuildServer):
 			       'local_callback_port': local_callback_port,
 			       'remote_server_port': remote_server_port,
 			       'remote_callback_port': remote_callback_port}
-		except (Exception, KeyboardInterrupt):
-			log.debug('Forcefully terminating SSH connection to the build server')
-			ssh_process.terminate()
-			raise
-		else:
+		finally:
 			log.debug('Waiting for SSH connection to the build server to close')
-			ssh_process.wait()
+			import time
+			start = time.time()
+			while ssh_process.poll() is None:
+				if time.time() - start > 5:
+					log.debug('Forcefully terminating SSH connection to the build server')
+					ssh_process.terminate()
+					break
+				else:
+					time.sleep(0.5)
 
 	def download(self, src, dst):
 		log.debug('Downloading file `{src}\' from '

From b067ada15e090177429db8d06b947fe2dcd179c4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 17:38:17 +0100
Subject: [PATCH 234/345] Introduce some awesome signal handling We can now
 press Ctrl+C remotely  while any subprocess of the bootstrapping process is
 running, previously SIGINTs weren't propagated to the bootstrapping process
 because there was a thread in between it all. Now the bootstrapping process
 is in it's own process group.

---
 bootstrapvz/base/main.py                     |  4 +-
 bootstrapvz/base/tasklist.py                 |  5 +-
 bootstrapvz/remote/build_servers/callback.py |  8 ---
 bootstrapvz/remote/build_servers/local.py    | 12 ++++-
 bootstrapvz/remote/build_servers/remote.py   |  6 +--
 bootstrapvz/remote/main.py                   | 34 ++----------
 bootstrapvz/remote/server.py                 | 57 +++++++++++++++++---
 tests/integration/tools/__init__.py          |  4 +-
 8 files changed, 71 insertions(+), 59 deletions(-)

diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py
index 55c3ec3..74d06aa 100644
--- a/bootstrapvz/base/main.py
+++ b/bootstrapvz/base/main.py
@@ -83,7 +83,7 @@ def setup_loggers(opts):
 	root.addHandler(console_handler)
 
 
-def run(manifest, debug=False, pause_on_error=False, dry_run=False, check_continue=None):
+def run(manifest, debug=False, pause_on_error=False, dry_run=False):
 	"""Runs the bootstrapping process
 
 	:params Manifest manifest: The manifest to run the bootstrapping process for
@@ -106,7 +106,7 @@ def run(manifest, debug=False, pause_on_error=False, dry_run=False, check_contin
 	log = logging.getLogger(__name__)
 	try:
 		# Run all the tasks the tasklist has gathered
-		tasklist.run(info=bootstrap_info, dry_run=dry_run, check_continue=check_continue)
+		tasklist.run(info=bootstrap_info, dry_run=dry_run)
 		# We're done! :-)
 		log.info('Successfully completed bootstrapping')
 	except (Exception, KeyboardInterrupt) as e:
diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 342896e..8e5dbb1 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -15,7 +15,7 @@ class TaskList(object):
 		self.tasks = tasks
 		self.tasks_completed = []
 
-	def run(self, info, dry_run=False, check_continue=None):
+	def run(self, info, dry_run=False):
 		"""Converts the taskgraph into a list and runs all tasks in that list
 
 		:param dict info: The bootstrap information object
@@ -27,9 +27,6 @@ class TaskList(object):
 		log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list))))
 
 		for task in task_list:
-			# Check if we should abort the run (used for asynchronous run abortion through remote building)
-			if callable(check_continue) and not check_continue():
-				raise TaskListError('Run was aborted.')
 			# Tasks are not required to have a description
 			if hasattr(task, 'description'):
 				log.info(task.description)
diff --git a/bootstrapvz/remote/build_servers/callback.py b/bootstrapvz/remote/build_servers/callback.py
index 79cf41b..2df5b82 100644
--- a/bootstrapvz/remote/build_servers/callback.py
+++ b/bootstrapvz/remote/build_servers/callback.py
@@ -12,7 +12,6 @@ class CallbackServer(object):
 		                           nathost='localhost', natport=remote_port,
 		                           unixsocket=None)
 		self.daemon.register(self)
-		self.abort = False
 
 	def __enter__(self):
 		def serve():
@@ -36,10 +35,3 @@ class CallbackServer(object):
 		record.extra = getattr(record, 'extra', {})
 		record.extra['source'] = 'remote'
 		log.handle(record)
-
-	@Pyro4.expose
-	def get_abort_run(self):
-		return self.abort
-
-	def abort_run(self):
-		self.abort = True
diff --git a/bootstrapvz/remote/build_servers/local.py b/bootstrapvz/remote/build_servers/local.py
index 8332995..0d29943 100644
--- a/bootstrapvz/remote/build_servers/local.py
+++ b/bootstrapvz/remote/build_servers/local.py
@@ -1,8 +1,16 @@
 from build_server import BuildServer
+from contextlib import contextmanager
 
 
 class LocalBuildServer(BuildServer):
 
-	def run(self, manifest):
+	@contextmanager
+	def connect(self):
+		yield LocalConnection()
+
+
+class LocalConnection(object):
+
+	def run(self, *args, **kwargs):
 		from bootstrapvz.base.main import run
-		return run(manifest)
+		return run(*args, **kwargs)
diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py
index 165fc92..7ee668a 100644
--- a/bootstrapvz/remote/build_servers/remote.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -25,7 +25,7 @@ class RemoteBuildServer(BuildServer):
 				        'remote_port': forwards['remote_callback_port']}
 				with CallbackServer(**args) as callback_server:
 					connection.set_callback_server(callback_server)
-					yield (connection, callback_server)
+					yield connection
 
 	@contextmanager
 	def spawn_server(self):
@@ -96,10 +96,6 @@ class RemoteBuildServer(BuildServer):
 		                  '--'] + command
 		log_check_call(ssh_cmd)
 
-	def run(self, manifest):
-		from bootstrapvz.remote.main import run
-		return run(manifest, self)
-
 
 @contextmanager
 def connect_pyro(host, port):
diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index 6508136..d007a20 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -40,10 +40,10 @@ def main():
 	register_deserialization_handlers()
 
 	# Everything has been set up, connect to the server and begin the bootstrapping process
-	run(manifest,
-	    build_server,
-	    debug=opts['--debug'],
-	    dry_run=opts['--dry-run'])
+	with build_server.connect() as connection:
+		connection.run(manifest,
+		               debug=opts['--debug'],
+		               dry_run=['--dry-run'])
 
 
 def get_opts():
@@ -68,29 +68,3 @@ Options:
   -h, --help          show this help
 	"""
 	return docopt(usage)
-
-
-def run(manifest, build_server, debug=False, dry_run=False):
-	"""Connects to the remote build server, starts an RPC daemin
-	on the other side and initiates a remote bootstrapping procedure
-	"""
-	bootstrap_info = None
-	with build_server.connect() as (connection, callback_server):
-		# Replace the standard SIGINT handler with a remote call to the server
-		# so that it may abort the run.
-		def abort(signum, frame):
-			import logging
-			logging.getLogger(__name__).warn('SIGINT received, asking remote to abort.')
-			callback_server.abort_run()
-		import signal
-		orig_sigint = signal.signal(signal.SIGINT, abort)
-
-		# Everything has been set up, begin the bootstrapping process
-		bootstrap_info = connection.run(manifest,
-		                                debug=debug,
-		                                # We can't pause the bootstrapping process remotely, yet...
-		                                pause_on_error=False,
-		                                dry_run=dry_run)
-		# Restore the old SIGINT handler
-		signal.signal(signal.SIGINT, orig_sigint)
-	return bootstrap_info
diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index 28e9e44..36c0b6e 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -57,18 +57,51 @@ class Server(object):
 	def start(self):
 		Pyro4.config.COMMTIMEOUT = 0.5
 		daemon = Pyro4.Daemon('localhost', port=int(self.listen_port), unixsocket=None)
-
 		daemon.register(self, 'server')
+
 		daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
 
 	@Pyro4.expose
-	def run(self, *args, **kwargs):
+	def run(self, manifest, debug=False, dry_run=False):
 
-		def abort_run():
-			return not self.callback_server.get_abort_run()
-		from bootstrapvz.base.main import run
-		kwargs['check_continue'] = abort_run
-		return run(*args, **kwargs)
+		def bootstrap(queue):
+			# setsid() creates a new session, making this process the group leader.
+			# We do that, so when the server calls killpg (kill process group)
+			# on us, it won't kill itself (this process was spawned from a
+			# thread under the server, meaning it's part of the same group).
+			# The process hierarchy looks like this:
+			# Pyro server (process - listening on a port)
+			# +- pool thread
+			# +- pool thread
+			# +- pool thread
+			# +- started thread (the one that got the "run()" call)
+			#    L bootstrap() process (us)
+			# Calling setsid() also fixes another problem:
+			#   SIGINTs sent to this process seem to be redirected
+			#   to the process leader. Since there is a thread between
+			#   us and the process leader, the signal will not be propagated
+			#   (signals are not propagated to threads), this means that any
+			#   subprocess we start (i.e. debootstrap) will not get a SIGINT.
+			import os
+			os.setsid()
+			from bootstrapvz.base.main import run
+			try:
+				bootstrap_info = run(manifest, debug=debug, dry_run=dry_run)
+				queue.put(bootstrap_info)
+			except (Exception, KeyboardInterrupt) as e:
+				queue.put(e)
+
+		from multiprocessing import Queue
+		from multiprocessing import Process
+		queue = Queue()
+		self.bootstrap_process = Process(target=bootstrap, args=(queue,))
+		self.bootstrap_process.start()
+		self.bootstrap_process.join()
+		del self.bootstrap_process
+		result = queue.get()
+		if isinstance(result, Exception):
+			raise result
+		return result
 
 	@Pyro4.expose
 	def set_callback_server(self, server):
@@ -82,4 +115,14 @@ class Server(object):
 
 	@Pyro4.expose
 	def stop(self):
+		if hasattr(self, 'bootstrap_process'):
+			log.warn('Sending SIGINT to bootstrapping process')
+			import os
+			import signal
+			os.killpg(self.bootstrap_process.pid, signal.SIGINT)
+			self.bootstrap_process.join()
+
+		# We can't send a SIGINT to the server,
+		# for some reason the Pyro4 shutdowns are rather unclean,
+		# throwing exceptions and such.
 		self.stop_serving = True
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 42bc46a..1d0a77f 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -17,7 +17,9 @@ def boot_manifest(manifest_data):
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(data=manifest_data)
 
-	bootstrap_info = build_server.run(manifest)
+	bootstrap_info = None
+	with build_server.connect() as connection:
+		bootstrap_info = connection.run(manifest)
 
 	from ..images import initialize_image
 	image = initialize_image(manifest, build_server, bootstrap_info)

From 686f9e423051fcaf649805c2aa4a8e8151c2b61a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 19:02:13 +0100
Subject: [PATCH 235/345] No need to remember the callback server

---
 bootstrapvz/remote/server.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index 36c0b6e..b221881 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -105,9 +105,8 @@ class Server(object):
 
 	@Pyro4.expose
 	def set_callback_server(self, server):
-		self.callback_server = server
-		self.log_forwarder.set_server(self.callback_server)
-		log.debug('Forwarding logs to the callback server now')
+		log.debug('Forwarding logs to the callback server')
+		self.log_forwarder.set_server(server)
 
 	@Pyro4.expose
 	def ping(self):

From 9505110d4ad313e7442b5a164abd5b5de3c6e2d9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 19:06:06 +0100
Subject: [PATCH 236/345] *first* start the callback server *then* the RPC
 daemon This way we don't stop logging before the daemon is shut down.

---
 bootstrapvz/remote/build_servers/remote.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py
index 7ee668a..a5b129a 100644
--- a/bootstrapvz/remote/build_servers/remote.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -19,11 +19,11 @@ class RemoteBuildServer(BuildServer):
 	@contextmanager
 	def connect(self):
 		with self.spawn_server() as forwards:
-			with connect_pyro('localhost', forwards['local_server_port']) as connection:
-				from callback import CallbackServer
-				args = {'listen_port': forwards['local_callback_port'],
-				        'remote_port': forwards['remote_callback_port']}
-				with CallbackServer(**args) as callback_server:
+			args = {'listen_port': forwards['local_callback_port'],
+			        'remote_port': forwards['remote_callback_port']}
+			from callback import CallbackServer
+			with CallbackServer(**args) as callback_server:
+				with connect_pyro('localhost', forwards['local_server_port']) as connection:
 					connection.set_callback_server(callback_server)
 					yield connection
 

From 4582ea1498c137d5f10eebaf612ecf9fb07af03b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 19:29:26 +0100
Subject: [PATCH 237/345] Better termination when exiting while connecting to
 the RPC daemon

---
 bootstrapvz/remote/build_servers/remote.py | 37 ++++++++++++----------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/bootstrapvz/remote/build_servers/remote.py b/bootstrapvz/remote/build_servers/remote.py
index a5b129a..2177550 100644
--- a/bootstrapvz/remote/build_servers/remote.py
+++ b/bootstrapvz/remote/build_servers/remote.py
@@ -103,23 +103,28 @@ def connect_pyro(host, port):
 	server_uri = 'PYRO:server@{host}:{port}'.format(host=host, port=port)
 	connection = Pyro4.Proxy(server_uri)
 
-	log.debug('Connecting to the RPC daemon')
-	remaining_retries = 5
-	while True:
-		try:
-			connection.ping()
-			break
-		except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError):
-			if remaining_retries > 0:
-				remaining_retries -= 1
-				from time import sleep
-				sleep(2)
-			else:
-				raise
+	log.debug('Connecting to RPC daemon')
 
+	connected = False
 	try:
+		remaining_retries = 5
+		while not connected:
+			try:
+				connection.ping()
+				connected = True
+			except (Pyro4.errors.ConnectionClosedError, Pyro4.errors.CommunicationError):
+				if remaining_retries > 0:
+					remaining_retries -= 1
+					from time import sleep
+					sleep(2)
+				else:
+					raise
+
 		yield connection
 	finally:
-		log.debug('Stopping the RPC daemon')
-		connection.stop()
-		connection._pyroRelease()
+		if connected:
+			log.debug('Stopping RPC daemon')
+			connection.stop()
+			connection._pyroRelease()
+		else:
+			log.warn('Unable to stop RPC daemon, it might still be running on the server')

From 51e9e29b24cacc251bf492aeed4397f1b04ea36f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 19:29:49 +0100
Subject: [PATCH 238/345] Move server control code to the top

---
 bootstrapvz/remote/server.py | 49 +++++++++++++++++++-----------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/bootstrapvz/remote/server.py b/bootstrapvz/remote/server.py
index b221881..bf37cbf 100644
--- a/bootstrapvz/remote/server.py
+++ b/bootstrapvz/remote/server.py
@@ -61,6 +61,32 @@ class Server(object):
 
 		daemon.requestLoop(loopCondition=lambda: not self.stop_serving)
 
+	@Pyro4.expose
+	def set_callback_server(self, server):
+		log.debug('Forwarding logs to the callback server')
+		self.log_forwarder.set_server(server)
+
+	@Pyro4.expose
+	def ping(self):
+		if hasattr(self, 'connection_timeout'):
+			self.connection_timeout.cancel()
+			del self.connection_timeout
+		return 'pong'
+
+	@Pyro4.expose
+	def stop(self):
+		if hasattr(self, 'bootstrap_process'):
+			log.warn('Sending SIGINT to bootstrapping process')
+			import os
+			import signal
+			os.killpg(self.bootstrap_process.pid, signal.SIGINT)
+			self.bootstrap_process.join()
+
+		# We can't send a SIGINT to the server,
+		# for some reason the Pyro4 shutdowns are rather unclean,
+		# throwing exceptions and such.
+		self.stop_serving = True
+
 	@Pyro4.expose
 	def run(self, manifest, debug=False, dry_run=False):
 
@@ -102,26 +128,3 @@ class Server(object):
 		if isinstance(result, Exception):
 			raise result
 		return result
-
-	@Pyro4.expose
-	def set_callback_server(self, server):
-		log.debug('Forwarding logs to the callback server')
-		self.log_forwarder.set_server(server)
-
-	@Pyro4.expose
-	def ping(self):
-		return 'pong'
-
-	@Pyro4.expose
-	def stop(self):
-		if hasattr(self, 'bootstrap_process'):
-			log.warn('Sending SIGINT to bootstrapping process')
-			import os
-			import signal
-			os.killpg(self.bootstrap_process.pid, signal.SIGINT)
-			self.bootstrap_process.join()
-
-		# We can't send a SIGINT to the server,
-		# for some reason the Pyro4 shutdowns are rather unclean,
-		# throwing exceptions and such.
-		self.stop_serving = True

From 3c642e0b02bba31d5de1c9c3757590746251a634 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 19:52:28 +0100
Subject: [PATCH 239/345] Fix serialization of DictClass using it in Queue.put
 somehow broke it

---
 bootstrapvz/base/bootstrapinfo.py | 34 ++++++++++++++++++-------------
 bootstrapvz/remote/__init__.py    |  1 +
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 0039253..9162b62 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -78,20 +78,6 @@ class BootstrapInformation(object):
 		:return: The manifest_vars dictionary
 		:rtype: dict
 		"""
-		class DictClass(dict):
-			"""Tiny extension of dict to allow setting and getting keys via attributes
-			"""
-			def __getattr__(self, name):
-				return self[name]
-
-			def __setattr__(self, name, value):
-				self[name] = value
-
-			def __delattr__(self, name):
-				del self[name]
-
-			def __getstate__(self):
-				return self.__dict__
 
 		def set_manifest_vars(obj, data):
 			"""Runs through the manifest and creates DictClasses for every key
@@ -139,3 +125,23 @@ class BootstrapInformation(object):
 	def __setstate__(self, state):
 		for key in state:
 			self.__dict__[key] = state[key]
+
+
+class DictClass(dict):
+	"""Tiny extension of dict to allow setting and getting keys via attributes
+	"""
+	def __getattr__(self, name):
+		return self[name]
+
+	def __setattr__(self, name, value):
+		self[name] = value
+
+	def __delattr__(self, name):
+		del self[name]
+
+	def __getstate__(self):
+		return self.__dict__
+
+	def __setstate__(self, state):
+		for key in state:
+			self[key] = state[key]
diff --git a/bootstrapvz/remote/__init__.py b/bootstrapvz/remote/__init__.py
index ad0049d..35303e2 100644
--- a/bootstrapvz/remote/__init__.py
+++ b/bootstrapvz/remote/__init__.py
@@ -6,6 +6,7 @@ log = logging.getLogger(__name__)
 
 supported_classes = ['bootstrapvz.base.manifest.Manifest',
                      'bootstrapvz.base.bootstrapinfo.BootstrapInformation',
+                     'bootstrapvz.base.bootstrapinfo.DictClass',
                      'bootstrapvz.common.fs.loopbackvolume.LoopbackVolume',
                      'bootstrapvz.common.fs.qemuvolume.QEMUVolume',
                      'bootstrapvz.common.fs.virtualdiskimage.VirtualDiskImage',

From e035f83edf5271d6438213ddb4524ac99012d2a6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 25 Jan 2015 20:00:57 +0100
Subject: [PATCH 240/345] Fix problem when (de-)serializing bootstrapinfo twice

---
 bootstrapvz/base/bootstrapinfo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 9162b62..f5dc0bc 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -118,7 +118,10 @@ class BootstrapInformation(object):
 		state = self.__dict__.copy()
 		exclude_keys = ['source_lists', 'preference_lists', 'packages']
 		for key in exclude_keys:
-			del state[key]
+			# We may have been serialized before,
+			# so don't fail if the keys do not exist
+			if key in state:
+				del state[key]
 		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
 		return state
 

From f63d3c73aa20d6d7eeed44626867cdd3f23e12a0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Wed, 4 Mar 2015 12:51:11 +0100
Subject: [PATCH 241/345] Use subn to count the actual replacements done in
 inline_replace()

---
 bootstrapvz/common/tools.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 2829c51..aa7157a 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -80,9 +80,8 @@ def inline_replace(file_path, pattern, subst):
 	import re
 	replacement_count = 0
 	for line in fileinput.input(files=file_path, inplace=True):
-		replacement = re.sub(pattern, subst, line)
-		if replacement != line:
-			replacement_count += 1
+		(replacement, count) = re.subn(pattern, subst, line)
+		replacement_count += count
 		print replacement,
 	return replacement_count
 

From d26ba8bea4a45339e35927f64b19bd66b0e69942 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Thu, 5 Mar 2015 17:15:13 +0100
Subject: [PATCH 242/345] Simplify exception throwing in sed_i

---
 bootstrapvz/common/exceptions.py |  6 +-----
 bootstrapvz/common/tools.py      | 17 +++++++----------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index 909d7d3..49d63fa 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -28,11 +28,7 @@ class TaskError(Exception):
 	pass
 
 
-class NoMatchesError(Exception):
-	pass
-
-
-class TooManyMatchesError(Exception):
+class UnexpectedNumMatchesError(Exception):
 	pass
 
 
diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index aa7157a..f0c30fd 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -63,16 +63,13 @@ def log_call(command, stdin=None, env=None, shell=False, cwd=None):
 
 def sed_i(file_path, pattern, subst, expected_replacements=1):
 	replacement_count = inline_replace(file_path, pattern, subst)
-	if replacement_count < expected_replacements:
-		from exceptions import NoMatchesError
-		msg = ('There were no matches for the expression `{exp}\' in the file `{path}\''
-		       .format(exp=pattern, path=file_path))
-		raise NoMatchesError(msg)
-	if replacement_count > expected_replacements:
-		from exceptions import TooManyMatchesError
-		msg = ('There were too many matches for the expression `{exp}\' in the file `{path}\''
-		       .format(exp=pattern, path=file_path))
-		raise TooManyMatchesError(msg)
+	if replacement_count != expected_replacements:
+		from exceptions import UnexpectedNumMatchesError
+		msg = ('There were {real} instead of {expected} matches for '
+		       'the expression `{exp}\' in the file `{path}\''
+		       .format(real=replacement_count, expected=expected_replacements,
+		               exp=pattern, path=file_path))
+		raise UnexpectedNumMatchesError(msg)
 
 
 def inline_replace(file_path, pattern, subst):

From ec96de3a0e8434e3fb0406e0586790baa3f14a80 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Fri, 6 Mar 2015 11:47:37 +0100
Subject: [PATCH 243/345] fix log_call logging when command is a string

---
 bootstrapvz/common/tools.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index f0c30fd..49fb425 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -22,7 +22,10 @@ def log_call(command, stdin=None, env=None, shell=False, cwd=None):
 
 	command_log = realpath(command[0]).replace('/', '.')
 	log = logging.getLogger(__name__ + command_log)
-	log.debug('Executing: {command}'.format(command=' '.join(command)))
+	if type(command) is list:
+		log.debug('Executing: {command}'.format(command=' '.join(command)))
+	else:
+		log.debug('Executing: {command}'.format(command=command))
 
 	process = subprocess.Popen(args=command, env=env, shell=shell, cwd=cwd,
 	                           stdin=subprocess.PIPE,

From 51bb3dd57f0e8820ab8b31cc97893b49c003d708 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 10 Mar 2015 23:05:12 +0100
Subject: [PATCH 244/345] Integration testing for EC2

---
 .../build_servers/build-servers-schema.yml    | 12 +++++++
 .../remote/build_servers/build_server.py      |  6 ++++
 tests/integration/ec2.py                      | 19 +++++++++++
 tests/integration/images/__init__.py          |  5 +++
 tests/integration/images/ami.py               | 33 +++++++++++++++++++
 tests/integration/instances/ec2.py            | 32 ++++++++++++++++++
 tests/integration/tools/__init__.py           |  4 +--
 7 files changed, 109 insertions(+), 2 deletions(-)
 create mode 100644 tests/integration/ec2.py
 create mode 100644 tests/integration/images/ami.py
 create mode 100644 tests/integration/instances/ec2.py

diff --git a/bootstrapvz/remote/build_servers/build-servers-schema.yml b/bootstrapvz/remote/build_servers/build-servers-schema.yml
index 20f0b85..077ee50 100644
--- a/bootstrapvz/remote/build_servers/build-servers-schema.yml
+++ b/bootstrapvz/remote/build_servers/build-servers-schema.yml
@@ -31,6 +31,18 @@ definitions:
     type: object
     properties:
       guest_additions: {$ref: '#/definitions/absolute_path'}
+      ec2-credentials:
+        required: [access-key, secret-key]
+        type: object
+        properties:
+          access-key: {type: string}
+          secret-key: {type: string}
+          certificate: {type: string}
+          private-key: {type: string}
+          user-id:
+            type: string
+            pattern: (^arn:aws:iam::\d*:user/\w.*$)|(^\d{4}-\d{4}-\d{4}$)
+        additional_properties: false
       apt_proxy:
         type: object
         properties:
diff --git a/bootstrapvz/remote/build_servers/build_server.py b/bootstrapvz/remote/build_servers/build_server.py
index bb57c49..59c51c1 100644
--- a/bootstrapvz/remote/build_servers/build_server.py
+++ b/bootstrapvz/remote/build_servers/build_server.py
@@ -14,4 +14,10 @@ class BuildServer(object):
 			manifest_data['provider']['guest_additions'] = self.build_settings['guest_additions']
 		if 'apt_proxy' in self.build_settings:
 			manifest_data.get('plugins', {})['apt_proxy'] = self.build_settings['apt_proxy']
+		if 'ec2-credentials' in self.build_settings:
+			if 'credentials' not in manifest_data['provider']:
+				manifest_data['provider']['credentials'] = {}
+			for key in ['access-key', 'secret-key', 'certificate', 'private-key', 'user-id']:
+				if key in self.build_settings['ec2-credentials']:
+					manifest_data['provider']['credentials'][key] = self.build_settings['ec2-credentials'][key]
 		return manifest_data
diff --git a/tests/integration/ec2.py b/tests/integration/ec2.py
new file mode 100644
index 0000000..b5db8d9
--- /dev/null
+++ b/tests/integration/ec2.py
@@ -0,0 +1,19 @@
+from manifests import merge_manifest_data
+from tools import boot_manifest
+
+partials = {'ec2': 'provider: {name: ec2}',
+            'ebs': 'volume: {backing: ebs}',
+            's3': 'volume: {backing: s3}',
+            'pvm': '{provider: {virtualization: pvm}, system: {bootloader: pvgrub}}',
+            'hvm-extlinux': '{provider: {virtualization: hvm}, system: {bootloader: extlinux}}',
+            'hvm-grub': '{provider: {virtualization: hvm}, system: {bootloader: grub}}',
+            }
+
+
+def test_unpartitioned_ebs_pvgrub_stable():
+	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py
index 041717f..313e098 100644
--- a/tests/integration/images/__init__.py
+++ b/tests/integration/images/__init__.py
@@ -4,3 +4,8 @@ def initialize_image(manifest, build_server, bootstrap_info):
 	if manifest.provider['name'] == 'virtualbox':
 		import vbox
 		return vbox.initialize_image(manifest, build_server, bootstrap_info)
+	if manifest.provider['name'] == 'ec2':
+		import ami
+		credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
+		               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
+		return ami.initialize_image(manifest, credentials, bootstrap_info)
diff --git a/tests/integration/images/ami.py b/tests/integration/images/ami.py
new file mode 100644
index 0000000..2f3404c
--- /dev/null
+++ b/tests/integration/images/ami.py
@@ -0,0 +1,33 @@
+from image import Image
+import logging
+from contextlib import contextmanager
+log = logging.getLogger(__name__)
+
+
+def initialize_image(manifest, credentials, bootstrap_info):
+	from boto.ec2 import connect_to_region
+	connection = connect_to_region(bootstrap_info._ec2['region'],
+	                               aws_access_key_id=credentials['access-key'],
+	                               aws_secret_access_key=credentials['secret-key'])
+	ami = connection.get_image(bootstrap_info._ec2['image'])
+	image = AmazonMachineImage(manifest, ami)
+	return image
+
+
+class AmazonMachineImage(Image):
+
+	def __init__(self, manifest, ami):
+		super(AmazonMachineImage, self).__init__(manifest)
+		self.ami = ami
+
+	def destroy(self):
+		log.debug('Deleting AMI')
+		self.ami.deregister(delete_snapshot=True)
+		del self.ami
+
+	@contextmanager
+	def get_instance(self, instance_type='t1.micro'):
+		from ..instances.ec2 import boot_image
+		name = 'bootstrap-vz test instance'
+		with boot_image(name, self.ami, instance_type) as instance:
+			yield instance
diff --git a/tests/integration/instances/ec2.py b/tests/integration/instances/ec2.py
new file mode 100644
index 0000000..a55e58a
--- /dev/null
+++ b/tests/integration/instances/ec2.py
@@ -0,0 +1,32 @@
+from contextlib import contextmanager
+from ..tools import waituntil
+import logging
+log = logging.getLogger(__name__)
+
+
+@contextmanager
+def boot_image(name, image, instance_type):
+	instance = None
+	try:
+		log.debug('Booting ec2 instance')
+		reservation = image.run(instance_type=instance_type)
+		[instance] = reservation.instances
+		instance.add_tag('Name', name)
+
+		def instance_running():
+			instance.update()
+			return instance.state == 'running'
+		if not waituntil(instance_running, timeout=120, interval=3):
+			raise EC2InstanceStartupException('Timeout while booting instance')
+
+		if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
+			raise EC2InstanceStartupException('Timeout while fetching console output')
+
+		yield instance
+	finally:
+		if instance is not None:
+			instance.terminate()
+
+
+class EC2InstanceStartupException(Exception):
+	pass
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 1d0a77f..f150602 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -7,7 +7,7 @@ register_deserialization_handlers()
 
 
 @contextmanager
-def boot_manifest(manifest_data):
+def boot_manifest(manifest_data, boot_vars={}):
 	from bootstrapvz.common.tools import load_data
 	build_servers = load_data('build-servers.yml')
 	from bootstrapvz.remote.build_servers import pick_build_server
@@ -24,7 +24,7 @@ def boot_manifest(manifest_data):
 	from ..images import initialize_image
 	image = initialize_image(manifest, build_server, bootstrap_info)
 	try:
-		with image.get_instance() as instance:
+		with image.get_instance(**boot_vars) as instance:
 			yield instance
 	finally:
 		image.destroy()

From 25051d4c04ce0458a15657645d2b072beddfce10 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 20:36:06 +0100
Subject: [PATCH 245/345] Improve __getstate__ for bootstrapinfo This approach
 may be a little hacked, but it works for now and if it breaks at some point
 in the future because of e.g. circular references that bridge will have to be
 crossed then

---
 bootstrapvz/base/bootstrapinfo.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index f5dc0bc..111c1d9 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -115,13 +115,22 @@ class BootstrapInformation(object):
 		return manifest_vars
 
 	def __getstate__(self):
-		state = self.__dict__.copy()
-		exclude_keys = ['source_lists', 'preference_lists', 'packages']
-		for key in exclude_keys:
-			# We may have been serialized before,
-			# so don't fail if the keys do not exist
-			if key in state:
-				del state[key]
+		from bootstrapvz.remote import supported_classes
+
+		def can_serialize(obj):
+			if hasattr(obj, '__class__') and hasattr(obj, '__module__'):
+				class_name = obj.__module__ + '.' + obj.__class__.__name__
+				return class_name in supported_classes or isinstance(obj, (BaseException, Exception))
+			return True
+
+		def filter_state(state):
+			if isinstance(state, dict):
+				return {key: filter_state(val) for key, val in state.items() if can_serialize(val)}
+			if isinstance(state, (set, tuple, list, frozenset)):
+				return type(state)(filter_state(val) for val in state if can_serialize(val))
+			return state
+
+		state = filter_state(self.__dict__)
 		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
 		return state
 

From a3fdeddd79a9224c1ed1428c68a689227273dda2 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 20:38:07 +0100
Subject: [PATCH 246/345] Rename virtualbox_tests.py to be less tautological

---
 tests/integration/{virtualbox_tests.py => virtualbox.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/integration/{virtualbox_tests.py => virtualbox.py} (100%)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox.py
similarity index 100%
rename from tests/integration/virtualbox_tests.py
rename to tests/integration/virtualbox.py

From 07e4c97c095a450e4183fab670df2bee7d6f758e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 20:38:33 +0100
Subject: [PATCH 247/345] Fix PEP8 nitpick

---
 bootstrapvz/common/tasks/packages.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/common/tasks/packages.py b/bootstrapvz/common/tasks/packages.py
index f6e41af..44263a5 100644
--- a/bootstrapvz/common/tasks/packages.py
+++ b/bootstrapvz/common/tasks/packages.py
@@ -49,7 +49,8 @@ class InstallPackages(Task):
 			log_check_call(['chroot', info.root,
 			                'apt-get', 'install',
 			                           '--no-install-recommends',
-			                           '--assume-yes'] + map(str, remote_packages),
+			                           '--assume-yes'] +
+			               map(str, remote_packages),
 			               env=env)
 		except CalledProcessError as e:
 			import logging

From b33577a0238ba6ec715627c5d8a4954f7a1287c0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 20:40:15 +0100
Subject: [PATCH 248/345] Remove ec2 default instance type

---
 tests/integration/images/ami.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/images/ami.py b/tests/integration/images/ami.py
index 2f3404c..58da13b 100644
--- a/tests/integration/images/ami.py
+++ b/tests/integration/images/ami.py
@@ -26,7 +26,7 @@ class AmazonMachineImage(Image):
 		del self.ami
 
 	@contextmanager
-	def get_instance(self, instance_type='t1.micro'):
+	def get_instance(self, instance_type):
 		from ..instances.ec2 import boot_image
 		name = 'bootstrap-vz test instance'
 		with boot_image(name, self.ami, instance_type) as instance:

From e9736f58a800eb0edd37e8b62589795c87ca3ca7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 20:44:06 +0100
Subject: [PATCH 249/345] Add _stable to vbox test names, that test Debian
 stable

---
 tests/integration/virtualbox.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/integration/virtualbox.py b/tests/integration/virtualbox.py
index a5930aa..4e58902 100644
--- a/tests/integration/virtualbox.py
+++ b/tests/integration/virtualbox.py
@@ -50,7 +50,7 @@ def test_gpt_grub_oldstable():
 		print(instance.console_output)
 
 
-def test_unpartitioned_extlinux():
+def test_unpartitioned_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -58,7 +58,7 @@ def test_unpartitioned_extlinux():
 		print(instance.console_output)
 
 
-def test_msdos_extlinux():
+def test_msdos_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -66,7 +66,7 @@ def test_msdos_extlinux():
 		print(instance.console_output)
 
 
-def test_gpt_extlinux():
+def test_gpt_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -74,7 +74,7 @@ def test_gpt_extlinux():
 		print(instance.console_output)
 
 
-def test_msdos_grub():
+def test_msdos_grub_stable():
 	std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
@@ -82,7 +82,7 @@ def test_msdos_grub():
 		print(instance.console_output)
 
 
-def test_gpt_grub():
+def test_gpt_grub_stable():
 	std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)

From 694f46923b2833a4cc32678c1184aae3cf436215 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 21:07:05 +0100
Subject: [PATCH 250/345] Fix snapshot deletion on AMI testing cleanup

---
 tests/integration/images/ami.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tests/integration/images/ami.py b/tests/integration/images/ami.py
index 58da13b..34714fa 100644
--- a/tests/integration/images/ami.py
+++ b/tests/integration/images/ami.py
@@ -9,20 +9,22 @@ def initialize_image(manifest, credentials, bootstrap_info):
 	connection = connect_to_region(bootstrap_info._ec2['region'],
 	                               aws_access_key_id=credentials['access-key'],
 	                               aws_secret_access_key=credentials['secret-key'])
-	ami = connection.get_image(bootstrap_info._ec2['image'])
-	image = AmazonMachineImage(manifest, ami)
+	image = AmazonMachineImage(manifest, bootstrap_info._ec2['image'], connection)
 	return image
 
 
 class AmazonMachineImage(Image):
 
-	def __init__(self, manifest, ami):
+	def __init__(self, manifest, image_id, connection):
 		super(AmazonMachineImage, self).__init__(manifest)
-		self.ami = ami
+		self.ami = connection.get_image(image_id)
+		self.connection = connection
 
 	def destroy(self):
 		log.debug('Deleting AMI')
-		self.ami.deregister(delete_snapshot=True)
+		self.ami.deregister()
+		for device, block_device_type in self.ami.block_device_mapping.items():
+			self.connection.delete_snapshot(block_device_type.snapshot_id)
 		del self.ami
 
 	@contextmanager

From 0c20acce1125ad274d70d57ee8693ecbb7acf1c9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 21:07:25 +0100
Subject: [PATCH 251/345] Fix positional argument handling in tox It's now
 possible to pass a single test through tox to nosetest

---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index 2257010..a8c18c3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -14,14 +14,14 @@ commands = flake8 bootstrapvz/ --exclude=minify_json.py {posargs}
 deps =
     nose
     nose-cov
-commands = nosetests -v tests/unit --with-coverage --cover-package=bootstrapvz --cover-inclusive {posargs}
+commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --verbose {posargs:tests/unit}
 
 [testenv:integration]
 deps =
     nose
     Pyro4 >= 4.30
     pyvbox >= 0.2.0
-commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive {posargs}
+commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --verbose {posargs:tests/integration}
 
 [testenv:docs]
 changedir = docs

From 29fd2d928a152c1f712b50427196328c8dede99b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 25 Mar 2015 21:07:55 +0100
Subject: [PATCH 252/345] Add tests for all ebs 64 bit pvgrub booted instances

---
 tests/integration/ec2.py | 72 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/tests/integration/ec2.py b/tests/integration/ec2.py
index b5db8d9..e3de576 100644
--- a/tests/integration/ec2.py
+++ b/tests/integration/ec2.py
@@ -10,6 +10,33 @@ partials = {'ec2': 'provider: {name: ec2}',
             }
 
 
+def test_unpartitioned_ebs_pvgrub_oldstable():
+	std_partials = ['base', 'oldstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_ebs_pvgrub_oldstable():
+	std_partials = ['base', 'oldstable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_ebs_pvgrub_oldstable():
+	std_partials = ['base', 'oldstable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
 def test_unpartitioned_ebs_pvgrub_stable():
 	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
 	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
@@ -17,3 +44,48 @@ def test_unpartitioned_ebs_pvgrub_stable():
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
+
+
+def test_msdos_ebs_pvgrub_stable():
+	std_partials = ['base', 'stable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_ebs_pvgrub_stable():
+	std_partials = ['base', 'stable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_unpartitioned_ebs_pvgrub_unstable():
+	std_partials = ['base', 'unstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_ebs_pvgrub_unstable():
+	std_partials = ['base', 'unstable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_ebs_pvgrub_unstable():
+	std_partials = ['base', 'unstable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't1.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)

From 59f19fe164530ac5034d17ed76cfffb685e31f4e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 7 Apr 2015 17:42:32 +0200
Subject: [PATCH 253/345] Rename ec2 testsuite to ec2_ebs_pvm

---
 tests/integration/{ec2.py => ec2_ebs_pvm.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/integration/{ec2.py => ec2_ebs_pvm.py} (100%)

diff --git a/tests/integration/ec2.py b/tests/integration/ec2_ebs_pvm.py
similarity index 100%
rename from tests/integration/ec2.py
rename to tests/integration/ec2_ebs_pvm.py

From c18e8800b5644ed156024c656ead80e023848f28 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Tue, 7 Apr 2015 18:15:17 +0200
Subject: [PATCH 254/345] Simplify ebs_pvm test suite

---
 tests/integration/ec2_ebs_pvm.py | 49 ++++++++++++++++----------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/tests/integration/ec2_ebs_pvm.py b/tests/integration/ec2_ebs_pvm.py
index e3de576..3c05e11 100644
--- a/tests/integration/ec2_ebs_pvm.py
+++ b/tests/integration/ec2_ebs_pvm.py
@@ -1,90 +1,91 @@
 from manifests import merge_manifest_data
 from tools import boot_manifest
 
-partials = {'ec2': 'provider: {name: ec2}',
-            'ebs': 'volume: {backing: ebs}',
-            's3': 'volume: {backing: s3}',
-            'pvm': '{provider: {virtualization: pvm}, system: {bootloader: pvgrub}}',
-            'hvm-extlinux': '{provider: {virtualization: hvm}, system: {bootloader: extlinux}}',
-            'hvm-grub': '{provider: {virtualization: hvm}, system: {bootloader: grub}}',
+partials = {'ebs_pvm': '''
+provider:
+  name: ec2
+  virtualization: pvm
+system: {bootloader: pvgrub}
+volume: {backing: ebs}
+'''
             }
 
 
-def test_unpartitioned_ebs_pvgrub_oldstable():
+def test_unpartitioned_oldstable():
 	std_partials = ['base', 'oldstable64', 'unpartitioned', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_msdos_ebs_pvgrub_oldstable():
+def test_msdos_oldstable():
 	std_partials = ['base', 'oldstable64', 'msdos', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_gpt_ebs_pvgrub_oldstable():
+def test_gpt_oldstable():
 	std_partials = ['base', 'oldstable64', 'gpt', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_unpartitioned_ebs_pvgrub_stable():
+def test_unpartitioned_stable():
 	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_msdos_ebs_pvgrub_stable():
+def test_msdos_stable():
 	std_partials = ['base', 'stable64', 'msdos', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_gpt_ebs_pvgrub_stable():
+def test_gpt_stable():
 	std_partials = ['base', 'stable64', 'gpt', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_unpartitioned_ebs_pvgrub_unstable():
+def test_unpartitioned_unstable():
 	std_partials = ['base', 'unstable64', 'unpartitioned', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_msdos_ebs_pvgrub_unstable():
+def test_msdos_unstable():
 	std_partials = ['base', 'unstable64', 'msdos', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:
 		print(instance.get_console_output().output)
 
 
-def test_gpt_ebs_pvgrub_unstable():
+def test_gpt_unstable():
 	std_partials = ['base', 'unstable64', 'gpt', 'single_partition', 'root_password']
-	custom_partials = [partials['ec2'], partials['ebs'], partials['pvm']]
+	custom_partials = [partials['ebs_pvm']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	boot_vars = {'instance_type': 't1.micro'}
 	with boot_manifest(manifest_data, boot_vars) as instance:

From 7f4d46a33078f74ffbd7fa2b221e8e2c1111bbfd Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:21:41 +0200
Subject: [PATCH 255/345] Fix serialization error when ManifestException is
 thrown ("__init__() takes at least 3 arguments (2 given)")

---
 bootstrapvz/common/exceptions.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py
index 49d63fa..4ea5725 100644
--- a/bootstrapvz/common/exceptions.py
+++ b/bootstrapvz/common/exceptions.py
@@ -6,6 +6,7 @@ class ManifestError(Exception):
 		self.message = message
 		self.manifest_path = manifest_path
 		self.data_path = data_path
+		self.args = (self.message, self.manifest_path, self.data_path)
 
 	def __str__(self):
 		if self.data_path is not None:
@@ -19,6 +20,7 @@ class TaskListError(Exception):
 	def __init__(self, message):
 		super(TaskListError, self).__init__(message)
 		self.message = message
+		self.args = (self.message,)
 
 	def __str__(self):
 		return 'Error in tasklist: ' + self.message

From e47d67bc4fda2455c1791875ef3b256f30e1ad8e Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:23:21 +0200
Subject: [PATCH 256/345] Launch ec2 test instances inside a VPC

---
 tests/integration/images/ami.py    | 25 ++++++----
 tests/integration/instances/ec2.py | 80 +++++++++++++++++++++++-------
 2 files changed, 76 insertions(+), 29 deletions(-)

diff --git a/tests/integration/images/ami.py b/tests/integration/images/ami.py
index 34714fa..895bf9c 100644
--- a/tests/integration/images/ami.py
+++ b/tests/integration/images/ami.py
@@ -5,31 +5,34 @@ log = logging.getLogger(__name__)
 
 
 def initialize_image(manifest, credentials, bootstrap_info):
-	from boto.ec2 import connect_to_region
-	connection = connect_to_region(bootstrap_info._ec2['region'],
-	                               aws_access_key_id=credentials['access-key'],
-	                               aws_secret_access_key=credentials['secret-key'])
-	image = AmazonMachineImage(manifest, bootstrap_info._ec2['image'], connection)
+	image = AmazonMachineImage(manifest, bootstrap_info._ec2['image'],
+	                           bootstrap_info._ec2['region'], credentials)
 	return image
 
 
 class AmazonMachineImage(Image):
 
-	def __init__(self, manifest, image_id, connection):
+	def __init__(self, manifest, image_id, region, credentials):
 		super(AmazonMachineImage, self).__init__(manifest)
-		self.ami = connection.get_image(image_id)
-		self.connection = connection
+
+		from boto.ec2 import connect_to_region as ec2_connect
+		self.ec2_connection = ec2_connect(region, aws_access_key_id=credentials['access-key'],
+		                                  aws_secret_access_key=credentials['secret-key'])
+		from boto.vpc import connect_to_region as vpc_connect
+		self.vpc_connection = vpc_connect(region, aws_access_key_id=credentials['access-key'],
+		                                  aws_secret_access_key=credentials['secret-key'])
+
+		self.ami = self.ec2_connection.get_image(image_id)
 
 	def destroy(self):
 		log.debug('Deleting AMI')
 		self.ami.deregister()
 		for device, block_device_type in self.ami.block_device_mapping.items():
-			self.connection.delete_snapshot(block_device_type.snapshot_id)
+			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
 		del self.ami
 
 	@contextmanager
 	def get_instance(self, instance_type):
 		from ..instances.ec2 import boot_image
-		name = 'bootstrap-vz test instance'
-		with boot_image(name, self.ami, instance_type) as instance:
+		with boot_image(self.ami, instance_type, self.ec2_connection, self.vpc_connection) as instance:
 			yield instance
diff --git a/tests/integration/instances/ec2.py b/tests/integration/instances/ec2.py
index a55e58a..f658856 100644
--- a/tests/integration/instances/ec2.py
+++ b/tests/integration/instances/ec2.py
@@ -5,27 +5,71 @@ log = logging.getLogger(__name__)
 
 
 @contextmanager
-def boot_image(name, image, instance_type):
-	instance = None
-	try:
-		log.debug('Booting ec2 instance')
-		reservation = image.run(instance_type=instance_type)
-		[instance] = reservation.instances
-		instance.add_tag('Name', name)
+def boot_image(image, instance_type, ec2_connection, vpc_connection):
 
-		def instance_running():
-			instance.update()
-			return instance.state == 'running'
-		if not waituntil(instance_running, timeout=120, interval=3):
-			raise EC2InstanceStartupException('Timeout while booting instance')
+	with create_env(ec2_connection, vpc_connection) as boot_env:
 
-		if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
-			raise EC2InstanceStartupException('Timeout while fetching console output')
+		def waituntil_instance_is(state):
+			def instance_has_state():
+				instance.update()
+				return instance.state == state
+			return waituntil(instance_has_state, timeout=600, interval=3)
 
-		yield instance
-	finally:
-		if instance is not None:
-			instance.terminate()
+		instance = None
+		try:
+			log.debug('Booting ec2 instance')
+			reservation = image.run(instance_type=instance_type,
+			                        subnet_id=boot_env['subnet_id'])
+			[instance] = reservation.instances
+			instance.add_tag('Name', 'bootstrap-vz test instance')
+
+			if not waituntil_instance_is('running'):
+				raise EC2InstanceStartupException('Timeout while booting instance')
+
+			if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
+				raise EC2InstanceStartupException('Timeout while fetching console output')
+
+			yield instance
+		finally:
+			if instance is not None:
+				log.debug('Terminating ec2 instance')
+				instance.terminate()
+				if not waituntil_instance_is('terminated'):
+					raise EC2InstanceStartupException('Timeout while terminating instance')
+				# wait a little longer, aws can be a little slow sometimes and think the instance is still running
+				import time
+				time.sleep(15)
+
+
+@contextmanager
+def create_env(ec2_connection, vpc_connection):
+
+	vpc_cidr = '10.0.0.0/28'
+	subnet_cidr = '10.0.0.0/28'
+
+	@contextmanager
+	def vpc():
+		log.debug('Creating VPC')
+		vpc = vpc_connection.create_vpc(vpc_cidr)
+		try:
+			yield vpc
+		finally:
+			log.debug('Deleting VPC')
+			vpc_connection.delete_vpc(vpc.id)
+
+	@contextmanager
+	def subnet(vpc):
+		log.debug('Creating subnet')
+		subnet = vpc_connection.create_subnet(vpc.id, subnet_cidr)
+		try:
+			yield subnet
+		finally:
+			log.debug('Deleting subnet')
+			vpc_connection.delete_subnet(subnet.id)
+
+	with vpc() as _vpc:
+		with subnet(_vpc) as _subnet:
+			yield {'subnet_id': _subnet.id}
 
 
 class EC2InstanceStartupException(Exception):

From b843cbe089d49af083ba4818928529e083888d9f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:23:47 +0200
Subject: [PATCH 257/345] Remove skipped tests that test functionality that'll
 probably never be implemented

---
 tests/integration/virtualbox.py | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/tests/integration/virtualbox.py b/tests/integration/virtualbox.py
index 4e58902..27e95ea 100644
--- a/tests/integration/virtualbox.py
+++ b/tests/integration/virtualbox.py
@@ -32,24 +32,6 @@ def test_gpt_extlinux_oldstable():
 		print(instance.console_output)
 
 
-def test_msdos_grub_oldstable():
-	raise SkipTest('grub install on squeeze is broken')
-	std_partials = ['base', 'oldstable64', 'grub', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
-	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with boot_manifest(manifest_data) as instance:
-		print(instance.console_output)
-
-
-def test_gpt_grub_oldstable():
-	raise SkipTest('grub install on squeeze is broken')
-	std_partials = ['base', 'oldstable64', 'grub', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
-	manifest_data = merge_manifest_data(std_partials, custom_partials)
-	with boot_manifest(manifest_data) as instance:
-		print(instance.console_output)
-
-
 def test_unpartitioned_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
 	custom_partials = [partials['vbox'], partials['vmdk']]

From b582bac853a7c9f45634b5ec3ebd71be6f35305b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:37:29 +0200
Subject: [PATCH 258/345] Generally deny installing grub on squeeze

---
 bootstrapvz/base/__init__.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index be224ce..39d88a1 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -16,7 +16,15 @@ def validate_manifest(data, validator, error):
 	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
+	from bootstrapvz.common.tools import get_codename
+	codename = get_codename(data['system']['release'])
+
 	# Check the bootloader/partitioning configuration.
 	# Doing this via the schema is a pain and does not output a useful error message.
-	if data['system']['bootloader'] == 'grub' and data['volume']['partitions']['type'] == 'none':
+	if data['system']['bootloader'] == 'grub':
+
+		if data['volume']['partitions']['type'] == 'none':
 			error('Grub cannot boot from unpartitioned disks', ['system', 'bootloader'])
+
+		if codename == 'squeeze':
+			error('Grub installation on squeeze is not supported', ['system', 'bootloader'])

From 80ac206fb4b00888c35f174fc2574569a6d8a91c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:52:46 +0200
Subject: [PATCH 259/345] Issue warning when specifying pre/successors across
 phases

---
 bootstrapvz/base/tasklist.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 8e5dbb1..c196d55 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -162,21 +162,29 @@ def check_ordering(task):
 	:raises TaskListError: If there is a conflict between task precedence and phase precedence
 	"""
 	for successor in task.successors:
-		# Run through all successors and check whether the phase of the task
-		# comes before the phase of a successor
+		# Run through all successors and throw an error if the phase of the task
+		# lies before the phase of a successor, log a warning if it lies after.
 		if task.phase > successor.phase:
 			msg = ("The task {task} is specified as running before {other}, "
 			       "but its phase '{phase}' lies after the phase '{other_phase}'"
 			       .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
 			raise TaskListError(msg)
+		if task.phase < successor.phase:
+			log.warn("The task {task} is specified as running before {other} "
+			         "although its phase '{phase}' already lies before the phase '{other_phase}'"
+			         .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
 	for predecessor in task.predecessors:
-		# Run through all predecessors and check whether the phase of the task
-		# comes after the phase of a predecessor
+		# Run through all successors and throw an error if the phase of the task
+		# lies after the phase of a predecessor, log a warning if it lies before.
 		if task.phase < predecessor.phase:
 			msg = ("The task {task} is specified as running after {other}, "
 			       "but its phase '{phase}' lies before the phase '{other_phase}'"
 			       .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
 			raise TaskListError(msg)
+		if task.phase > predecessor.phase:
+			log.warn("The task {task} is specified as running after {other} "
+			         "although its phase '{phase}' already lies after the phase '{other_phase}'"
+			         .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
 
 
 def strongly_connected_components(graph):

From 4ad1d516243e601bf31842b473a53d6dca442ce1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:53:31 +0200
Subject: [PATCH 260/345] Take @ssgelm's advice in #155 and copy the mount
 table df warnings no more :-)

---
 bootstrapvz/common/task_groups.py      |  2 ++
 bootstrapvz/common/tasks/filesystem.py | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index d4417e9..90bff08 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -74,6 +74,8 @@ boot_partition_group = [filesystem.CreateBootMountDir,
 mounting_group = [filesystem.CreateMountDir,
                   filesystem.MountRoot,
                   filesystem.MountSpecials,
+                  filesystem.CopyMountTable,
+                  filesystem.RemoveMountTable,
                   filesystem.UnmountRoot,
                   filesystem.DeleteMountDir,
                   ]
diff --git a/bootstrapvz/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py
index a6cb782..19acbfe 100644
--- a/bootstrapvz/common/tasks/filesystem.py
+++ b/bootstrapvz/common/tasks/filesystem.py
@@ -115,6 +115,18 @@ class MountSpecials(Task):
 		root.add_mount('none', 'dev/pts', ['--types', 'devpts'])
 
 
+class CopyMountTable(Task):
+	description = 'Copying mtab from host system'
+	phase = phases.os_installation
+	predecessors = [MountSpecials]
+
+	@classmethod
+	def run(cls, info):
+		import shutil
+		import os.path
+		shutil.copy('/proc/mounts', os.path.join(info.root, 'etc/mtab'))
+
+
 class UnmountRoot(Task):
 	description = 'Unmounting the bootstrap volume'
 	phase = phases.volume_unmounting
@@ -125,6 +137,17 @@ class UnmountRoot(Task):
 		info.volume.partition_map.root.unmount()
 
 
+class RemoveMountTable(Task):
+	description = 'Removing mtab'
+	phase = phases.volume_unmounting
+	successors = [UnmountRoot]
+
+	@classmethod
+	def run(cls, info):
+		import os
+		os.remove(os.path.join(info.root, 'etc/mtab'))
+
+
 class DeleteMountDir(Task):
 	description = 'Deleting mountpoint for the bootstrap volume'
 	phase = phases.volume_unmounting

From 8364f824b0f381adbde8789f10d8cc217397b8fa Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 21:59:41 +0200
Subject: [PATCH 261/345] Fix typo

---
 bootstrapvz/providers/ec2/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index 2a5dd65..ed54df5 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -54,7 +54,7 @@ def validate_manifest(data, validator, error):
 		if bootloader != 'extlinux':
 			error('HVM AMIs currently only work with extlinux as a bootloader', ['system', 'bootloader'])
 		if bootloader == 'extlinux' and partition_type not in ['none', 'msdos']:
-			error('HVM AMIs booted with extlinux currently work with unpartitioned or msdos partitions volumes',
+			error('HVM AMIs booted with extlinux currently work with unpartitioned or msdos partitioned volumes',
 			      ['volume', 'partitions', 'type'])
 
 	if backing == 's3':

From 411578a4984c3c1a164710b9dd121572ea8a3cdd Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 23:15:02 +0200
Subject: [PATCH 262/345] *Always* use link_dm_node() when install grub 1.99
 Grub install fails when in a chrooted environment, it has nothing to do with
 the volume being a loopback volume

---
 bootstrapvz/common/fs/__init__.py            |  9 +++---
 bootstrapvz/common/tasks/grub.py             | 32 ++++++--------------
 bootstrapvz/plugins/prebootstrapped/tasks.py | 11 +++----
 3 files changed, 20 insertions(+), 32 deletions(-)

diff --git a/bootstrapvz/common/fs/__init__.py b/bootstrapvz/common/fs/__init__.py
index 694846e..c393ea2 100644
--- a/bootstrapvz/common/fs/__init__.py
+++ b/bootstrapvz/common/fs/__init__.py
@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 
 
 def get_partitions():
@@ -16,7 +17,8 @@ def get_partitions():
 	return matches
 
 
-def remount(volume, fn):
+@contextmanager
+def unmounted(volume):
 	from bootstrapvz.base.fs.partitionmaps.none import NoPartitions
 
 	p_map = volume.partition_map
@@ -24,9 +26,8 @@ def remount(volume, fn):
 	p_map.root.unmount()
 	if not isinstance(p_map, NoPartitions):
 		p_map.unmap(volume)
-		result = fn()
+		yield
 		p_map.map(volume)
 	else:
-		result = fn()
+		yield
 	p_map.root.mount(destination=root_dir)
-	return result
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index b528712..9aabba3 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -39,26 +39,16 @@ class InstallGrub_1_99(Task):
 
 	@classmethod
 	def run(cls, info):
-
-		from ..fs import remount
 		p_map = info.volume.partition_map
 
-		def link_fn():
+		# GRUB screws up when installing in chrooted environments
+		# so we fake a real harddisk with dmsetup.
+		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
+		from ..fs import unmounted
+		with unmounted(info.volume):
 			info.volume.link_dm_node()
 			if isinstance(p_map, partitionmaps.none.NoPartitions):
 				p_map.root.device_path = info.volume.device_path
-
-		def unlink_fn():
-			info.volume.unlink_dm_node()
-			if isinstance(p_map, partitionmaps.none.NoPartitions):
-				p_map.root.device_path = info.volume.device_path
-
-		# GRUB cannot deal with installing to loopback devices
-		# so we fake a real harddisk with dmsetup.
-		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
-		from ..fs.loopbackvolume import LoopbackVolume
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, link_fn)
 		try:
 			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
 			device_map_path = os.path.join(info.root, 'boot/grub/device.map')
@@ -77,13 +67,11 @@ class InstallGrub_1_99(Task):
 			# Install grub
 			log_check_call(['chroot', info.root, 'grub-install', device_path])
 			log_check_call(['chroot', info.root, 'update-grub'])
-		except Exception:
-			if isinstance(info.volume, LoopbackVolume):
-				remount(info.volume, unlink_fn)
-			raise
-
-		if isinstance(info.volume, LoopbackVolume):
-			remount(info.volume, unlink_fn)
+		finally:
+			with unmounted(info.volume):
+				info.volume.unlink_dm_node()
+				if isinstance(p_map, partitionmaps.none.NoPartitions):
+					p_map.root.device_path = info.volume.device_path
 
 
 class InstallGrub_2(Task):
diff --git a/bootstrapvz/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py
index 4711e46..df09782 100644
--- a/bootstrapvz/plugins/prebootstrapped/tasks.py
+++ b/bootstrapvz/plugins/prebootstrapped/tasks.py
@@ -4,7 +4,7 @@ from bootstrapvz.common.tasks import volume
 from bootstrapvz.common.tasks import packages
 from bootstrapvz.providers.virtualbox.tasks import guest_additions
 from bootstrapvz.providers.ec2.tasks import ebs
-from bootstrapvz.common.fs import remount
+from bootstrapvz.common.fs import unmounted
 from shutil import copyfile
 import os.path
 import time
@@ -19,9 +19,9 @@ class Snapshot(Task):
 
 	@classmethod
 	def run(cls, info):
-		def mk_snapshot():
-			return info.volume.snapshot()
-		snapshot = remount(info.volume, mk_snapshot)
+		snapshot = None
+		with unmounted(info.volume):
+			snapshot = info.volume.snapshot()
 		msg = 'A snapshot of the bootstrapped volume was created. ID: ' + snapshot.id
 		log.info(msg)
 
@@ -55,9 +55,8 @@ class CopyImage(Task):
 		loopback_backup_name = 'volume-{id}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension)
 		destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name)
 
-		def mk_snapshot():
+		with unmounted(info.volume):
 			copyfile(info.volume.image_path, destination)
-		remount(info.volume, mk_snapshot)
 		msg = 'A copy of the bootstrapped volume was created. Path: ' + destination
 		log.info(msg)
 

From 736852a95961a31975821c99532718aac2903b03 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 23:16:22 +0200
Subject: [PATCH 263/345] Enable grub for hvm AMIs

---
 bootstrapvz/providers/ec2/__init__.py         | 2 --
 bootstrapvz/providers/ec2/manifest-schema.yml | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index ed54df5..2d13675 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -51,8 +51,6 @@ def validate_manifest(data, validator, error):
 	if virtualization == 'hvm':
 		if backing != 'ebs':
 			error('HVM AMIs currently only work when they are EBS backed', ['volume', 'backing'])
-		if bootloader != 'extlinux':
-			error('HVM AMIs currently only work with extlinux as a bootloader', ['system', 'bootloader'])
 		if bootloader == 'extlinux' and partition_type not in ['none', 'msdos']:
 			error('HVM AMIs booted with extlinux currently work with unpartitioned or msdos partitioned volumes',
 			      ['volume', 'partitions', 'type'])
diff --git a/bootstrapvz/providers/ec2/manifest-schema.yml b/bootstrapvz/providers/ec2/manifest-schema.yml
index 40fc332..696cfa7 100644
--- a/bootstrapvz/providers/ec2/manifest-schema.yml
+++ b/bootstrapvz/providers/ec2/manifest-schema.yml
@@ -31,6 +31,7 @@ properties:
         type: string
         enum:
         - pvgrub
+        - grub
         - extlinux
   volume:
     type: object

From 50297d790ce79f5596fa21cb7440e8690e4999b1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 8 Apr 2015 23:28:41 +0200
Subject: [PATCH 264/345] Fix problem with 1 MiB too large volume when
 combining gpt and grub

---
 bootstrapvz/base/fs/partitionmaps/gpt.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py
index 3113ceb..44f8385 100644
--- a/bootstrapvz/base/fs/partitionmaps/gpt.py
+++ b/bootstrapvz/base/fs/partitionmaps/gpt.py
@@ -27,7 +27,7 @@ class GPTPartitionMap(AbstractPartitionMap):
 			# If we are using the grub bootloader we need to create an unformatted partition
 			# at the beginning of the map. Its size is 1007kb, which seems to be chosen so that
 			# primary gpt + grub = 1024KiB
-			# The 34 sectors for the primary gpt will be subtracted later on
+			# The 1 MiB will be subtracted later on, once we know what the subsequent partition is
 			from ..partitions.unformatted import UnformattedPartition
 			self.grub_boot = UnformattedPartition(Sectors('1MiB', sector_size), last_partition())
 			self.partitions.append(self.grub_boot)
@@ -63,6 +63,15 @@ class GPTPartitionMap(AbstractPartitionMap):
 			self.root.size -= partition_gap
 		self.partitions.append(self.root)
 
+		if hasattr(self, 'grub_boot'):
+			# Mark the grub partition as a bios_grub partition
+			self.grub_boot.flags.append('bios_grub')
+			# Subtract the grub partition size from the subsequent partition
+			self.partitions[1].size -= self.grub_boot.size
+		else:
+			# Not using grub, mark the boot partition or root as bootable
+			getattr(self, 'boot', self.root).flags.append('legacy_boot')
+
 		# The first and last 34 sectors are reserved for the primary/secondary GPT
 		primary_gpt_size = Sectors(34, sector_size)
 		self.partitions[0].pad_start += primary_gpt_size
@@ -72,13 +81,6 @@ class GPTPartitionMap(AbstractPartitionMap):
 		self.partitions[-1].pad_end += secondary_gpt_size
 		self.partitions[-1].size -= secondary_gpt_size
 
-		if hasattr(self, 'grub_boot'):
-			# Mark the grub partition as a bios_grub partition
-			self.grub_boot.flags.append('bios_grub')
-		else:
-			# Not using grub, mark the boot partition or root as bootable
-			getattr(self, 'boot', self.root).flags.append('legacy_boot')
-
 		super(GPTPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):

From 97ad69df5ec3a8de0e1997cd583dfd1d2fd7ac3a Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 17:49:29 +0200
Subject: [PATCH 265/345] extlinux now works with gpt on hvm instances

---
 bootstrapvz/providers/ec2/__init__.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index 2d13675..bb14977 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -51,9 +51,6 @@ def validate_manifest(data, validator, error):
 	if virtualization == 'hvm':
 		if backing != 'ebs':
 			error('HVM AMIs currently only work when they are EBS backed', ['volume', 'backing'])
-		if bootloader == 'extlinux' and partition_type not in ['none', 'msdos']:
-			error('HVM AMIs booted with extlinux currently work with unpartitioned or msdos partitioned volumes',
-			      ['volume', 'partitions', 'type'])
 
 	if backing == 's3':
 		if partition_type != 'none':

From 9ad79cbf60a1a1206c43c9b5c8675ee7bd529de0 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 18:18:49 +0200
Subject: [PATCH 266/345] Remove some unnecessary dependencies across phases

---
 bootstrapvz/common/tasks/bootstrap.py         | 2 --
 bootstrapvz/plugins/google_cloud_sdk/tasks.py | 2 --
 bootstrapvz/plugins/pip_install/tasks.py      | 4 ----
 3 files changed, 8 deletions(-)

diff --git a/bootstrapvz/common/tasks/bootstrap.py b/bootstrapvz/common/tasks/bootstrap.py
index be9803d..e656f63 100644
--- a/bootstrapvz/common/tasks/bootstrap.py
+++ b/bootstrapvz/common/tasks/bootstrap.py
@@ -80,7 +80,6 @@ class Bootstrap(Task):
 class IncludePackagesInBootstrap(Task):
 	description = 'Add packages in the bootstrap phase'
 	phase = phases.preparation
-	successors = [Bootstrap]
 
 	@classmethod
 	def run(cls, info):
@@ -92,7 +91,6 @@ class IncludePackagesInBootstrap(Task):
 class ExcludePackagesInBootstrap(Task):
 	description = 'Remove packages from bootstrap phase'
 	phase = phases.preparation
-	successors = [Bootstrap]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/google_cloud_sdk/tasks.py b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
index c59cf97..33c21a2 100644
--- a/bootstrapvz/plugins/google_cloud_sdk/tasks.py
+++ b/bootstrapvz/plugins/google_cloud_sdk/tasks.py
@@ -1,6 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import workspace
 from bootstrapvz.common.tools import log_check_call
 import os
 
@@ -66,7 +65,6 @@ class InstallCloudSDK(Task):
 class RemoveCloudSDKTarball(Task):
 	description = 'Remove tarball for Cloud SDK'
 	phase = phases.system_cleaning
-	successors = [workspace.DeleteWorkspace]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/pip_install/tasks.py b/bootstrapvz/plugins/pip_install/tasks.py
index a4e6eb5..98b6bfa 100644
--- a/bootstrapvz/plugins/pip_install/tasks.py
+++ b/bootstrapvz/plugins/pip_install/tasks.py
@@ -1,7 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import network
-from bootstrapvz.common.tasks import packages
 from bootstrapvz.common.tasks import apt
 
 
@@ -9,7 +7,6 @@ class AddPipPackage(Task):
 	description = 'Adding `pip\' and Co. to the image packages'
 	phase = phases.preparation
 	predecessors = [apt.AddDefaultSources]
-	successors = [packages.InstallPackages]
 
 	@classmethod
 	def run(cls, info):
@@ -20,7 +17,6 @@ class AddPipPackage(Task):
 class PipInstallCommand(Task):
 	description = 'Install python packages from pypi with pip'
 	phase = phases.system_modification
-	successors = [network.RemoveDNSInfo]
 
 	@classmethod
 	def run(cls, info):

From caf5d6af15bdd1e867535ac9587d933cff7677c9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 18:19:13 +0200
Subject: [PATCH 267/345] Add salt dependencies in the right phase

---
 bootstrapvz/plugins/salt/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/salt/tasks.py b/bootstrapvz/plugins/salt/tasks.py
index d44cbc3..6c2d3f1 100644
--- a/bootstrapvz/plugins/salt/tasks.py
+++ b/bootstrapvz/plugins/salt/tasks.py
@@ -10,7 +10,7 @@ import urllib
 
 class InstallSaltDependencies(Task):
 	description = 'Add depended packages for salt-minion'
-	phase = phases.package_installation
+	phase = phases.preparation
 	predecessors = [apt.AddDefaultSources]
 
 	@classmethod

From 9101b2e9eca6cda978a9ed66826abfc239f225e1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 18:23:08 +0200
Subject: [PATCH 268/345] Add a little helpful not to tasklist ordering warning

---
 bootstrapvz/base/tasklist.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index c196d55..7617a15 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -171,7 +171,8 @@ def check_ordering(task):
 			raise TaskListError(msg)
 		if task.phase < successor.phase:
 			log.warn("The task {task} is specified as running before {other} "
-			         "although its phase '{phase}' already lies before the phase '{other_phase}'"
+			         "although its phase '{phase}' already lies before the phase '{other_phase}' "
+			         "(or the task has been placed in the wrong phase)"
 			         .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
 	for predecessor in task.predecessors:
 		# Run through all successors and throw an error if the phase of the task
@@ -183,7 +184,8 @@ def check_ordering(task):
 			raise TaskListError(msg)
 		if task.phase > predecessor.phase:
 			log.warn("The task {task} is specified as running after {other} "
-			         "although its phase '{phase}' already lies after the phase '{other_phase}'"
+			         "although its phase '{phase}' already lies after the phase '{other_phase}' "
+			         "(or the task has been placed in the wrong phase)"
 			         .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
 
 

From f4851ac06d4becf89ba2bee07dca3da3862b8aeb Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 19:26:04 +0200
Subject: [PATCH 269/345] fix typo in comment

---
 bootstrapvz/base/fs/partitions/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 3e5cbab..fbf9220 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -94,5 +94,5 @@ class BasePartition(AbstractPartition):
 		self.device_path = e.device_path
 
 	def _before_unmap(self, e):
-		# When unmapped, the device_path ifnromation becomes invalid, so we delete it
+		# When unmapped, the device_path information becomes invalid, so we delete it
 		self.device_path = None

From 3c5d385a6975fabaa7429a8d35e2f0cc7e560da9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 20:01:30 +0200
Subject: [PATCH 270/345] Fix grub boot on ec2 hvm jessie

---
 bootstrapvz/base/fs/partitions/base.py | 30 ++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index fbf9220..6837278 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -31,6 +31,8 @@ class BasePartition(AbstractPartition):
 		self.previous = previous
 		# List of flags that parted should put on the partition
 		self.flags = []
+		# Path to symlink in /dev/disk/by-uuid (manually maintained by this class)
+		self.disk_by_uuid_path = None
 		super(BasePartition, self).__init__(size, filesystem, format_command)
 
 	def create(self, volume):
@@ -71,6 +73,25 @@ class BasePartition(AbstractPartition):
 		"""
 		self.fsm.map(device_path=device_path)
 
+	def link_uuid(self):
+		# /lib/udev/rules.d/60-kpartx.rules does not create symlinks in /dev/disk/by-{uuid,label}
+		# This patch would fix that: http://www.redhat.com/archives/dm-devel/2013-July/msg00080.html
+		# For now we just do the uuid part ourselves.
+		# This is mainly to fix a problem in update-grub where /etc/grub.d/10_linux
+		# checks if the $GRUB_DEVICE_UUID exists in /dev/disk/by-uuid and falls
+		# back to $GRUB_DEVICE if it doesn't.
+		# $GRUB_DEVICE is /dev/mapper/xvd{f,g...}# (on ec2), opposed to /dev/xvda# when booting.
+		# Creating the symlink ensures that grub consistently uses
+		# $GRUB_DEVICE_UUID when creating /boot/grub/grub.cfg
+		import os
+		self.disk_by_uuid_path = os.path.join('/dev/disk/by-uuid', self.get_uuid())
+		os.symlink(self.device_path, self.disk_by_uuid_path)
+
+	def unlink_uuid(self):
+		import os
+		os.remove(self.disk_by_uuid_path)
+		self.disk_by_uuid_path = None
+
 	def _before_create(self, e):
 		"""Creates the partition
 		"""
@@ -92,7 +113,16 @@ class BasePartition(AbstractPartition):
 	def _before_map(self, e):
 		# Set the device path
 		self.device_path = e.device_path
+		if e.src == 'unmapped_fmt':
+			# Only link the uuid if the partition is formatted
+			self.link_uuid()
+
+	def _after_format(self, e):
+		# We do this after formatting because there otherwise would be no UUID
+		self.link_uuid()
 
 	def _before_unmap(self, e):
 		# When unmapped, the device_path information becomes invalid, so we delete it
 		self.device_path = None
+		if e.src == 'formatted':
+			self.unlink_uuid()

From 0b49943383cbeabec1735bab250fa15feba6829d Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 20:38:15 +0200
Subject: [PATCH 271/345] Add some logging to the test harness

---
 tests/integration/tools/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index f150602..052dcbd 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -1,5 +1,7 @@
 from contextlib import contextmanager
 from bootstrapvz.remote import register_deserialization_handlers
+import logging
+log = logging.getLogger(__name__)
 
 # Register deserialization handlers for objects
 # that will pass between server and client
@@ -18,9 +20,12 @@ def boot_manifest(manifest_data, boot_vars={}):
 	manifest = Manifest(data=manifest_data)
 
 	bootstrap_info = None
+	log.info('Connecting to build server')
 	with build_server.connect() as connection:
+		log.info('Building manifest')
 		bootstrap_info = connection.run(manifest)
 
+	log.info('Creating and booting instance')
 	from ..images import initialize_image
 	image = initialize_image(manifest, build_server, bootstrap_info)
 	try:

From 287c5441cea141b4d7ae9e0495c5f417b542b7ce Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 20:39:06 +0200
Subject: [PATCH 272/345] All ebs backed hvm image combinations are now
 working!

---
 tests/integration/ec2_ebs_hvm.py | 129 +++++++++++++++++++++++++++++++
 1 file changed, 129 insertions(+)
 create mode 100644 tests/integration/ec2_ebs_hvm.py

diff --git a/tests/integration/ec2_ebs_hvm.py b/tests/integration/ec2_ebs_hvm.py
new file mode 100644
index 0000000..5cda8cd
--- /dev/null
+++ b/tests/integration/ec2_ebs_hvm.py
@@ -0,0 +1,129 @@
+from manifests import merge_manifest_data
+from tools import boot_manifest
+
+partials = {'ebs_hvm': '''
+provider:
+  name: ec2
+  virtualization: hvm
+volume: {backing: ebs}
+''',
+            'extlinux': 'system: {bootloader: extlinux}',
+            'grub': 'system: {bootloader: grub}',
+            }
+
+
+def test_unpartitioned_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_extlinux_oldstable():
+	std_partials = ['base', 'oldstable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_unpartitioned_extlinux_stable():
+	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_extlinux_stable():
+	std_partials = ['base', 'stable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_extlinux_stable():
+	std_partials = ['base', 'stable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_grub_stable():
+	std_partials = ['base', 'stable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['grub']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_grub_stable():
+	std_partials = ['base', 'stable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['grub']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_unpartitioned_extlinux_unstable():
+	std_partials = ['base', 'unstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_extlinux_unstable():
+	std_partials = ['base', 'unstable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_extlinux_unstable():
+	std_partials = ['base', 'unstable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['extlinux']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_msdos_grub_unstable():
+	std_partials = ['base', 'unstable64', 'msdos', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['grub']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_gpt_grub_unstable():
+	std_partials = ['base', 'unstable64', 'gpt', 'single_partition', 'root_password']
+	custom_partials = [partials['ebs_hvm'], partials['grub']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 't2.micro'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)

From 6726df1c9105a3275c0784535f9062f090eeeed8 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 9 Apr 2015 23:20:56 +0200
Subject: [PATCH 273/345] Simplify test harness architecture by reducing the
 amount of interfacing between generic and provider specific code

---
 tests/integration/images/__init__.py          | 11 ----
 tests/integration/images/ami.py               | 38 -----------
 tests/integration/images/image.py             |  6 --
 tests/integration/images/vbox.py              | 64 -------------------
 tests/integration/instances/instance.py       | 16 -----
 .../{instances => providers}/__init__.py      |  0
 .../ec2.py => providers/ec2/__init__.py}      | 33 ++++++++--
 tests/integration/providers/ec2/images.py     | 19 ++++++
 .../providers/virtualbox/__init__.py          | 55 ++++++++++++++++
 .../integration/providers/virtualbox/image.py | 27 ++++++++
 .../virtualbox/instance.py}                   | 32 +++-------
 tests/integration/tools/__init__.py           | 11 ++--
 12 files changed, 144 insertions(+), 168 deletions(-)
 delete mode 100644 tests/integration/images/__init__.py
 delete mode 100644 tests/integration/images/ami.py
 delete mode 100644 tests/integration/images/image.py
 delete mode 100644 tests/integration/images/vbox.py
 delete mode 100644 tests/integration/instances/instance.py
 rename tests/integration/{instances => providers}/__init__.py (100%)
 rename tests/integration/{instances/ec2.py => providers/ec2/__init__.py} (58%)
 create mode 100644 tests/integration/providers/ec2/images.py
 create mode 100644 tests/integration/providers/virtualbox/__init__.py
 create mode 100644 tests/integration/providers/virtualbox/image.py
 rename tests/integration/{instances/vbox.py => providers/virtualbox/instance.py} (88%)

diff --git a/tests/integration/images/__init__.py b/tests/integration/images/__init__.py
deleted file mode 100644
index 313e098..0000000
--- a/tests/integration/images/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-def initialize_image(manifest, build_server, bootstrap_info):
-	if manifest.provider['name'] == 'virtualbox':
-		import vbox
-		return vbox.initialize_image(manifest, build_server, bootstrap_info)
-	if manifest.provider['name'] == 'ec2':
-		import ami
-		credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
-		               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
-		return ami.initialize_image(manifest, credentials, bootstrap_info)
diff --git a/tests/integration/images/ami.py b/tests/integration/images/ami.py
deleted file mode 100644
index 895bf9c..0000000
--- a/tests/integration/images/ami.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from image import Image
-import logging
-from contextlib import contextmanager
-log = logging.getLogger(__name__)
-
-
-def initialize_image(manifest, credentials, bootstrap_info):
-	image = AmazonMachineImage(manifest, bootstrap_info._ec2['image'],
-	                           bootstrap_info._ec2['region'], credentials)
-	return image
-
-
-class AmazonMachineImage(Image):
-
-	def __init__(self, manifest, image_id, region, credentials):
-		super(AmazonMachineImage, self).__init__(manifest)
-
-		from boto.ec2 import connect_to_region as ec2_connect
-		self.ec2_connection = ec2_connect(region, aws_access_key_id=credentials['access-key'],
-		                                  aws_secret_access_key=credentials['secret-key'])
-		from boto.vpc import connect_to_region as vpc_connect
-		self.vpc_connection = vpc_connect(region, aws_access_key_id=credentials['access-key'],
-		                                  aws_secret_access_key=credentials['secret-key'])
-
-		self.ami = self.ec2_connection.get_image(image_id)
-
-	def destroy(self):
-		log.debug('Deleting AMI')
-		self.ami.deregister()
-		for device, block_device_type in self.ami.block_device_mapping.items():
-			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
-		del self.ami
-
-	@contextmanager
-	def get_instance(self, instance_type):
-		from ..instances.ec2 import boot_image
-		with boot_image(self.ami, instance_type, self.ec2_connection, self.vpc_connection) as instance:
-			yield instance
diff --git a/tests/integration/images/image.py b/tests/integration/images/image.py
deleted file mode 100644
index 5260afa..0000000
--- a/tests/integration/images/image.py
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-class Image(object):
-
-	def __init__(self, manifest):
-		self.manifest = manifest
diff --git a/tests/integration/images/vbox.py b/tests/integration/images/vbox.py
deleted file mode 100644
index f8d7803..0000000
--- a/tests/integration/images/vbox.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from image import Image
-import virtualbox
-import logging
-from contextlib import contextmanager
-log = logging.getLogger(__name__)
-
-
-def initialize_image(manifest, build_server, bootstrap_info):
-	from bootstrapvz.remote.build_servers.local import LocalBuildServer
-	if isinstance(build_server, LocalBuildServer):
-		image_path = bootstrap_info.volume.image_path
-	else:
-		import tempfile
-		handle, image_path = tempfile.mkstemp()
-		import os
-		os.close(handle)
-		try:
-			build_server.download(bootstrap_info.volume.image_path, image_path)
-		except (Exception, KeyboardInterrupt):
-			os.remove(image_path)
-			raise
-		finally:
-			build_server.delete(bootstrap_info.volume.image_path)
-	image = VirtualBoxImage(manifest, image_path)
-	return image
-
-
-class VirtualBoxImage(Image):
-
-	def __init__(self, manifest, image_path):
-		super(VirtualBoxImage, self).__init__(manifest)
-		self.image_path = image_path
-		self.vbox = virtualbox.VirtualBox()
-
-	def open(self):
-		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
-		self.medium = self.vbox.open_medium(self.image_path,  # location
-		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
-		                                    virtualbox.library.AccessMode.read_only,  # access_mode
-		                                    False)  # force_new_uuid
-
-	def close(self):
-		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
-		self.medium.close()
-
-	def destroy(self):
-		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
-		import os
-		os.remove(self.image_path)
-		del self.image_path
-
-	@contextmanager
-	def get_instance(self):
-		import hashlib
-		image_hash = hashlib.sha1(self.image_path).hexdigest()
-		name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
-
-		self.open()
-		try:
-			from ..instances.vbox import boot_image
-			with boot_image(name, self) as instance:
-				yield instance
-		finally:
-			self.close()
diff --git a/tests/integration/instances/instance.py b/tests/integration/instances/instance.py
deleted file mode 100644
index 049fc0f..0000000
--- a/tests/integration/instances/instance.py
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-class Instance(object):
-
-	def __init__(self, name, image):
-		self.name = name
-		self.image = image
-
-	def boot(self):
-		pass
-
-	def shutdown(self):
-		pass
-
-	def destroy(self):
-		pass
diff --git a/tests/integration/instances/__init__.py b/tests/integration/providers/__init__.py
similarity index 100%
rename from tests/integration/instances/__init__.py
rename to tests/integration/providers/__init__.py
diff --git a/tests/integration/instances/ec2.py b/tests/integration/providers/ec2/__init__.py
similarity index 58%
rename from tests/integration/instances/ec2.py
rename to tests/integration/providers/ec2/__init__.py
index f658856..62ef7ce 100644
--- a/tests/integration/instances/ec2.py
+++ b/tests/integration/providers/ec2/__init__.py
@@ -1,11 +1,36 @@
 from contextlib import contextmanager
-from ..tools import waituntil
+from tests.integration.tools import waituntil
 import logging
 log = logging.getLogger(__name__)
 
 
 @contextmanager
-def boot_image(image, instance_type, ec2_connection, vpc_connection):
+def boot_image(manifest, build_server, bootstrap_info, instance_type=None):
+
+	credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
+	               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
+	from boto.ec2 import connect_to_region as ec2_connect
+	ec2_connection = ec2_connect(bootstrap_info._ec2['region'],
+	                             aws_access_key_id=credentials['access-key'],
+	                             aws_secret_access_key=credentials['secret-key'])
+	from boto.vpc import connect_to_region as vpc_connect
+	vpc_connection = vpc_connect(bootstrap_info._ec2['region'],
+	                             aws_access_key_id=credentials['access-key'],
+	                             aws_secret_access_key=credentials['secret-key'])
+
+	if manifest.volume['backing'] == 'ebs':
+		from images import EBSImage
+		image = EBSImage(bootstrap_info._ec2['image'], ec2_connection)
+
+	try:
+		with run_instance(image, instance_type, ec2_connection, vpc_connection) as instance:
+			yield instance
+	finally:
+		image.destroy()
+
+
+@contextmanager
+def run_instance(image, instance_type, ec2_connection, vpc_connection):
 
 	with create_env(ec2_connection, vpc_connection) as boot_env:
 
@@ -18,8 +43,8 @@ def boot_image(image, instance_type, ec2_connection, vpc_connection):
 		instance = None
 		try:
 			log.debug('Booting ec2 instance')
-			reservation = image.run(instance_type=instance_type,
-			                        subnet_id=boot_env['subnet_id'])
+			reservation = image.ami.run(instance_type=instance_type,
+			                            subnet_id=boot_env['subnet_id'])
 			[instance] = reservation.instances
 			instance.add_tag('Name', 'bootstrap-vz test instance')
 
diff --git a/tests/integration/providers/ec2/images.py b/tests/integration/providers/ec2/images.py
new file mode 100644
index 0000000..b2932fc
--- /dev/null
+++ b/tests/integration/providers/ec2/images.py
@@ -0,0 +1,19 @@
+import logging
+log = logging.getLogger(__name__)
+
+
+class AmazonMachineImage(object):
+
+	def __init__(self, image_id, ec2_connection):
+		self.ec2_connection = ec2_connection
+		self.ami = self.ec2_connection.get_image(image_id)
+
+
+class EBSImage(AmazonMachineImage):
+
+	def destroy(self):
+		log.debug('Deleting AMI')
+		self.ami.deregister()
+		for device, block_device_type in self.ami.block_device_mapping.items():
+			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
+		del self.ami
diff --git a/tests/integration/providers/virtualbox/__init__.py b/tests/integration/providers/virtualbox/__init__.py
new file mode 100644
index 0000000..f94fb26
--- /dev/null
+++ b/tests/integration/providers/virtualbox/__init__.py
@@ -0,0 +1,55 @@
+from contextlib import contextmanager
+import logging
+log = logging.getLogger(__name__)
+
+
+@contextmanager
+def boot_image(manifest, build_server, bootstrap_info):
+	from bootstrapvz.remote.build_servers.local import LocalBuildServer
+	if isinstance(build_server, LocalBuildServer):
+		image_path = bootstrap_info.volume.image_path
+	else:
+		import tempfile
+		handle, image_path = tempfile.mkstemp()
+		import os
+		os.close(handle)
+		try:
+			build_server.download(bootstrap_info.volume.image_path, image_path)
+		except (Exception, KeyboardInterrupt):
+			os.remove(image_path)
+			raise
+		finally:
+			build_server.delete(bootstrap_info.volume.image_path)
+
+	from image import VirtualBoxImage
+	image = VirtualBoxImage(image_path)
+
+	import hashlib
+	image_hash = hashlib.sha1(image_path).hexdigest()
+	instance_name = 'bootstrap-vz-{hash}'.format(hash=image_hash[:8])
+
+	try:
+		image.open()
+		try:
+			with run_instance(image, instance_name, manifest) as instance:
+				yield instance
+		finally:
+			image.close()
+	finally:
+		image.destroy()
+
+
+@contextmanager
+def run_instance(image, instance_name, manifest):
+	from instance import VirtualBoxInstance
+	instance = VirtualBoxInstance(image, instance_name,
+	                              manifest.system['architecture'], manifest.system['release'])
+	try:
+		instance.create()
+		try:
+			instance.boot()
+			yield instance
+		finally:
+			instance.shutdown()
+	finally:
+		instance.destroy()
diff --git a/tests/integration/providers/virtualbox/image.py b/tests/integration/providers/virtualbox/image.py
new file mode 100644
index 0000000..7d51eb8
--- /dev/null
+++ b/tests/integration/providers/virtualbox/image.py
@@ -0,0 +1,27 @@
+import virtualbox
+import logging
+log = logging.getLogger(__name__)
+
+
+class VirtualBoxImage(object):
+
+	def __init__(self, image_path):
+		self.image_path = image_path
+		self.vbox = virtualbox.VirtualBox()
+
+	def open(self):
+		log.debug('Opening vbox medium `{path}\''.format(path=self.image_path))
+		self.medium = self.vbox.open_medium(self.image_path,  # location
+		                                    virtualbox.library.DeviceType.hard_disk,  # device_type
+		                                    virtualbox.library.AccessMode.read_only,  # access_mode
+		                                    False)  # force_new_uuid
+
+	def close(self):
+		log.debug('Closing vbox medium `{path}\''.format(path=self.image_path))
+		self.medium.close()
+
+	def destroy(self):
+		log.debug('Deleting vbox image `{path}\''.format(path=self.image_path))
+		import os
+		os.remove(self.image_path)
+		del self.image_path
diff --git a/tests/integration/instances/vbox.py b/tests/integration/providers/virtualbox/instance.py
similarity index 88%
rename from tests/integration/instances/vbox.py
rename to tests/integration/providers/virtualbox/instance.py
index 1ab9927..fa520e6 100644
--- a/tests/integration/instances/vbox.py
+++ b/tests/integration/providers/virtualbox/instance.py
@@ -1,32 +1,20 @@
-from instance import Instance
 import virtualbox
 from contextlib import contextmanager
-from ..tools import waituntil
+from tests.integration.tools import waituntil
 import logging
 log = logging.getLogger(__name__)
 
 
-@contextmanager
-def boot_image(name, image):
-	instance = VirtualBoxInstance(name, image)
-	try:
-		instance.create()
-		try:
-			instance.boot()
-			yield instance
-		finally:
-			instance.shutdown()
-	finally:
-		instance.destroy()
-
-
-class VirtualBoxInstance(Instance):
+class VirtualBoxInstance(object):
 
 	cpus = 1
 	memory = 256
 
-	def __init__(self, name, image):
-		super(VirtualBoxInstance, self).__init__(name, image)
+	def __init__(self, image, name, arch, release):
+		self.image = image
+		self.name = name
+		self.arch = arch
+		self.release = release
 		self.vbox = virtualbox.VirtualBox()
 		manager = virtualbox.Manager()
 		self.session = manager.get_session()
@@ -35,7 +23,7 @@ class VirtualBoxInstance(Instance):
 		log.debug('Creating vbox machine `{name}\''.format(name=self.name))
 		# create machine
 		os_type = {'x86': 'Debian',
-		           'amd64': 'Debian_64'}.get(self.image.manifest.system['architecture'])
+		           'amd64': 'Debian_64'}.get(self.arch)
 		self.machine = self.vbox.create_machine(settings_file='', name=self.name,
 		                                        groups=[], os_type_id=os_type, flags='')
 		self.machine.cpu_count = self.cpus
@@ -71,12 +59,12 @@ class VirtualBoxInstance(Instance):
 	def boot(self):
 		log.debug('Booting vbox machine `{name}\''.format(name=self.name))
 		self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1)
-		from ..tools import read_from_socket
+		from tests.integration.tools import read_from_socket
 		# Gotta figure out a more reliable way to check when the system is done booting.
 		# Maybe bootstrapped unit test images should have a startup script that issues
 		# a callback to the host.
 		from bootstrapvz.common.tools import get_codename
-		if get_codename(self.image.manifest.system['release']) in ['squeeze', 'wheezy']:
+		if get_codename(self.release) in ['squeeze', 'wheezy']:
 			termination_string = 'INIT: Entering runlevel: 2'
 		else:
 			termination_string = 'Debian GNU/Linux'
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index 052dcbd..d2c9945 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -26,13 +26,10 @@ def boot_manifest(manifest_data, boot_vars={}):
 		bootstrap_info = connection.run(manifest)
 
 	log.info('Creating and booting instance')
-	from ..images import initialize_image
-	image = initialize_image(manifest, build_server, bootstrap_info)
-	try:
-		with image.get_instance(**boot_vars) as instance:
-			yield instance
-	finally:
-		image.destroy()
+	import importlib
+	provider_module = importlib.import_module('tests.integration.providers.' + manifest.provider['name'])
+	with provider_module.boot_image(manifest, build_server, bootstrap_info, **boot_vars) as instance:
+		yield instance
 
 
 def waituntil(predicate, timeout=5, interval=0.05):

From 53c9eb572e84f855a53f8a584c5de05d8cc50416 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 10 Apr 2015 00:17:00 +0200
Subject: [PATCH 274/345] Support testing of EC2 S3 backed instances

---
 .../remote/build_servers/build_server.py      |  3 +++
 tests/integration/providers/ec2/__init__.py   | 25 ++++++++++++++++++
 tests/integration/providers/ec2/images.py     |  8 ++++++
 tests/integration/tools/__init__.py           | 26 ++++++++++++-------
 4 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/bootstrapvz/remote/build_servers/build_server.py b/bootstrapvz/remote/build_servers/build_server.py
index 59c51c1..0cdf6a3 100644
--- a/bootstrapvz/remote/build_servers/build_server.py
+++ b/bootstrapvz/remote/build_servers/build_server.py
@@ -20,4 +20,7 @@ class BuildServer(object):
 			for key in ['access-key', 'secret-key', 'certificate', 'private-key', 'user-id']:
 				if key in self.build_settings['ec2-credentials']:
 					manifest_data['provider']['credentials'][key] = self.build_settings['ec2-credentials'][key]
+		if 's3-region' in self.build_settings and manifest_data['volume']['backing'] == 's3':
+			if 'region' not in manifest_data['image']:
+				manifest_data['image']['region'] = self.build_settings['s3-region']
 		return manifest_data
diff --git a/tests/integration/providers/ec2/__init__.py b/tests/integration/providers/ec2/__init__.py
index 62ef7ce..eda4b1f 100644
--- a/tests/integration/providers/ec2/__init__.py
+++ b/tests/integration/providers/ec2/__init__.py
@@ -4,6 +4,28 @@ import logging
 log = logging.getLogger(__name__)
 
 
+@contextmanager
+def prepare_bootstrap(manifest, build_server):
+	if manifest.volume['backing'] == 's3':
+		credentials = {'access-key': build_server.build_settings['ec2-credentials']['access-key'],
+		               'secret-key': build_server.build_settings['ec2-credentials']['secret-key']}
+		from boto.s3 import connect_to_region as s3_connect
+		s3_connection = s3_connect(manifest.image['region'],
+		                           aws_access_key_id=credentials['access-key'],
+		                           aws_secret_access_key=credentials['secret-key'])
+		log.debug('Creating S3 bucket')
+		bucket = s3_connection.create_bucket(manifest.image['bucket'], location=manifest.image['region'])
+		try:
+			yield
+		finally:
+			log.debug('Deleting S3 bucket')
+			for item in bucket.list():
+				bucket.delete_key(item.key)
+			s3_connection.delete_bucket(manifest.image['bucket'])
+	else:
+			yield
+
+
 @contextmanager
 def boot_image(manifest, build_server, bootstrap_info, instance_type=None):
 
@@ -21,6 +43,9 @@ def boot_image(manifest, build_server, bootstrap_info, instance_type=None):
 	if manifest.volume['backing'] == 'ebs':
 		from images import EBSImage
 		image = EBSImage(bootstrap_info._ec2['image'], ec2_connection)
+	if manifest.volume['backing'] == 's3':
+		from images import S3Image
+		image = S3Image(bootstrap_info._ec2['image'], ec2_connection)
 
 	try:
 		with run_instance(image, instance_type, ec2_connection, vpc_connection) as instance:
diff --git a/tests/integration/providers/ec2/images.py b/tests/integration/providers/ec2/images.py
index b2932fc..6375f8a 100644
--- a/tests/integration/providers/ec2/images.py
+++ b/tests/integration/providers/ec2/images.py
@@ -17,3 +17,11 @@ class EBSImage(AmazonMachineImage):
 		for device, block_device_type in self.ami.block_device_mapping.items():
 			self.ec2_connection.delete_snapshot(block_device_type.snapshot_id)
 		del self.ami
+
+
+class S3Image(AmazonMachineImage):
+
+	def destroy(self):
+		log.debug('Deleting AMI')
+		self.ami.deregister()
+		del self.ami
diff --git a/tests/integration/tools/__init__.py b/tests/integration/tools/__init__.py
index d2c9945..1e82230 100644
--- a/tests/integration/tools/__init__.py
+++ b/tests/integration/tools/__init__.py
@@ -19,17 +19,20 @@ def boot_manifest(manifest_data, boot_vars={}):
 	from bootstrapvz.base.manifest import Manifest
 	manifest = Manifest(data=manifest_data)
 
-	bootstrap_info = None
-	log.info('Connecting to build server')
-	with build_server.connect() as connection:
-		log.info('Building manifest')
-		bootstrap_info = connection.run(manifest)
-
-	log.info('Creating and booting instance')
 	import importlib
 	provider_module = importlib.import_module('tests.integration.providers.' + manifest.provider['name'])
-	with provider_module.boot_image(manifest, build_server, bootstrap_info, **boot_vars) as instance:
-		yield instance
+
+	prepare_bootstrap = getattr(provider_module, 'prepare_bootstrap', noop)
+	with prepare_bootstrap(manifest, build_server):
+		bootstrap_info = None
+		log.info('Connecting to build server')
+		with build_server.connect() as connection:
+			log.info('Building manifest')
+			bootstrap_info = connection.run(manifest)
+
+		log.info('Creating and booting instance')
+		with provider_module.boot_image(manifest, build_server, bootstrap_info, **boot_vars) as instance:
+			yield instance
 
 
 def waituntil(predicate, timeout=5, interval=0.05):
@@ -79,3 +82,8 @@ def read_from_socket(socket_path, termination_string, timeout, read_timeout=0.5)
 				raise SocketReadTimeout(msg)
 		console.close()
 		return output
+
+
+@contextmanager
+def noop(*args, **kwargs):
+	yield

From bfe58dd4a8caf3af0af0036b4f0a0dce108d21b9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Fri, 10 Apr 2015 01:11:54 +0200
Subject: [PATCH 275/345] Implement PR #201 by @jszwedko

---
 bootstrapvz/providers/ec2/tasks/ami.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 4e2da26..752d807 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -71,8 +71,7 @@ class UploadImage(Task):
 		                '--access-key', info.credentials['access-key'],
 		                '--secret-key', info.credentials['secret-key'],
 		                '--url', s3_url,
-		                '--region', info._ec2['region'],
-		                '--ec2cert', cert_ec2])
+		                '--region', info._ec2['region']])
 
 
 class RemoveBundle(Task):

From a4bd52bf9e32c90abaf2c73400baab713a865548 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Fri, 10 Apr 2015 14:40:47 +0200
Subject: [PATCH 276/345] Check for termination string in ec2 tests Pretty much
 copy&paste from vbox

---
 tests/integration/providers/ec2/__init__.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tests/integration/providers/ec2/__init__.py b/tests/integration/providers/ec2/__init__.py
index eda4b1f..f794463 100644
--- a/tests/integration/providers/ec2/__init__.py
+++ b/tests/integration/providers/ec2/__init__.py
@@ -48,14 +48,14 @@ def boot_image(manifest, build_server, bootstrap_info, instance_type=None):
 		image = S3Image(bootstrap_info._ec2['image'], ec2_connection)
 
 	try:
-		with run_instance(image, instance_type, ec2_connection, vpc_connection) as instance:
+		with run_instance(image, manifest, instance_type, ec2_connection, vpc_connection) as instance:
 			yield instance
 	finally:
 		image.destroy()
 
 
 @contextmanager
-def run_instance(image, instance_type, ec2_connection, vpc_connection):
+def run_instance(image, manifest, instance_type, ec2_connection, vpc_connection):
 
 	with create_env(ec2_connection, vpc_connection) as boot_env:
 
@@ -79,6 +79,19 @@ def run_instance(image, instance_type, ec2_connection, vpc_connection):
 			if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
 				raise EC2InstanceStartupException('Timeout while fetching console output')
 
+			from bootstrapvz.common.tools import get_codename
+			if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+				termination_string = 'INIT: Entering runlevel: 2'
+			else:
+				termination_string = 'Debian GNU/Linux'
+
+			console_output = instance.get_console_output().output
+			if termination_string not in console_output:
+				last_lines = '\n'.join(console_output.split('\n')[-50:])
+				message = ('The instance did not boot properly.\n'
+				           'Last 50 lines of console output:\n{output}'.format(output=last_lines))
+				raise EC2InstanceStartupException(message)
+
 			yield instance
 		finally:
 			if instance is not None:

From d81ac0972f2ff66081b348c651552ce9f71fc9de Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 13:54:39 +0200
Subject: [PATCH 277/345] Simpler checks for manifest support

---
 bootstrapvz/providers/ec2/__init__.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index bb14977..ec7a91e 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -48,17 +48,14 @@ def validate_manifest(data, validator, error):
 	if virtualization == 'pvm' and bootloader != 'pvgrub':
 		error('Paravirtualized AMIs only support pvgrub as a bootloader', ['system', 'bootloader'])
 
-	if virtualization == 'hvm':
-		if backing != 'ebs':
+	if backing != 'ebs' and virtualization == 'hvm':
 			error('HVM AMIs currently only work when they are EBS backed', ['volume', 'backing'])
 
-	if backing == 's3':
-		if partition_type != 'none':
+	if backing == 's3' and partition_type != 'none':
 			error('S3 backed AMIs currently only work with unpartitioned volumes', ['system', 'bootloader'])
 
-	if enhanced_networking == 'simple':
-		if virtualization != 'hvm':
-			error('Enhanced networking currently only works with HVM virtualization', ['provider', 'virtualization'])
+	if enhanced_networking == 'simple' and virtualization != 'hvm':
+			error('Enhanced networking only works with HVM virtualization', ['provider', 'virtualization'])
 
 
 def resolve_tasks(taskset, manifest):

From c4637af0cf40222a8e745b7dd2f2c0ce9c64b494 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 11 Apr 2015 13:54:59 +0200
Subject: [PATCH 278/345] Integration testing for S3 backed PVM instances

---
 tests/integration/ec2_s3_pvm.py | 41 +++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 tests/integration/ec2_s3_pvm.py

diff --git a/tests/integration/ec2_s3_pvm.py b/tests/integration/ec2_s3_pvm.py
new file mode 100644
index 0000000..da64923
--- /dev/null
+++ b/tests/integration/ec2_s3_pvm.py
@@ -0,0 +1,41 @@
+from manifests import merge_manifest_data
+from tools import boot_manifest
+import random
+
+s3_bucket_name = '{id:x}'.format(id=random.randrange(16 ** 16))
+partials = {'s3_pvm': '''
+provider:
+  name: ec2
+  virtualization: pvm
+image: {bucket: ''' + s3_bucket_name + '''}
+system: {bootloader: pvgrub}
+volume: {backing: s3}
+'''
+            }
+
+
+def test_unpartitioned_oldstable():
+	std_partials = ['base', 'oldstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['s3_pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 'm1.small'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_unpartitioned_stable():
+	std_partials = ['base', 'stable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['s3_pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 'm1.small'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)
+
+
+def test_unpartitioned_unstable():
+	std_partials = ['base', 'unstable64', 'unpartitioned', 'root_password']
+	custom_partials = [partials['s3_pvm']]
+	manifest_data = merge_manifest_data(std_partials, custom_partials)
+	boot_vars = {'instance_type': 'm1.small'}
+	with boot_manifest(manifest_data, boot_vars) as instance:
+		print(instance.get_console_output().output)

From 3c17c756a1a14b680c32232785b547a508c38aca Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 08:56:07 +0200
Subject: [PATCH 279/345] Fix typo in docs

---
 bootstrapvz/base/fs/partitions/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 6837278..3906948 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -69,7 +69,7 @@ class BasePartition(AbstractPartition):
 	def map(self, device_path):
 		"""Maps the partition to a device_path
 
-		:param str device_path: The device patht his partition should be mapped to
+		:param str device_path: The device path this partition should be mapped to
 		"""
 		self.fsm.map(device_path=device_path)
 

From d9adb293eb862c282ed43049ffe3e5696c6e7df3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 08:56:15 +0200
Subject: [PATCH 280/345] Simplify enhanced networking check

---
 bootstrapvz/providers/ec2/__init__.py  | 2 +-
 bootstrapvz/providers/ec2/tasks/ami.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py
index ec7a91e..23e3c45 100644
--- a/bootstrapvz/providers/ec2/__init__.py
+++ b/bootstrapvz/providers/ec2/__init__.py
@@ -105,7 +105,7 @@ def resolve_tasks(taskset, manifest):
 		                ])
 		taskset.discard(filesystem.FStab)
 
-	if 'enhanced_networking' in manifest.provider and manifest.provider['enhanced_networking'] == 'simple':
+	if manifest.provider.get('enhanced_networking', None) == 'simple':
 		taskset.update([kernel.AddDKMSPackages,
 		                tasks.network.InstallEnhancedNetworking,
 		                kernel.UpdateInitramfs])
diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py
index 752d807..4218294 100644
--- a/bootstrapvz/providers/ec2/tasks/ami.py
+++ b/bootstrapvz/providers/ec2/tasks/ami.py
@@ -121,7 +121,7 @@ class RegisterAMI(Task):
 			registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'],
 			                                                          info.manifest.system['architecture']])
 
-		if 'enhanced_networking' in info.manifest.provider and info.manifest.provider['enhanced_networking'] == 'simple':
+		if info.manifest.provider.get('enhanced_networking', None) == 'simple':
 			registration_params['sriov_net_support'] = 'simple'
 
 		info._ec2['image'] = info._ec2['connection'].register_image(**registration_params)

From f1e4d34eb97fb9c991f9642fd3a12e0fc58df200 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 09:40:09 +0200
Subject: [PATCH 281/345] Fix error mentioned in #152 (Pyro4 shouldn't be a
 dependency)

---
 bootstrapvz/base/tasklist.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py
index 7617a15..2af2b1d 100644
--- a/bootstrapvz/base/tasklist.py
+++ b/bootstrapvz/base/tasklist.py
@@ -117,7 +117,8 @@ def get_all_tasks():
 	# Get a generator that returns all classes in the package
 	import os.path
 	pkg_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
-	classes = get_all_classes(pkg_path, 'bootstrapvz.')
+	exclude_pkgs = ['bootstrapvz.base', 'bootstrapvz.remote']
+	classes = get_all_classes(pkg_path, 'bootstrapvz.', exclude_pkgs)
 
 	# lambda function to check whether a class is a task (excluding the superclass Task)
 	def is_task(obj):
@@ -126,11 +127,12 @@ def get_all_tasks():
 	return filter(is_task, classes)  # Only return classes that are tasks
 
 
-def get_all_classes(path=None, prefix=''):
+def get_all_classes(path=None, prefix='', excludes=[]):
 	""" Given a path to a package, this function retrieves all the classes in it
 
 	:param str path: Path to the package
 	:param str prefix: Name of the package followed by a dot
+	:param list excludes: List of str matching module names that should be ignored
 	:return: A generator that yields classes
 	:rtype: generator
 	:raises Exception: If a module cannot be inspected.
@@ -139,10 +141,13 @@ def get_all_classes(path=None, prefix=''):
 	import importlib
 	import inspect
 
-	def walk_error(module):
-		raise Exception('Unable to inspect module ' + module)
+	def walk_error(module_name):
+		if not any(map(lambda excl: module_name.startswith(excl), excludes)):
+			raise Exception('Unable to inspect module ' + module_name)
 	walker = pkgutil.walk_packages([path], prefix, walk_error)
 	for _, module_name, _ in walker:
+		if any(map(lambda excl: module_name.startswith(excl), excludes)):
+			continue
 		module = importlib.import_module(module_name)
 		classes = inspect.getmembers(module, inspect.isclass)
 		for class_name, obj in classes:

From a2cf28b054617a6773eeb2a2f8b609b185603fe9 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 10:13:16 +0200
Subject: [PATCH 282/345] Revert 03efa0f (#210) It seems the bug was only
 present on my old dev bootstrapping machine

This reverts commit 03efa0ffdf1ee0d3cf81905e6e4d9f5f6aa62143.
---
 bootstrapvz/base/fs/partitions/base.py | 30 --------------------------
 1 file changed, 30 deletions(-)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 3906948..67234e0 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -31,8 +31,6 @@ class BasePartition(AbstractPartition):
 		self.previous = previous
 		# List of flags that parted should put on the partition
 		self.flags = []
-		# Path to symlink in /dev/disk/by-uuid (manually maintained by this class)
-		self.disk_by_uuid_path = None
 		super(BasePartition, self).__init__(size, filesystem, format_command)
 
 	def create(self, volume):
@@ -73,25 +71,6 @@ class BasePartition(AbstractPartition):
 		"""
 		self.fsm.map(device_path=device_path)
 
-	def link_uuid(self):
-		# /lib/udev/rules.d/60-kpartx.rules does not create symlinks in /dev/disk/by-{uuid,label}
-		# This patch would fix that: http://www.redhat.com/archives/dm-devel/2013-July/msg00080.html
-		# For now we just do the uuid part ourselves.
-		# This is mainly to fix a problem in update-grub where /etc/grub.d/10_linux
-		# checks if the $GRUB_DEVICE_UUID exists in /dev/disk/by-uuid and falls
-		# back to $GRUB_DEVICE if it doesn't.
-		# $GRUB_DEVICE is /dev/mapper/xvd{f,g...}# (on ec2), opposed to /dev/xvda# when booting.
-		# Creating the symlink ensures that grub consistently uses
-		# $GRUB_DEVICE_UUID when creating /boot/grub/grub.cfg
-		import os
-		self.disk_by_uuid_path = os.path.join('/dev/disk/by-uuid', self.get_uuid())
-		os.symlink(self.device_path, self.disk_by_uuid_path)
-
-	def unlink_uuid(self):
-		import os
-		os.remove(self.disk_by_uuid_path)
-		self.disk_by_uuid_path = None
-
 	def _before_create(self, e):
 		"""Creates the partition
 		"""
@@ -113,16 +92,7 @@ class BasePartition(AbstractPartition):
 	def _before_map(self, e):
 		# Set the device path
 		self.device_path = e.device_path
-		if e.src == 'unmapped_fmt':
-			# Only link the uuid if the partition is formatted
-			self.link_uuid()
-
-	def _after_format(self, e):
-		# We do this after formatting because there otherwise would be no UUID
-		self.link_uuid()
 
 	def _before_unmap(self, e):
 		# When unmapped, the device_path information becomes invalid, so we delete it
 		self.device_path = None
-		if e.src == 'formatted':
-			self.unlink_uuid()

From f397ba5f0879f1e20b62e6f98efce844337538b7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 21:28:36 +0200
Subject: [PATCH 283/345] Yeah... 94bd938 really wasn't very smart Next time I
 should probably try to run tox after doing any kind of rename like this :-)

---
 tests/integration/{ec2_ebs_hvm.py => ec2_ebs_hvm_tests.py} | 0
 tests/integration/{ec2_ebs_pvm.py => ec2_ebs_pvm_tests.py} | 0
 tests/integration/{ec2_s3_pvm.py => ec2_s3_pvm_tests.py}   | 0
 tests/integration/{virtualbox.py => virtualbox_tests.py}   | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/integration/{ec2_ebs_hvm.py => ec2_ebs_hvm_tests.py} (100%)
 rename tests/integration/{ec2_ebs_pvm.py => ec2_ebs_pvm_tests.py} (100%)
 rename tests/integration/{ec2_s3_pvm.py => ec2_s3_pvm_tests.py} (100%)
 rename tests/integration/{virtualbox.py => virtualbox_tests.py} (100%)

diff --git a/tests/integration/ec2_ebs_hvm.py b/tests/integration/ec2_ebs_hvm_tests.py
similarity index 100%
rename from tests/integration/ec2_ebs_hvm.py
rename to tests/integration/ec2_ebs_hvm_tests.py
diff --git a/tests/integration/ec2_ebs_pvm.py b/tests/integration/ec2_ebs_pvm_tests.py
similarity index 100%
rename from tests/integration/ec2_ebs_pvm.py
rename to tests/integration/ec2_ebs_pvm_tests.py
diff --git a/tests/integration/ec2_s3_pvm.py b/tests/integration/ec2_s3_pvm_tests.py
similarity index 100%
rename from tests/integration/ec2_s3_pvm.py
rename to tests/integration/ec2_s3_pvm_tests.py
diff --git a/tests/integration/virtualbox.py b/tests/integration/virtualbox_tests.py
similarity index 100%
rename from tests/integration/virtualbox.py
rename to tests/integration/virtualbox_tests.py

From 5a46a0b1ab9c8a2f0f8bc04163932e014eac6b87 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 21:31:03 +0200
Subject: [PATCH 284/345] Yup, same goes for bytes.py....

---
 tests/unit/{bytes.py => bytes_tests.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/unit/{bytes.py => bytes_tests.py} (100%)

diff --git a/tests/unit/bytes.py b/tests/unit/bytes_tests.py
similarity index 100%
rename from tests/unit/bytes.py
rename to tests/unit/bytes_tests.py

From f210bfc13287d90498ef895ee7b23816cd4b2403 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sun, 12 Apr 2015 17:05:21 -0300
Subject: [PATCH 285/345] Add `nose-cov` dependency to tox integration tests

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index a8c18c3..a843467 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,6 +19,7 @@ commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusi
 [testenv:integration]
 deps =
     nose
+    nose-cov
     Pyro4 >= 4.30
     pyvbox >= 0.2.0
 commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --verbose {posargs:tests/integration}

From f9eb11b5cbcd40fda7b874d072af281880552f15 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 12 Apr 2015 22:48:11 +0200
Subject: [PATCH 286/345] Add html output to integration tests It's near
 impossible to scroll through the nosetest output...

---
 .gitignore | 1 +
 tox.ini    | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 3422ad6..dfcda55 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@
 /.coverage
 /.tox/
 /build-servers.yml
+/integration.html
diff --git a/tox.ini b/tox.ini
index a843467..d0a1ba1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,9 +20,10 @@ commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusi
 deps =
     nose
     nose-cov
+    nose-htmloutput
     Pyro4 >= 4.30
     pyvbox >= 0.2.0
-commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --verbose {posargs:tests/integration}
+commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --with-html --html-file=integration.html --verbose {posargs:tests/integration}
 
 [testenv:docs]
 changedir = docs

From 61dc50ae23cd834f607719b4001c868bbf991133 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:00:35 +0200
Subject: [PATCH 287/345] Add list of supported builds

---
 docs/index.rst            |  1 +
 docs/supported_builds.rst | 73 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 docs/supported_builds.rst

diff --git a/docs/index.rst b/docs/index.rst
index 4056a36..058b355 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -8,6 +8,7 @@ bootstrap-vz
    manifest
    providers/index
    plugins/index
+   supported_builds
    logging
    changelog
    developers/index
diff --git a/docs/supported_builds.rst b/docs/supported_builds.rst
new file mode 100644
index 0000000..3b03ba7
--- /dev/null
+++ b/docs/supported_builds.rst
@@ -0,0 +1,73 @@
+Supported builds
+================
+
+The following is a list of supported manifest combinations.
+
+Note that grub cannot boot from unpartitioned volumes.
+
+Additionally grub installation is not supported on *squeeze*.
+This is not a technical limitation, but simply stems from a
+lack of motivation to implement support for it.
+
+Azure
+-----
+
+TODO
+
+
+EC2
+---
+
+EBS (wheezy & jessie)
+~~~~~~~~~~~~~~~~~~~~~
+
+========================== ================= ================= =================
+ Bootloader / Partitioning        none              msdos            gpt
+========================== ================= ================= =================
+ pvgrub (paravirtualized)      supported         supported         supported
+ extlinux (hvm)                supported         supported         supported
+ grub (hvm)                 *not supported*      supported         supported
+========================== ================= ================= =================
+
+EBS (squeeze)
+~~~~~~~~~~~~~
+
+========================== ================= ================= =================
+ Bootloader / Partitioning        none              msdos            gpt
+========================== ================= ================= =================
+ pvgrub (paravirtualized)      supported         supported         supported
+ extlinux (hvm)                supported         supported         supported
+ grub (hvm)                 *not supported*  *not implemented* *not implemented*
+========================== ================= ================= =================
+
+S3 (all releases)
+~~~~~~~~~~~~~~~~~
+
+========================== ================= ================= =================
+ Bootloader / Partitioning        none              msdos            gpt
+========================== ================= ================= =================
+ pvgrub (paravirtualized)      supported     *not implemented* *not implemented*
+ extlinux (hvm)            *not implemented* *not implemented* *not implemented*
+ grub (hvm)                 *not supported*  *not implemented* *not implemented*
+========================== ================= ================= =================
+
+GCE
+---
+
+TODO
+
+KVM
+---
+
+TODO
+
+
+VirtualBox
+----------
+
+========================== ================= ================= =================
+ Bootloader / Partitioning        none              msdos            gpt
+========================== ================= ================= =================
+ extlinux                      supported         supported         supported
+ grub                       *not supported*      supported         supported
+========================== ================= ================= =================

From e3f511f8e87bd02c9eb02c81f538535dd4171880 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:15:28 +0200
Subject: [PATCH 288/345] Fix inclusion of main README by using "self" in
 toctree

---
 README.rst     | 2 +-
 docs/index.rst | 8 ++++----
 docs/intro.rst | 1 -
 3 files changed, 5 insertions(+), 6 deletions(-)
 delete mode 100644 docs/intro.rst

diff --git a/README.rst b/README.rst
index ac2ce78..7a4f041 100644
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,4 @@
-Introduction
+bootstrap-vz
 ============
 
 bootstrap-vz is a bootstrapping framework for Debian.
diff --git a/docs/index.rst b/docs/index.rst
index 058b355..f972b25 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,10 +1,8 @@
-bootstrap-vz
-============
-
 .. toctree::
    :maxdepth: 1
+   :hidden:
 
-   intro
+   self
    manifest
    providers/index
    plugins/index
@@ -12,3 +10,5 @@ bootstrap-vz
    logging
    changelog
    developers/index
+
+.. include:: ../README.rst
diff --git a/docs/intro.rst b/docs/intro.rst
deleted file mode 100644
index 72a3355..0000000
--- a/docs/intro.rst
+++ /dev/null
@@ -1 +0,0 @@
-.. include:: ../README.rst

From 03f8d231e2141b89bb88584e4196c822bcbf5b09 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:26:21 +0200
Subject: [PATCH 289/345] Move developers intro into bootstrapvz/ as README

---
 bootstrapvz/README.rst    | 49 +++++++++++++++++++++++++++++++++++
 docs/developers/index.rst | 54 +++------------------------------------
 2 files changed, 52 insertions(+), 51 deletions(-)
 create mode 100644 bootstrapvz/README.rst

diff --git a/bootstrapvz/README.rst b/bootstrapvz/README.rst
new file mode 100644
index 0000000..2ca112d
--- /dev/null
+++ b/bootstrapvz/README.rst
@@ -0,0 +1,49 @@
+How bootstrap-vz works
+----------------------
+
+Tasks
+~~~~~
+At its core bootstrap-vz is based on tasks that perform units of work.
+By keeping those tasks small and with a solid structure built around
+them a high degree of flexibility can be achieved. To ensure that
+tasks are executed in the right order, each task is placed in a
+dependency graph where directed edges dictate precedence. Each task is
+a simple class that defines its predecessor tasks and successor tasks
+via attributes. Here is an example:
+
+::
+
+    class MapPartitions(Task):
+      description = 'Mapping volume partitions'
+      phase = phases.volume_preparation
+      predecessors = [PartitionVolume]
+      successors = [filesystem.Format]
+
+      @classmethod
+      def run(cls, info):
+         info.volume.partition_map.map(info.volume)
+
+In this case the attributes define that the task at hand should run
+after the ``PartitionVolume`` task — i.e. after volume has been
+partitioned (``predecessors``) — but before formatting each
+partition (``successors``).
+It is also placed in the ``volume_preparation`` phase.
+Phases are ordered and group tasks together. All tasks in a phase are
+run before proceeding with the tasks in the next phase. They are a way
+of avoiding the need to list 50 different tasks as predecessors and
+successors.
+
+The final task list that will be executed is computed by enumerating
+all tasks in the package, placing them in the graph and
+`sorting them topologically <http://en.wikipedia.org/wiki/Topological_sort>`_.
+Subsequently the list returned is filtered to contain only the tasks the
+provider and the plugins added to the taskset.
+
+
+System abstractions
+~~~~~~~~~~~~~~~~~~~
+There are several abstractions in bootstrap-vz that make it possible
+to generalize things like volume creation, partitioning, mounting and
+package installation. As a rule these abstractions are located in the
+``base/`` folder, where the manifest parsing and task ordering algorithm
+are placed as well.
diff --git a/docs/developers/index.rst b/docs/developers/index.rst
index 5f1cdea..54f2f0d 100644
--- a/docs/developers/index.rst
+++ b/docs/developers/index.rst
@@ -1,61 +1,13 @@
 Developers
-============
+==========
 
 .. toctree::
    :maxdepth: 1
+   :hidden:
 
    contributing
    switches
    api/index
    taskoverview
 
-
-How bootstrap-vz works
-----------------------
-
-Tasks
-~~~~~
-At its core bootstrap-vz is based on tasks that perform units of work.
-By keeping those tasks small and with a solid structure built around
-them a high degree of flexibility can be achieved. To ensure that
-tasks are executed in the right order, each task is placed in a
-dependency graph where directed edges dictate precedence. Each task is
-a simple class that defines its predecessor tasks and successor tasks
-via attributes. Here is an example:
-
-::
-
-    class MapPartitions(Task):
-      description = 'Mapping volume partitions'
-      phase = phases.volume_preparation
-      predecessors = [PartitionVolume]
-      successors = [filesystem.Format]
-    
-      @classmethod
-      def run(cls, info):
-         info.volume.partition_map.map(info.volume)
-
-In this case the attributes define that the task at hand should run
-after the ``PartitionVolume`` task — i.e. after volume has been
-partitioned (``predecessors``) — but before formatting each
-partition (``successors``).
-It is also placed in the ``volume_preparation`` phase.
-Phases are ordered and group tasks together. All tasks in a phase are
-run before proceeding with the tasks in the next phase. They are a way
-of avoiding the need to list 50 different tasks as predecessors and
-successors.
-
-The final task list that will be executed is computed by enumerating
-all tasks in the package, placing them in the graph and
-`sorting them topologically <http://en.wikipedia.org/wiki/Topological_sort>`_.
-Subsequently the list returned is filtered to contain only the tasks the
-provider and the plugins added to the taskset.
-
-
-System abstractions
-~~~~~~~~~~~~~~~~~~~
-There are several abstractions in bootstrap-vz that make it possible
-to generalize things like volume creation, partitioning, mounting and
-package installation. As a rule these abstractions are located in the
-``base/`` folder, where the manifest parsing and task ordering algorithm
-are placed as well.
+.. include:: ../../bootstrapvz/README.rst

From 01f5a2db0427e4f928ff4d06dff59f3bc138c548 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:30:41 +0200
Subject: [PATCH 290/345] Add structure for testing docs

---
 docs/developers/index.rst                     |  1 +
 docs/developers/testing/index.rst             | 10 ++++++++++
 docs/developers/testing/integration_tests.rst |  1 +
 docs/developers/testing/unit_tests.rst        |  1 +
 tests/README.rst                              |  2 ++
 tests/integration/README.rst                  |  2 ++
 tests/unit/README.rst                         |  2 ++
 7 files changed, 19 insertions(+)
 create mode 100644 docs/developers/testing/index.rst
 create mode 100644 docs/developers/testing/integration_tests.rst
 create mode 100644 docs/developers/testing/unit_tests.rst
 create mode 100644 tests/README.rst
 create mode 100644 tests/integration/README.rst
 create mode 100644 tests/unit/README.rst

diff --git a/docs/developers/index.rst b/docs/developers/index.rst
index 54f2f0d..27e7d8a 100644
--- a/docs/developers/index.rst
+++ b/docs/developers/index.rst
@@ -8,6 +8,7 @@ Developers
    contributing
    switches
    api/index
+   testing/index
    taskoverview
 
 .. include:: ../../bootstrapvz/README.rst
diff --git a/docs/developers/testing/index.rst b/docs/developers/testing/index.rst
new file mode 100644
index 0000000..d65dfb0
--- /dev/null
+++ b/docs/developers/testing/index.rst
@@ -0,0 +1,10 @@
+Testing
+=======
+
+.. toctree::
+   :maxdepth: 1
+
+   unit_tests
+   integration_tests
+
+.. include:: ../../../tests/README.rst
diff --git a/docs/developers/testing/integration_tests.rst b/docs/developers/testing/integration_tests.rst
new file mode 100644
index 0000000..aea3bd7
--- /dev/null
+++ b/docs/developers/testing/integration_tests.rst
@@ -0,0 +1 @@
+.. include:: ../../../tests/integration/README.rst
diff --git a/docs/developers/testing/unit_tests.rst b/docs/developers/testing/unit_tests.rst
new file mode 100644
index 0000000..afb6a91
--- /dev/null
+++ b/docs/developers/testing/unit_tests.rst
@@ -0,0 +1 @@
+.. include:: ../../../tests/unit/README.rst
diff --git a/tests/README.rst b/tests/README.rst
new file mode 100644
index 0000000..07a3b69
--- /dev/null
+++ b/tests/README.rst
@@ -0,0 +1,2 @@
+The testing framework consists of two parts:
+The unit tests and the integration tests
diff --git a/tests/integration/README.rst b/tests/integration/README.rst
new file mode 100644
index 0000000..a93edec
--- /dev/null
+++ b/tests/integration/README.rst
@@ -0,0 +1,2 @@
+Integration tests
+=================
diff --git a/tests/unit/README.rst b/tests/unit/README.rst
new file mode 100644
index 0000000..a251ed9
--- /dev/null
+++ b/tests/unit/README.rst
@@ -0,0 +1,2 @@
+Unit tests
+==========

From a7d5c93fa140522fd4e1ab0f494b4513e31356d4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:32:19 +0200
Subject: [PATCH 291/345] Add README for remote bootstrapping

---
 bootstrapvz/remote/README.rst | 2 ++
 docs/index.rst                | 1 +
 docs/remote_bootstrapping.rst | 1 +
 3 files changed, 4 insertions(+)
 create mode 100644 bootstrapvz/remote/README.rst
 create mode 100644 docs/remote_bootstrapping.rst

diff --git a/bootstrapvz/remote/README.rst b/bootstrapvz/remote/README.rst
new file mode 100644
index 0000000..5091606
--- /dev/null
+++ b/bootstrapvz/remote/README.rst
@@ -0,0 +1,2 @@
+Remote bootstrapping
+====================
diff --git a/docs/index.rst b/docs/index.rst
index f972b25..b95a98c 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -8,6 +8,7 @@
    plugins/index
    supported_builds
    logging
+   remote_bootstrapping
    changelog
    developers/index
 
diff --git a/docs/remote_bootstrapping.rst b/docs/remote_bootstrapping.rst
new file mode 100644
index 0000000..c1251fc
--- /dev/null
+++ b/docs/remote_bootstrapping.rst
@@ -0,0 +1 @@
+.. include:: ../bootstrapvz/remote/README.rst

From 7d0b9f5bc547b3c5a46345ff6809a0d74dd5c73c Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:34:46 +0200
Subject: [PATCH 292/345] Flatten docs structure

---
 docs/{developers => }/api/base/fs.rst            | 0
 docs/{developers => }/api/base/index.rst         | 0
 docs/{developers => }/api/base/pkg.rst           | 0
 docs/{developers => }/api/common/fs.rst          | 0
 docs/{developers => }/api/common/index.rst       | 0
 docs/{developers => }/api/common/tasks/index.rst | 0
 docs/{developers => }/api/index.rst              | 3 ++-
 docs/developers/index.rst                        | 2 --
 docs/developers/testing/integration_tests.rst    | 1 -
 docs/developers/testing/unit_tests.rst           | 1 -
 docs/index.rst                                   | 2 ++
 docs/{developers => }/testing/index.rst          | 3 ++-
 docs/testing/integration_tests.rst               | 1 +
 docs/testing/unit_tests.rst                      | 1 +
 14 files changed, 8 insertions(+), 6 deletions(-)
 rename docs/{developers => }/api/base/fs.rst (100%)
 rename docs/{developers => }/api/base/index.rst (100%)
 rename docs/{developers => }/api/base/pkg.rst (100%)
 rename docs/{developers => }/api/common/fs.rst (100%)
 rename docs/{developers => }/api/common/index.rst (100%)
 rename docs/{developers => }/api/common/tasks/index.rst (100%)
 rename docs/{developers => }/api/index.rst (65%)
 delete mode 100644 docs/developers/testing/integration_tests.rst
 delete mode 100644 docs/developers/testing/unit_tests.rst
 rename docs/{developers => }/testing/index.rst (63%)
 create mode 100644 docs/testing/integration_tests.rst
 create mode 100644 docs/testing/unit_tests.rst

diff --git a/docs/developers/api/base/fs.rst b/docs/api/base/fs.rst
similarity index 100%
rename from docs/developers/api/base/fs.rst
rename to docs/api/base/fs.rst
diff --git a/docs/developers/api/base/index.rst b/docs/api/base/index.rst
similarity index 100%
rename from docs/developers/api/base/index.rst
rename to docs/api/base/index.rst
diff --git a/docs/developers/api/base/pkg.rst b/docs/api/base/pkg.rst
similarity index 100%
rename from docs/developers/api/base/pkg.rst
rename to docs/api/base/pkg.rst
diff --git a/docs/developers/api/common/fs.rst b/docs/api/common/fs.rst
similarity index 100%
rename from docs/developers/api/common/fs.rst
rename to docs/api/common/fs.rst
diff --git a/docs/developers/api/common/index.rst b/docs/api/common/index.rst
similarity index 100%
rename from docs/developers/api/common/index.rst
rename to docs/api/common/index.rst
diff --git a/docs/developers/api/common/tasks/index.rst b/docs/api/common/tasks/index.rst
similarity index 100%
rename from docs/developers/api/common/tasks/index.rst
rename to docs/api/common/tasks/index.rst
diff --git a/docs/developers/api/index.rst b/docs/api/index.rst
similarity index 65%
rename from docs/developers/api/index.rst
rename to docs/api/index.rst
index 1c7c3c7..f00ef99 100644
--- a/docs/developers/api/index.rst
+++ b/docs/api/index.rst
@@ -3,7 +3,8 @@ API
 
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
+   :hidden:
 
    base/index
    common/index
diff --git a/docs/developers/index.rst b/docs/developers/index.rst
index 27e7d8a..3569604 100644
--- a/docs/developers/index.rst
+++ b/docs/developers/index.rst
@@ -7,8 +7,6 @@ Developers
 
    contributing
    switches
-   api/index
-   testing/index
    taskoverview
 
 .. include:: ../../bootstrapvz/README.rst
diff --git a/docs/developers/testing/integration_tests.rst b/docs/developers/testing/integration_tests.rst
deleted file mode 100644
index aea3bd7..0000000
--- a/docs/developers/testing/integration_tests.rst
+++ /dev/null
@@ -1 +0,0 @@
-.. include:: ../../../tests/integration/README.rst
diff --git a/docs/developers/testing/unit_tests.rst b/docs/developers/testing/unit_tests.rst
deleted file mode 100644
index afb6a91..0000000
--- a/docs/developers/testing/unit_tests.rst
+++ /dev/null
@@ -1 +0,0 @@
-.. include:: ../../../tests/unit/README.rst
diff --git a/docs/index.rst b/docs/index.rst
index b95a98c..bdca053 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -11,5 +11,7 @@
    remote_bootstrapping
    changelog
    developers/index
+   api/index
+   testing/index
 
 .. include:: ../README.rst
diff --git a/docs/developers/testing/index.rst b/docs/testing/index.rst
similarity index 63%
rename from docs/developers/testing/index.rst
rename to docs/testing/index.rst
index d65dfb0..42c091e 100644
--- a/docs/developers/testing/index.rst
+++ b/docs/testing/index.rst
@@ -3,8 +3,9 @@ Testing
 
 .. toctree::
    :maxdepth: 1
+   :hidden:
 
    unit_tests
    integration_tests
 
-.. include:: ../../../tests/README.rst
+.. include:: ../../tests/README.rst
diff --git a/docs/testing/integration_tests.rst b/docs/testing/integration_tests.rst
new file mode 100644
index 0000000..73d05c4
--- /dev/null
+++ b/docs/testing/integration_tests.rst
@@ -0,0 +1 @@
+.. include:: ../../tests/integration/README.rst
diff --git a/docs/testing/unit_tests.rst b/docs/testing/unit_tests.rst
new file mode 100644
index 0000000..418ca76
--- /dev/null
+++ b/docs/testing/unit_tests.rst
@@ -0,0 +1 @@
+.. include:: ../../tests/unit/README.rst

From e80ad46972b055a19b142ca93873086abefcbdb7 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:39:14 +0200
Subject: [PATCH 293/345] Repeat index/README pattern for plugins & providers

---
 bootstrapvz/plugins/README.rst   |  8 ++++++++
 bootstrapvz/providers/README.rst |  8 ++++++++
 docs/plugins/index.rst           | 14 +++-----------
 docs/providers/index.rst         |  3 +++
 4 files changed, 22 insertions(+), 11 deletions(-)
 create mode 100644 bootstrapvz/plugins/README.rst
 create mode 100644 bootstrapvz/providers/README.rst

diff --git a/bootstrapvz/plugins/README.rst b/bootstrapvz/plugins/README.rst
new file mode 100644
index 0000000..df46d3c
--- /dev/null
+++ b/bootstrapvz/plugins/README.rst
@@ -0,0 +1,8 @@
+Plugins are a key feature of bootstrap-vz. Despite their small size
+(most plugins do not exceed 100 source lines of code) they can modify
+the behavior of bootstrapped systems to a great extent.
+
+Below you will find documentation for all plugins available for
+bootstrap-vz. If you cannot find what you are looking for, consider
+`developing it yourself <http://bootstrap-vz.readthedocs.org>`__ and
+contribute to this list!
diff --git a/bootstrapvz/providers/README.rst b/bootstrapvz/providers/README.rst
new file mode 100644
index 0000000..4ffd326
--- /dev/null
+++ b/bootstrapvz/providers/README.rst
@@ -0,0 +1,8 @@
+Providers in bootstrap-vz represent various cloud providers and virtual machines.
+
+bootstrap-vz is an extensible platform with loose coupling and a significant
+amount of tooling, which allows for painless implementation of new providers.
+
+The virtualbox provider for example is implemented in only 89 lines of python,
+since most of the building blocks are a part of the common task library.
+Only the kernel and guest additions installation are specific to that provider.
diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst
index 208c1d3..1d62e4f 100644
--- a/docs/plugins/index.rst
+++ b/docs/plugins/index.rst
@@ -1,19 +1,11 @@
-
 Plugins
 =======
 
-Plugins are a key feature of bootstrap-vz. Despite their small size
-(most plugins do not exceed 100 source lines of code) they can modify
-the behavior of bootstrapped systems to a great extent.
-
-Below you will find documentation for all plugins available for
-bootstrap-vz. If you cannot find what you are looking for, consider
-`developing it yourself <http://bootstrap-vz.readthedocs.org>`__ and
-contribute to this list!
-
-
 .. toctree::
    :maxdepth: 1
+   :hidden:
    :glob:
 
    *
+
+.. include:: ../../bootstrapvz/plugins/README.rst
diff --git a/docs/providers/index.rst b/docs/providers/index.rst
index 7bf1ed5..5b11a93 100644
--- a/docs/providers/index.rst
+++ b/docs/providers/index.rst
@@ -3,6 +3,9 @@ Providers
 
 .. toctree::
    :maxdepth: 1
+   :hidden:
    :glob:
 
    *
+
+.. include:: ../../bootstrapvz/providers/README.rst

From 8fddd9e374806e3ff2875553e9745f31d8764f02 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 16 Apr 2015 23:56:24 +0200
Subject: [PATCH 294/345] Clarify first sentence in main README

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 7a4f041..5df80d6 100644
--- a/README.rst
+++ b/README.rst
@@ -1,8 +1,8 @@
 bootstrap-vz
 ============
 
-bootstrap-vz is a bootstrapping framework for Debian.
-It is is specifically targeted at bootstrapping systems for virtualized environments.
+bootstrap-vz is a bootstrapping framework for Debian that creates ready-to-boot
+images able to run on a number of cloud providers and virtual machines.
 bootstrap-vz runs without any user intervention and
 generates ready-to-boot images for a number of virtualization
 platforms.

From 501e6ad97bf471c2905d77debbd43d5956af1152 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 13:05:58 +0200
Subject: [PATCH 295/345] This is cool. Relative links that link between
 folders/files in the github repo are converted to relative links in the
 documentation.

---
 CONTRIBUTING.rst                            |  6 +-
 README.rst                                  | 16 ++--
 bootstrapvz/providers/ec2/README.rst        |  2 +-
 bootstrapvz/providers/virtualbox/README.rst |  2 +-
 docs/conf.py                                | 14 ++--
 docs/replace_rtd_links.py                   | 89 +++++++++++++++++++--
 manifests/README.rst                        |  8 +-
 7 files changed, 107 insertions(+), 30 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 9c853f5..d5ab31a 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -6,7 +6,7 @@ Sending pull requests
 
 Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow:
 
-+ Read the `development guidelines <http://bootstrap-vz.readthedocs.org/en/master/contributing.html#development-guidelines>`__
++ Read the `development guidelines <#development-guidelines>`__
 + Fork this repository.
 + Make any changes you want/need.
 + Check the coding style of your changes using `tox <http://tox.readthedocs.org/>`__ by running `tox -e flake8`
@@ -14,7 +14,7 @@ Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic w
   This check will be repeated by `Travis CI <https://travis-ci.org/andsens/bootstrap-vz>`__
   once you send a pull request, so it's better if you check this beforehand.
 + If the change is significant (e.g. a new plugin, manifest setting or security fix)
-  add your name and contribution to the `changelog <http://bootstrap-vz.readthedocs.org/en/master/changelog.html>`__.
+  add your name and contribution to the `changelog <CHANGELOG.rst>`__.
 + Commit your changes.
 + Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
   of work, but there's no reason to have many little commits just because of corrected typos.
@@ -107,7 +107,7 @@ value to the bootstrap-vz codebase.
   This allows other tasks to interleave with the control-flow and add extended
   functionality (e.g. because volume creation and mounting are two
   separate tasks, `the prebootstrapped plugin
-  <http://bootstrap-vz.readthedocs.org/en/master/plugins/prebootstrapped.html>`__
+  <bootstrapvz/plugins/prebootstrapped>`__
   can replace the volume creation task with a task of its own that
   creates a volume from a snapshot instead, but still reuse the mount task).
 
diff --git a/README.rst b/README.rst
index 5df80d6..8b47a45 100644
--- a/README.rst
+++ b/README.rst
@@ -7,7 +7,7 @@ bootstrap-vz runs without any user intervention and
 generates ready-to-boot images for a number of virtualization
 platforms.
 Its aim is to provide a reproducable bootstrapping process using
-`manifests <http://bootstrap-vz.readthedocs.org/en/master/manifest.html>`__
+`manifests <manifests>`__
 as well as supporting a high degree of customizability through plugins.
 
 bootstrap-vz was coded from scratch in python once the bash script
@@ -21,11 +21,11 @@ Documentation
 The documentation for bootstrap-vz is available at
 `bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org/en/master>`__.
 There, you can discover `what the
-dependencies <http://bootstrap-vz.readthedocs.org/en/master/index.html#dependencies>`__
+dependencies <#dependencies>`__
 for a specific cloud provider are, `see a list of available
-plugins <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__ and
+plugins <bootstrapvz/plugins>`__ and
 learn `how you create a
-manifest <http://bootstrap-vz.readthedocs.org/en/master/manifest.html>`__.
+manifest <manifests>`__.
 
 Installation
 ------------
@@ -68,7 +68,7 @@ VirtualBox Vagrant
     root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies
     root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml
 
-If you want to use the `minimize\_size <http://bootstrap-vz.readthedocs.org/en/master/plugins/minimize_size.html>`__
+If you want to use the `minimize\_size <bootstrapvz/plugins/minimize_size>`__
 plugin, you will have to install the ``zerofree`` package and `VMWare
 Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
 as well.
@@ -113,7 +113,7 @@ Dependencies
 ------------
 
 bootstrap-vz has a number of dependencies depending on the target
-platform and `the selected plugins <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__.
+platform and `the selected plugins <bootstrapvz/plugins>`__.
 At a bare minimum the following python libraries are needed:
 
 * `termcolor <https://pypi.python.org/pypi/termcolor>`__
@@ -142,7 +142,7 @@ Contributing
 ------------
 
 Contribution guidelines are described in the documentation under
-`Contributing <http://bootstrap-vz.readthedocs.org/en/master/developers/contributing.html>`__.
+`Contributing <CONTRIBUTING.rst>`__.
 There's also
-`a topic <http://bootstrap-vz.readthedocs.org/en/master/developers/contributing.html#coding-style>`__
+`a topic <CONTRIBUTING.rst#coding-style>`__
 regarding the coding style.
diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst
index 05b63e8..c968f48 100644
--- a/bootstrapvz/providers/ec2/README.rst
+++ b/bootstrapvz/providers/ec2/README.rst
@@ -7,7 +7,7 @@ once it is done and registers it as an AMI. EBS volume backing only
 works on an EC2 host while S3 backed volumes *should* work locally (at
 this time however they do not, a fix is in the works).
 
-Unless `the cloud-init plugin <http://bootstrap-vz.readthedocs.org/en/master/plugins/cloud_init.html>`__
+Unless `the cloud-init plugin <../../plugins/cloud_init>`__
 is used, special startup scripts will be installed that automatically fetch the
 configured authorized\_key from the instance metadata and save or run
 any userdata supplied (if the userdata begins with ``#!`` it will be
diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst
index 154199b..37300ff 100644
--- a/bootstrapvz/providers/virtualbox/README.rst
+++ b/bootstrapvz/providers/virtualbox/README.rst
@@ -9,4 +9,4 @@ interoperability (e.g.
 *should* support vdi files, but since they have no identifier URL not
 even VirtualBox itself can import them). VirtualBox Guest Additions can
 be installed automatically if the ISO is `provided in the
-manifest <http://bootstrap-vz.readthedocs.org/en/master/manifest.html#bootstrapper>`__.
+manifest <../../../manifests#bootstrapper>`__.
diff --git a/docs/conf.py b/docs/conf.py
index f6deef4..b43fedd 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -272,17 +272,19 @@ import os.path
 for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'):
 	provider_name = os.path.basename(os.path.dirname(readme_path))
 	include_path = os.path.join('providers', provider_name + '.rst')
-	path_to_readme = os.path.join('../../bootstrapvz/providers', provider_name, 'README.rst')
-	with open(include_path, 'w') as include:
-		include.write('.. include:: ' + path_to_readme)
+	if not os.path.exists(include_path):
+		path_to_readme = os.path.join('../../bootstrapvz/providers', provider_name, 'README.rst')
+		with open(include_path, 'w') as include:
+			include.write('.. include:: ' + path_to_readme)
 
 
 for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'):
 	plugin_name = os.path.basename(os.path.dirname(readme_path))
 	include_path = os.path.join('plugins', plugin_name + '.rst')
-	path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst')
-	with open(include_path, 'w') as include:
-		include.write('.. include:: ' + path_to_readme)
+	if not os.path.exists(include_path):
+		path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst')
+		with open(include_path, 'w') as include:
+			include.write('.. include:: ' + path_to_readme)
 
 
 # -- Create task overview graph data --------------------------------------
diff --git a/docs/replace_rtd_links.py b/docs/replace_rtd_links.py
index c18d1c4..1303d9f 100644
--- a/docs/replace_rtd_links.py
+++ b/docs/replace_rtd_links.py
@@ -1,19 +1,94 @@
+import re
+
 
 def setup(app):
 	app.connect('doctree-resolved', replace_rtd_links)
-
 	return {'version': '0.1'}
 
+# Maps from files in docs/ to folders/files in repo
+includes_mapping = {
+	r'^index$': r'',
+	r'^(providers|plugins)/index$': r'bootstrapvz/\1/',
+	r'^(providers|plugins)/(?!index)([^/]+)$': r'bootstrapvz/\1/\2/',
+	r'^manifest$': r'manifests/',
+	r'^testing/index$': r'testing/',
+	r'^testing/(?!index)([^/]+)_tests$': r'testing/\1/',
+	r'^remote_bootstrapping$': r'bootstrapvz/remote/',
+	r'^developers/index$': r'bootstrapvz/',
+	r'^developers/contributing$': r'CONTRIBUTING.rst',
+	r'^changelog$': r'CHANGELOG.rst',
+}
+
+
+# Maps from links in repo to files/folders in docs/
+links_mapping = {
+	r'^$': r'',
+	r'^bootstrapvz/(providers|plugins)$': r'\1',
+	r'^bootstrapvz/(providers|plugins)/([^/]+)$': r'\1/\2.html',
+	r'^testing$': r'testing',
+	r'^manifests$': r'manifest.html',
+	r'^testing/([^/]+)$': r'testing/\1_tests.html',
+	r'^bootstrapvz/remote$': r'remote_bootstrapping.html',
+	r'^bootstrapvz$': r'developers',
+	r'^CONTRIBUTING\.rst$': r'developers/contributing.html',
+	r'^CHANGELOG\.rst$': r'changelog.html',
+}
+
+for key, val in includes_mapping.items():
+	del includes_mapping[key]
+	includes_mapping[re.compile(key)] = val
+
+for key, val in links_mapping.items():
+	del links_mapping[key]
+	links_mapping[re.compile(key)] = val
+
+
+def find_original(path):
+	for key, val in includes_mapping.items():
+		if re.match(key, path):
+			return re.sub(key, val, path)
+	return None
+
+
+def find_docs_link(link):
+	try:
+		# Preserve anchor when doing lookups
+		link, anchor = link.split('#', 1)
+		anchor = '#' + anchor
+	except ValueError:
+		# No anchor, keep the original link
+		anchor = ''
+	for key, val in links_mapping.items():
+		if re.match(key, link):
+			return re.sub(key, val, link) + anchor
+	return None
+
 
 def replace_rtd_links(app, doctree, fromdocname):
-	from docutils import nodes
-	import re
+	# Convert relative links in repo into relative links in docs.
+	# We do this by first figuring out whether the current document
+	# has been included from outside docs/ and only continue if so.
+	# Next we take the repo path matching the current document
+	# (lookup through 'includes_mapping'), tack the link onto the dirname
+	# of that path and normalize it using os.path.normpath.
+	# The result is the path to a document/folder in the repo.
+	# We then convert this path into one that works in the documentation
+	# (lookup through 'links_mapping').
+	# If a mapping is found we, create a relative link from the current document.
+
+	from docutils import nodes
+	import os.path
+	original_path = find_original(fromdocname)
+	if original_path is None:
+		return
 
-	rtd_baseurl = 'http://bootstrap-vz.readthedocs.org/en/master/'
-	search = re.compile('^' + re.escape(rtd_baseurl) + '(.*)$')
 	for node in doctree.traverse(nodes.reference):
 		if 'refuri' not in node:
 			continue
-		if not node['refuri'].startswith(rtd_baseurl):
+		if node['refuri'].startswith('http'):
 			continue
-		node['refuri'] = re.sub(search, r'\1', node['refuri'])
+		abs_link = os.path.normpath(os.path.join(os.path.dirname(original_path), node['refuri']))
+		docs_link = find_docs_link(abs_link)
+		if docs_link is None:
+			continue
+		node['refuri'] = os.path.relpath(docs_link, os.path.dirname(fromdocname))
diff --git a/manifests/README.rst b/manifests/README.rst
index d644979..df598a3 100644
--- a/manifests/README.rst
+++ b/manifests/README.rst
@@ -39,7 +39,7 @@ name of the provider itself.
 -  **``name``**: target virtualization platform of the installation
    *``required``*
 
-Consult the `providers <http://bootstrap-vz.readthedocs.org/en/master/providers>`__ section of the documentation
+Consult the `providers <../bootstrapvz/providers>`__ section of the documentation
 for a list of valid values.
 
 Bootstrapper
@@ -76,7 +76,7 @@ are 4 possible settings:
    Accepts a list of package names.
    *``optional``*
 -  **``guest_additions``**: This setting is only relevant for the
-   `virtualbox provider <http://bootstrap-vz.readthedocs.org/en/master/providers/virtualbox.html>`__.
+   `virtualbox provider <../bootstrapvz/providers/virtualbox.html>`__.
    It specifies the path to the VirtualBox Guest Additions ISO, which, when specified,
    will be mounted and used to install the VirtualBox Guest Additions.
    *``optional``*
@@ -216,7 +216,7 @@ boot, root and swap.
       optional setting overrides the command bootstrap-vz would normally
       use to format the partition. The command is specified as a string
       array where each option/argument is an item in that array (much
-      like the `image\_commands <http://bootstrap-vz.readthedocs.org/en/master/plugins/image_commands.html>`__ plugin).
+      like the `image\_commands <../bootstrapvz/plugins/image_commands.html>`__ plugin).
       *``optional``* The following variables are available:
    -  **``{fs}``**: The filesystem of the partition.
    -  **``{device_path}``**: The device path of the partition.
@@ -237,5 +237,5 @@ Plugins
 ~~~~~~~
 
 The plugins section is a map of plugin names to whatever configuration a
-plugin requires. Go to the `plugin section <http://bootstrap-vz.readthedocs.org/en/master/plugins>`__
+plugin requires. Go to the `plugin section <../bootstrapvz/plugins>`__
 of the documentation, to see the configuration for a specific plugin.

From b42b788a450eb1aa2a4d1aa0de6a3a8af2f14d04 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 13:29:10 +0200
Subject: [PATCH 296/345] Rename replace_rtd_links to transform_github_links
 since that's what's actually happening now

---
 docs/conf.py                                             | 2 +-
 docs/{replace_rtd_links.py => transform_github_links.py} | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename docs/{replace_rtd_links.py => transform_github_links.py} (96%)

diff --git a/docs/conf.py b/docs/conf.py
index b43fedd..a53c41e 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -31,7 +31,7 @@ sys.path.insert(0, os.path.abspath(os.pardir))
 extensions = ['sphinx.ext.coverage',
               'sphinx.ext.autodoc',
               'sphinx.ext.linkcode',
-              'docs.replace_rtd_links',
+              'docs.transform_github_links',
               ]
 
 # Add any paths that contain templates here, relative to this directory.
diff --git a/docs/replace_rtd_links.py b/docs/transform_github_links.py
similarity index 96%
rename from docs/replace_rtd_links.py
rename to docs/transform_github_links.py
index 1303d9f..daf31f6 100644
--- a/docs/replace_rtd_links.py
+++ b/docs/transform_github_links.py
@@ -2,7 +2,7 @@ import re
 
 
 def setup(app):
-	app.connect('doctree-resolved', replace_rtd_links)
+	app.connect('doctree-resolved', transform_github_links)
 	return {'version': '0.1'}
 
 # Maps from files in docs/ to folders/files in repo
@@ -64,7 +64,7 @@ def find_docs_link(link):
 	return None
 
 
-def replace_rtd_links(app, doctree, fromdocname):
+def transform_github_links(app, doctree, fromdocname):
 	# Convert relative links in repo into relative links in docs.
 	# We do this by first figuring out whether the current document
 	# has been included from outside docs/ and only continue if so.

From 8064cddb1dbbf6ffc0f0f1b5397e91725238e311 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 13:40:00 +0200
Subject: [PATCH 297/345] Add README to docs

---
 README.rst                        | 11 +++++------
 docs/README.rst                   |  5 +++++
 docs/developers/documentation.rst |  1 +
 docs/developers/index.rst         |  1 +
 docs/transform_github_links.py    |  2 ++
 5 files changed, 14 insertions(+), 6 deletions(-)
 create mode 100644 docs/README.rst
 create mode 100644 docs/developers/documentation.rst

diff --git a/README.rst b/README.rst
index 8b47a45..a4cccf3 100644
--- a/README.rst
+++ b/README.rst
@@ -20,12 +20,11 @@ Documentation
 
 The documentation for bootstrap-vz is available at
 `bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org/en/master>`__.
-There, you can discover `what the
-dependencies <#dependencies>`__
-for a specific cloud provider are, `see a list of available
-plugins <bootstrapvz/plugins>`__ and
-learn `how you create a
-manifest <manifests>`__.
+There, you can discover `what the dependencies <#dependencies>`__ for
+a specific cloud provider are, `see a list of available plugins <bootstrapvz/plugins>`__
+and learn `how you create a manifest <manifests>`__.
+
+Note to developers: `The documenation <docs>`__ is generated in a rather peculiar and nifty way.
 
 Installation
 ------------
diff --git a/docs/README.rst b/docs/README.rst
new file mode 100644
index 0000000..2b41dad
--- /dev/null
+++ b/docs/README.rst
@@ -0,0 +1,5 @@
+:orphan:
+
+Documentation
+=============
+
diff --git a/docs/developers/documentation.rst b/docs/developers/documentation.rst
new file mode 100644
index 0000000..72a3355
--- /dev/null
+++ b/docs/developers/documentation.rst
@@ -0,0 +1 @@
+.. include:: ../README.rst
diff --git a/docs/developers/index.rst b/docs/developers/index.rst
index 3569604..fa243d4 100644
--- a/docs/developers/index.rst
+++ b/docs/developers/index.rst
@@ -6,6 +6,7 @@ Developers
    :hidden:
 
    contributing
+   documentation
    switches
    taskoverview
 
diff --git a/docs/transform_github_links.py b/docs/transform_github_links.py
index daf31f6..2bded08 100644
--- a/docs/transform_github_links.py
+++ b/docs/transform_github_links.py
@@ -16,6 +16,7 @@ includes_mapping = {
 	r'^remote_bootstrapping$': r'bootstrapvz/remote/',
 	r'^developers/index$': r'bootstrapvz/',
 	r'^developers/contributing$': r'CONTRIBUTING.rst',
+	r'^developers/documentation$': r'docs/',
 	r'^changelog$': r'CHANGELOG.rst',
 }
 
@@ -31,6 +32,7 @@ links_mapping = {
 	r'^bootstrapvz/remote$': r'remote_bootstrapping.html',
 	r'^bootstrapvz$': r'developers',
 	r'^CONTRIBUTING\.rst$': r'developers/contributing.html',
+	r'^docs$': r'developers/documentation.html',
 	r'^CHANGELOG\.rst$': r'changelog.html',
 }
 

From 3a1e3c7feb4ea0c92c8c081a5ff88ce6a59c94db Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 14:37:50 +0200
Subject: [PATCH 298/345] Account for anchorlinks inside the same document

---
 docs/transform_github_links.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/transform_github_links.py b/docs/transform_github_links.py
index 2bded08..e136eda 100644
--- a/docs/transform_github_links.py
+++ b/docs/transform_github_links.py
@@ -93,4 +93,8 @@ def transform_github_links(app, doctree, fromdocname):
 		docs_link = find_docs_link(abs_link)
 		if docs_link is None:
 			continue
-		node['refuri'] = os.path.relpath(docs_link, os.path.dirname(fromdocname))
+		# special handling for when we link inside the same document
+		if docs_link.startswith('#'):
+			node['refuri'] = docs_link
+		else:
+			node['refuri'] = os.path.relpath(docs_link, os.path.dirname(fromdocname))

From b46858d9fc4ad451e3ec814ac966b5b3bb0de099 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 14:38:05 +0200
Subject: [PATCH 299/345] Add documentation about the documentation

---
 CONTRIBUTING.rst | 62 +++++++++++++++++++++++++++---------------------
 README.rst       |  2 +-
 docs/README.rst  | 39 ++++++++++++++++++++++++++++++
 3 files changed, 75 insertions(+), 28 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index d5ab31a..30d5b95 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -1,24 +1,24 @@
 Contributing
 ============
 
+
 Sending pull requests
 ---------------------
-
 Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow:
 
-+ Read the `development guidelines <#development-guidelines>`__
-+ Fork this repository.
-+ Make any changes you want/need.
-+ Check the coding style of your changes using `tox <http://tox.readthedocs.org/>`__ by running `tox -e flake8`
+* Read the `development guidelines <#development-guidelines>`__
+* Fork this repository.
+* Make any changes you want/need.
+* Check the coding style of your changes using `tox <http://tox.readthedocs.org/>`__ by running `tox -e flake8`
   and fix any warnings that may appear.
   This check will be repeated by `Travis CI <https://travis-ci.org/andsens/bootstrap-vz>`__
   once you send a pull request, so it's better if you check this beforehand.
-+ If the change is significant (e.g. a new plugin, manifest setting or security fix)
+* If the change is significant (e.g. a new plugin, manifest setting or security fix)
   add your name and contribution to the `changelog <CHANGELOG.rst>`__.
-+ Commit your changes.
-+ Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
+* Commit your changes.
+* Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
   of work, but there's no reason to have many little commits just because of corrected typos.
-+ Push to your fork, preferably on a topic branch.
+* Push to your fork, preferably on a topic branch.
 
 From here on there are two paths to consider:
 
@@ -31,8 +31,6 @@ If it is a bug/security fix:
 
 * Send a pull request to the `master` branch.
 
---
-
 Please try to be very descriptive about your changes when you write a pull request, stating what it does, why
 it is needed, which use cases this change covers etc.
 You may be asked to rebase your work on the current branch state, so it can be merged cleanly.
@@ -54,7 +52,7 @@ these guidelines are not rules , they are advice on how to better add
 value to the bootstrap-vz codebase.
 
 
-+ **The manifest should always fully describe the resulting image. The
+* **The manifest should always fully describe the resulting image. The
   outcome of a bootstrapping process should never depend on settings
   specified elsewhere.**
 
@@ -65,7 +63,7 @@ value to the bootstrap-vz codebase.
   for example can be reproduced using the manifests available
   in the manifest directory of bootstrap-vz.
 
-+ **The bootstrapper should always be able to run fully unattended.**
+* **The bootstrapper should always be able to run fully unattended.**
   
   For end users, this guideline minimizes the risk of errors. Any
   required input would also be in direct conflict with the previous
@@ -76,7 +74,7 @@ value to the bootstrap-vz codebase.
   process multiple times though, any prompts in the middle of that
   process may significantly slow down the development speed.
 
-+ **The bootstrapper should only need as much setup as the manifest
+* **The bootstrapper should only need as much setup as the manifest
   requires.**
 
   Having to shuffle specific paths on the host into place
@@ -88,7 +86,7 @@ value to the bootstrap-vz codebase.
   the VirtualBox Guest Additions ISO or tools like ``parted`` that
   need to be installed on the host.
 
-+ **Roll complexity into which tasks are added to the tasklist.**
+* **Roll complexity into which tasks are added to the tasklist.**
 
   If a ``run()`` function checks whether it should do any work or simply be
   skipped, consider doing that check in ``resolve_tasks()`` instead and
@@ -99,7 +97,7 @@ value to the bootstrap-vz codebase.
   conclude that the file has either been overwritten or that the
   search & replace does not work correctly.
 
-+ **Control flow should be directed from the task graph.**
+* **Control flow should be directed from the task graph.**
 
   Avoid creating complicated ``run()`` functions. If necessary, split up
   a function into two semantically separate tasks.
@@ -111,28 +109,28 @@ value to the bootstrap-vz codebase.
   can replace the volume creation task with a task of its own that
   creates a volume from a snapshot instead, but still reuse the mount task).
 
-+ **Task classes should be treated as decorated run() functions, they 
+* **Task classes should be treated as decorated run() functions, they 
   should not have any state**
 
   Thats what the BootstrapInformation object is for.
 
-+ **Only add stuff to the BootstrapInformation object when really necessary.**
+* **Only add stuff to the BootstrapInformation object when really necessary.**
 
   This is mainly to avoid clutter.
 
-+ **Use a json-schema to check for allowed settings**
+* **Use a json-schema to check for allowed settings**
   The json-schema may be verbose but it keeps the bulk of check work outside the
   python code, which is a big plus when it comes to readability. This of
   course only applies bas long as the checks are simple. You can of
   course fall back to doing the check in python when that solution is
   considerably less complex.
 
-+ **When invoking external programs, use long options whenever possible**
+* **When invoking external programs, use long options whenever possible**
 
   This makes the commands a lot easier to understand, since
   the option names usually hint at what they do.
 
-+ **When invoking external programs, don't use full paths, rely on ``$PATH``**
+* **When invoking external programs, don't use full paths, rely on ``$PATH``**
 
   This increases robustness when executable locations change.
   Example: Use ``log_call(['wget', ...])`` instead of ``log_call(['/usr/bin/wget', ...])``.
@@ -143,16 +141,26 @@ Coding style
 bootstrap-vz is coded to comply closely with the PEP8 style
 guidelines. There however a few exceptions:
 
-+ Max line length is 110 chars, not 80.
-+ Multiple assignments may be aligned with spaces so that the = match
+* Max line length is 110 chars, not 80.
+* Multiple assignments may be aligned with spaces so that the = match
   vertically.
-+ Ignore ``E101``: Indent with tabs and align with spaces
-+ Ignore ``E221 & E241``: Alignment of assignments
-+ Ignore ``E501``: The max line length is not 80 characters
-+ Ignore ``W191``: Indent with tabs not spaces
+* Ignore ``E101``: Indent with tabs and align with spaces
+* Ignore ``E221 & E241``: Alignment of assignments
+* Ignore ``E501``: The max line length is not 80 characters
+* Ignore ``W191``: Indent with tabs not spaces
 
 The codebase can be checked for any violations quite easily, since those rules are already specified in the
 `tox <http://tox.readthedocs.org/>`__ configuration file.
 ::
 
     tox -e flake8
+
+
+Documentation
+-------------
+When developing a provider or plugin, make sure to update/create the README.rst
+located in provider/plugin folder.
+Any links to other rst files should be relative and work, when viewed on github.
+For information on `how to build the documentation <docs#building>`_ and how
+the various parts fit together,
+refer to `the documentation about the documentation <docs>`__ :-)
diff --git a/README.rst b/README.rst
index a4cccf3..2f0dbb5 100644
--- a/README.rst
+++ b/README.rst
@@ -24,7 +24,7 @@ There, you can discover `what the dependencies <#dependencies>`__ for
 a specific cloud provider are, `see a list of available plugins <bootstrapvz/plugins>`__
 and learn `how you create a manifest <manifests>`__.
 
-Note to developers: `The documenation <docs>`__ is generated in a rather peculiar and nifty way.
+Note to developers: `The documentaion <docs>`__ is generated in a rather peculiar and nifty way.
 
 Installation
 ------------
diff --git a/docs/README.rst b/docs/README.rst
index 2b41dad..08ab12b 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -1,5 +1,44 @@
 :orphan:
 
+
 Documentation
 =============
+Both the end-user and developer documentation is combined into a single sphinx
+build (the two were previously split between github pages and sphinx).
 
+
+Building
+--------
+To build the documentation, simply run ``tox -e docs`` in the project root.
+Serving the docs through http can be achieved by subsequently running
+``(cd docs/_build/html; python -m SimpleHTTPServer 8080)`` and accessing them
+on ``http://localhost:8080/``.
+
+
+READMEs
+-------
+Many of the folders in the project have a README.rst which describes
+the purpose of the contents in that folder.
+These files are automatically included when building the documentation,
+through use of the `include`__ directive.
+
+__ http://docutils.sourceforge.net/docs/ref/rst/directives.html#including-an-external-document-fragment
+
+Include files for the providers and plugins are autogenerated
+through the sphinx conf.py script.
+
+
+Links
+-----
+All links in rst files outside of ``docs/`` (but also ``docs/README.rst``) that
+link to other rst files are relative and reference folder names when the link
+would point at a README.rst otherwise. This is done to take advantage of the
+github feature where README files are displayed when viewing its parent folder.
+When accessing the ``manifests/`` folder for example, the documentation for how
+manifests work is displayed at the bottom.
+
+When sphinx generates the documentation, these relative links are
+automatically converted into relative links that work inside the generated
+html pages instead.
+If you are interested in how this works, take a look at the
+link transformation module in ``docs/transform_github_links``.

From fe68d20829f7950851240977c49afce3170044e3 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 17:04:50 +0200
Subject: [PATCH 300/345] Better formatting for CONTRIBUTING.rst

---
 CONTRIBUTING.rst | 133 +++++++++++++++++++++++++----------------------
 1 file changed, 71 insertions(+), 62 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 30d5b95..e46da26 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -52,88 +52,97 @@ these guidelines are not rules , they are advice on how to better add
 value to the bootstrap-vz codebase.
 
 
-* **The manifest should always fully describe the resulting image. The
-  outcome of a bootstrapping process should never depend on settings
-  specified elsewhere.**
+The manifest should always fully describe the resulting image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The outcome of a bootstrapping process should never depend on settings
+specified elsewhere.
 
-  This allows others to easily reproduce any
-  setup other people are running and makes it possible to share
-  manifests. `The official debian EC2 images <https:/aws.amazon.com/marketplace/seller-
-  profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5>`__
-  for example can be reproduced using the manifests available
-  in the manifest directory of bootstrap-vz.
+This allows others to easily reproduce any setup other people are running
+and makes it possible to share manifests.
+`The official debian EC2 images`__ for example can be reproduced
+using the manifests available in the manifest directory of bootstrap-vz.
 
-* **The bootstrapper should always be able to run fully unattended.**
-  
-  For end users, this guideline minimizes the risk of errors. Any
-  required input would also be in direct conflict with the previous
-  guideline that the manifest should always fully describe the resulting
-  image.
+__ https:/aws.amazon.com/marketplace/seller-profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5
 
-  Additionally developers may have to run the bootstrap
-  process multiple times though, any prompts in the middle of that
-  process may significantly slow down the development speed.
+The bootstrapper should always be able to run fully unattended
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+For end users, this guideline minimizes the risk of errors. Any
+required input would also be in direct conflict with the previous
+guideline that the manifest should always fully describe the resulting
+image.
 
-* **The bootstrapper should only need as much setup as the manifest
-  requires.**
+Additionally developers may have to run the bootstrap
+process multiple times though, any prompts in the middle of that
+process may significantly slow down the development speed.
 
-  Having to shuffle specific paths on the host into place
-  (e.g. ``/target`` has to be created manually) to get the bootstrapper
-  running is going to increase the rate of errors made by users.
-  Aim for minimal setup.
 
-  Exceptions are of course things such as the path to
-  the VirtualBox Guest Additions ISO or tools like ``parted`` that
-  need to be installed on the host.
+The bootstrapper should only need as much setup as the manifest requires
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Having to shuffle specific paths on the host into place
+(e.g. ``/target`` has to be created manually) to get the bootstrapper
+running is going to increase the rate of errors made by users.
+Aim for minimal setup.
 
-* **Roll complexity into which tasks are added to the tasklist.**
+Exceptions are of course things such as the path to
+the VirtualBox Guest Additions ISO or tools like ``parted`` that
+need to be installed on the host.
 
-  If a ``run()`` function checks whether it should do any work or simply be
-  skipped, consider doing that check in ``resolve_tasks()`` instead and
-  avoid adding that task alltogether. This allows people looking at the
-  tasklist in the logfile to determine what work has been performed. If
-  a task says it will modify a file but then bails , a developer may get
-  confused when looking at that file after bootstrapping. He could
-  conclude that the file has either been overwritten or that the
-  search & replace does not work correctly.
 
-* **Control flow should be directed from the task graph.**
+Roll complexity into which tasks are added to the tasklist
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+If a ``run()`` function checks whether it should do any work or simply be
+skipped, consider doing that check in ``resolve_tasks()`` instead and
+avoid adding that task alltogether. This allows people looking at the
+tasklist in the logfile to determine what work has been performed.
 
-  Avoid creating complicated ``run()`` functions. If necessary, split up
-  a function into two semantically separate tasks.
+If a task says it will modify a file but then bails , a developer may get
+confused when looking at that file after bootstrapping. He could
+conclude that the file has either been overwritten or that the
+search & replace does not work correctly.
 
-  This allows other tasks to interleave with the control-flow and add extended
-  functionality (e.g. because volume creation and mounting are two
-  separate tasks, `the prebootstrapped plugin
-  <bootstrapvz/plugins/prebootstrapped>`__
-  can replace the volume creation task with a task of its own that
-  creates a volume from a snapshot instead, but still reuse the mount task).
 
-* **Task classes should be treated as decorated run() functions, they 
-  should not have any state**
+Control flow should be directed from the task graph
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Avoid creating complicated ``run()`` functions. If necessary, split up
+a function into two semantically separate tasks.
 
-  Thats what the BootstrapInformation object is for.
+This allows other tasks to interleave with the control-flow and add extended
+functionality (e.g. because volume creation and mounting are two
+separate tasks, `the prebootstrapped plugin
+<bootstrapvz/plugins/prebootstrapped>`__
+can replace the volume creation task with a task of its own that
+creates a volume from a snapshot instead, but still reuse the mount task).
 
-* **Only add stuff to the BootstrapInformation object when really necessary.**
 
-  This is mainly to avoid clutter.
+Task classes should be treated as decorated run() functions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Tasks should not have any state, thats what the
+BootstrapInformation object is for.
 
-* **Use a json-schema to check for allowed settings**
-  The json-schema may be verbose but it keeps the bulk of check work outside the
-  python code, which is a big plus when it comes to readability. This of
-  course only applies bas long as the checks are simple. You can of
-  course fall back to doing the check in python when that solution is
-  considerably less complex.
+Only add stuff to the BootstrapInformation object when really necessary
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This is mainly to avoid clutter.
 
-* **When invoking external programs, use long options whenever possible**
 
-  This makes the commands a lot easier to understand, since
-  the option names usually hint at what they do.
+Use a json-schema to check for allowed settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The json-schema may be verbose but it keeps the bulk of check work outside the
+python code, which is a big plus when it comes to readability.
+This only applies bas long as the checks are simple.
+You can of course fall back to doing the check in python when that solution is
+considerably less complex.
 
-* **When invoking external programs, don't use full paths, rely on ``$PATH``**
 
-  This increases robustness when executable locations change.
-  Example: Use ``log_call(['wget', ...])`` instead of ``log_call(['/usr/bin/wget', ...])``.
+When invoking external programs, use long options whenever possible
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This makes the commands a lot easier to understand, since
+the option names usually hint at what they do.
+
+
+When invoking external programs, don't use full paths, rely on ``$PATH``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This increases robustness when executable locations change.
+Example: Use ``log_call(['wget', ...])`` instead of ``log_call(['/usr/bin/wget', ...])``.
 
 
 Coding style

From 134e283748fc28c9f191068984129a0267afa587 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 17:06:17 +0200
Subject: [PATCH 301/345] Better formatting for manifests readme

---
 manifests/README.rst | 156 +++++++++++++++++++++----------------------
 1 file changed, 77 insertions(+), 79 deletions(-)

diff --git a/manifests/README.rst b/manifests/README.rst
index df598a3..af473a2 100644
--- a/manifests/README.rst
+++ b/manifests/README.rst
@@ -1,9 +1,7 @@
 Manifest
 ========
-
-| The manifest file is the primary way to interact with bootstrap-vz.
-| Every configuration and customization of a Debian installation is
-  specified in this file.
+The manifest file is the primary way to interact with bootstrap-vz.
+Every configuration and customization of a Debian installation is specified in this file.
 
 The manifest format is YAML or JSON. It is near impossible to run the
 bootstrapper with an invalid configuration, since every part of the
@@ -23,7 +21,7 @@ bootstrapper (e.g. all python date formatting variables are available).
 Any reference uses dots to specify a path to the desired manifest
 setting. Not all settings support this though, to see whether embedding
 a manifest variable in a setting is possible, look for the
-*``manifest vars``* label.
+``manifest vars`` label.
 
 Sections
 --------
@@ -36,8 +34,8 @@ Provider
 The provider section contains all provider specific settings and the
 name of the provider itself.
 
--  **``name``**: target virtualization platform of the installation
-   *``required``*
+-  ``name``: target virtualization platform of the installation
+   ``required``
 
 Consult the `providers <../bootstrapvz/providers>`__ section of the documentation
 for a list of valid values.
@@ -48,10 +46,10 @@ Bootstrapper
 This section concerns the bootstrapper itself and its behavior. There
 are 4 possible settings:
 
--  **``workspace``**: Path to where the bootstrapper should place images
+-  ``workspace``: Path to where the bootstrapper should place images
    and intermediate files. Any volumes will be mounted under that path.
-   *``required``*
--  **``tarball``**: debootstrap has the option to download all the
+   ``required``
+-  ``tarball``: debootstrap has the option to download all the
    software and pack it up in a tarball. When starting the actual
    bootstrapping process, debootstrap can then be pointed at that
    tarball and use it instead of downloading anything from the internet.
@@ -60,26 +58,26 @@ are 4 possible settings:
    whether it should create a new tarball or not. It will search for and
    use an available tarball if it already exists, regardless of this
    setting.
-   *``optional``*
+   ``optional``
    Valid values: ``true, false``
    Default: ``false``
--  **``mirror``**: The mirror debootstrap should download software from.
+-  ``mirror``: The mirror debootstrap should download software from.
    It is advisable to specify a mirror close to your location (or the
    location of the host you are bootstrapping on), to decrease latency
    and improve bandwidth. If not specified, `the configured aptitude
    mirror URL <#packages>`__ is used.
-   *``optional``*
--  **``include_packages``**: Extra packages to be installed during
+   ``optional``
+-  ``include_packages``: Extra packages to be installed during
    bootstrap. Accepts a list of package names.
-   *``optional``*
--  **``exclude_packages``**: Packages to exclude during bootstrap phase.
+   ``optional``
+-  ``exclude_packages``: Packages to exclude during bootstrap phase.
    Accepts a list of package names.
-   *``optional``*
--  **``guest_additions``**: This setting is only relevant for the
+   ``optional``
+-  ``guest_additions``: This setting is only relevant for the
    `virtualbox provider <../bootstrapvz/providers/virtualbox.html>`__.
    It specifies the path to the VirtualBox Guest Additions ISO, which, when specified,
    will be mounted and used to install the VirtualBox Guest Additions.
-   *``optional``*
+   ``optional``
 
 Image
 ~~~~~
@@ -87,23 +85,23 @@ Image
 The image section configures anything pertaining directly to the image
 that will be created.
 
--  **``name``**: The name of the resulting image.
+-  ``name``: The name of the resulting image.
    When bootstrapping cloud images, this would be the name visible in
    the interface when booting up new instances.
    When bootstrapping for VirtualBox or kvm, it's the filename of the
    image.
-   *``required``*
-   *``manifest vars``*
--  **``description``**: Description of the image. Where this setting is
+   ``required``
+   ``manifest vars``
+-  ``description``: Description of the image. Where this setting is
    used depends highly on which provider is set. At the moment it is
    only used for AWS images.
-   **``required for ec2 provider``**
-   *``manifest vars``*
--  **``bucket``**: When bootstrapping an S3 backed image for AWS, this
+   ``required for ec2 provider``
+   ``manifest vars``
+-  ``bucket``: When bootstrapping an S3 backed image for AWS, this
    will be the bucket where the image is uploaded to.
-   **``required for S3 backing``**
--  **``region``**: Region in which the AMI should be registered.
-   **``required for S3 backing``**
+   ``required for S3 backing``
+-  ``region``: Region in which the AMI should be registered.
+   ``required for S3 backing``
 
 System
 ~~~~~~
@@ -111,30 +109,30 @@ System
 This section defines anything that pertains directly to the bootstrapped
 system and does not fit under any other section.
 
--  **``architecture``**: The architecture of the system.
+-  ``architecture``: The architecture of the system.
    Valid values: ``i386, amd64``
-   *``required``*
--  **``bootloader``**: The bootloader for the system. Depending on the
+   ``required``
+-  ``bootloader``: The bootloader for the system. Depending on the
    bootmethod of the virtualization platform, the options may be
    restricted.
    Valid values: ``grub, extlinux, pv-grub``
-   *``required``*
--  **``charmap``**: The default charmap of the system.
-   Valid values: Any valid charmap like ``UTF-8``, ``ISO-8859-*`` or
+   ``required``
+-  ``charmap``: The default charmap of the system.
+   Valid values: Any valid charmap like ``UTF-8``, ``ISO-8859-`` or
    ``GBK``.
-   *``required``*
--  **``hostname``**: hostname to preconfigure the system with.
-   *``optional``*
--  **``locale``**: The default locale of the system.
+   ``required``
+-  ``hostname``: hostname to preconfigure the system with.
+   ``optional``
+-  ``locale``: The default locale of the system.
    Valid values: Any locale mentioned in ``/etc/locale.gen``
-   *``required``*
--  **``release``**: Defines which debian release should be bootstrapped.
+   ``required``
+-  ``release``: Defines which debian release should be bootstrapped.
    Valid values: ``squeeze``, ``wheezy``, ``jessie``, ``sid``,
    ``oldstable``, ``stable``, ``testing``, ``unstable``
-   *``required``*
--  **``timezone``**: Timezone of the system.
+   ``required``
+-  ``timezone``: Timezone of the system.
    Valid values: Any filename from ``/usr/share/zoneinfo``
-   *``required``*
+   ``required``
 
 Packages
 ~~~~~~~~
@@ -142,10 +140,10 @@ Packages
 The packages section allows you to install custom packages from a
 variety of sources.
 
--  **``install``**: A list of strings that specify which packages should
+-  ``install``: A list of strings that specify which packages should
    be installed. Valid values: package names optionally followed by a
    ``/target`` or paths to local ``.deb`` files.
--  **``install_standard``**: Defines if the packages of the
+-  ``install_standard``: Defines if the packages of the
    ``"Standard System Utilities"`` option of the Debian installer,
    provided by `tasksel <https://wiki.debian.org/tasksel>`__, should be
    installed or not. The problem is that with just ``debootstrap``, the
@@ -153,36 +151,36 @@ variety of sources.
    machine that will not be used interactively, but otherwise it is nice
    to have at hand tools like ``bash-completion``, ``less``, ``locate``,
    etc.
-   *``optional``*
+   ``optional``
    Valid values: ``true``, ``false``
    Default: ``false``
--  **``mirror``**: The default aptitude mirror.
-   *``optional``*
+-  ``mirror``: The default aptitude mirror.
+   ``optional``
    Default: ``http://http.debian.net/debian``
--  **``sources``**: A map of additional sources that should be added to
+-  ``sources``: A map of additional sources that should be added to
    the aptitude sources list. The key becomes the filename in
    ``/etc/apt/sources.list.d/`` (with ``.list`` appended to it), while
    the value is an array with each entry being a line.
-   *``optional``*
--  **``components``**: A list of components that should be added to the
+   ``optional``
+-  ``components``: A list of components that should be added to the
    default apt sources. For example ``contrib`` or ``non-free``
-   *``optional``*
+   ``optional``
    Default: ``['main']``
--  **``trusted-keys``**: List of paths to ``.gpg`` keyrings that should
+-  ``trusted-keys``: List of paths to ``.gpg`` keyrings that should
    be added to the aptitude keyring of trusted signatures for
    repositories.
-   *``optional``*
--  **``preferences``**: Allows you to pin packages through `apt
+   ``optional``
+-  ``preferences``: Allows you to pin packages through `apt
    preferences <https://wiki.debian.org/AptPreferences>`__. The setting
    is an object where the key is the preference filename in
    ``/etc/apt/preferences.d/``. The key ``main`` is special and refers
    to the file ``/etc/apt/preferences``, which will be overwritten if
    specified.
-   *``optional``*
+   ``optional``
    The values are objects with three keys:
--  **``package``**: The package to pin (wildcards allowed)
--  **``pin``**: The release to pin the package to.
--  **``pin-priority``**: The priority of this pin.
+-  ``package``: The package to pin (wildcards allowed)
+-  ``pin``: The release to pin the package to.
+-  ``pin-priority``: The priority of this pin.
 
 Volume
 ~~~~~~
@@ -193,45 +191,45 @@ volumes using either the gpt or msdos scheme. At most, there are only
 three partitions with predefined roles configurable though. They are
 boot, root and swap.
 
--  **``backing``**: Specifies the volume backing. This setting is very
+-  ``backing``: Specifies the volume backing. This setting is very
    provider specific.
    Valid values: ``ebs``, ``s3``, ``vmdk``, ``vdi``, ``raw``
-   *``required``*
--  **``partitions``**: A map of the partitions that should be created on
+   ``required``
+-  ``partitions``: A map of the partitions that should be created on
    the volume.
--  **``type``**: The partitioning scheme to use. When using ``none``,
+-  ``type``: The partitioning scheme to use. When using ``none``,
    only root can be specified as a partition.
    Valid values: ``none``, ``gpt``, ``msdos``
-   *``required``*
--  **``root``**: Configuration of the root partition. *``required``*
+   ``required``
+-  ``root``: Configuration of the root partition. ``required``
 
-   -  **``size``**: The size of the partition. Valid values: Any
+   -  ``size``: The size of the partition. Valid values: Any
       datasize specification up to TB (e.g. 5KiB, 1MB, 6TB).
-      *``required``*
-   -  **``filesystem``**: The filesystem of the partition. When choosing
+      ``required``
+   -  ``filesystem``: The filesystem of the partition. When choosing
       ``xfs``, the ``xfsprogs`` package will need to be installed.
       Valid values: ``ext2``, ``ext3``, ``ext4``, ``xfs``
-      *``required``*
-   -  **``format_command``**: Command to format the partition with. This
+      ``required``
+   -  ``format_command``: Command to format the partition with. This
       optional setting overrides the command bootstrap-vz would normally
       use to format the partition. The command is specified as a string
       array where each option/argument is an item in that array (much
       like the `image\_commands <../bootstrapvz/plugins/image_commands.html>`__ plugin).
-      *``optional``* The following variables are available:
-   -  **``{fs}``**: The filesystem of the partition.
-   -  **``{device_path}``**: The device path of the partition.
-   -  **``{size}``**: The size of the partition.
+      ``optional`` The following variables are available:
+   -  ``{fs}``: The filesystem of the partition.
+   -  ``{device_path}``: The device path of the partition.
+   -  ``{size}``: The size of the partition.
 
    The default command used by boostrap-vz is
    ``['mkfs.{fs}', '{device_path}']``.
 
-   -  **``boot``**: Configuration of the boot partition. The three
+   -  ``boot``: Configuration of the boot partition. The three
       settings equal those of the root partition.
-      *``optional``*
-   -  **``swap``**: Configuration of the swap partition. Since the swap
+      ``optional``
+   -  ``swap``: Configuration of the swap partition. Since the swap
       partition has its own filesystem you can only specify the size for
       this partition.
-      *``optional``*
+      ``optional``
 
 Plugins
 ~~~~~~~

From 594bd5773bae90e98e13dc804cd723f4f10a1fb4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 17:09:20 +0200
Subject: [PATCH 302/345] Fix links in manifest README

---
 manifests/README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/manifests/README.rst b/manifests/README.rst
index af473a2..bff6c7b 100644
--- a/manifests/README.rst
+++ b/manifests/README.rst
@@ -74,7 +74,7 @@ are 4 possible settings:
    Accepts a list of package names.
    ``optional``
 -  ``guest_additions``: This setting is only relevant for the
-   `virtualbox provider <../bootstrapvz/providers/virtualbox.html>`__.
+   `virtualbox provider <../bootstrapvz/providers/virtualbox>`__.
    It specifies the path to the VirtualBox Guest Additions ISO, which, when specified,
    will be mounted and used to install the VirtualBox Guest Additions.
    ``optional``
@@ -214,7 +214,7 @@ boot, root and swap.
       optional setting overrides the command bootstrap-vz would normally
       use to format the partition. The command is specified as a string
       array where each option/argument is an item in that array (much
-      like the `image\_commands <../bootstrapvz/plugins/image_commands.html>`__ plugin).
+      like the `image\_commands <../bootstrapvz/plugins/image_commands>`__ plugin).
       ``optional`` The following variables are available:
    -  ``{fs}``: The filesystem of the partition.
    -  ``{device_path}``: The device path of the partition.

From db3d2f191a4ab3e0da37426a18ba54c2c7669a38 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 19:23:50 +0200
Subject: [PATCH 303/345] Fix links to tests/ READMEs

---
 docs/transform_github_links.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/transform_github_links.py b/docs/transform_github_links.py
index e136eda..7fc3469 100644
--- a/docs/transform_github_links.py
+++ b/docs/transform_github_links.py
@@ -11,8 +11,8 @@ includes_mapping = {
 	r'^(providers|plugins)/index$': r'bootstrapvz/\1/',
 	r'^(providers|plugins)/(?!index)([^/]+)$': r'bootstrapvz/\1/\2/',
 	r'^manifest$': r'manifests/',
-	r'^testing/index$': r'testing/',
-	r'^testing/(?!index)([^/]+)_tests$': r'testing/\1/',
+	r'^testing/index$': r'tests/',
+	r'^testing/(?!index)([^/]+)_tests$': r'tests/\1/',
 	r'^remote_bootstrapping$': r'bootstrapvz/remote/',
 	r'^developers/index$': r'bootstrapvz/',
 	r'^developers/contributing$': r'CONTRIBUTING.rst',
@@ -26,9 +26,9 @@ links_mapping = {
 	r'^$': r'',
 	r'^bootstrapvz/(providers|plugins)$': r'\1',
 	r'^bootstrapvz/(providers|plugins)/([^/]+)$': r'\1/\2.html',
-	r'^testing$': r'testing',
+	r'^tests$': r'testing',
 	r'^manifests$': r'manifest.html',
-	r'^testing/([^/]+)$': r'testing/\1_tests.html',
+	r'^tests/([^/]+)$': r'testing/\1_tests.html',
 	r'^bootstrapvz/remote$': r'remote_bootstrapping.html',
 	r'^bootstrapvz$': r'developers',
 	r'^CONTRIBUTING\.rst$': r'developers/contributing.html',

From 24f0f071e7a7841ead9e4ea170c6422686bc7891 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 19:24:13 +0200
Subject: [PATCH 304/345] Include bootstrap-vz-{remote,server} in setup.py

---
 setup.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index e68979a..4d8493d 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,10 @@ setup(name='bootstrap-vz',
       version=find_version(os.path.join(os.path.dirname(__file__), 'bootstrapvz/__init__.py')),
       packages=find_packages(),
       include_package_data=True,
-      entry_points={'console_scripts': ['bootstrap-vz = bootstrapvz.base:main']},
+      entry_points={'console_scripts': ['bootstrap-vz = bootstrapvz.base:main',
+                                        'bootstrap-vz-remote = bootstrapvz.remote.main:main',
+                                        'bootstrap-vz-server = bootstrapvz.remote.server:main',
+                                        ]},
       install_requires=['termcolor >= 1.1.0',
                         'fysom >= 1.0.15',
                         'jsonschema >= 2.3.0',

From d1c135791e651efea0dd742c5a54a1eab17b1c05 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 19:27:56 +0200
Subject: [PATCH 305/345] Add note about Ctrl+C

---
 README.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.rst b/README.rst
index 2f0dbb5..1555143 100644
--- a/README.rst
+++ b/README.rst
@@ -56,6 +56,12 @@ package and ``kpartx``:
 
     root@host:~# apt-get install parted kpartx
 
+Note that you can always abort a bootstrapping process by pressing
+``Ctrl+C``, bootstrap-vz will then initiate a cleanup/rollback process,
+where volumes are detached/deleted and temporary files removed, pressing
+``Ctrl+C`` a second time shortcuts that procedure, halts the cleanup and
+quits the process.
+
 VirtualBox Vagrant
 ~~~~~~~~~~~~~~~~~~
 

From d496c028cdc94de285bf26f529b8d72bf94ec6f1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 19 Apr 2015 19:28:13 +0200
Subject: [PATCH 306/345] Document the remote bootstrapping procedure

---
 bootstrapvz/remote/README.rst | 169 ++++++++++++++++++++++++++++++++++
 tests/integration/README.rst  |   3 +
 2 files changed, 172 insertions(+)

diff --git a/bootstrapvz/remote/README.rst b/bootstrapvz/remote/README.rst
index 5091606..69335c7 100644
--- a/bootstrapvz/remote/README.rst
+++ b/bootstrapvz/remote/README.rst
@@ -1,2 +1,171 @@
 Remote bootstrapping
 ====================
+
+bootstrap-vz is able to bootstrap images not only on the machine
+on which it is invoked, but also on remote machines that have bootstrap-vz
+installed.
+
+This is helpful when you create manifests on your own workstation, but have a
+beefed up remote build server which can create images quickly.
+There may also be situations where you want to build multiple manifests that
+have different providers and require the host machines to be running on
+that provider (e.g. EBS backed AMIs can only be created on EC2 instances),
+when doing this multiple times SSHing into the machines and copying the
+manifests can be a hassle.
+
+Lastly, the main motivation for supporting remote bootstrapping is the
+automation of `integration testing <../../tests/integration>`__.
+As you will see `further down <#bootstrap-vz-remote>`__,
+bootstrap-vz is able to select which build server is required
+for a specific test and run the bootstrapping procedure on said server.
+
+
+bootstrap-vz-remote
+-------------------
+Normally you'd use ``bootstrap-vz`` to start a bootstrapping process.
+When bootstrapping remotely simply use ``bootstrap-vz-remote`` instead,
+it takes the same arguments plus a few additional ones:
+
+* ``--servers <path>``: Path to a list of build-servers
+  (see `build-servers.yml <#build-servers-yml>`__ for more info)
+* ``--name <name>``: Selects a specific build-server from the list
+  of build-servers
+* ``--release <release>``: Restricts the autoselection of build-servers
+  to the ones with the specified release
+
+Much like when bootstrapping directly, you can press ``Ctrl+C`` at any time
+to abort the bootstrapping process.
+The remote process will receive the keyboard interrupt signal
+and begin cleaning up - pressing ``Ctrl+C`` a second time will abort that as
+well and kill the connection immediately.
+
+Note that there is also a ``bootstrap-vz-server``, this file is not meant to be
+invoked directly by the user, but is instead launched by bootstrap-vz on the
+remote server when connecting to it.
+
+
+Dependencies
+------------
+For the remote bootstrapping procedure to work, you will need to install
+bootstrap-vz as well as the ``sudo`` command on the remote machine.
+Also make sure that all the needed dependencies for bootstrapping your image
+are installed.
+
+Locally the pip package `Pyro4`__ is needed.
+
+__ https://pypi.python.org/pypi/Pyro4
+
+
+
+build-servers.yml
+-----------------
+The file ``build-servers.yml`` informs bootstrap-vz about the different
+build servers you have at your disposal.
+In its simplest form you can just add your own machine like this:
+
+.. code:: yaml
+
+  local:
+    type: local
+    can_bootstrap: [virtualbox]
+    release: jessie
+    build_settings: {}
+
+``type`` specifies how bootstrap-vz should connect to the build-server.
+``local`` simply means that it will call the bootstrapping procedure directly,
+no new process is spawned.
+
+``can_bootstrap`` tells bootstrap-vz for which providers this machine is capable
+of building images. With the exception of the EC2 provider,
+the accepted values match the accepted provider names in the manifest.
+For EC2 you can specify ``ec2-s3`` and/or ``ec2-ebs``.
+``ec2-ebs`` specifies that the machine in question can bootstrap EBS backed
+images and should only be used when the it is located on EC2.
+``ec2-s3`` signifies that the machine is capable of bootstrapping S3 backed
+images.
+
+Beyond being a string, the value of ``release`` is not enforced in any way.
+It's only current use is for ``bootstrap-vz-remote`` where you can restrict
+which build-server should be autoselected.
+
+
+Remote settings
+~~~~~~~~~~~~~~~
+The other (and more interesting) setting for ``type`` is ``ssh``,
+which requires a few more configuration settings:
+
+.. code:: yaml
+
+  local_vm:
+    type: ssh
+    can_bootstrap:
+      - virtualbox
+      - ec2-s3
+    release: wheezy
+    # remote settings below here
+    address: 127.0.0.1
+    port: 2222
+    username: admin
+    keyfile: path_to_private_key_file
+    server_bin: /root/bootstrap/bootstrap-vz-server
+
+
+The last 5 settings specify how bootstrap-vz can connect
+to the remote build-server.
+While the initial handshake is achieved through SSH, bootstrap-vz mainly
+communicates with its counterpart through RPC (the communication port is
+automatically forwarded through an SSH tunnel).
+``address``, ``port``, ``username`` and ``keyfile`` are hopefully
+self explanatory (remote machine address, SSH port, login name and path to
+private SSH key file).
+
+``server_bin`` refers to the `aboved mentioned <#bootstrap-vz-remote>`__
+bootstrap-vz-server executable. This is the command bootstrap-vz executes
+on the remote machine to start the RPC server.
+
+Be aware that there are a few limitations as to what bootstrap-vz is able to
+deal with, regarding the remote machine setup (in time they may be fixed
+by a benevolent contributor):
+
+* The login user must be able to execute sudo without a password
+* The private key file must be added to the ssh-agent before invocation
+  (alternatively it may not be password protected)
+* The server must already be part of the known_hosts list
+  (bootstrap-vz uses ``ssh`` directly and cannot handle interactive prompts)
+
+
+Build settings
+~~~~~~~~~~~~~~
+The build settings allow you to override specific manifest properties.
+This is useful when for example the VirtualBox guest additions ISO is located
+at ``/root/guest_additions.iso`` on server 1, while server 2 has it at
+``/root/images/vbox.iso``.
+
+.. code:: yaml
+
+  local:
+    type: local
+    can_bootstrap:
+      - virtualbox
+      - ec2-s3
+    release: jessie
+    build_settings:
+      guest_additions: /root/images/VBoxGuestAdditions.iso
+      apt_proxy:
+        address: 127.0.0.1
+        port: 3142
+      ec2-credentials:
+        access-key: AFAKEACCESSKEYFORAWS
+        secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva
+        certificate: /root/manifests/cert.pem
+        private-key: /root/manifests/pk.pem
+        user-id: 1234-1234-1234
+      s3-region: eu-west-1
+
+* ``guest_additions`` specifies the path to the VirtualBox guest additions ISO
+  on the remote machine.
+* ``apt_proxy`` sets the configuration for the `apt_proxy plugin <../plugins/apt_proxy>`.
+* ``ec2-credentials`` contains all the settings you know from EC2 manifests,
+  note that when running `integration tests <../../tests/integration>`__,
+  these credentials are also used when running instances.
+* ``s3-region`` overrides the s3 bucket region when bootstrapping S3 backed images.
diff --git a/tests/integration/README.rst b/tests/integration/README.rst
index a93edec..883f78f 100644
--- a/tests/integration/README.rst
+++ b/tests/integration/README.rst
@@ -1,2 +1,5 @@
 Integration tests
 =================
+
+When running integration tests, the framework will look for ``build-servers.yml``
+at the root of the repo and raise an error if it is not found.

From 5942c665bca183b6baec1ffadf421b0d3a8ea95b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Mon, 20 Apr 2015 13:04:17 +0200
Subject: [PATCH 307/345] Fix some indentation and links

---
 README.rst       | 36 ++++++++++++++++++------------------
 tests/README.rst |  7 ++++++-
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/README.rst b/README.rst
index 1555143..81b0768 100644
--- a/README.rst
+++ b/README.rst
@@ -24,7 +24,8 @@ There, you can discover `what the dependencies <#dependencies>`__ for
 a specific cloud provider are, `see a list of available plugins <bootstrapvz/plugins>`__
 and learn `how you create a manifest <manifests>`__.
 
-Note to developers: `The documentaion <docs>`__ is generated in a rather peculiar and nifty way.
+Note to developers: `The documentaion <docs>`__ is generated in
+a rather peculiar and nifty way.
 
 Installation
 ------------
@@ -65,7 +66,7 @@ quits the process.
 VirtualBox Vagrant
 ~~~~~~~~~~~~~~~~~~
 
-::
+.. code:: sh
 
     user@host:~$ sudo -i # become root
     root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo
@@ -73,10 +74,10 @@ VirtualBox Vagrant
     root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies
     root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml
 
-If you want to use the `minimize\_size <bootstrapvz/plugins/minimize_size>`__
-plugin, you will have to install the ``zerofree`` package and `VMWare
-Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
-as well.
+If you want to use the `minimize\_size <bootstrapvz/plugins/minimize_size>`__ plugin,
+you will have to install the ``zerofree`` package and `VMWare Workstation`__ as well.
+
+__ https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0
 
 Amazon EC2 EBS backed AMI
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -109,11 +110,12 @@ occurred you can simply correct the problem that caused it and rerun
 everything, there will be no leftovers from the previous run (as always
 there are of course rare/unlikely exceptions to that rule). The error
 messages should always give you a strong hint at what is wrong, if that
-is not the case please consider `opening an
-issue <https://github.com/andsens/bootstrap-vz/issues>`__ and attach
+is not the case please consider `opening an issue`__ and attach
 both the error message and your manifest (preferably as a gist or
 similar).
 
+__ https://github.com/andsens/bootstrap-vz/issues
+
 Dependencies
 ------------
 
@@ -127,9 +129,9 @@ At a bare minimum the following python libraries are needed:
 * `docopt <https://pypi.python.org/pypi/docopt>`__
 * `pyyaml <https://pypi.python.org/pypi/pyyaml>`__
 
-To bootstrap Debian itself
-`debootstrap <https://packages.debian.org/wheezy/debootstrap>`__
-is needed as well.
+To bootstrap Debian itself `debootstrap`__ is needed as well.
+
+__ https://packages.debian.org/wheezy/debootstrap
 
 Any other requirements are dependent upon the manifest configuration
 and are detailed in the corresponding sections of the documentation.
@@ -140,14 +142,12 @@ Developers
 ----------
 
 The API documentation, development guidelines and an explanation of
-bootstrap-vz internals can be found at
-`bootstrap-vz.readthedocs.org <http://bootstrap-vz.readthedocs.org/en/master/developers>`__.
+bootstrap-vz internals can be found at `bootstrap-vz.readthedocs.org`__.
+
+__ http://bootstrap-vz.readthedocs.org/en/master/developers
 
 Contributing
 ------------
 
-Contribution guidelines are described in the documentation under
-`Contributing <CONTRIBUTING.rst>`__.
-There's also
-`a topic <CONTRIBUTING.rst#coding-style>`__
-regarding the coding style.
+Contribution guidelines are described in the documentation under `Contributing <CONTRIBUTING.rst>`__.
+There's also a topic regarding `the coding style <CONTRIBUTING.rst#coding-style>`__.
diff --git a/tests/README.rst b/tests/README.rst
index 07a3b69..808c60b 100644
--- a/tests/README.rst
+++ b/tests/README.rst
@@ -1,2 +1,7 @@
 The testing framework consists of two parts:
-The unit tests and the integration tests
+The unit tests and the integration tests.
+
+The `unit tests <unit>`__ are responsible for testing individual
+parts of bootstrap-vz,
+while the `integration tests <integration>`__ test entire manifests by
+bootstrapping and booting them.

From 7a1187c6399f7ae099e35ad0a363194135e14663 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Mon, 20 Apr 2015 16:25:35 +0200
Subject: [PATCH 308/345] Simplify virtualbox test manifest composition

---
 tests/integration/virtualbox_tests.py | 31 +++++++++++++--------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/tests/integration/virtualbox_tests.py b/tests/integration/virtualbox_tests.py
index 27e95ea..3453444 100644
--- a/tests/integration/virtualbox_tests.py
+++ b/tests/integration/virtualbox_tests.py
@@ -2,15 +2,14 @@ from manifests import merge_manifest_data
 from tools import boot_manifest
 from unittest.case import SkipTest
 
-partials = {'vbox': 'provider: {name: virtualbox}',
-            'vdi': 'volume: {backing: vdi}',
-            'vmdk': 'volume: {backing: vmdk}',
+partials = {'vdi': '{provider: {name: virtualbox}, volume: {backing: vdi}}',
+            'vmdk': '{provider: {name: virtualbox}, volume: {backing: vmdk}}',
             }
 
 
 def test_unpartitioned_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'unpartitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -18,7 +17,7 @@ def test_unpartitioned_extlinux_oldstable():
 
 def test_msdos_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -26,7 +25,7 @@ def test_msdos_extlinux_oldstable():
 
 def test_gpt_extlinux_oldstable():
 	std_partials = ['base', 'oldstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -34,7 +33,7 @@ def test_gpt_extlinux_oldstable():
 
 def test_unpartitioned_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -42,7 +41,7 @@ def test_unpartitioned_extlinux_stable():
 
 def test_msdos_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -50,7 +49,7 @@ def test_msdos_extlinux_stable():
 
 def test_gpt_extlinux_stable():
 	std_partials = ['base', 'stable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -58,7 +57,7 @@ def test_gpt_extlinux_stable():
 
 def test_msdos_grub_stable():
 	std_partials = ['base', 'stable64', 'grub', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -66,7 +65,7 @@ def test_msdos_grub_stable():
 
 def test_gpt_grub_stable():
 	std_partials = ['base', 'stable64', 'grub', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -74,7 +73,7 @@ def test_gpt_grub_stable():
 
 def test_unpartitioned_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'unpartitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -82,7 +81,7 @@ def test_unpartitioned_extlinux_unstable():
 
 def test_msdos_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -90,7 +89,7 @@ def test_msdos_extlinux_unstable():
 
 def test_gpt_extlinux_unstable():
 	std_partials = ['base', 'unstable64', 'extlinux', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -98,7 +97,7 @@ def test_gpt_extlinux_unstable():
 
 def test_msdos_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'msdos', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)
@@ -106,7 +105,7 @@ def test_msdos_grub_unstable():
 
 def test_gpt_grub_unstable():
 	std_partials = ['base', 'unstable64', 'grub', 'gpt', 'partitioned', 'root_password']
-	custom_partials = [partials['vbox'], partials['vmdk']]
+	custom_partials = [partials['vmdk']]
 	manifest_data = merge_manifest_data(std_partials, custom_partials)
 	with boot_manifest(manifest_data) as instance:
 		print(instance.console_output)

From 84de1de00f1a397688b778691fe1380dd7aa1165 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <aim@secoya.dk>
Date: Mon, 20 Apr 2015 16:25:54 +0200
Subject: [PATCH 309/345] Docs for integration testing

---
 docs/testing/index.rst                      |  1 +
 docs/testing/integration_test_providers.rst |  1 +
 tests/README.rst                            |  5 +-
 tests/integration/README.rst                | 68 +++++++++++++++++++++
 tests/integration/providers/README.rst      |  2 +
 5 files changed, 74 insertions(+), 3 deletions(-)
 create mode 100644 docs/testing/integration_test_providers.rst
 create mode 100644 tests/integration/providers/README.rst

diff --git a/docs/testing/index.rst b/docs/testing/index.rst
index 42c091e..5c0b339 100644
--- a/docs/testing/index.rst
+++ b/docs/testing/index.rst
@@ -7,5 +7,6 @@ Testing
 
    unit_tests
    integration_tests
+   integration_test_providers
 
 .. include:: ../../tests/README.rst
diff --git a/docs/testing/integration_test_providers.rst b/docs/testing/integration_test_providers.rst
new file mode 100644
index 0000000..c81153e
--- /dev/null
+++ b/docs/testing/integration_test_providers.rst
@@ -0,0 +1 @@
+.. include:: ../../tests/integration/providers/README.rst
diff --git a/tests/README.rst b/tests/README.rst
index 808c60b..da36feb 100644
--- a/tests/README.rst
+++ b/tests/README.rst
@@ -2,6 +2,5 @@ The testing framework consists of two parts:
 The unit tests and the integration tests.
 
 The `unit tests <unit>`__ are responsible for testing individual
-parts of bootstrap-vz,
-while the `integration tests <integration>`__ test entire manifests by
-bootstrapping and booting them.
+parts of bootstrap-vz, while the `integration tests <integration>`__ test
+entire manifests by bootstrapping and booting them.
diff --git a/tests/integration/README.rst b/tests/integration/README.rst
index 883f78f..2e720e4 100644
--- a/tests/integration/README.rst
+++ b/tests/integration/README.rst
@@ -1,5 +1,73 @@
 Integration tests
 =================
+`Integration tests`__ test
+bootstrap-vz in its entirety.
+This testing includes building images from manifests and
+creating/booting said images.
+
+__ http://en.wikipedia.org/wiki/Integration_testing
+
+Since hardcoding manifests for each test, bootstrapping them and booting the
+resulting images is too much code for a single test, a testing harness has
+been developed that reduces each test to it's bare essentials:
+
+* Combine available `manifest partials <manifest-partials>`__ into a single manifest
+* Boot an instance from a manifest
+* Run tests on the booted instance
+
+In order for the integration testing harness to be able to bootstrap it must
+know about your `build-servers <../../bootstrapvz/remote#build-servers-yml>`__.
+Depending on the manifest that is bootstrapped, the harness chooses
+a fitting build-server, connects to it and starts the bootstrapping process.
 
 When running integration tests, the framework will look for ``build-servers.yml``
 at the root of the repo and raise an error if it is not found.
+
+
+Manifest combinations
+---------------------
+The tests mainly focus on varying key parts of an image
+(e.g. partitioning, Debian release, bootloader, ec2 backing, ec2 virtualization method)
+that have been problem areas.
+Essentially the tests are the cartesian product of these key parts.
+
+
+Aborting a test
+---------------
+You can press ``Ctrl+C`` at any time during the testing to abort -
+the harness will automatically clean up any temporary resources and shut down
+running instances. Pressing ``Ctrl+C`` a second time stops the cleanup and quits
+immediately.
+
+
+Manifest partials
+-----------------
+Instead of creating manifests from scratch for each single test, reusable parts
+are factored out into partials in the manifest folder.
+This allows code like this:
+
+.. code:: python
+
+	partials = {'vdi': '{provider: {name: virtualbox}, volume: {backing: vdi}}',
+	            'vmdk': '{provider: {name: virtualbox}, volume: {backing: vmdk}}',
+	            }
+
+	def test_unpartitioned_extlinux_oldstable():
+		std_partials = ['base', 'stable64', 'extlinux', 'unpartitioned', 'root_password']
+		custom_partials = [partials['vmdk']]
+		manifest_data = merge_manifest_data(std_partials, custom_partials)
+
+The code above produces a manifest for Debian stable 64-bit unpartitioned
+virtualbox VMDK image.
+``root_password`` is a special partial in that the actual password is
+randomly generated on load.
+
+
+Missing parts
+-------------
+The integration testing harness is in no way complete.
+
+* It still has no support for providers other than virtualbox and EC2.
+* Creating an SSH connection to a booted instance is cumbersome and does not
+  happen in any of the tests - this would be particularly useful when manifests
+  are to be tested beyond whether they boot up.
diff --git a/tests/integration/providers/README.rst b/tests/integration/providers/README.rst
new file mode 100644
index 0000000..9452950
--- /dev/null
+++ b/tests/integration/providers/README.rst
@@ -0,0 +1,2 @@
+Integration test providers
+==========================

From ad8d817c5241314217f0429fc18c94a22b99c087 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 22 Apr 2015 08:53:59 +0200
Subject: [PATCH 310/345] Add missing hashmark to link

---
 tests/integration/README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/README.rst b/tests/integration/README.rst
index 2e720e4..01c4f0f 100644
--- a/tests/integration/README.rst
+++ b/tests/integration/README.rst
@@ -11,7 +11,7 @@ Since hardcoding manifests for each test, bootstrapping them and booting the
 resulting images is too much code for a single test, a testing harness has
 been developed that reduces each test to it's bare essentials:
 
-* Combine available `manifest partials <manifest-partials>`__ into a single manifest
+* Combine available `manifest partials <#manifest-partials>`__ into a single manifest
 * Boot an instance from a manifest
 * Run tests on the booted instance
 

From 4d561274e4efb8689644390ee8faa8ffce9292d6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Thu, 23 Apr 2015 23:30:39 +0200
Subject: [PATCH 311/345] Make bootstrap-vz-remote actually work Wow, that's
 quite an oversight...

---
 bootstrapvz/remote/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/remote/main.py b/bootstrapvz/remote/main.py
index d007a20..9613088 100644
--- a/bootstrapvz/remote/main.py
+++ b/bootstrapvz/remote/main.py
@@ -43,7 +43,7 @@ def main():
 	with build_server.connect() as connection:
 		connection.run(manifest,
 		               debug=opts['--debug'],
-		               dry_run=['--dry-run'])
+		               dry_run=opts['--dry-run'])
 
 
 def get_opts():

From cba0a99d0a5ab0c73de18ced58c4aca31fcadfea Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Thu, 23 Apr 2015 23:19:16 -0300
Subject: [PATCH 312/345] Fix `truncate` arguments

`truncate` doesn't expect the parameter `create` that was used
previously by `qemu-img`. This made a sparse file named "create" to
appear at current working directory every time the command as executed.

This closes #212.
---
 bootstrapvz/common/fs/loopbackvolume.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/common/fs/loopbackvolume.py b/bootstrapvz/common/fs/loopbackvolume.py
index 2db9edb..248f3d7 100644
--- a/bootstrapvz/common/fs/loopbackvolume.py
+++ b/bootstrapvz/common/fs/loopbackvolume.py
@@ -12,7 +12,7 @@ class LoopbackVolume(Volume):
 	def _before_create(self, e):
 		self.image_path = e.image_path
 		size_opt = '--size={mib}M'.format(mib=self.size.bytes.get_qty_in('MiB'))
-		log_check_call(['truncate', 'create', size_opt, self.image_path])
+		log_check_call(['truncate', size_opt, self.image_path])
 
 	def _before_attach(self, e):
 		[self.loop_device_path] = log_check_call(['losetup', '--show', '--find', self.image_path])

From c95259c9c4cef1031fa23398ca40110b2b5b728d Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 25 Apr 2015 11:43:45 +0200
Subject: [PATCH 313/345] Update changelog

---
 CHANGELOG.rst | 126 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 80 insertions(+), 46 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 420540c..c65bc86 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,77 +1,111 @@
 Changelog
 =========
 
-2014-11-23:
+2015-04-25
+----------
+Anders Ingemann (work started 2014-08-31, merged on 2015-04-25):
+	* Introduce `remote bootstrapping <bootstrapvz/remote>`__
+	* Introduce `integration testing <tests/integration>`__ (for VirtualBox and EC2)
+	* Merge the end-user documentation into the sphinx docs
+	  (plugin & provider docs are now located in their respective folders as READMEs)
+	* Include READMEs in sphinx docs and transform their links
+	* Docs for integration testing
+	* Document the remote bootstrapping procedure
+	* Add documentation about the documentation
+	* Add list of supported builds to the docs
+	* Add html output to integration tests
+	* Implement PR #201 by @jszwedko (bump required euca2ools version)
+	* grub now works on jessie
+	* extlinux is now running on jessie
+	* Issue warning when specifying pre/successors across phases (but still error out if it's a conflict)
+	* Add salt dependencies in the right phase
+	* extlinux now works with GPT on HVM instances
+	* Take @ssgelm's advice in #155 and copy the mount table -- df warnings no more
+	* Generally deny installing grub on squeeze (too much of a hassle to get working, PRs welcome)
+	* Add 1 sector gap between partitions on GPT
+	* Add new task: DeterminKernelVersion, this can potentially fix a lot of small problems
+	* Disable getty processes on jessie through logind config
+	* Partition volumes by sectors instead of bytes
+	  This allows for finer grained control over the partition sizes and gaps
+	  Add new Sectors unit, enhance Bytes unit, add unit tests for both
+	* Don't require qemu for raw volumes, use `truncate` instead
+	* Fix #179: Disabling getty processes task fails half the time
+	* Split grub and extlinux installs into separate modules
+	* Fix extlinux config for squeeze
+	* Fix #136: Make extlinux output boot messages to the serial console
+	* Extend sed_i to raise Exceptions when the expected amount of replacements is not met
 
+2014-11-23
+----------
 Noah Fontes:
-	+ Add support for enhanced networking on EC2 images
-
-2014-07-12:
+	* Add support for enhanced networking on EC2 images
 
+2014-07-12
+----------
 Tiago Ilieve:
-	+ Fixes #96: AddBackports is now a common task
-
-2014-07-09:
+	* Fixes #96: AddBackports is now a common task
 
+2014-07-09
+----------
 Anders Ingemann:
-	+ Allow passing data into the manifest
-	+ Refactor logging setup to be more modular
-	+ Convert every JSON file to YAML
-	+ Convert "provider" into provider specific section
-
-2014-07-02:
+	* Allow passing data into the manifest
+	* Refactor logging setup to be more modular
+	* Convert every JSON file to YAML
+	* Convert "provider" into provider specific section
 
+2014-07-02
+----------
 Vladimir Vitkov:
-	+ Improve grub options to work better with virtual machines
-
-2014-06-30:
+	* Improve grub options to work better with virtual machines
 
+2014-06-30
+----------
 Tomasz Rybak:
-	+ Return information about created image
-
-2014-06-22:
+	* Return information about created image
 
+2014-06-22
+----------
 Victor Marmol:
-	+ Enable the memory cgroup for the Docker plugin
-
-2014-06-19:
+	* Enable the memory cgroup for the Docker plugin
 
+2014-06-19
+----------
 Tiago Ilieve:
-	+ Fixes #94: allow stable/oldstable as release name on manifest
+	* Fixes #94: allow stable/oldstable as release name on manifest
 
 Vladimir Vitkov:
-	+ Improve ami listing performance
-
-2014-06-07:
+	* Improve ami listing performance
 
+2014-06-07
+----------
 Tiago Ilieve:
-	+ Download `gsutil` tarball to workspace instead of working directory
-	+ Fixes #97: remove raw disk image created by GCE after build
-
-2014-06-06:
+	* Download `gsutil` tarball to workspace instead of working directory
+	* Fixes #97: remove raw disk image created by GCE after build
 
+2014-06-06
+----------
 Ilya Margolin:
-	+ pip_install plugin
-
-2014-05-23:
+	* pip_install plugin
 
+2014-05-23
+----------
 Tiago Ilieve:
-	+ Fixes #95: check if the specified APT proxy server can be reached
-
-2014-05-04:
+	* Fixes #95: check if the specified APT proxy server can be reached
 
+2014-05-04
+----------
 Dhananjay Balan:
-	+ Salt minion installation & configuration plugin
-	+ Expose debootstrap --include-packages and --exclude-packages options to manifest
-
-2014-05-03:
+	* Salt minion installation & configuration plugin
+	* Expose debootstrap --include-packages and --exclude-packages options to manifest
 
+2014-05-03
+----------
 Anders Ingemann:
-	+ Require hostname setting for vagrant plugin
-	+ Fixes #14: S3 images can now be bootstrapped outside EC2.
-	+ Added enable_agent option to puppet plugin
-
-2014-05-02:
+	* Require hostname setting for vagrant plugin
+	* Fixes #14: S3 images can now be bootstrapped outside EC2.
+	* Added enable_agent option to puppet plugin
 
+2014-05-02
+----------
 Tomasz Rybak:
-	+ Added Google Compute Engine Provider
+	* Added Google Compute Engine Provider

From 8f03987593c6da259055596a899f4cd9bb77040e Mon Sep 17 00:00:00 2001
From: Jonas Bergler <jbergler@meraki.com>
Date: Wed, 26 Nov 2014 17:06:53 +0000
Subject: [PATCH 314/345] Fix installation of vbox guest additions.

Credits for this idea go to @myhro who suggested emulating uname.

I cleaned his changes up somewhat and moved the script into a
separate file to make things easier to look at.

I did a test build of wheezy with my changes and the modules were
installed correctly.

root@localhost:/home/vagrant# dkms status
vboxguest, 4.3.20, 3.2.0-4-amd64, x86_64: installed

root@localhost:/home/vagrant# lsmod | egrep ^vbox
vboxsf                 33359  0
vboxvideo              12437  0
vboxguest             162115  1 vboxsf
---
 .../assets/install_guest_additions.sh         | 16 +++++++++
 .../virtualbox/tasks/guest_additions.py       | 35 ++++++++++++++++---
 2 files changed, 46 insertions(+), 5 deletions(-)
 create mode 100644 bootstrapvz/providers/virtualbox/assets/install_guest_additions.sh

diff --git a/bootstrapvz/providers/virtualbox/assets/install_guest_additions.sh b/bootstrapvz/providers/virtualbox/assets/install_guest_additions.sh
new file mode 100644
index 0000000..3774c5d
--- /dev/null
+++ b/bootstrapvz/providers/virtualbox/assets/install_guest_additions.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+function uname {
+  if [[ $1 == '-r' ]]; then
+    echo "KERNEL_VERSION"
+    return 0
+  elif [[ $1 == '-m' ]]; then
+    echo "KERNEL_ARCH"
+    return 0
+  else
+    $(which uname) $@
+  fi
+}
+export -f uname
+
+INSTALL_SCRIPT --nox11
diff --git a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
index e4e3890..55aed80 100644
--- a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
+++ b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
@@ -2,6 +2,9 @@ from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks.packages import InstallPackages
 from bootstrapvz.common.exceptions import TaskError
+import os
+
+assets = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets'))
 
 
 class CheckGuestAdditionsPath(Task):
@@ -10,7 +13,6 @@ class CheckGuestAdditionsPath(Task):
 
 	@classmethod
 	def run(cls, info):
-		import os.path
 		guest_additions_path = info.manifest.provider['guest_additions']
 		if not os.path.exists(guest_additions_path):
 			msg = 'The file {file} does not exist.'.format(file=guest_additions_path)
@@ -27,12 +29,19 @@ class AddGuestAdditionsPackages(Task):
 		info.packages.add('bzip2')
 		info.packages.add('build-essential')
 		info.packages.add('dkms')
+
 		kernel_headers_pkg = 'linux-headers-'
 		if info.manifest.system['architecture'] == 'i386':
+			arch = 'i686'
 			kernel_headers_pkg += '686-pae'
 		else:
+			arch = 'x86_64'
 			kernel_headers_pkg += 'amd64'
 		info.packages.add(kernel_headers_pkg)
+		info.kernel = {
+			'arch': arch,
+			'headers_pkg': kernel_headers_pkg,
+		}
 
 
 class InstallGuestAdditions(Task):
@@ -42,19 +51,35 @@ class InstallGuestAdditions(Task):
 
 	@classmethod
 	def run(cls, info):
-		import os
+		from bootstrapvz.common.tools import log_call, log_check_call
+		for line in log_check_call(['chroot', info.root, 'apt-cache', 'show', info.kernel['headers_pkg']]):
+			key, value = line.split(':')
+			if key.strip() == 'Depends':
+				kernel_version = value.strip().split('linux-headers-')[-1]
+				break
+
 		guest_additions_path = info.manifest.provider['guest_additions']
 		mount_dir = 'mnt/guest_additions'
 		mount_path = os.path.join(info.root, mount_dir)
 		os.mkdir(mount_path)
 		root = info.volume.partition_map.root
 		root.add_mount(guest_additions_path, mount_path, ['-o', 'loop'])
+                install_script = os.path.join('/', mount_dir, 'VBoxLinuxAdditions.run')
+                install_wrapper_name = 'install_guest_additions.sh'
+                install_wrapper = open(os.path.join(assets, install_wrapper_name)) \
+                        .read() \
+                        .replace("KERNEL_VERSION", kernel_version) \
+                        .replace("KERNEL_ARCH", info.kernel['arch']) \
+                        .replace("INSTALL_SCRIPT", install_script)
+		install_wrapper_path = os.path.join(info.root, install_wrapper_name)
+		with open(install_wrapper_path, 'w') as f:
+			f.write(install_wrapper + '\n')
 
-		install_script = os.path.join('/', mount_dir, 'VBoxLinuxAdditions.run')
 		# Don't check the return code of the scripts here, because 1 not necessarily means they have failed
-		from bootstrapvz.common.tools import log_call
-		log_call(['chroot', info.root, install_script, '--nox11'])
+		log_call(['chroot', info.root, 'bash', '/' + install_wrapper_name])
+
 		# VBoxService process could be running, as it is not affected by DisableDaemonAutostart
 		log_call(['chroot', info.root, 'service', 'vboxadd-service', 'stop'])
 		root.remove_mount(mount_path)
 		os.rmdir(mount_path)
+		os.remove(install_wrapper_path)

From 940a089933fbcd057fd330ebe07d00f2b2443a4d Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 25 Apr 2015 16:37:27 -0300
Subject: [PATCH 315/345] Little indentation fix (spaces to tabs)

---
 bootstrapvz/providers/virtualbox/tasks/guest_additions.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
index 55aed80..350ce4a 100644
--- a/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
+++ b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py
@@ -64,9 +64,9 @@ class InstallGuestAdditions(Task):
 		os.mkdir(mount_path)
 		root = info.volume.partition_map.root
 		root.add_mount(guest_additions_path, mount_path, ['-o', 'loop'])
-                install_script = os.path.join('/', mount_dir, 'VBoxLinuxAdditions.run')
-                install_wrapper_name = 'install_guest_additions.sh'
-                install_wrapper = open(os.path.join(assets, install_wrapper_name)) \
+		install_script = os.path.join('/', mount_dir, 'VBoxLinuxAdditions.run')
+		install_wrapper_name = 'install_guest_additions.sh'
+		install_wrapper = open(os.path.join(assets, install_wrapper_name)) \
                         .read() \
                         .replace("KERNEL_VERSION", kernel_version) \
                         .replace("KERNEL_ARCH", info.kernel['arch']) \

From 22760c76db0843082609efeb62e32f351f02c059 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 25 Apr 2015 19:24:55 -0300
Subject: [PATCH 316/345] Revert a2cf28b (#213)

Looks like we may or may not need to create those uuid links by
ourselves. This patch also adds a verification to not create/delete the
link if it already exists or not, respectively.
---
 bootstrapvz/base/fs/partitions/base.py | 31 ++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 67234e0..696a92a 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -1,3 +1,4 @@
+import os
 from abstract import AbstractPartition
 from bootstrapvz.common.sectors import Sectors
 
@@ -31,6 +32,8 @@ class BasePartition(AbstractPartition):
 		self.previous = previous
 		# List of flags that parted should put on the partition
 		self.flags = []
+		# Path to symlink in /dev/disk/by-uuid (manually maintained by this class)
+		self.disk_by_uuid_path = None
 		super(BasePartition, self).__init__(size, filesystem, format_command)
 
 	def create(self, volume):
@@ -71,6 +74,25 @@ class BasePartition(AbstractPartition):
 		"""
 		self.fsm.map(device_path=device_path)
 
+	def link_uuid(self):
+		# /lib/udev/rules.d/60-kpartx.rules does not create symlinks in /dev/disk/by-{uuid,label}
+		# This patch would fix that: http://www.redhat.com/archives/dm-devel/2013-July/msg00080.html
+		# For now we just do the uuid part ourselves.
+		# This is mainly to fix a problem in update-grub where /etc/grub.d/10_linux
+		# checks if the $GRUB_DEVICE_UUID exists in /dev/disk/by-uuid and falls
+		# back to $GRUB_DEVICE if it doesn't.
+		# $GRUB_DEVICE is /dev/mapper/xvd{f,g...}# (on ec2), opposed to /dev/xvda# when booting.
+		# Creating the symlink ensures that grub consistently uses
+		# $GRUB_DEVICE_UUID when creating /boot/grub/grub.cfg
+		self.disk_by_uuid_path = os.path.join('/dev/disk/by-uuid', self.get_uuid())
+		if not os.path.exists(self.disk_by_uuid_path):
+			os.symlink(self.device_path, self.disk_by_uuid_path)
+
+	def unlink_uuid(self):
+		if os.path.isfile(self.disk_by_uuid_path):
+			os.remove(self.disk_by_uuid_path)
+		self.disk_by_uuid_path = None
+
 	def _before_create(self, e):
 		"""Creates the partition
 		"""
@@ -92,7 +114,16 @@ class BasePartition(AbstractPartition):
 	def _before_map(self, e):
 		# Set the device path
 		self.device_path = e.device_path
+		if e.src == 'unmapped_fmt':
+			# Only link the uuid if the partition is formatted
+			self.link_uuid()
+
+	def _after_format(self, e):
+		# We do this after formatting because there otherwise would be no UUID
+		self.link_uuid()
 
 	def _before_unmap(self, e):
 		# When unmapped, the device_path information becomes invalid, so we delete it
 		self.device_path = None
+		if e.src == 'formatted':
+			self.unlink_uuid()

From 313155869147a4370d3131ffbe0b810be48bfe47 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 25 Apr 2015 21:07:48 -0300
Subject: [PATCH 317/345] Fix msdos partition/filesystem types definition

This closes #142.
---
 bootstrapvz/base/fs/partitions/base.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py
index 696a92a..df60712 100644
--- a/bootstrapvz/base/fs/partitions/base.py
+++ b/bootstrapvz/base/fs/partitions/base.py
@@ -97,9 +97,18 @@ class BasePartition(AbstractPartition):
 		"""Creates the partition
 		"""
 		from bootstrapvz.common.tools import log_check_call
-		# The create command is failry simple, start and end are just Bytes objects coerced into strings
-		create_command = ('mkpart primary {start} {end}'
-		                  .format(start=str(self.get_start() + self.pad_start),
+		# The create command is fairly simple:
+		# - fs_type is the partition filesystem, as defined by parted:
+		#   fs-type can be one of "fat16", "fat32", "ext2", "HFS", "linux-swap",
+		#   "NTFS", "reiserfs", or "ufs".
+		# - start and end are just Bytes objects coerced into strings
+		if self.filesystem == 'swap':
+			fs_type = 'linux-swap'
+		else:
+			fs_type = 'ext2'
+		create_command = ('mkpart primary {fs_type} {start} {end}'
+		                  .format(fs_type=fs_type,
+		                          start=str(self.get_start() + self.pad_start),
 		                          end=str(self.get_end() - self.pad_end)))
 		# Create the partition
 		log_check_call(['parted', '--script', '--align', 'none', e.volume.device_path,

From d1197ca82cc347dd6b0ab0e8aaa48e60325c4048 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 26 Apr 2015 10:39:54 +0200
Subject: [PATCH 318/345] Jessie was release, update codename mapping

---
 bootstrapvz/common/release-codenames.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/common/release-codenames.yml b/bootstrapvz/common/release-codenames.yml
index 218488d..43b49fe 100644
--- a/bootstrapvz/common/release-codenames.yml
+++ b/bootstrapvz/common/release-codenames.yml
@@ -2,15 +2,15 @@
 # This is a mapping of Debian release names to their respective codenames
 
 unstable:  sid
-testing:   jessie
-stable:    wheezy
-oldstable: squeeze
+# testing: -- unknown atm
+stable:    jessie
+oldstable: wheezy
 
 jessie:  jessie
 wheezy:  wheezy
 squeeze: squeeze
 
-# The following release names are not supported, but included fir completeness sake
+# The following release names are not supported, but included for completeness sake
 lenny:  lenny
 etch:   etch
 sarge:  sarge

From a28b8cee8b94bd3844199819de2f6b39c5527de0 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sun, 26 Apr 2015 18:53:53 -0300
Subject: [PATCH 319/345] Add codename of the next testing release

The Debian 9, following 8 "Jessie", will be named "Stretch":

https://wiki.debian.org/DebianStretch
https://lists.debian.org/debian-devel-announce/2014/11/msg00005.html
---
 bootstrapvz/common/release-codenames.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/common/release-codenames.yml b/bootstrapvz/common/release-codenames.yml
index 43b49fe..cfc2bc6 100644
--- a/bootstrapvz/common/release-codenames.yml
+++ b/bootstrapvz/common/release-codenames.yml
@@ -2,7 +2,7 @@
 # This is a mapping of Debian release names to their respective codenames
 
 unstable:  sid
-# testing: -- unknown atm
+testing:   stretch
 stable:    jessie
 oldstable: wheezy
 

From af3302124b422bb49b5dc5ad660d933ad2fcec4f Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sun, 26 Apr 2015 22:46:53 -0300
Subject: [PATCH 320/345] Update changelog

---
 CHANGELOG.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index c65bc86..bf2f801 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -35,6 +35,18 @@ Anders Ingemann (work started 2014-08-31, merged on 2015-04-25):
 	* Fix #136: Make extlinux output boot messages to the serial console
 	* Extend sed_i to raise Exceptions when the expected amount of replacements is not met
 
+Jonas Bergler:
+    * Fixes #145: Fix installation of vbox guest additions.
+
+Tiago Ilieve:
+    * Fixes #142: msdos partition type incorrect for swap partition (Linux)
+
+2015-04-23
+----------
+
+Tiago Ilieve:
+    * Fixes #212: Sparse file is created on the current directory
+
 2014-11-23
 ----------
 Noah Fontes:

From 4120260a994a4cfda988651da5bc48a19c796289 Mon Sep 17 00:00:00 2001
From: John Kristensen <john@jerrykan.com>
Date: Mon, 27 Apr 2015 12:36:27 +1000
Subject: [PATCH 321/345] Add authentication support to the apt proxy plugin

Add username and password settings to the APT Proxy plugin so that
users who are behind an authenticating proxy can still use bootstrap-vz
without having to jump through hoops. If either the username or password
are not set, then no authentication is used.
---
 bootstrapvz/plugins/apt_proxy/README.rst          |  6 ++++++
 bootstrapvz/plugins/apt_proxy/manifest-schema.yml |  2 ++
 bootstrapvz/plugins/apt_proxy/tasks.py            | 14 ++++++++++++--
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst
index 33711a7..6bc1e83 100644
--- a/bootstrapvz/plugins/apt_proxy/README.rst
+++ b/bootstrapvz/plugins/apt_proxy/README.rst
@@ -14,6 +14,12 @@ Settings
     *``required``*
 -  ``port``: The port (integer) of the proxy server.
     *``required``*
+-  ``username``: The username for authentication against the proxy server.
+   This is ignored if ``password`` is not also set.
+   *``optional``*
+-  ``password``: The password for authentication against the proxy server.
+   This is ignored if ``username`` is not also set.
+   *``optional``*
 -  ``persistent``: Whether the proxy configuration file should remain on
    the machine or not.
    Valid values: true, false
diff --git a/bootstrapvz/plugins/apt_proxy/manifest-schema.yml b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
index 5f3f051..efa7875 100644
--- a/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
+++ b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
@@ -10,8 +10,10 @@ properties:
         type: object
         properties:
           address: {type: string}
+          password: {type: string}
           port: {type: integer}
           persistent: {type: boolean}
+          username: {type: string}
         required:
         - address
         - port
diff --git a/bootstrapvz/plugins/apt_proxy/tasks.py b/bootstrapvz/plugins/apt_proxy/tasks.py
index 6447b70..f6823e5 100644
--- a/bootstrapvz/plugins/apt_proxy/tasks.py
+++ b/bootstrapvz/plugins/apt_proxy/tasks.py
@@ -34,11 +34,21 @@ class SetAptProxy(Task):
 	@classmethod
 	def run(cls, info):
 		proxy_path = os.path.join(info.root, 'etc/apt/apt.conf.d/02proxy')
+		proxy_username = info.manifest.plugins['apt_proxy'].get('username')
+		proxy_password = info.manifest.plugins['apt_proxy'].get('password')
 		proxy_address = info.manifest.plugins['apt_proxy']['address']
 		proxy_port = info.manifest.plugins['apt_proxy']['port']
+
+		if None not in (proxy_username, proxy_password):
+			proxy_auth = '{username}:{password}@'.format(
+				username=proxy_username, password=proxy_password)
+		else:
+			proxy_auth = ''
+
 		with open(proxy_path, 'w') as proxy_file:
-			proxy_file.write('Acquire::http {{ Proxy "http://{address}:{port}"; }};\n'
-			                 .format(address=proxy_address, port=proxy_port))
+			proxy_file.write(
+				'Acquire::http {{ Proxy "http://{auth}{address}:{port}"; }};\n'
+				.format(auth=proxy_auth, address=proxy_address, port=proxy_port))
 
 
 class RemoveAptProxy(Task):

From c6087f398d3e621d528af5d6fe3b9e4320e56e75 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.ilieve@sisqualis.com.br>
Date: Mon, 27 Apr 2015 10:30:32 -0300
Subject: [PATCH 322/345] Update changelog

---
 CHANGELOG.rst | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index bf2f801..7c476d7 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,6 +1,11 @@
 Changelog
 =========
 
+2015-04-27
+----------
+John Kristensen:
+    * Add authentication support to the apt proxy plugin
+
 2015-04-25
 ----------
 Anders Ingemann (work started 2014-08-31, merged on 2015-04-25):
@@ -43,7 +48,6 @@ Tiago Ilieve:
 
 2015-04-23
 ----------
-
 Tiago Ilieve:
     * Fixes #212: Sparse file is created on the current directory
 

From e409115b254822dcb9ab8f12c3ff82649990f241 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Tue, 28 Apr 2015 01:21:07 -0300
Subject: [PATCH 323/345] Docs: fix wrongfully escaped terms

---
 bootstrapvz/plugins/admin_user/README.rst     |  2 +-
 bootstrapvz/plugins/apt_proxy/README.rst      | 10 +++++-----
 bootstrapvz/plugins/cloud_init/README.rst     |  6 +++---
 bootstrapvz/plugins/docker_daemon/README.rst  |  2 +-
 bootstrapvz/plugins/image_commands/README.rst |  4 ++--
 bootstrapvz/plugins/minimize_size/README.rst  |  4 ++--
 bootstrapvz/plugins/ntp/README.rst            |  2 +-
 bootstrapvz/plugins/puppet/README.rst         |  6 +++---
 bootstrapvz/plugins/root_password/README.rst  |  2 +-
 bootstrapvz/plugins/salt/README.rst           |  8 ++++----
 .../plugins/unattended_upgrades/README.rst    |  6 +++---
 bootstrapvz/providers/ec2/README.rst          | 20 +++++++++----------
 12 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/bootstrapvz/plugins/admin_user/README.rst b/bootstrapvz/plugins/admin_user/README.rst
index 3506f92..8da68d1 100644
--- a/bootstrapvz/plugins/admin_user/README.rst
+++ b/bootstrapvz/plugins/admin_user/README.rst
@@ -9,4 +9,4 @@ username specified.
 Settings
 ~~~~~~~~
 
--  ``username``: The username of the account to create. *``required``*
+-  ``username``: The username of the account to create. ``required``
diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst
index 6bc1e83..7e42c3e 100644
--- a/bootstrapvz/plugins/apt_proxy/README.rst
+++ b/bootstrapvz/plugins/apt_proxy/README.rst
@@ -11,17 +11,17 @@ Settings
 ~~~~~~~~
 
 -  ``address``: The IP or host of the proxy server.
-    *``required``*
+    ``required``
 -  ``port``: The port (integer) of the proxy server.
-    *``required``*
+    ``required``
 -  ``username``: The username for authentication against the proxy server.
    This is ignored if ``password`` is not also set.
-   *``optional``*
+   ``optional``
 -  ``password``: The password for authentication against the proxy server.
    This is ignored if ``username`` is not also set.
-   *``optional``*
+   ``optional``
 -  ``persistent``: Whether the proxy configuration file should remain on
    the machine or not.
    Valid values: true, false
    Default: ``false``
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/cloud_init/README.rst b/bootstrapvz/plugins/cloud_init/README.rst
index fa36e1f..b5dc6c8 100644
--- a/bootstrapvz/plugins/cloud_init/README.rst
+++ b/bootstrapvz/plugins/cloud_init/README.rst
@@ -12,12 +12,12 @@ Settings
 ~~~~~~~~
 
 -  ``username``: The username of the account to create.
-   *``required``*
+   ``required``
 -  ``disable_modules``: A list of strings specifying which cloud-init
    modules should be disabled.
-   *``optional``*
+   ``optional``
 -  ``metadata_sources``: A string that sets the
    `datasources <http://cloudinit.readthedocs.org/en/latest/topics/datasources.html>`__
    that cloud-init should try fetching metadata from. The source is
    automatically set when using the ec2 provider.
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/docker_daemon/README.rst b/bootstrapvz/plugins/docker_daemon/README.rst
index e210435..c559ab1 100644
--- a/bootstrapvz/plugins/docker_daemon/README.rst
+++ b/bootstrapvz/plugins/docker_daemon/README.rst
@@ -15,4 +15,4 @@ Settings
 -  ``version``: Selects the docker version to install. To select the
    latest version simply omit this setting.
    Default: ``latest``
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/image_commands/README.rst
index 106ad22..5ed7e65 100644
--- a/bootstrapvz/plugins/image_commands/README.rst
+++ b/bootstrapvz/plugins/image_commands/README.rst
@@ -15,5 +15,5 @@ Settings
    (to circumvent this, simply put the entire command in a single
    string). In addition to the manifest variables ``{root}`` is also
    available. It points at the root of the image volume.
-   *``required``*
-   *``manifest vars``*
+   ``required``
+   ``manifest vars``
diff --git a/bootstrapvz/plugins/minimize_size/README.rst b/bootstrapvz/plugins/minimize_size/README.rst
index e275237..bb7860c 100644
--- a/bootstrapvz/plugins/minimize_size/README.rst
+++ b/bootstrapvz/plugins/minimize_size/README.rst
@@ -29,9 +29,9 @@ Settings
    zeroes, so the volume could be better shrunk after this.
    Valid values: true, false
    Default: false
-   *``optional``*
+   ``optional``
 -  ``shrink``: Whether the volume should be shrunk. This setting works
    best in conjunction with the zerofree tool.
    Valid values: true, false
    Default: false
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/ntp/README.rst b/bootstrapvz/plugins/ntp/README.rst
index 40421b1..f90c81a 100644
--- a/bootstrapvz/plugins/ntp/README.rst
+++ b/bootstrapvz/plugins/ntp/README.rst
@@ -9,4 +9,4 @@ Settings
 
 -  ``servers``: A list of strings specifying which servers should be
    used to synchronize the machine clock.
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/puppet/README.rst b/bootstrapvz/plugins/puppet/README.rst
index 203e0b3..530694f 100644
--- a/bootstrapvz/plugins/puppet/README.rst
+++ b/bootstrapvz/plugins/puppet/README.rst
@@ -16,9 +16,9 @@ Settings
 ~~~~~~~~
 
 -  ``manifest``: Path to the puppet manifest that should be applied.
-   *``optional``*
+   ``optional``
 -  ``assets``: Path to puppet assets. The contents will be copied into
    ``/etc/puppet`` on the image. Any existing files will be overwritten.
-   *``optional``*
+   ``optional``
 -  ``enable_agent``: Whether the puppet agent daemon should be enabled.
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/root_password/README.rst b/bootstrapvz/plugins/root_password/README.rst
index a24d904..c2f37c8 100644
--- a/bootstrapvz/plugins/root_password/README.rst
+++ b/bootstrapvz/plugins/root_password/README.rst
@@ -8,4 +8,4 @@ Settings
 ~~~~~~~~
 
 -  ``password``: The password for the root user.
-   *``required``*
+   ``required``
diff --git a/bootstrapvz/plugins/salt/README.rst b/bootstrapvz/plugins/salt/README.rst
index acb71ab..cb48c4b 100644
--- a/bootstrapvz/plugins/salt/README.rst
+++ b/bootstrapvz/plugins/salt/README.rst
@@ -11,16 +11,16 @@ Settings
 -  ``install_source``: Source to install salt codebase from. ``stable``
    for current stable, ``daily`` for installing the daily build, and
    ``git`` to install from git repository.
-   *``required``*
+   ``required``
 -  ``version``: Only needed if you are installing from ``git``.
    \ ``develop`` to install current development head, or provide any tag
    name or commit hash from `salt
    repo <https://github.com/saltstack/salt>`__
-   *``optional``*
+   ``optional``
 -  ``master``: Salt master FQDN or IP
-   *``optional``*
+   ``optional``
 -  ``grains``: Set `salt
    grains <http://docs.saltstack.com/en/latest/topics/targeting/grains.html>`__
    for this minion. Accepts a map with grain name as key and the grain
    data as value.
-   *``optional``*
+   ``optional``
diff --git a/bootstrapvz/plugins/unattended_upgrades/README.rst b/bootstrapvz/plugins/unattended_upgrades/README.rst
index 3339b7b..7caded2 100644
--- a/bootstrapvz/plugins/unattended_upgrades/README.rst
+++ b/bootstrapvz/plugins/unattended_upgrades/README.rst
@@ -10,9 +10,9 @@ Settings
 ~~~~~~~~
 
 -  ``update_interval``: Days between running ``apt-get update``.
-   *``required``*
+   ``required``
 -  ``download_interval``: Days between running
    ``apt-get upgrade --download-only``
-   *``required``*
+   ``required``
 -  ``upgrade_interval``: Days between installing any security upgrades.
-   *``required``*
+   ``required``
diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst
index c968f48..184f236 100644
--- a/bootstrapvz/providers/ec2/README.rst
+++ b/bootstrapvz/providers/ec2/README.rst
@@ -24,28 +24,28 @@ and secret key, which are needed for bootstraping EBS backed instances.
 The settings describes below should be placed in the ``credentials`` key
 under the ``provider`` section.
 
--  **``access-key``**: AWS access-key.
+-  ``access-key``: AWS access-key.
    May also be supplied via the environment variable
    ``$AWS_ACCESS_KEY``
-   *``required for EBS & S3 backing``*
--  **``secret-key``**: AWS secret-key.
+   ``required for EBS & S3 backing``
+-  ``secret-key``: AWS secret-key.
    May also be supplied via the environment variable
    ``$AWS_SECRET_KEY``
-   *``required for EBS & S3 backing``*
--  **``certificate``**: Path to the AWS user certificate. Used for
+   ``required for EBS & S3 backing``
+-  ``certificate``: Path to the AWS user certificate. Used for
    uploading the image to an S3 bucket.
    May also be supplied via the environment variable
    ``$AWS_CERTIFICATE``
-   *``required for S3 backing``*
--  **``private-key``**: Path to the AWS private key. Used for uploading
+   ``required for S3 backing``
+-  ``private-key``: Path to the AWS private key. Used for uploading
    the image to an S3 bucket.
    May also be supplied via the environment variable
    ``$AWS_PRIVATE_KEY``
-   *``required for S3 backing``*
--  **``user-id``**: AWS user ID. Used for uploading the image to an S3
+   ``required for S3 backing``
+-  ``user-id``: AWS user ID. Used for uploading the image to an S3
    bucket.
    May also be supplied via the environment variable ``$AWS_USER_ID``
-   *``required for S3 backing``*
+   ``required for S3 backing``
 
 Example:
 

From b3cda5e8597cba368f76e74664260028a97243b5 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Tue, 28 Apr 2015 01:29:41 -0300
Subject: [PATCH 324/345] Docs: fix `apt_proxy` plugin's README

`address` and `port` description were being rendered as bold because of
an extra space indenting its `required` note.

Also escaped `true`/`false` terms for the `persistent` option.
---
 bootstrapvz/plugins/apt_proxy/README.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst
index 7e42c3e..75c2c16 100644
--- a/bootstrapvz/plugins/apt_proxy/README.rst
+++ b/bootstrapvz/plugins/apt_proxy/README.rst
@@ -11,9 +11,9 @@ Settings
 ~~~~~~~~
 
 -  ``address``: The IP or host of the proxy server.
-    ``required``
+   ``required``
 -  ``port``: The port (integer) of the proxy server.
-    ``required``
+   ``required``
 -  ``username``: The username for authentication against the proxy server.
    This is ignored if ``password`` is not also set.
    ``optional``
@@ -22,6 +22,6 @@ Settings
    ``optional``
 -  ``persistent``: Whether the proxy configuration file should remain on
    the machine or not.
-   Valid values: true, false
-   Default: ``false``
+   Valid values: ``true``, ``false``
+   Default: ``false``.
    ``optional``

From d9e9014a8f4855ec88ee027e240531f064c6850b Mon Sep 17 00:00:00 2001
From: Jonh Wendell <jonh.wendell@gmail.com>
Date: Tue, 28 Apr 2015 17:03:54 -0300
Subject: [PATCH 325/345] root_password: Enable SSH root login

Jessie comes with root login disabled for passwords, however,
if the user is using this plugin, chances are high that they
need to login via SSH as root.
---
 bootstrapvz/common/tasks/ssh.py               | 11 +++++++++++
 bootstrapvz/plugins/root_password/__init__.py |  1 +
 2 files changed, 12 insertions(+)

diff --git a/bootstrapvz/common/tasks/ssh.py b/bootstrapvz/common/tasks/ssh.py
index 354e95b..7ea7f2d 100644
--- a/bootstrapvz/common/tasks/ssh.py
+++ b/bootstrapvz/common/tasks/ssh.py
@@ -51,6 +51,17 @@ class DisableSSHPasswordAuthentication(Task):
 		sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no')
 
 
+class PermitSSHRootLogin(Task):
+	description = 'Permitting SSH root login'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		from ..tools import sed_i
+		sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
+		sed_i(sshd_config_path, '^PermitRootLogin .*', 'PermitRootLogin yes')
+
+
 class DisableSSHDNSLookup(Task):
 	description = 'Disabling sshd remote host name lookup'
 	phase = phases.system_modification
diff --git a/bootstrapvz/plugins/root_password/__init__.py b/bootstrapvz/plugins/root_password/__init__.py
index 850c6d2..4eadcd7 100644
--- a/bootstrapvz/plugins/root_password/__init__.py
+++ b/bootstrapvz/plugins/root_password/__init__.py
@@ -10,4 +10,5 @@ def resolve_tasks(taskset, manifest):
 	from bootstrapvz.common.tasks import ssh
 	from tasks import SetRootPassword
 	taskset.discard(ssh.DisableSSHPasswordAuthentication)
+	taskset.add(ssh.PermitSSHRootLogin)
 	taskset.add(SetRootPassword)

From af1fab40e0e9fafdce1912f62558a32a93fe8676 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 08:40:48 +0200
Subject: [PATCH 326/345] Only disable root login on squeeze & wheezy (jessie
 has it disabled per default)

---
 bootstrapvz/plugins/admin_user/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bootstrapvz/plugins/admin_user/__init__.py b/bootstrapvz/plugins/admin_user/__init__.py
index b506b0d..a2efa16 100644
--- a/bootstrapvz/plugins/admin_user/__init__.py
+++ b/bootstrapvz/plugins/admin_user/__init__.py
@@ -12,8 +12,11 @@ def resolve_tasks(taskset, manifest):
 	if initd.AddEC2InitScripts in taskset:
 		taskset.add(tasks.AdminUserCredentials)
 
+	from bootstrapvz.common.tools import get_codename
+	if get_codename(manifest.system['release']) in ['wheezy', 'squeeze']:
+		taskset.update([tasks.DisableRootLogin])
+
 	taskset.update([tasks.AddSudoPackage,
 	                tasks.CreateAdminUser,
 	                tasks.PasswordlessSudo,
-	                tasks.DisableRootLogin,
 	                ])

From a34423c1a11a01d3aea6a3ddaafe90ed628c2563 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 20:38:03 +0200
Subject: [PATCH 327/345] Rename sectors tests so it is actually run

---
 tests/unit/{sectors.py => sectors_tests.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/unit/{sectors.py => sectors_tests.py} (100%)

diff --git a/tests/unit/sectors.py b/tests/unit/sectors_tests.py
similarity index 100%
rename from tests/unit/sectors.py
rename to tests/unit/sectors_tests.py

From 40f66d48ccba2f755b9f397c54a7b51bc689c7a6 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 20:46:37 +0200
Subject: [PATCH 328/345] Don't test sectors with unimplemented abs()

---
 tests/unit/sectors_tests.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/unit/sectors_tests.py b/tests/unit/sectors_tests.py
index 7a86ae9..25ebdc0 100644
--- a/tests/unit/sectors_tests.py
+++ b/tests/unit/sectors_tests.py
@@ -8,7 +8,6 @@ std_secsz = Bytes(512)
 
 
 def test_init_with_int():
-	eq_(4, abs(Sectors(4, std_secsz)))
 	secsize = 4096
 	eq_(Sectors('1MiB', secsize), Sectors(256, secsize))
 

From c8ddd7de4558eda216c618362b419617b311b224 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 20:46:53 +0200
Subject: [PATCH 329/345] Flip compairson for whether sector size is equal in
 Sectors

---
 bootstrapvz/common/sectors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bootstrapvz/common/sectors.py b/bootstrapvz/common/sectors.py
index 92db8f7..d658140 100644
--- a/bootstrapvz/common/sectors.py
+++ b/bootstrapvz/common/sectors.py
@@ -134,7 +134,7 @@ class Sectors(object):
 		if isinstance(other, (int, long)):
 			return Sectors(self.bytes / other, self.sector_size)
 		if isinstance(other, Sectors):
-			if self.sector_size != other.sector_size:
+			if self.sector_size == other.sector_size:
 				return self.bytes / other.bytes
 			else:
 				raise UnitError('Cannot divide sectors with different sector sizes')

From 2d3a0a0ce35e17dc183cf353ace2aed84f20accc Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 21:28:38 +0200
Subject: [PATCH 330/345] The change in d9e9014 failed when SSH was not
 installed

---
 bootstrapvz/common/tasks/ssh.py               | 31 ++++++++++++++++---
 bootstrapvz/plugins/admin_user/__init__.py    |  3 +-
 bootstrapvz/plugins/admin_user/tasks.py       | 20 ------------
 bootstrapvz/plugins/root_password/__init__.py |  2 +-
 4 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/bootstrapvz/common/tasks/ssh.py b/bootstrapvz/common/tasks/ssh.py
index 7ea7f2d..9ad8f21 100644
--- a/bootstrapvz/common/tasks/ssh.py
+++ b/bootstrapvz/common/tasks/ssh.py
@@ -51,15 +51,36 @@ class DisableSSHPasswordAuthentication(Task):
 		sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no')
 
 
-class PermitSSHRootLogin(Task):
-	description = 'Permitting SSH root login'
+class EnableRootLogin(Task):
+	description = 'Disabling SSH login for root'
 	phase = phases.system_modification
 
 	@classmethod
 	def run(cls, info):
-		from ..tools import sed_i
-		sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
-		sed_i(sshd_config_path, '^PermitRootLogin .*', 'PermitRootLogin yes')
+		sshdconfig_path = os.path.join(info.root, 'etc/ssh/sshd_config')
+		if os.path.exists(sshdconfig_path):
+			from bootstrapvz.common.tools import sed_i
+			sed_i(sshdconfig_path, 'PermitRootLogin .*', 'PermitRootLogin yes')
+		else:
+			import logging
+			logging.getLogger(__name__).warn('The OpenSSH server has not been installed, '
+			                                 'not enabling SSH root login.')
+
+
+class DisableRootLogin(Task):
+	description = 'Disabling SSH login for root'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		sshdconfig_path = os.path.join(info.root, 'etc/ssh/sshd_config')
+		if os.path.exists(sshdconfig_path):
+			from bootstrapvz.common.tools import sed_i
+			sed_i(sshdconfig_path, 'PermitRootLogin .*', 'PermitRootLogin no')
+		else:
+			import logging
+			logging.getLogger(__name__).warn('The OpenSSH server has not been installed, '
+			                                 'not disabling SSH root login.')
 
 
 class DisableSSHDNSLookup(Task):
diff --git a/bootstrapvz/plugins/admin_user/__init__.py b/bootstrapvz/plugins/admin_user/__init__.py
index a2efa16..d27003f 100644
--- a/bootstrapvz/plugins/admin_user/__init__.py
+++ b/bootstrapvz/plugins/admin_user/__init__.py
@@ -8,13 +8,14 @@ def validate_manifest(data, validator, error):
 
 def resolve_tasks(taskset, manifest):
 	import tasks
+	from bootstrapvz.common.tasks import ssh
 	from bootstrapvz.providers.ec2.tasks import initd
 	if initd.AddEC2InitScripts in taskset:
 		taskset.add(tasks.AdminUserCredentials)
 
 	from bootstrapvz.common.tools import get_codename
 	if get_codename(manifest.system['release']) in ['wheezy', 'squeeze']:
-		taskset.update([tasks.DisableRootLogin])
+		taskset.update([ssh.DisableRootLogin])
 
 	taskset.update([tasks.AddSudoPackage,
 	                tasks.CreateAdminUser,
diff --git a/bootstrapvz/plugins/admin_user/tasks.py b/bootstrapvz/plugins/admin_user/tasks.py
index 5266f27..91b5235 100644
--- a/bootstrapvz/plugins/admin_user/tasks.py
+++ b/bootstrapvz/plugins/admin_user/tasks.py
@@ -54,23 +54,3 @@ class AdminUserCredentials(Task):
 		getcreds_path = os.path.join(info.root, 'etc/init.d/ec2-get-credentials')
 		username = info.manifest.plugins['admin_user']['username']
 		sed_i(getcreds_path, 'username=\'root\'', 'username=\'{username}\''.format(username=username))
-
-
-class DisableRootLogin(Task):
-	description = 'Disabling SSH login for root'
-	phase = phases.system_modification
-
-	@classmethod
-	def run(cls, info):
-		from subprocess import CalledProcessError
-		from bootstrapvz.common.tools import log_check_call
-		try:
-			log_check_call(['chroot', info.root,
-			                'dpkg-query', '-W', 'openssh-server'])
-			from bootstrapvz.common.tools import sed_i
-			sshdconfig_path = os.path.join(info.root, 'etc/ssh/sshd_config')
-			sed_i(sshdconfig_path, 'PermitRootLogin yes', 'PermitRootLogin no')
-		except CalledProcessError:
-			import logging
-			logging.getLogger(__name__).warn('The OpenSSH server has not been installed, '
-			                                 'not disabling SSH root login.')
diff --git a/bootstrapvz/plugins/root_password/__init__.py b/bootstrapvz/plugins/root_password/__init__.py
index 4eadcd7..f97e9f8 100644
--- a/bootstrapvz/plugins/root_password/__init__.py
+++ b/bootstrapvz/plugins/root_password/__init__.py
@@ -10,5 +10,5 @@ def resolve_tasks(taskset, manifest):
 	from bootstrapvz.common.tasks import ssh
 	from tasks import SetRootPassword
 	taskset.discard(ssh.DisableSSHPasswordAuthentication)
-	taskset.add(ssh.PermitSSHRootLogin)
+	taskset.add(ssh.EnableRootLogin)
 	taskset.add(SetRootPassword)

From 71c7d445ad7fff265febda36426a7a4ca19cef94 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 20:55:55 +0200
Subject: [PATCH 331/345] Fix #217, by introducing class for comparison of
 releases

---
 bootstrapvz/base/__init__.py                  | 10 ++-
 bootstrapvz/base/bootstrapinfo.py             |  3 -
 bootstrapvz/base/manifest-schema.yml          | 11 +--
 bootstrapvz/base/manifest.py                  |  2 +
 bootstrapvz/common/release-codenames.yml      | 23 -------
 bootstrapvz/common/releases.py                | 68 +++++++++++++++++++
 bootstrapvz/common/task_groups.py             |  6 +-
 bootstrapvz/common/tasks/apt.py               |  6 +-
 bootstrapvz/common/tasks/boot.py              |  6 +-
 bootstrapvz/common/tasks/extlinux.py          |  3 +-
 bootstrapvz/common/tasks/initd.py             |  3 +-
 bootstrapvz/common/tasks/network.py           |  2 +-
 bootstrapvz/common/tasks/ssh.py               |  6 +-
 bootstrapvz/common/tools.py                   |  9 ---
 bootstrapvz/plugins/admin_user/__init__.py    |  4 +-
 bootstrapvz/plugins/cloud_init/__init__.py    |  4 +-
 bootstrapvz/plugins/cloud_init/tasks.py       |  3 +-
 bootstrapvz/plugins/docker_daemon/__init__.py |  7 +-
 bootstrapvz/plugins/opennebula/__init__.py    |  4 +-
 bootstrapvz/plugins/opennebula/tasks.py       |  3 +-
 bootstrapvz/providers/azure/tasks/packages.py |  2 +-
 bootstrapvz/providers/ec2/tasks/network.py    |  3 +-
 bootstrapvz/providers/ec2/tasks/packages.py   |  5 +-
 bootstrapvz/providers/gce/tasks/packages.py   |  2 +-
 bootstrapvz/providers/kvm/tasks/packages.py   |  2 +-
 .../providers/virtualbox/tasks/packages.py    |  2 +-
 tests/integration/providers/ec2/__init__.py   |  4 +-
 .../providers/virtualbox/__init__.py          |  2 +-
 .../providers/virtualbox/instance.py          |  4 +-
 tests/unit/releases_tests.py                  | 39 +++++++++++
 30 files changed, 164 insertions(+), 84 deletions(-)
 delete mode 100644 bootstrapvz/common/release-codenames.yml
 create mode 100644 bootstrapvz/common/releases.py
 create mode 100644 tests/unit/releases_tests.py

diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py
index 39d88a1..4bcf580 100644
--- a/bootstrapvz/base/__init__.py
+++ b/bootstrapvz/base/__init__.py
@@ -16,8 +16,12 @@ def validate_manifest(data, validator, error):
 	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
 
-	from bootstrapvz.common.tools import get_codename
-	codename = get_codename(data['system']['release'])
+	from bootstrapvz.common.releases import get_release
+	from bootstrapvz.common.releases import squeeze
+	release = get_release(data['system']['release'])
+
+	if release < squeeze:
+		error('Only Debian squeeze and later is supported', ['system', 'release'])
 
 	# Check the bootloader/partitioning configuration.
 	# Doing this via the schema is a pain and does not output a useful error message.
@@ -26,5 +30,5 @@ def validate_manifest(data, validator, error):
 		if data['volume']['partitions']['type'] == 'none':
 			error('Grub cannot boot from unpartitioned disks', ['system', 'bootloader'])
 
-		if codename == 'squeeze':
+		if release == squeeze:
 			error('Grub installation on squeeze is not supported', ['system', 'bootloader'])
diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py
index 111c1d9..2990d24 100644
--- a/bootstrapvz/base/bootstrapinfo.py
+++ b/bootstrapvz/base/bootstrapinfo.py
@@ -31,9 +31,6 @@ class BootstrapInformation(object):
 		# The default apt mirror
 		self.apt_mirror = self.manifest.packages.get('mirror', 'http://http.debian.net/debian')
 
-		from bootstrapvz.common.tools import get_codename
-		self.release_codename = get_codename(self.manifest.system['release'])
-
 		# Create the manifest_vars dictionary
 		self.manifest_vars = self.__create_manifest_vars(self.manifest, {'apt_mirror': self.apt_mirror})
 
diff --git a/bootstrapvz/base/manifest-schema.yml b/bootstrapvz/base/manifest-schema.yml
index 229feb6..fb596eb 100644
--- a/bootstrapvz/base/manifest-schema.yml
+++ b/bootstrapvz/base/manifest-schema.yml
@@ -52,16 +52,7 @@ properties:
         type: string
         pattern: ^\S+$
       locale: {type: string}
-      release:
-        enum:
-        - squeeze
-        - wheezy
-        - jessie
-        - sid
-        - oldstable
-        - stable
-        - testing
-        - unstable
+      release: {type: string}
       timezone: {type: string}
     required:
     - release
diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py
index 02ddae8..2e04d42 100644
--- a/bootstrapvz/base/manifest.py
+++ b/bootstrapvz/base/manifest.py
@@ -103,6 +103,8 @@ class Manifest(object):
 		self.image        = self.data['image']
 		self.volume       = self.data['volume']
 		self.system       = self.data['system']
+		from bootstrapvz.common.releases import get_release
+		self.release      = get_release(self.system['release'])
 		# The packages and plugins section is not required
 		self.packages     = self.data['packages'] if 'packages' in self.data else {}
 		self.plugins      = self.data['plugins'] if 'plugins' in self.data else {}
diff --git a/bootstrapvz/common/release-codenames.yml b/bootstrapvz/common/release-codenames.yml
deleted file mode 100644
index cfc2bc6..0000000
--- a/bootstrapvz/common/release-codenames.yml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-# This is a mapping of Debian release names to their respective codenames
-
-unstable:  sid
-testing:   stretch
-stable:    jessie
-oldstable: wheezy
-
-jessie:  jessie
-wheezy:  wheezy
-squeeze: squeeze
-
-# The following release names are not supported, but included for completeness sake
-lenny:  lenny
-etch:   etch
-sarge:  sarge
-woody:  woody
-potato: potato
-slink:  slink
-hamm:   hamm
-bo:     bo
-rex:    rex
-buzz:   buz
diff --git a/bootstrapvz/common/releases.py b/bootstrapvz/common/releases.py
new file mode 100644
index 0000000..558b4b1
--- /dev/null
+++ b/bootstrapvz/common/releases.py
@@ -0,0 +1,68 @@
+
+
+class _Release(object):
+	def __init__(self, codename, version):
+		self.codename = codename
+		self.version = version
+
+	def __cmp__(self, other):
+		return self.version - other.version
+
+	def __str__(self):
+		return self.codename
+
+	def __getstate__(self):
+		state = self.__dict__.copy()
+		state['__class__'] = self.__module__ + '.' + self.__class__.__name__
+		return state
+
+	def __setstate__(self, state):
+		for key in state:
+			self.__dict__[key] = state[key]
+
+
+class _ReleaseAlias(_Release):
+	def __init__(self, alias, release):
+		self.alias = alias
+		self.release = release
+		super(_ReleaseAlias, self).__init__(self.release.codename, self.release.version)
+
+	def __str__(self):
+		return self.alias
+
+
+sid = _Release('sid', 10)
+stretch = _Release('stretch', 9)
+jessie = _Release('jessie', 8)
+wheezy = _Release('wheezy', 7)
+squeeze = _Release('squeeze', 6.0)
+lenny = _Release('lenny', 5.0)
+etch = _Release('etch', 4.0)
+sarge = _Release('sarge', 3.1)
+woody = _Release('woody', 3.0)
+potato = _Release('potato', 2.2)
+slink = _Release('slink', 2.1)
+hamm = _Release('hamm', 2.0)
+bo = _Release('bo', 1.3)
+rex = _Release('rex', 1.2)
+buzz = _Release('buzz', 1.1)
+
+unstable = _ReleaseAlias('unstable', sid)
+testing = _ReleaseAlias('testing', stretch)
+stable = _ReleaseAlias('stable', jessie)
+oldstable = _ReleaseAlias('oldstable', wheezy)
+
+
+def get_release(release_name):
+	"""Normalizes the release codenames
+	This allows tasks to query for release codenames rather than 'stable', 'unstable' etc.
+	"""
+	from . import releases
+	release = getattr(releases, release_name, None)
+	if release is None or not isinstance(release, _Release):
+		raise UnknownReleaseException('The release `{name}\' is unknown'.format(name=release))
+	return release
+
+
+class UnknownReleaseException(Exception):
+	pass
diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py
index 90bff08..ab463f0 100644
--- a/bootstrapvz/common/task_groups.py
+++ b/bootstrapvz/common/task_groups.py
@@ -136,18 +136,18 @@ locale_group = [locale.LocaleBootstrapPackage,
 
 
 def get_bootloader_group(manifest):
-	from bootstrapvz.common.tools import get_codename
+	from bootstrapvz.common.releases import jessie
 	group = []
 	if manifest.system['bootloader'] == 'grub':
 		group.extend([grub.AddGrubPackage,
 		              grub.ConfigureGrub])
-		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+		if manifest.release < jessie:
 			group.append(grub.InstallGrub_1_99)
 		else:
 			group.append(grub.InstallGrub_2)
 	if manifest.system['bootloader'] == 'extlinux':
 		group.append(extlinux.AddExtlinuxPackage)
-		if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+		if manifest.release < jessie:
 			group.extend([extlinux.ConfigureExtlinux,
 			              extlinux.InstallExtlinux])
 		else:
diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 6572674..2ee8384 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -24,12 +24,13 @@ class AddDefaultSources(Task):
 
 	@classmethod
 	def run(cls, info):
+		from bootstrapvz.common.releases import sid
 		include_src = info.manifest.packages.get('include-source-type', False)
 		components = ' '.join(info.manifest.packages.get('components', ['main']))
 		info.source_lists.add('main', 'deb     {apt_mirror} {system.release} ' + components)
 		if include_src:
 			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} ' + components)
-		if info.release_codename != 'sid':
+		if info.manifest.release != sid:
 			info.source_lists.add('main', 'deb     http://security.debian.org/  {system.release}/updates ' + components)
 			if include_src:
 				info.source_lists.add('main', 'deb-src http://security.debian.org/  {system.release}/updates ' + components)
@@ -45,10 +46,11 @@ class AddBackports(Task):
 
 	@classmethod
 	def run(cls, info):
+		from bootstrapvz.common.releases import sid
 		if info.source_lists.target_exists('{system.release}-backports'):
 			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
 			logging.getLogger(__name__).info(msg)
-		elif info.release_codename == 'sid':
+		elif info.manifest.release == sid:
 			logging.getLogger(__name__).info('There are no backports for sid/unstable')
 		else:
 			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')
diff --git a/bootstrapvz/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py
index 4cc121c..afa257f 100644
--- a/bootstrapvz/common/tasks/boot.py
+++ b/bootstrapvz/common/tasks/boot.py
@@ -34,9 +34,9 @@ class DisableGetTTYs(Task):
 
 	@classmethod
 	def run(cls, info):
-		# Forward compatible check for jessie,
-		# we should probably get some version numbers up in dis bitch
-		if info.release_codename in ['squeeze', 'wheezy']:
+		# Forward compatible check for jessie
+		from bootstrapvz.common.releases import jessie
+		if info.manifest.release < jessie:
 			from ..tools import sed_i
 			inittab_path = os.path.join(info.root, 'etc/inittab')
 			tty1 = '1:2345:respawn:/sbin/getty 38400 tty1'
diff --git a/bootstrapvz/common/tasks/extlinux.py b/bootstrapvz/common/tasks/extlinux.py
index 1a3b8b5..5848a43 100644
--- a/bootstrapvz/common/tasks/extlinux.py
+++ b/bootstrapvz/common/tasks/extlinux.py
@@ -27,7 +27,8 @@ class ConfigureExtlinux(Task):
 
 	@classmethod
 	def run(cls, info):
-		if info.release_codename == 'squeeze':
+		from bootstrapvz.common.releases import squeeze
+		if info.manifest.release == squeeze:
 			# On squeeze /etc/default/extlinux is generated when running extlinux-update
 			log_check_call(['chroot', info.root,
 			                'extlinux-update'])
diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py
index c145465..5e43fa5 100644
--- a/bootstrapvz/common/tasks/initd.py
+++ b/bootstrapvz/common/tasks/initd.py
@@ -44,8 +44,9 @@ class RemoveHWClock(Task):
 
 	@classmethod
 	def run(cls, info):
+		from bootstrapvz.common.releases import squeeze
 		info.initd['disable'].append('hwclock.sh')
-		if info.release_codename == 'squeeze':
+		if info.manifest.release == squeeze:
 			info.initd['disable'].append('hwclockfirst.sh')
 
 
diff --git a/bootstrapvz/common/tasks/network.py b/bootstrapvz/common/tasks/network.py
index 6bec727..c073867 100644
--- a/bootstrapvz/common/tasks/network.py
+++ b/bootstrapvz/common/tasks/network.py
@@ -47,7 +47,7 @@ class ConfigureNetworkIF(Task):
 	def run(cls, info):
 		network_config_path = os.path.join(os.path.dirname(__file__), 'network-configuration.yml')
 		from ..tools import config_get
-		if_config = config_get(network_config_path, [info.release_codename])
+		if_config = config_get(network_config_path, [info.manifest.release.codename])
 
 		interfaces_path = os.path.join(info.root, 'etc/network/interfaces')
 		with open(interfaces_path, 'a') as interfaces:
diff --git a/bootstrapvz/common/tasks/ssh.py b/bootstrapvz/common/tasks/ssh.py
index 9ad8f21..6d3f88c 100644
--- a/bootstrapvz/common/tasks/ssh.py
+++ b/bootstrapvz/common/tasks/ssh.py
@@ -30,7 +30,8 @@ class AddSSHKeyGeneration(Task):
 		try:
 			log_check_call(['chroot', info.root,
 			                'dpkg-query', '-W', 'openssh-server'])
-			if info.release_codename == 'squeeze':
+			from bootstrapvz.common.releases import squeeze
+			if info.manifest.release == squeeze:
 				install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'squeeze/generate-ssh-hostkeys')
 			else:
 				install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'generate-ssh-hostkeys')
@@ -102,7 +103,8 @@ class ShredHostkeys(Task):
 	def run(cls, info):
 		ssh_hostkeys = ['ssh_host_dsa_key',
 		                'ssh_host_rsa_key']
-		if info.release_codename != 'squeeze':
+		from bootstrapvz.common.releases import wheezy
+		if info.manifest.release >= wheezy:
 			ssh_hostkeys.append('ssh_host_ecdsa_key')
 
 		private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
diff --git a/bootstrapvz/common/tools.py b/bootstrapvz/common/tools.py
index 49fb425..5df7ff3 100644
--- a/bootstrapvz/common/tools.py
+++ b/bootstrapvz/common/tools.py
@@ -118,15 +118,6 @@ def config_get(path, config_path):
 	return config
 
 
-def get_codename(release):
-	"""Normalizes the release codenames
-	This allows tasks to query for release codenames rather than 'stable', 'unstable' etc.
-	"""
-	release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.yml')
-	from bootstrapvz.common.tools import config_get
-	return config_get(release_codenames_path, [release])
-
-
 def copy_tree(from_path, to_path):
 	from shutil import copy
 	for abs_prefix, dirs, files in os.walk(from_path):
diff --git a/bootstrapvz/plugins/admin_user/__init__.py b/bootstrapvz/plugins/admin_user/__init__.py
index d27003f..11888de 100644
--- a/bootstrapvz/plugins/admin_user/__init__.py
+++ b/bootstrapvz/plugins/admin_user/__init__.py
@@ -13,8 +13,8 @@ def resolve_tasks(taskset, manifest):
 	if initd.AddEC2InitScripts in taskset:
 		taskset.add(tasks.AdminUserCredentials)
 
-	from bootstrapvz.common.tools import get_codename
-	if get_codename(manifest.system['release']) in ['wheezy', 'squeeze']:
+	from bootstrapvz.common.releases import jessie
+	if manifest.release < jessie:
 		taskset.update([ssh.DisableRootLogin])
 
 	taskset.update([tasks.AddSudoPackage,
diff --git a/bootstrapvz/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py
index f01b02f..7a59d40 100644
--- a/bootstrapvz/plugins/cloud_init/__init__.py
+++ b/bootstrapvz/plugins/cloud_init/__init__.py
@@ -13,8 +13,8 @@ def resolve_tasks(taskset, manifest):
 	from bootstrapvz.common.tasks import initd
 	from bootstrapvz.common.tasks import ssh
 
-	from bootstrapvz.common.tools import get_codename
-	if get_codename(manifest.system['release']) == 'wheezy':
+	from bootstrapvz.common.releases import wheezy
+	if manifest.release == wheezy:
 		taskset.add(apt.AddBackports)
 
 	taskset.update([tasks.SetMetadataSource,
diff --git a/bootstrapvz/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py
index 39bb7be..60ba51f 100644
--- a/bootstrapvz/plugins/cloud_init/tasks.py
+++ b/bootstrapvz/plugins/cloud_init/tasks.py
@@ -15,7 +15,8 @@ class AddCloudInitPackages(Task):
 	@classmethod
 	def run(cls, info):
 		target = None
-		if info.release_codename == 'wheezy':
+		from bootstrapvz.common.releases import wheezy
+		if info.manifest.release == wheezy:
 			target = '{system.release}-backports'
 		info.packages.add('cloud-init', target)
 		info.packages.add('sudo')
diff --git a/bootstrapvz/plugins/docker_daemon/__init__.py b/bootstrapvz/plugins/docker_daemon/__init__.py
index 637163e..3a72b6b 100644
--- a/bootstrapvz/plugins/docker_daemon/__init__.py
+++ b/bootstrapvz/plugins/docker_daemon/__init__.py
@@ -1,13 +1,14 @@
 import os.path
 import tasks
 from bootstrapvz.common.tasks import apt
-from bootstrapvz.common.tools import get_codename
+from bootstrapvz.common.releases import wheezy
 
 
 def validate_manifest(data, validator, error):
 	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
 	validator(data, schema_path)
-	if get_codename(data['system']['release']) == 'wheezy':
+	from bootstrapvz.common.releases import get_release
+	if get_release(data['system']['release']) == wheezy:
 		# prefs is a generator of apt preferences across files in the manifest
 		prefs = (item for vals in data.get('packages', {}).get('preferences', {}).values() for item in vals)
 		if not any('linux-image' in item['package'] and 'wheezy-backports' in item['pin'] for item in prefs):
@@ -16,7 +17,7 @@ def validate_manifest(data, validator, error):
 
 
 def resolve_tasks(taskset, manifest):
-	if get_codename(manifest.system['release']) == 'wheezy':
+	if manifest.release == wheezy:
 		taskset.add(apt.AddBackports)
 	taskset.add(tasks.AddDockerDeps)
 	taskset.add(tasks.AddDockerBinary)
diff --git a/bootstrapvz/plugins/opennebula/__init__.py b/bootstrapvz/plugins/opennebula/__init__.py
index f41e21b..8681d18 100644
--- a/bootstrapvz/plugins/opennebula/__init__.py
+++ b/bootstrapvz/plugins/opennebula/__init__.py
@@ -3,7 +3,7 @@
 def resolve_tasks(taskset, manifest):
 	import tasks
 	from bootstrapvz.common.tasks import apt
-	from bootstrapvz.common.tools import get_codename
-	if get_codename(manifest.system['release']) == 'wheezy':
+	from bootstrapvz.common.releases import wheezy
+	if manifest.release == wheezy:
 		taskset.add(apt.AddBackports)
 	taskset.update([tasks.AddONEContextPackage])
diff --git a/bootstrapvz/plugins/opennebula/tasks.py b/bootstrapvz/plugins/opennebula/tasks.py
index d37e9be..33050dc 100644
--- a/bootstrapvz/plugins/opennebula/tasks.py
+++ b/bootstrapvz/plugins/opennebula/tasks.py
@@ -11,6 +11,7 @@ class AddONEContextPackage(Task):
 	@classmethod
 	def run(cls, info):
 		target = None
-		if info.release_codename == 'wheezy':
+		from bootstrapvz.common.releases import wheezy
+		if info.manifest.release == wheezy:
 			target = '{system.release}-backports'
 		info.packages.add('opennebula-context', target)
diff --git a/bootstrapvz/providers/azure/tasks/packages.py b/bootstrapvz/providers/azure/tasks/packages.py
index b7d2558..daed133 100644
--- a/bootstrapvz/providers/azure/tasks/packages.py
+++ b/bootstrapvz/providers/azure/tasks/packages.py
@@ -19,7 +19,7 @@ class DefaultPackages(Task):
 		import os.path
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
-		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
 		                                                   info.manifest.system['architecture']])
 		info.packages.add(kernel_package)
 
diff --git a/bootstrapvz/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py
index d358f2b..5c7bf81 100644
--- a/bootstrapvz/providers/ec2/tasks/network.py
+++ b/bootstrapvz/providers/ec2/tasks/network.py
@@ -13,7 +13,8 @@ class EnableDHCPCDDNS(Task):
 	def run(cls, info):
 		# The dhcp client that ships with debian sets the DNS servers per default.
 		# For dhcpcd in Wheezy and earlier we need to configure it to do that.
-		if info.release_codename not in {'jessie', 'sid'}:
+		from bootstrapvz.common.releases import wheezy
+		if info.manifest.release <= wheezy:
 			from bootstrapvz.common.tools import sed_i
 			dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
 			sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')
diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py
index a85f7ed..079d112 100644
--- a/bootstrapvz/providers/ec2/tasks/packages.py
+++ b/bootstrapvz/providers/ec2/tasks/packages.py
@@ -12,7 +12,8 @@ class DefaultPackages(Task):
 	def run(cls, info):
 		info.packages.add('file')  # Needed for the init scripts
 		# isc-dhcp-client doesn't work properly with ec2
-		if info.release_codename in {'jessie', 'sid'}:
+		from bootstrapvz.common.releases import jessie
+		if info.manifest.release >= jessie:
 			info.packages.add('dhcpcd5')
 		else:
 			info.packages.add('dhcpcd')
@@ -23,6 +24,6 @@ class DefaultPackages(Task):
 		import os.path
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
-		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
 		                                                   info.manifest.system['architecture']])
 		info.packages.add(kernel_package)
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index d746af0..b67ca54 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -24,7 +24,7 @@ class DefaultPackages(Task):
 
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
-		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
 		                                                   info.manifest.system['architecture']])
 		info.packages.add(kernel_package)
 
diff --git a/bootstrapvz/providers/kvm/tasks/packages.py b/bootstrapvz/providers/kvm/tasks/packages.py
index 7498c53..30d3eda 100644
--- a/bootstrapvz/providers/kvm/tasks/packages.py
+++ b/bootstrapvz/providers/kvm/tasks/packages.py
@@ -13,6 +13,6 @@ class DefaultPackages(Task):
 		import os.path
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
-		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
 		                                                   info.manifest.system['architecture']])
 		info.packages.add(kernel_package)
diff --git a/bootstrapvz/providers/virtualbox/tasks/packages.py b/bootstrapvz/providers/virtualbox/tasks/packages.py
index ccdb419..d8a192d 100644
--- a/bootstrapvz/providers/virtualbox/tasks/packages.py
+++ b/bootstrapvz/providers/virtualbox/tasks/packages.py
@@ -13,6 +13,6 @@ class DefaultPackages(Task):
 		import os.path
 		kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
 		from bootstrapvz.common.tools import config_get
-		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
 		                                                   info.manifest.system['architecture']])
 		info.packages.add(kernel_package)
diff --git a/tests/integration/providers/ec2/__init__.py b/tests/integration/providers/ec2/__init__.py
index f794463..39c044f 100644
--- a/tests/integration/providers/ec2/__init__.py
+++ b/tests/integration/providers/ec2/__init__.py
@@ -79,8 +79,8 @@ def run_instance(image, manifest, instance_type, ec2_connection, vpc_connection)
 			if not waituntil(lambda: instance.get_console_output().output is not None, timeout=600, interval=3):
 				raise EC2InstanceStartupException('Timeout while fetching console output')
 
-			from bootstrapvz.common.tools import get_codename
-			if get_codename(manifest.system['release']) in ['squeeze', 'wheezy']:
+			from bootstrapvz.common.releases import wheezy
+			if manifest.release <= wheezy:
 				termination_string = 'INIT: Entering runlevel: 2'
 			else:
 				termination_string = 'Debian GNU/Linux'
diff --git a/tests/integration/providers/virtualbox/__init__.py b/tests/integration/providers/virtualbox/__init__.py
index f94fb26..f9460da 100644
--- a/tests/integration/providers/virtualbox/__init__.py
+++ b/tests/integration/providers/virtualbox/__init__.py
@@ -43,7 +43,7 @@ def boot_image(manifest, build_server, bootstrap_info):
 def run_instance(image, instance_name, manifest):
 	from instance import VirtualBoxInstance
 	instance = VirtualBoxInstance(image, instance_name,
-	                              manifest.system['architecture'], manifest.system['release'])
+	                              manifest.system['architecture'], manifest.release)
 	try:
 		instance.create()
 		try:
diff --git a/tests/integration/providers/virtualbox/instance.py b/tests/integration/providers/virtualbox/instance.py
index fa520e6..5ab28da 100644
--- a/tests/integration/providers/virtualbox/instance.py
+++ b/tests/integration/providers/virtualbox/instance.py
@@ -63,8 +63,8 @@ class VirtualBoxInstance(object):
 		# Gotta figure out a more reliable way to check when the system is done booting.
 		# Maybe bootstrapped unit test images should have a startup script that issues
 		# a callback to the host.
-		from bootstrapvz.common.tools import get_codename
-		if get_codename(self.release) in ['squeeze', 'wheezy']:
+		from bootstrapvz.common.releases import wheezy
+		if self.release <= wheezy:
 			termination_string = 'INIT: Entering runlevel: 2'
 		else:
 			termination_string = 'Debian GNU/Linux'
diff --git a/tests/unit/releases_tests.py b/tests/unit/releases_tests.py
new file mode 100644
index 0000000..df7d588
--- /dev/null
+++ b/tests/unit/releases_tests.py
@@ -0,0 +1,39 @@
+from nose.tools import raises
+from bootstrapvz.common import releases
+
+
+def test_gt():
+	assert releases.wheezy > releases.squeeze
+
+
+def test_lt():
+	assert releases.wheezy < releases.stretch
+
+
+def test_eq():
+	assert releases.wheezy == releases.wheezy
+
+
+def test_neq():
+	assert releases.wheezy != releases.jessie
+
+
+def test_identity():
+	assert releases.wheezy is releases.wheezy
+
+
+def test_not_identity():
+	assert releases.wheezy is not releases.stable
+	assert releases.stable is not releases.jessie
+
+
+def test_alias():
+	assert releases.oldstable == releases.wheezy
+	assert releases.stable == releases.jessie
+	assert releases.testing == releases.stretch
+	assert releases.unstable == releases.sid
+
+
+@raises(releases.UnknownReleaseException)
+def test_bogus_releasename():
+	releases.get_release('nemo')

From f1ce58299485f5683ea0ad8b6deb4c6200c7a619 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 23:31:09 +0200
Subject: [PATCH 332/345] Compare with "unstable" instead of "sid"

---
 bootstrapvz/common/tasks/apt.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 2ee8384..53c39f0 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -46,11 +46,11 @@ class AddBackports(Task):
 
 	@classmethod
 	def run(cls, info):
-		from bootstrapvz.common.releases import sid
+		from bootstrapvz.common.releases import unstable
 		if info.source_lists.target_exists('{system.release}-backports'):
 			msg = ('{system.release}-backports target already exists').format(**info.manifest_vars)
 			logging.getLogger(__name__).info(msg)
-		elif info.manifest.release == sid:
+		elif info.manifest.release == unstable:
 			logging.getLogger(__name__).info('There are no backports for sid/unstable')
 		else:
 			info.source_lists.add('backports', 'deb     {apt_mirror} {system.release}-backports main')

From 1f6f23e68038345c6c18dd3d12112a90a8cd8528 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Wed, 29 Apr 2015 23:39:55 +0200
Subject: [PATCH 333/345] Fix #104: Don't verify default target when adding
 packages

---
 bootstrapvz/base/pkg/packagelist.py                | 10 ++++------
 bootstrapvz/common/tasks/apt.py                    |  5 +++++
 bootstrapvz/common/tasks/extlinux.py               |  2 --
 bootstrapvz/common/tasks/filesystem.py             |  2 --
 bootstrapvz/common/tasks/grub.py                   |  2 --
 bootstrapvz/common/tasks/packages.py               |  1 -
 bootstrapvz/common/tasks/ssh.py                    |  2 --
 bootstrapvz/plugins/admin_user/tasks.py            |  2 --
 bootstrapvz/plugins/ansible/tasks.py               |  2 --
 bootstrapvz/plugins/chef/tasks.py                  |  2 --
 bootstrapvz/plugins/cloud_init/tasks.py            |  2 +-
 bootstrapvz/plugins/opennebula/tasks.py            |  2 +-
 bootstrapvz/plugins/pip_install/tasks.py           |  2 --
 bootstrapvz/plugins/puppet/tasks.py                |  2 --
 bootstrapvz/plugins/salt/tasks.py                  |  2 --
 bootstrapvz/plugins/unattended_upgrades/tasks.py   |  2 --
 bootstrapvz/plugins/vagrant/tasks.py               |  2 --
 bootstrapvz/providers/azure/tasks/packages.py      |  2 --
 bootstrapvz/providers/ec2/tasks/network.py         |  2 --
 bootstrapvz/providers/ec2/tasks/packages.py        |  2 --
 bootstrapvz/providers/gce/tasks/packages.py        |  3 +--
 bootstrapvz/providers/kvm/tasks/packages.py        |  2 --
 bootstrapvz/providers/virtualbox/tasks/packages.py |  2 --
 23 files changed, 12 insertions(+), 45 deletions(-)

diff --git a/bootstrapvz/base/pkg/packagelist.py b/bootstrapvz/base/pkg/packagelist.py
index 7bd1908..5d5c0c7 100644
--- a/bootstrapvz/base/pkg/packagelist.py
+++ b/bootstrapvz/base/pkg/packagelist.py
@@ -87,12 +87,10 @@ class PackageList(object):
 			# The package has already been added, skip the checks below
 			return
 
-		# Check if the target exists in the sources list, raise a PackageError if not
-		check_target = target
-		if check_target is None:
-			check_target = self.default_target
-		if not self.source_lists.target_exists(check_target):
-			msg = ('The target release {target} was not found in the sources list').format(target=check_target)
+		# Check if the target exists (unless it's the default target) in the sources list
+		# raise a PackageError if does not
+		if target not in (None, self.default_target) and not self.source_lists.target_exists(target):
+			msg = ('The target release {target} was not found in the sources list').format(target=target)
 			raise PackageError(msg)
 
 		# Note that we maintain the target value even if it is none.
diff --git a/bootstrapvz/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py
index 53c39f0..68a5cc6 100644
--- a/bootstrapvz/common/tasks/apt.py
+++ b/bootstrapvz/common/tasks/apt.py
@@ -87,6 +87,11 @@ class WriteSources(Task):
 
 	@classmethod
 	def run(cls, info):
+		if not info.source_lists.target_exists(info.manifest.system['release']):
+			import logging
+			log = logging.getLogger(__name__)
+			log.warn('No default target has been specified in the sources list, '
+			         'installing packages may fail')
 		for name, sources in info.source_lists.sources.iteritems():
 			if name == 'main':
 				list_path = os.path.join(info.root, 'etc/apt/sources.list')
diff --git a/bootstrapvz/common/tasks/extlinux.py b/bootstrapvz/common/tasks/extlinux.py
index 5848a43..25da011 100644
--- a/bootstrapvz/common/tasks/extlinux.py
+++ b/bootstrapvz/common/tasks/extlinux.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from .. import phases
 from ..tools import log_check_call
-import apt
 import filesystem
 import kernel
 from bootstrapvz.base.fs import partitionmaps
@@ -11,7 +10,6 @@ import os
 class AddExtlinuxPackage(Task):
 	description = 'Adding extlinux package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py
index 19acbfe..9f348e0 100644
--- a/bootstrapvz/common/tasks/filesystem.py
+++ b/bootstrapvz/common/tasks/filesystem.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from .. import phases
 from ..tools import log_check_call
-import apt
 import bootstrap
 import host
 import volume
@@ -51,7 +50,6 @@ class TuneVolumeFS(Task):
 class AddXFSProgs(Task):
 	description = 'Adding `xfsprogs\' to the image packages'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/common/tasks/grub.py b/bootstrapvz/common/tasks/grub.py
index 9aabba3..5fe60b2 100644
--- a/bootstrapvz/common/tasks/grub.py
+++ b/bootstrapvz/common/tasks/grub.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from .. import phases
 from ..tools import log_check_call
-import apt
 import filesystem
 import kernel
 from bootstrapvz.base.fs import partitionmaps
@@ -11,7 +10,6 @@ import os.path
 class AddGrubPackage(Task):
 	description = 'Adding grub package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/common/tasks/packages.py b/bootstrapvz/common/tasks/packages.py
index 44263a5..662ae11 100644
--- a/bootstrapvz/common/tasks/packages.py
+++ b/bootstrapvz/common/tasks/packages.py
@@ -7,7 +7,6 @@ from ..tools import log_check_call
 class AddManifestPackages(Task):
 	description = 'Adding packages from the manifest'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/common/tasks/ssh.py b/bootstrapvz/common/tasks/ssh.py
index 6d3f88c..c3f14c5 100644
--- a/bootstrapvz/common/tasks/ssh.py
+++ b/bootstrapvz/common/tasks/ssh.py
@@ -3,14 +3,12 @@ from .. import phases
 from ..tools import log_check_call
 import os.path
 from . import assets
-import apt
 import initd
 
 
 class AddOpenSSHPackage(Task):
 	description = 'Adding openssh package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/admin_user/tasks.py b/bootstrapvz/plugins/admin_user/tasks.py
index 91b5235..ae7f221 100644
--- a/bootstrapvz/plugins/admin_user/tasks.py
+++ b/bootstrapvz/plugins/admin_user/tasks.py
@@ -1,14 +1,12 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks.initd import InstallInitScripts
-from bootstrapvz.common.tasks import apt
 import os
 
 
 class AddSudoPackage(Task):
 	description = 'Adding `sudo\' to the image packages'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/ansible/tasks.py b/bootstrapvz/plugins/ansible/tasks.py
index 9d1c893..100e69d 100644
--- a/bootstrapvz/plugins/ansible/tasks.py
+++ b/bootstrapvz/plugins/ansible/tasks.py
@@ -1,6 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 import os
 
 
@@ -23,7 +22,6 @@ class CheckPlaybookPath(Task):
 class AddPackages(Task):
     description = 'Making sure python is installed'
     phase = phases.preparation
-    predecessors = [apt.AddDefaultSources]
 
     @classmethod
     def run(cls, info):
diff --git a/bootstrapvz/plugins/chef/tasks.py b/bootstrapvz/plugins/chef/tasks.py
index 2f103bf..2c02adf 100644
--- a/bootstrapvz/plugins/chef/tasks.py
+++ b/bootstrapvz/plugins/chef/tasks.py
@@ -1,6 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 import os
 
 
@@ -23,7 +22,6 @@ class CheckAssetsPath(Task):
 class AddPackages(Task):
 	description = 'Add chef package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py
index 60ba51f..2947214 100644
--- a/bootstrapvz/plugins/cloud_init/tasks.py
+++ b/bootstrapvz/plugins/cloud_init/tasks.py
@@ -10,7 +10,7 @@ import os.path
 class AddCloudInitPackages(Task):
 	description = 'Adding cloud-init package and sudo'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources, apt.AddBackports]
+	predecessors = [apt.AddBackports]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/opennebula/tasks.py b/bootstrapvz/plugins/opennebula/tasks.py
index 33050dc..1dcac7e 100644
--- a/bootstrapvz/plugins/opennebula/tasks.py
+++ b/bootstrapvz/plugins/opennebula/tasks.py
@@ -6,7 +6,7 @@ from bootstrapvz.common import phases
 class AddONEContextPackage(Task):
 	description = 'Adding the OpenNebula context package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources, apt.AddBackports]
+	predecessors = [apt.AddBackports]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/pip_install/tasks.py b/bootstrapvz/plugins/pip_install/tasks.py
index 98b6bfa..c6ff89b 100644
--- a/bootstrapvz/plugins/pip_install/tasks.py
+++ b/bootstrapvz/plugins/pip_install/tasks.py
@@ -1,12 +1,10 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 
 
 class AddPipPackage(Task):
 	description = 'Adding `pip\' and Co. to the image packages'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/puppet/tasks.py b/bootstrapvz/plugins/puppet/tasks.py
index 02d98ab..05df26f 100644
--- a/bootstrapvz/plugins/puppet/tasks.py
+++ b/bootstrapvz/plugins/puppet/tasks.py
@@ -1,6 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tools import sed_i
 import os
 
@@ -40,7 +39,6 @@ class CheckManifestPath(Task):
 class AddPackages(Task):
 	description = 'Add puppet package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/salt/tasks.py b/bootstrapvz/plugins/salt/tasks.py
index 6c2d3f1..008a678 100644
--- a/bootstrapvz/plugins/salt/tasks.py
+++ b/bootstrapvz/plugins/salt/tasks.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import packages
-from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.tools import sed_i
 import os
@@ -11,7 +10,6 @@ import urllib
 class InstallSaltDependencies(Task):
 	description = 'Add depended packages for salt-minion'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/unattended_upgrades/tasks.py b/bootstrapvz/plugins/unattended_upgrades/tasks.py
index 1299588..dd24fb5 100644
--- a/bootstrapvz/plugins/unattended_upgrades/tasks.py
+++ b/bootstrapvz/plugins/unattended_upgrades/tasks.py
@@ -1,12 +1,10 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 
 
 class AddUnattendedUpgradesPackage(Task):
 	description = 'Adding `unattended-upgrades\' to the image packages'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py
index 9f673cf..ead8569 100644
--- a/bootstrapvz/plugins/vagrant/tasks.py
+++ b/bootstrapvz/plugins/vagrant/tasks.py
@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import workspace
-from bootstrapvz.common.tasks import apt
 import os
 import shutil
 
@@ -39,7 +38,6 @@ class CreateVagrantBoxDir(Task):
 class AddPackages(Task):
 	description = 'Add packages that vagrant depends on'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/azure/tasks/packages.py b/bootstrapvz/providers/azure/tasks/packages.py
index daed133..363f5fe 100644
--- a/bootstrapvz/providers/azure/tasks/packages.py
+++ b/bootstrapvz/providers/azure/tasks/packages.py
@@ -1,13 +1,11 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tasks.packages import InstallPackages
 
 
 class DefaultPackages(Task):
 	description = 'Adding image packages required for Azure'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py
index 5c7bf81..efc8a56 100644
--- a/bootstrapvz/providers/ec2/tasks/network.py
+++ b/bootstrapvz/providers/ec2/tasks/network.py
@@ -1,6 +1,5 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 from bootstrapvz.common.tasks import kernel
 import os.path
 
@@ -23,7 +22,6 @@ class EnableDHCPCDDNS(Task):
 class AddBuildEssentialPackage(Task):
 	description = 'Adding build-essential package'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py
index 079d112..e02cd60 100644
--- a/bootstrapvz/providers/ec2/tasks/packages.py
+++ b/bootstrapvz/providers/ec2/tasks/packages.py
@@ -1,12 +1,10 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 
 
 class DefaultPackages(Task):
 	description = 'Adding image packages required for EC2'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py
index b67ca54..2928a6d 100644
--- a/bootstrapvz/providers/gce/tasks/packages.py
+++ b/bootstrapvz/providers/gce/tasks/packages.py
@@ -8,7 +8,6 @@ import os
 class DefaultPackages(Task):
 	description = 'Adding image packages required for GCE'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
@@ -32,7 +31,7 @@ class DefaultPackages(Task):
 class ReleasePackages(Task):
 	description = 'Adding release-specific packages required for GCE'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources, apt.AddBackports, DefaultPackages]
+	predecessors = [apt.AddBackports, DefaultPackages]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/kvm/tasks/packages.py b/bootstrapvz/providers/kvm/tasks/packages.py
index 30d3eda..9fe6bbe 100644
--- a/bootstrapvz/providers/kvm/tasks/packages.py
+++ b/bootstrapvz/providers/kvm/tasks/packages.py
@@ -1,12 +1,10 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 
 
 class DefaultPackages(Task):
 	description = 'Adding image packages required for kvm'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/providers/virtualbox/tasks/packages.py b/bootstrapvz/providers/virtualbox/tasks/packages.py
index d8a192d..8bae0b8 100644
--- a/bootstrapvz/providers/virtualbox/tasks/packages.py
+++ b/bootstrapvz/providers/virtualbox/tasks/packages.py
@@ -1,12 +1,10 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import apt
 
 
 class DefaultPackages(Task):
 	description = 'Adding image packages required for virtualbox'
 	phase = phases.preparation
-	predecessors = [apt.AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):

From f10fe8efd6b018065c637319232a30c4ae109677 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 11:51:17 +0200
Subject: [PATCH 334/345] Document tox positional arguments

---
 tests/README.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/README.rst b/tests/README.rst
index da36feb..c14730c 100644
--- a/tests/README.rst
+++ b/tests/README.rst
@@ -4,3 +4,19 @@ The unit tests and the integration tests.
 The `unit tests <unit>`__ are responsible for testing individual
 parts of bootstrap-vz, while the `integration tests <integration>`__ test
 entire manifests by bootstrapping and booting them.
+
+Selecting tests
+---------------
+To run one specific test suite simply append the module path to tox:
+
+.. code:: sh
+
+	$ tox -e unit tests.unit.releases_tests
+
+Specific tests can be selected by appending the function name with a colon
+to the modulepath -- to run more than one tests, simply attach more arguments.
+
+
+.. code:: sh
+
+	$ tox -e unit tests.unit.releases_tests:test_lt tests.unit.releases_tests:test_eq

From df224e61a4ed4dfb000b8c4be5a2de4f5cf809d4 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:15:21 +0200
Subject: [PATCH 335/345] Fix modification box.ovf in vagrant plugin

---
 bootstrapvz/plugins/vagrant/assets/box.ovf | 2 +-
 bootstrapvz/plugins/vagrant/tasks.py       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/plugins/vagrant/assets/box.ovf b/bootstrapvz/plugins/vagrant/assets/box.ovf
index b990dfd..9f8d7f7 100644
--- a/bootstrapvz/plugins/vagrant/assets/box.ovf
+++ b/bootstrapvz/plugins/vagrant/assets/box.ovf
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Envelope ovf:version="1.0" xml:lang="en-US" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:vbox="http://www.virtualbox.org/ovf/machine">
   <References>
-    <File ovf:href="[disk_name]" ovf:id="file1"/>
+    <File ovf:href="[DISK_NAME]" ovf:id="file1"/>
   </References>
   <DiskSection>
     <Info>List of the virtual disks used in the package</Info>
diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py
index ead8569..90ad443 100644
--- a/bootstrapvz/plugins/vagrant/tasks.py
+++ b/bootstrapvz/plugins/vagrant/tasks.py
@@ -184,7 +184,7 @@ class PackageBox(Task):
 		[disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces)
 		attr(disk, 'ovf:capacity', info.volume.size.bytes.get_qty_in('B'))
 		attr(disk, 'ovf:format', info.volume.ovf_uri)
-		attr(disk, 'ovf:uuid', volume_uuid)
+		attr(disk, 'vbox:uuid', volume_uuid)
 
 		[system] = root.findall('./ovf:VirtualSystem', namespaces)
 		attr(system, 'ovf:id', info._vagrant['box_name'])
@@ -205,7 +205,7 @@ class PackageBox(Task):
 		[device_img] = machine.findall('./ovf:StorageControllers'
 		                               '/ovf:StorageController[@name="SATA Controller"]'
 		                               '/ovf:AttachedDevice/ovf:Image', namespaces)
-		attr(device_img, 'ovf:uuid', '{' + str(volume_uuid) + '}')
+		attr(device_img, 'uuid', '{' + str(volume_uuid) + '}')
 
 		template.write(destination, xml_declaration=True)  # , default_namespace=namespaces['ovf']
 

From e783927ea0f580dea67152bc51ef336503339a7f Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:16:00 +0200
Subject: [PATCH 336/345] Fix #139. Adjust Debian OS type depending on
 architecture

---
 bootstrapvz/plugins/vagrant/assets/box.ovf |  6 +++---
 bootstrapvz/plugins/vagrant/tasks.py       | 11 +++++++++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/bootstrapvz/plugins/vagrant/assets/box.ovf b/bootstrapvz/plugins/vagrant/assets/box.ovf
index 9f8d7f7..1bcb6e1 100644
--- a/bootstrapvz/plugins/vagrant/assets/box.ovf
+++ b/bootstrapvz/plugins/vagrant/assets/box.ovf
@@ -15,10 +15,10 @@
   </NetworkSection>
   <VirtualSystem ovf:id="[BOXNAME]">
     <Info>A virtual machine</Info>
-    <OperatingSystemSection ovf:id="96">
+    <OperatingSystemSection ovf:id="[OS_ID]">
       <Info>The kind of installed guest operating system</Info>
-      <Description>Debian_64</Description>
-      <vbox:OSType ovf:required="false">Debian_64</vbox:OSType>
+      <Description>[OS_DESCRIPTION]</Description>
+      <vbox:OSType ovf:required="false">[OS_TYPE]</vbox:OSType>
     </OperatingSystemSection>
     <VirtualHardwareSection>
       <Info>Virtual hardware requirements for a virtual machine</Info>
diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py
index 90ad443..776f1c3 100644
--- a/bootstrapvz/plugins/vagrant/tasks.py
+++ b/bootstrapvz/plugins/vagrant/tasks.py
@@ -189,6 +189,17 @@ class PackageBox(Task):
 		[system] = root.findall('./ovf:VirtualSystem', namespaces)
 		attr(system, 'ovf:id', info._vagrant['box_name'])
 
+		# Set the operating system
+		[os_section] = system.findall('./ovf:OperatingSystemSection', namespaces)
+		os_info = {'i386': {'id': 96, 'name': 'Debian'},
+		           'amd64': {'id': 96, 'name': 'Debian_64'}
+		           }.get(info.manifest.system['architecture'])
+		attr(os_section, 'ovf:id', os_info['id'])
+		[os_desc] = os_section.findall('./ovf:Description', namespaces)
+		os_desc.text = os_info['name']
+		[os_type] = os_section.findall('./vbox:OSType', namespaces)
+		os_type.text = os_info['name']
+
 		[sysid] = system.findall('./ovf:VirtualHardwareSection/ovf:System/'
 		                         'vssd:VirtualSystemIdentifier', namespaces)
 		sysid.text = info._vagrant['box_name']

From a23c9936b6fabf2489ce54fae666110747f61641 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:29:34 +0200
Subject: [PATCH 337/345] Fix #32: Extend image_commands docs

---
 bootstrapvz/plugins/image_commands/README.rst | 21 +++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/image_commands/README.rst
index 5ed7e65..fd4e579 100644
--- a/bootstrapvz/plugins/image_commands/README.rst
+++ b/bootstrapvz/plugins/image_commands/README.rst
@@ -11,9 +11,22 @@ Settings
 
 -  ``commands``: A list of lists containing strings. Each top-level item
    is a single command, while the strings inside each list comprise
-   parts of a command. This allows for proper shell argument escaping
-   (to circumvent this, simply put the entire command in a single
-   string). In addition to the manifest variables ``{root}`` is also
-   available. It points at the root of the image volume.
+   parts of a command. This allows for proper shell argument escaping.
+   To circumvent escaping, simply put the entire command in a single
+   string, the command will additionally be evaluated in a shell
+   (e.g. globbing will work).
+   In addition to the manifest variables ``{root}`` is also available.
+   It points at the root of the image volume.
    ``required``
    ``manifest vars``
+
+Example
+~~~~~~~
+
+Create an empty `index.html` in `/var/www` and delete all locales except english.
+.. code:: yaml
+
+  image_commands:
+      commands:
+        - [touch, '{root}/var/www/index.html']
+        - ['rm -rf /usr/share/locale/[^en]*']

From 2948badf47121e57a125a63342fab22d4bf4b7b1 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:29:55 +0200
Subject: [PATCH 338/345] Fix #204, user_modification phase has been introduced

---
 bootstrapvz/common/phases.py                | 2 ++
 bootstrapvz/plugins/ansible/tasks.py        | 2 +-
 bootstrapvz/plugins/file_copy/tasks.py      | 8 ++++----
 bootstrapvz/plugins/image_commands/tasks.py | 4 ++--
 bootstrapvz/plugins/puppet/tasks.py         | 2 +-
 5 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/bootstrapvz/common/phases.py b/bootstrapvz/common/phases.py
index e83feab..99f39a7 100644
--- a/bootstrapvz/common/phases.py
+++ b/bootstrapvz/common/phases.py
@@ -7,6 +7,7 @@ volume_mounting = Phase('Volume mounting', 'Mounting bootstrap volume')
 os_installation = Phase('OS installation', 'Installing the operating system')
 package_installation = Phase('Package installation', 'Installing software')
 system_modification = Phase('System modification', 'Modifying configuration files, adding resources, etc.')
+user_modification = Phase('User modification', 'Running user specified modifications')
 system_cleaning = Phase('System cleaning', 'Removing sensitive data, temporary files and other leftovers')
 volume_unmounting = Phase('Volume unmounting', 'Unmounting the bootstrap volume')
 image_registration = Phase('Image registration', 'Uploading/Registering with the provider')
@@ -19,6 +20,7 @@ order = [preparation,
          os_installation,
          package_installation,
          system_modification,
+         user_modification,
          system_cleaning,
          volume_unmounting,
          image_registration,
diff --git a/bootstrapvz/plugins/ansible/tasks.py b/bootstrapvz/plugins/ansible/tasks.py
index 100e69d..4dd675d 100644
--- a/bootstrapvz/plugins/ansible/tasks.py
+++ b/bootstrapvz/plugins/ansible/tasks.py
@@ -30,7 +30,7 @@ class AddPackages(Task):
 
 class RunAnsiblePlaybook(Task):
     description = 'Running ansible playbooks'
-    phase = phases.system_modification
+    phase = phases.user_modification
 
     @classmethod
     def run(cls, info):
diff --git a/bootstrapvz/plugins/file_copy/tasks.py b/bootstrapvz/plugins/file_copy/tasks.py
index 06c5dd8..90e1f08 100644
--- a/bootstrapvz/plugins/file_copy/tasks.py
+++ b/bootstrapvz/plugins/file_copy/tasks.py
@@ -37,8 +37,8 @@ def modify_path(info, path, entry):
 
 
 class MkdirCommand(Task):
-	description = 'copy files into the image'
-	phase = phases.system_modification
+	description = 'Creating directories requested by user'
+	phase = phases.user_modification
 
 	@classmethod
 	def run(cls, info):
@@ -51,8 +51,8 @@ class MkdirCommand(Task):
 
 
 class FileCopyCommand(Task):
-	description = 'copy files into the image'
-	phase = phases.system_modification
+	description = 'Copying user specified files into the image'
+	phase = phases.user_modification
 	predecessors = [MkdirCommand]
 
 	@classmethod
diff --git a/bootstrapvz/plugins/image_commands/tasks.py b/bootstrapvz/plugins/image_commands/tasks.py
index 2ea50e9..5ab2b27 100644
--- a/bootstrapvz/plugins/image_commands/tasks.py
+++ b/bootstrapvz/plugins/image_commands/tasks.py
@@ -3,8 +3,8 @@ from bootstrapvz.common import phases
 
 
 class ImageExecuteCommand(Task):
-	description = 'Execute command in the image'
-	phase = phases.system_modification
+	description = 'Executing commands in the image'
+	phase = phases.user_modification
 
 	@classmethod
 	def run(cls, info):
diff --git a/bootstrapvz/plugins/puppet/tasks.py b/bootstrapvz/plugins/puppet/tasks.py
index 05df26f..1efac12 100644
--- a/bootstrapvz/plugins/puppet/tasks.py
+++ b/bootstrapvz/plugins/puppet/tasks.py
@@ -57,7 +57,7 @@ class CopyPuppetAssets(Task):
 
 class ApplyPuppetManifest(Task):
 	description = 'Applying puppet manifest'
-	phase = phases.system_modification
+	phase = phases.user_modification
 	predecessors = [CopyPuppetAssets]
 
 	@classmethod

From a9b162442785eaaaaae24bb36488da1341007083 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:33:11 +0200
Subject: [PATCH 339/345] fixup! Fix #32: Extend image_commands docs

---
 bootstrapvz/plugins/image_commands/README.md | 28 --------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 bootstrapvz/plugins/image_commands/README.md

diff --git a/bootstrapvz/plugins/image_commands/README.md b/bootstrapvz/plugins/image_commands/README.md
deleted file mode 100644
index e4ffb8b..0000000
--- a/bootstrapvz/plugins/image_commands/README.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# Image script plugin
-
-This plugin gives the possibility to the user to execute commands.
-
-Plugin is defined in the manifest file, plugin section with:
-
-    "image_commands": {
-        "commands": [ [ "touch", "/var/www/index.html" ]],
-    }
-
-The *commands* element is an array of commands. Each command is an array describing the executable and its arguments.
-
-If you need shell expansion of wildcards, like __\*__, just put the entire command as a single entry:
-
-    "image_commands": {
-        "commands": [ [ "rm -f /tmp/*" ]],
-    }
-
-Command is executed in current context. It is possible to use variables to access the image or execute chroot commands in the image.
-
-Available variables are:
-  {root} : image mount point (to copy files for example or chroot commands)
-
-Example:
-
-    [[ "touch", "{root}/var/www/hello" ],
-    [ "/usr/sbin/chroot", "{root}", "touch", "/var/www/hello"]]
-

From fa8da6171df4abbacdf8c0db8e37bd89c14a831b Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:36:14 +0200
Subject: [PATCH 340/345] Fix #99: Rename image_commands to commands

---
 .../plugins/{image_commands => commands}/README.rst       | 8 ++++----
 .../plugins/{image_commands => commands}/__init__.py      | 0
 .../{image_commands => commands}/manifest-schema.yml      | 4 ++--
 bootstrapvz/plugins/{image_commands => commands}/tasks.py | 2 +-
 manifests/README.rst                                      | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)
 rename bootstrapvz/plugins/{image_commands => commands}/README.rst (87%)
 rename bootstrapvz/plugins/{image_commands => commands}/__init__.py (100%)
 rename bootstrapvz/plugins/{image_commands => commands}/manifest-schema.yml (91%)
 rename bootstrapvz/plugins/{image_commands => commands}/tasks.py (85%)

diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/commands/README.rst
similarity index 87%
rename from bootstrapvz/plugins/image_commands/README.rst
rename to bootstrapvz/plugins/commands/README.rst
index fd4e579..882f64b 100644
--- a/bootstrapvz/plugins/image_commands/README.rst
+++ b/bootstrapvz/plugins/commands/README.rst
@@ -26,7 +26,7 @@ Example
 Create an empty `index.html` in `/var/www` and delete all locales except english.
 .. code:: yaml
 
-  image_commands:
-      commands:
-        - [touch, '{root}/var/www/index.html']
-        - ['rm -rf /usr/share/locale/[^en]*']
+  commands:
+    commands:
+      - [touch, '{root}/var/www/index.html']
+      - ['rm -rf /usr/share/locale/[^en]*']
diff --git a/bootstrapvz/plugins/image_commands/__init__.py b/bootstrapvz/plugins/commands/__init__.py
similarity index 100%
rename from bootstrapvz/plugins/image_commands/__init__.py
rename to bootstrapvz/plugins/commands/__init__.py
diff --git a/bootstrapvz/plugins/image_commands/manifest-schema.yml b/bootstrapvz/plugins/commands/manifest-schema.yml
similarity index 91%
rename from bootstrapvz/plugins/image_commands/manifest-schema.yml
rename to bootstrapvz/plugins/commands/manifest-schema.yml
index da1ad8d..5292ea2 100644
--- a/bootstrapvz/plugins/image_commands/manifest-schema.yml
+++ b/bootstrapvz/plugins/commands/manifest-schema.yml
@@ -3,7 +3,7 @@ $schema: http://json-schema.org/draft-04/schema#
 properties:
   plugins:
     properties:
-      image_commands:
+      commands:
         properties:
           commands:
             items:
@@ -17,7 +17,7 @@ properties:
         - commands
         type: object
     required:
-    - image_commands
+    - commands
     type: object
 required:
 - plugins
diff --git a/bootstrapvz/plugins/image_commands/tasks.py b/bootstrapvz/plugins/commands/tasks.py
similarity index 85%
rename from bootstrapvz/plugins/image_commands/tasks.py
rename to bootstrapvz/plugins/commands/tasks.py
index 5ab2b27..2dd6eda 100644
--- a/bootstrapvz/plugins/image_commands/tasks.py
+++ b/bootstrapvz/plugins/commands/tasks.py
@@ -9,7 +9,7 @@ class ImageExecuteCommand(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.common.tools import log_check_call
-		for raw_command in info.manifest.plugins['image_commands']['commands']:
+		for raw_command in info.manifest.plugins['commands']['commands']:
 			command = map(lambda part: part.format(root=info.root, **info.manifest_vars), raw_command)
 			shell = len(command) == 1
 			log_check_call(command, shell=shell)
diff --git a/manifests/README.rst b/manifests/README.rst
index bff6c7b..da2c08a 100644
--- a/manifests/README.rst
+++ b/manifests/README.rst
@@ -214,7 +214,7 @@ boot, root and swap.
       optional setting overrides the command bootstrap-vz would normally
       use to format the partition. The command is specified as a string
       array where each option/argument is an item in that array (much
-      like the `image\_commands <../bootstrapvz/plugins/image_commands>`__ plugin).
+      like the `commands <../bootstrapvz/plugins/commands>`__ plugin).
       ``optional`` The following variables are available:
    -  ``{fs}``: The filesystem of the partition.
    -  ``{device_path}``: The device path of the partition.

From a7f2327e42efe2070f7028b7caaff8df8f5d1d10 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sat, 2 May 2015 12:45:55 +0200
Subject: [PATCH 341/345] Litter the manifest schemas with
 additionalProperties:false This pretty much eliminates the possiblity of
 undiscovered typos in the manifest

---
 bootstrapvz/base/manifest-schema.yml          |  9 ++++++--
 .../plugins/admin_user/manifest-schema.yml    |  1 +
 .../plugins/ansible/manifest-schema.yml       |  4 ++--
 .../plugins/apt_proxy/manifest-schema.yml     |  5 ++--
 bootstrapvz/plugins/chef/manifest-schema.yml  |  4 ++--
 .../plugins/cloud_init/manifest-schema.yml    |  1 +
 .../plugins/commands/manifest-schema.yml      | 17 ++++++--------
 .../plugins/docker_daemon/manifest-schema.yml | 23 ++++++++++---------
 .../plugins/file_copy/manifest-schema.yml     |  1 +
 .../plugins/minimize_size/manifest-schema.yml |  1 +
 bootstrapvz/plugins/ntp/manifest-schema.yml   |  1 +
 .../plugins/pip_install/manifest-schema.yml   |  1 +
 .../prebootstrapped/manifest-schema.yml       |  1 +
 .../plugins/root_password/manifest-schema.yml |  1 +
 bootstrapvz/plugins/salt/manifest-schema.yml  |  1 +
 .../unattended_upgrades/manifest-schema.yml   |  1 +
 .../plugins/vagrant/manifest-schema.yml       |  4 +++-
 17 files changed, 45 insertions(+), 31 deletions(-)

diff --git a/bootstrapvz/base/manifest-schema.yml b/bootstrapvz/base/manifest-schema.yml
index fb596eb..ed4b3f0 100644
--- a/bootstrapvz/base/manifest-schema.yml
+++ b/bootstrapvz/base/manifest-schema.yml
@@ -9,6 +9,7 @@ properties:
     properties:
       name: {type: string}
     required: [name]
+    additionalProperties: true
   bootstrapper:
     type: object
     properties:
@@ -31,12 +32,14 @@ properties:
       workspace:
         $ref: '#/definitions/path'
     required: [workspace]
+    additionalProperties: false
   image:
     type: object
     properties:
       name: {type: string}
     required: [name]
   system:
+    type: object
     properties:
       architecture:
         enum: [i386, amd64]
@@ -61,7 +64,7 @@ properties:
     - timezone
     - locale
     - charmap
-    type: object
+    additionalProperties: false
   packages:
     type: object
     properties:
@@ -128,6 +131,7 @@ properties:
         - $ref: '#/definitions/no_partitions'
         - $ref: '#/definitions/partition_table'
     required: [partitions]
+    additionalProperties: false
 definitions:
   absolute_path:
     type: string
@@ -153,9 +157,9 @@ definitions:
         type: array
       size: {$ref: '#/definitions/bytes'}
     required: [size, filesystem]
+    additionalProperties: false
   partition_table:
     type: object
-    additionalProperties: false
     properties:
       boot: {$ref: '#/definitions/partition'}
       root: {$ref: '#/definitions/partition'}
@@ -166,6 +170,7 @@ definitions:
         required: [size]
       type: {enum: [msdos, gpt]}
     required: [root]
+    additionalProperties: false
   path:
     type: string
     pattern: ^[^\0]+$
diff --git a/bootstrapvz/plugins/admin_user/manifest-schema.yml b/bootstrapvz/plugins/admin_user/manifest-schema.yml
index 31e8d82..02cce0a 100644
--- a/bootstrapvz/plugins/admin_user/manifest-schema.yml
+++ b/bootstrapvz/plugins/admin_user/manifest-schema.yml
@@ -11,3 +11,4 @@ properties:
         properties:
           username: {type: string}
         required: [username]
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/ansible/manifest-schema.yml b/bootstrapvz/plugins/ansible/manifest-schema.yml
index 0fc2f57..dc99679 100644
--- a/bootstrapvz/plugins/ansible/manifest-schema.yml
+++ b/bootstrapvz/plugins/ansible/manifest-schema.yml
@@ -21,8 +21,8 @@ properties:
             host: {type: string}
             minItems: 1
           playbook: {$ref: '#/definitions/absolute_path'}
-        required:
-        - playbook
+        required: [playbook]
+        additionalProperties: false
 definitions:
   absolute_path:
     pattern: ^/[^\0]+$
diff --git a/bootstrapvz/plugins/apt_proxy/manifest-schema.yml b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
index efa7875..3f35a09 100644
--- a/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
+++ b/bootstrapvz/plugins/apt_proxy/manifest-schema.yml
@@ -14,6 +14,5 @@ properties:
           port: {type: integer}
           persistent: {type: boolean}
           username: {type: string}
-        required:
-        - address
-        - port
+        required: [address, port]
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/chef/manifest-schema.yml b/bootstrapvz/plugins/chef/manifest-schema.yml
index 58fd02f..da44dd8 100644
--- a/bootstrapvz/plugins/chef/manifest-schema.yml
+++ b/bootstrapvz/plugins/chef/manifest-schema.yml
@@ -11,8 +11,8 @@ properties:
         properties:
           assets:
             $ref: '#/definitions/absolute_path'
-        required:
-        - assets
+        required: [assets]
+        additionalProperties: false
 definitions:
   absolute_path:
     pattern: ^/[^\0]+$
diff --git a/bootstrapvz/plugins/cloud_init/manifest-schema.yml b/bootstrapvz/plugins/cloud_init/manifest-schema.yml
index ea7f64c..1950c21 100644
--- a/bootstrapvz/plugins/cloud_init/manifest-schema.yml
+++ b/bootstrapvz/plugins/cloud_init/manifest-schema.yml
@@ -28,3 +28,4 @@ properties:
             items: {type: string}
             uniqueItems: true
         required: [username]
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/commands/manifest-schema.yml b/bootstrapvz/plugins/commands/manifest-schema.yml
index 5292ea2..d868c8f 100644
--- a/bootstrapvz/plugins/commands/manifest-schema.yml
+++ b/bootstrapvz/plugins/commands/manifest-schema.yml
@@ -1,9 +1,13 @@
 ---
 $schema: http://json-schema.org/draft-04/schema#
+title: Image commands plugin manifest
+type: object
 properties:
   plugins:
+    type: object
     properties:
       commands:
+        type: object
         properties:
           commands:
             items:
@@ -13,13 +17,6 @@ properties:
               type: array
             minItems: 1
             type: array
-        required:
-        - commands
-        type: object
-    required:
-    - commands
-    type: object
-required:
-- plugins
-title: Image commands plugin manifest
-type: object
+        required: [commands]
+        additionalProperties: false
+    required: [commands]
diff --git a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
index 2e5f1f0..62928d3 100644
--- a/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
+++ b/bootstrapvz/plugins/docker_daemon/manifest-schema.yml
@@ -3,17 +3,6 @@ $schema: http://json-schema.org/draft-04/schema#
 title: Install Docker plugin manifest
 type: object
 properties:
-  plugins:
-    type: object
-    properties:
-      docker_daemon:
-        type: object
-        properties:
-          version:
-            pattern: '^\d\.\d{1,2}\.\d$'
-            type: string
-          docker_opts:
-            type: string
   system:
     type: object
     properties:
@@ -26,3 +15,15 @@ properties:
           enum:
           - squeeze
           - oldstable
+  plugins:
+    type: object
+    properties:
+      docker_daemon:
+        type: object
+        properties:
+          version:
+            pattern: '^\d\.\d{1,2}\.\d$'
+            type: string
+          docker_opts:
+            type: string
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/file_copy/manifest-schema.yml b/bootstrapvz/plugins/file_copy/manifest-schema.yml
index 25bdb12..0a65dde 100644
--- a/bootstrapvz/plugins/file_copy/manifest-schema.yml
+++ b/bootstrapvz/plugins/file_copy/manifest-schema.yml
@@ -35,6 +35,7 @@ properties:
         required:
         - files
         type: object
+        additionalProperties: false
     required:
     - file_copy
     type: object
diff --git a/bootstrapvz/plugins/minimize_size/manifest-schema.yml b/bootstrapvz/plugins/minimize_size/manifest-schema.yml
index fc84249..5a36be8 100644
--- a/bootstrapvz/plugins/minimize_size/manifest-schema.yml
+++ b/bootstrapvz/plugins/minimize_size/manifest-schema.yml
@@ -10,6 +10,7 @@ properties:
           zerofree:
             type: boolean
         type: object
+        additionalProperties: false
     type: object
 title: Minimize size plugin manifest
 type: object
diff --git a/bootstrapvz/plugins/ntp/manifest-schema.yml b/bootstrapvz/plugins/ntp/manifest-schema.yml
index 2303d1c..170095e 100644
--- a/bootstrapvz/plugins/ntp/manifest-schema.yml
+++ b/bootstrapvz/plugins/ntp/manifest-schema.yml
@@ -13,3 +13,4 @@ properties:
             type: array
             items: {type: string}
             minItems: 1
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/pip_install/manifest-schema.yml b/bootstrapvz/plugins/pip_install/manifest-schema.yml
index df3b922..36ae5d0 100644
--- a/bootstrapvz/plugins/pip_install/manifest-schema.yml
+++ b/bootstrapvz/plugins/pip_install/manifest-schema.yml
@@ -15,3 +15,4 @@ properties:
               type: string
             minItems: 1
             uniqueItems: true
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml b/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
index 68fbbe4..d01820c 100644
--- a/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
+++ b/bootstrapvz/plugins/prebootstrapped/manifest-schema.yml
@@ -23,3 +23,4 @@ properties:
         properties:
           image: {type: string}
           snapshot: {type: string}
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/root_password/manifest-schema.yml b/bootstrapvz/plugins/root_password/manifest-schema.yml
index 7267807..f91ef63 100644
--- a/bootstrapvz/plugins/root_password/manifest-schema.yml
+++ b/bootstrapvz/plugins/root_password/manifest-schema.yml
@@ -11,3 +11,4 @@ properties:
         properties:
           password: {type: string}
         required: [password]
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/salt/manifest-schema.yml b/bootstrapvz/plugins/salt/manifest-schema.yml
index 43b6b3a..6b99428 100644
--- a/bootstrapvz/plugins/salt/manifest-schema.yml
+++ b/bootstrapvz/plugins/salt/manifest-schema.yml
@@ -22,3 +22,4 @@ properties:
           master: {type: string}
           version: {type: string}
         required: [install_source]
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
index 7e5fc0b..a0a7568 100644
--- a/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
+++ b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.yml
@@ -16,3 +16,4 @@ properties:
         - update_interval
         - download_interval
         - upgrade_interval
+        additionalProperties: false
diff --git a/bootstrapvz/plugins/vagrant/manifest-schema.yml b/bootstrapvz/plugins/vagrant/manifest-schema.yml
index c27172f..08c6544 100644
--- a/bootstrapvz/plugins/vagrant/manifest-schema.yml
+++ b/bootstrapvz/plugins/vagrant/manifest-schema.yml
@@ -21,4 +21,6 @@ properties:
   plugins:
     type: object
     properties:
-      vagrant: {type: object}
+      vagrant:
+        type: object
+        additionalProperties: false

From ebbba2310b7a83301acff2d9f2912dab458f76c1 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 2 May 2015 12:22:17 -0300
Subject: [PATCH 342/345] Update changelog

---
 CHANGELOG.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 7c476d7..9e35c93 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+2015-05-02
+----------
+Anders Ingemann:
+	* Fix #32: Add image_commands example
+	* Fix #99: rename image_commands to commands
+	* Fix #139: Vagrant / Virtualbox provider should set ostype when 32 bits selected
+	* Fix #204: Create a new phase where user modification tasks can run
+
+2015-04-29
+----------
+Anders Ingemann:
+	* Fix #104: Don't verify default target when adding packages
+	* Fix #217: Implement get_version() function in common.tools
+
+2015-04-28
+----------
+Jonh Wendell:
+	* root_password: Enable SSH root login
+
 2015-04-27
 ----------
 John Kristensen:

From 1f052d3c1f7cbdecba7eed45bf6879bb4a5b7b78 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 2 May 2015 12:23:00 -0300
Subject: [PATCH 343/345] CHANGELOG: use tabs instead of spaces

---
 CHANGELOG.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 9e35c93..0d17e6e 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -23,7 +23,7 @@ Jonh Wendell:
 2015-04-27
 ----------
 John Kristensen:
-    * Add authentication support to the apt proxy plugin
+	* Add authentication support to the apt proxy plugin
 
 2015-04-25
 ----------
@@ -60,15 +60,15 @@ Anders Ingemann (work started 2014-08-31, merged on 2015-04-25):
 	* Extend sed_i to raise Exceptions when the expected amount of replacements is not met
 
 Jonas Bergler:
-    * Fixes #145: Fix installation of vbox guest additions.
+	* Fixes #145: Fix installation of vbox guest additions.
 
 Tiago Ilieve:
-    * Fixes #142: msdos partition type incorrect for swap partition (Linux)
+	* Fixes #142: msdos partition type incorrect for swap partition (Linux)
 
 2015-04-23
 ----------
 Tiago Ilieve:
-    * Fixes #212: Sparse file is created on the current directory
+	* Fixes #212: Sparse file is created on the current directory
 
 2014-11-23
 ----------

From 3cfba0898399288ab118ae861a12ef0d8b50abe3 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 2 May 2015 13:11:14 -0300
Subject: [PATCH 344/345] Fix mentions to the `commands` plugin name

---
 bootstrapvz/plugins/commands/README.rst          | 9 ++++-----
 bootstrapvz/plugins/commands/manifest-schema.yml | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/bootstrapvz/plugins/commands/README.rst b/bootstrapvz/plugins/commands/README.rst
index 882f64b..1e31635 100644
--- a/bootstrapvz/plugins/commands/README.rst
+++ b/bootstrapvz/plugins/commands/README.rst
@@ -1,10 +1,9 @@
-Image commands
+Commands
 --------------
 
-The image commands plugin allows you to run arbitrary commands during
-the bootstrap process. The commands are run at an indeterminate point
-*after* packages have been installed, but *before* the volume has been
-unmounted.
+This plugin allows you to run arbitrary commands during the bootstrap process.
+The commands are run at an indeterminate point *after* packages have been
+installed, but *before* the volume has been unmounted.
 
 Settings
 ~~~~~~~~
diff --git a/bootstrapvz/plugins/commands/manifest-schema.yml b/bootstrapvz/plugins/commands/manifest-schema.yml
index d868c8f..0d7c2e5 100644
--- a/bootstrapvz/plugins/commands/manifest-schema.yml
+++ b/bootstrapvz/plugins/commands/manifest-schema.yml
@@ -1,6 +1,6 @@
 ---
 $schema: http://json-schema.org/draft-04/schema#
-title: Image commands plugin manifest
+title: Commands plugin manifest
 type: object
 properties:
   plugins:

From b4dd36e8cfbd4fe0099685caccbf94ce60537279 Mon Sep 17 00:00:00 2001
From: Tiago Ilieve <tiago.myhro@gmail.com>
Date: Sat, 2 May 2015 13:21:02 -0300
Subject: [PATCH 345/345] Update contributing docs

Removes mentions to the old branching strategy and `gh-pages`.
---
 CONTRIBUTING.rst | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e46da26..e615fb8 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -19,26 +19,16 @@ Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic w
 * Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units
   of work, but there's no reason to have many little commits just because of corrected typos.
 * Push to your fork, preferably on a topic branch.
-
-From here on there are two paths to consider:
-
-If your patch is a new feature, e.g.: plugin, provider, etc. then:
-
-* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make
-  sure that the code is stable.
-
-If it is a bug/security fix:
-
 * Send a pull request to the `master` branch.
 
 Please try to be very descriptive about your changes when you write a pull request, stating what it does, why
-it is needed, which use cases this change covers etc.
+it is needed, which use cases this change covers, etc.
 You may be asked to rebase your work on the current branch state, so it can be merged cleanly.
 If you push a new commit to your pull request you will have to add a new comment to the PR,
 provided that you want us notified. Github will otherwise not send a notification.
 
-Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they
-concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`.
+Be aware that your modifications need to be properly documented. Please take a look at the
+`documentation section <#documentation>`__ to see how to do that.
 
 Happy hacking! :-)