Merge branch 'WIP-partitioning' into python

README.md

@@ -14,10 +14,17 @@ Pull requests are also welcome!
 Dependencies
 ------------
 You will need to run debian wheezy with **python 2.7** and **debootstrap** installed.
+Other depencies include:
+* qemu-utils
+* parted
+* grub2
+* euca2ools
+* xfsprogs (If you want to use XFS as a filesystem)
 Also the following python libraries are required:
 * **boto**
 * **jsomschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip)
 * **termcolor**
+* **fysom**
 
 Bootstrapping instance store AMIs requires **euca2ools** to be installed.
 

base/bootstrapinfo.py

@@ -3,6 +3,11 @@
 class BootstrapInformation(object):
 	def __init__(self, manifest=None, debug=False):
 		self.manifest = manifest
+		from fs import load_volume
+		self.volume = load_volume(self.manifest.volume)
 		self.debug = debug
 		import random
 		self.run_id = random.randrange(16 ** 8)
+		import os.path
+		workspace_dirname = '{id:x}'.format(id=self.run_id)
+		self.workspace = os.path.join(manifest.bootstrapper['workspace'], workspace_dirname)

base/fs/__init__.py

@@ -0,0 +1,20 @@
+
+
+def load_volume(data):
+	from common.fs.loopbackvolume import LoopbackVolume
+	from providers.ec2.ebsvolume import EBSVolume
+	from providers.virtualbox.volume import VirtualBoxVolume
+	from partitionmaps.gpt import GPTPartitionMap
+	from partitionmaps.mbr import MBRPartitionMap
+	from partitionmaps.none import NoPartitions
+	partition_maps = {'none': NoPartitions,
+	                  'gpt': GPTPartitionMap,
+	                  'mbr': MBRPartitionMap,
+	                  }
+	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'])
+	volume_backings = {'raw': LoopbackVolume,
+	                   's3':  LoopbackVolume,
+	                   'vdi': VirtualBoxVolume,
+	                   'ebs': EBSVolume
+	                   }
+	return volume_backings.get(data['backing'])(partition_map)

base/fs/exceptions.py

@@ -0,0 +1,8 @@
+
+
+class VolumeError(Exception):
+	pass
+
+
+class PartitionError(Exception):
+	pass

base/fs/partitionmaps/abstract.py

@@ -0,0 +1,78 @@
+from abc import ABCMeta
+from abc import abstractmethod
+from common.tools import log_check_call
+from common.fsm_proxy import FSMProxy
+from ..exceptions import PartitionError
+
+
+class AbstractPartitionMap(FSMProxy):
+
+	__metaclass__ = ABCMeta
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'},
+	          {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'},
+	          {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'},
+	          ]
+
+	def __init__(self):
+		cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}}
+		super(AbstractPartitionMap, self).__init__(cfg)
+
+	def is_blocking(self):
+		return self.fsm.current == 'mapped'
+
+	def get_total_size(self):
+		return sum(p.size for p in self.partitions)
+
+	def create(self, volume):
+		self.fsm.create(volume=volume)
+
+	@abstractmethod
+	def _before_create(self, event):
+		pass
+
+	def map(self, volume):
+		self.fsm.map(volume=volume)
+
+	def _before_map(self, event):
+		volume = event.volume
+		try:
+			mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path])
+			import re
+			regexp = re.compile('^(?P<name>.+[^\d](?P<p_idx>\d+)) : '
+			                    '(?P<start_blk>\d) (?P<num_blks>\d+) '
+			                    '{device_path} (?P<blk_offset>\d+)$'
+			                    .format(device_path=volume.device_path))
+			log_check_call(['/sbin/kpartx', '-a', volume.device_path])
+			import os.path
+			for mapping in mappings:
+				match = regexp.match(mapping)
+				if match is None:
+					raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping))
+				partition_path = os.path.join('/dev/mapper', match.group('name'))
+				p_idx = int(match.group('p_idx'))-1
+				self.partitions[p_idx].map(partition_path)
+
+			for idx, partition in enumerate(self.partitions):
+				if partition.fsm.current not in ['mapped', 'formatted']:
+					raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1))
+
+		except PartitionError as e:
+			for partition in self.partitions:
+				if not partition.fsm.can('unmap'):
+					partition.unmap()
+			log_check_call(['/sbin/kpartx', '-d', volume.device_path])
+			raise e
+
+	def unmap(self, volume):
+		self.fsm.unmap(volume=volume)
+
+	def _before_unmap(self, event):
+		volume = event.volume
+		for partition in self.partitions:
+			if partition.fsm.cannot('unmap'):
+				msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
+				raise PartitionError(msg)
+		log_check_call(['/sbin/kpartx', '-d', volume.device_path])
+		for partition in self.partitions:
+			partition.unmap()

base/fs/partitionmaps/gpt.py

@@ -0,0 +1,36 @@
+from abstract import AbstractPartitionMap
+from ..partitions.gpt import GPTPartition
+from ..partitions.gpt_swap import GPTSwapPartition
+from common.tools import log_check_call
+
+
+class GPTPartitionMap(AbstractPartitionMap):
+
+	def __init__(self, data):
+		self.partitions = []
+		if 'boot' in data:
+			self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None)
+			self.partitions.append(self.boot)
+		self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root',
+		                         getattr(self, 'boot', None))
+		self.partitions.append(self.root)
+		if 'swap' in data:
+			self.swap = GPTSwapPartition(data['swap']['size'], self.root)
+			self.partitions.append(self.swap)
+
+		super(GPTPartitionMap, self).__init__()
+
+	def _before_create(self, event):
+		volume = event.volume
+		log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path,
+		                '--', 'mklabel', 'gpt'])
+		for partition in self.partitions:
+			partition.create(volume)
+
+		boot_idx = self.root.get_index()
+		if hasattr(self, 'boot'):
+			boot_idx = self.boot.get_index()
+		log_check_call(['/sbin/parted', '--script', volume.device_path,
+		                '--', 'set ' + str(boot_idx) + ' boot on'])
+		log_check_call(['/sbin/parted', '--script', volume.device_path,
+		                '--', 'set ' + str(boot_idx) + ' bios_grub on'])

base/fs/partitionmaps/mbr.py

@@ -0,0 +1,37 @@
+from abstract import AbstractPartitionMap
+from ..partitions.mbr import MBRPartition
+from ..partitions.mbr_swap import MBRSwapPartition
+from common.tools import log_check_call
+
+
+class MBRPartitionMap(AbstractPartitionMap):
+
+	def __init__(self, data):
+		self.partitions = []
+		if 'boot' in data:
+			self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None)
+			self.partitions.append(self.boot)
+		self.root = MBRPartition(data['root']['size'], data['root']['filesystem'],
+		                         getattr(self, 'boot', None))
+		self.partitions.append(self.root)
+		if 'swap' in data:
+			self.swap = MBRSwapPartition(data['swap']['size'], self.root)
+			self.partitions.append(self.swap)
+
+		super(MBRPartitionMap, self).__init__()
+
+	def get_total_size(self):
+		return sum(p.size for p in self.partitions) + 1  # Post-MBR gap for embedding grub
+
+	def _before_create(self, event):
+		volume = event.volume
+		log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path,
+		                '--', 'mklabel', 'msdos'])
+		for partition in self.partitions:
+			partition.create(volume)
+
+		boot_idx = self.root.get_index()
+		if hasattr(self, 'boot'):
+			boot_idx = self.boot.get_index()
+		log_check_call(['/sbin/parted', '--script', volume.device_path,
+		                '--', 'set ' + str(boot_idx) + ' boot on'])

base/fs/partitionmaps/none.py

@@ -0,0 +1,15 @@
+from ..partitions.single import SinglePartition
+
+
+class NoPartitions(object):
+
+	def __init__(self, data):
+		root = data['root']
+		self.root = SinglePartition(root['size'], root['filesystem'])
+		self.partitions = [self.root]
+
+	def is_blocking(self):
+		return self.root.fsm == 'mounted'
+
+	def get_total_size(self):
+		return self.root.size

base/fs/partitions/abstract.py

@@ -0,0 +1,41 @@
+from abc import ABCMeta
+from common.tools import log_check_call
+from common.fsm_proxy import FSMProxy
+
+
+class AbstractPartition(FSMProxy):
+
+	__metaclass__ = ABCMeta
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'},
+	          {'name': 'format', 'src': 'created', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          ]
+
+	def __init__(self, size, filesystem):
+		self.size          = size
+		self.filesystem    = filesystem
+		self.device_path   = None
+
+		cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}}
+		super(AbstractPartition, self).__init__(cfg)
+
+	def get_uuid(self):
+		[uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path])
+		return uuid
+
+	def _before_format(self, e):
+		mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem)
+		log_check_call([mkfs, self.device_path])
+
+	def mount(self, destination):
+		self.fsm.mount(destination=destination)
+
+	def _before_mount(self, e):
+		log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination])
+		self.mount_dir = e.destination
+
+	def _before_unmount(self, e):
+		log_check_call(['/bin/umount', self.mount_dir])
+		del self.mount_dir

base/fs/partitions/base.py

@@ -0,0 +1,43 @@
+from abstract import AbstractPartition
+
+
+class BasePartition(AbstractPartition):
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'},
+	          {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'},
+	          {'name': 'format', 'src': 'mapped', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'},
+
+	          {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'},
+	          {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'},
+	          ]
+
+	def __init__(self, size, filesystem, previous):
+		self.previous = previous
+		super(BasePartition, self).__init__(size, filesystem)
+
+	def create(self, volume):
+		self.fsm.create(volume=volume)
+
+	def get_index(self):
+		if self.previous is None:
+			return 1
+		else:
+			return self.previous.get_index()+1
+
+	def get_start(self):
+		if self.previous is None:
+			return 0
+		else:
+			return self.previous.get_start() + self.previous.size
+
+	def map(self, device_path):
+		self.fsm.map(device_path=device_path)
+
+	def _before_map(self, e):
+		self.device_path = e.device_path
+
+	def _before_unmap(self, e):
+		self.device_path = None

base/fs/partitions/gpt.py

@@ -0,0 +1,19 @@
+from common.tools import log_check_call
+from base import BasePartition
+
+
+class GPTPartition(BasePartition):
+
+	def __init__(self, size, filesystem, name, previous):
+		self.name = name
+		super(GPTPartition, self).__init__(size, filesystem, previous)
+
+	def _before_create(self, e):
+		start = self.get_start()
+		# {name} only works for gpt, for msdos that becomes the part-type (primary, extended, logical)
+		parted_command = ('mkpart primary {start}MiB {end}MiB'
+		                  .format(name=self.name,
+		                          start=str(start),
+		                          end=str(start + self.size)))
+		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
+		                '--', parted_command])

base/fs/partitions/gpt_swap.py

@@ -0,0 +1,11 @@
+from common.tools import log_check_call
+from gpt import GPTPartition
+
+
+class GPTSwapPartition(GPTPartition):
+
+	def __init__(self, size, previous):
+		super(GPTSwapPartition, self).__init__(size, 'swap', 'swap', previous)
+
+	def _before_format(self, e):
+		log_check_call(['/sbin/mkswap', self.device_path])

base/fs/partitions/mbr.py

@@ -0,0 +1,19 @@
+from common.tools import log_check_call
+from base import BasePartition
+
+
+class MBRPartition(BasePartition):
+
+	def get_start(self):
+		if self.previous is None:
+			return 1  # Post-MBR gap for embedding grub
+		else:
+			return self.previous.get_start() + self.previous.size
+
+	def _before_create(self, e):
+		start = self.get_start()
+		parted_command = ('mkpart primary {start}MiB {end}MiB'
+		                  .format(start=str(start),
+		                          end=str(start + self.size)))
+		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,
+		                '--', parted_command])

base/fs/partitions/mbr_swap.py

@@ -0,0 +1,11 @@
+from common.tools import log_check_call
+from mbr import MBRPartition
+
+
+class MBRSwapPartition(MBRPartition):
+
+	def __init__(self, size, previous):
+		super(MBRSwapPartition, self).__init__(size, 'swap', previous)
+
+	def _before_format(self, e):
+		log_check_call(['/sbin/mkswap', self.device_path])

base/fs/partitions/single.py

@@ -0,0 +1,5 @@
+from abstract import AbstractPartition
+
+
+class SinglePartition(AbstractPartition):
+		pass

base/fs/volume.py

@@ -0,0 +1,64 @@
+from abc import ABCMeta
+from common.fsm_proxy import FSMProxy
+from common.tools import log_check_call
+from exceptions import VolumeError
+from partitionmaps.none import NoPartitions
+
+
+class Volume(FSMProxy):
+
+	__metaclass__ = ABCMeta
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'},
+	          {'name': 'attach', 'src': 'detached', 'dst': 'attached'},
+	          {'name': 'detach', 'src': 'attached', 'dst': 'detached'},
+	          {'name': 'delete', 'src': 'detached', 'dst': 'deleted'},
+	          ]
+
+	def __init__(self, partition_map):
+		self.device_path = None
+		self.specials_mounted = False
+		self.partition_map = partition_map
+		self.size = self.partition_map.get_total_size()
+
+		callbacks = {'onbeforedetach': self._check_blocking}
+		if isinstance(self.partition_map, NoPartitions):
+			def set_dev_path(e):
+				self.partition_map.root.device_path = self.device_path
+			callbacks['onafterattach'] = set_dev_path
+
+		cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks}
+		super(Volume, self).__init__(cfg)
+
+	def _after_create(self, e):
+		if isinstance(self.partition_map, NoPartitions):
+			self.partition_map.root.create()
+
+	def can_mount_specials(self):
+		return self.fsm.current == 'attached'
+
+	def mount_specials(self):
+		if self.specials_mounted:
+			raise VolumeError('The special devices are already mounted')
+		root = self.partition_map.root.mount_dir
+		log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts'])
+		self.specials_mounted = True
+
+	def unmount_specials(self):
+		if not self.specials_mounted:
+			raise VolumeError('The special devices are not mounted')
+		root = self.partition_map.root.mount_dir
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc'])
+		log_check_call(['/bin/umount', '{root}/dev'.format(root=root)])
+		self.specials_mounted = False
+
+	def _check_blocking(self, e):
+		if self.partition_map.is_blocking():
+			raise VolumeError('The partitionmap prevents the detach procedure')
+		if self.specials_mounted:
+			raise VolumeError('The special devices are mounted and prevent the detaching procedure')

base/main.py

@@ -15,6 +15,10 @@ def get_args():
 	parser = ArgumentParser(description='Bootstrap Debian for the cloud.')
 	parser.add_argument('--debug', action='store_true',
 	                    help='Print debugging information')
+	parser.add_argument('--pause-on-error', action='store_true',
+	                    help='Pause on error, before rollback')
+	parser.add_argument('--dry-run', action='store_true',
+	                    help='Dont\'t actually run the tasks')
 	parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST')
 	return parser.parse_args()
 
@@ -33,10 +37,12 @@ def run(args):
 	bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug)
 
 	try:
-		tasklist.run(bootstrap_info)
+		tasklist.run(info=bootstrap_info, dry_run=args.dry_run)
 		log.info('Successfully completed bootstrapping')
 	except (Exception, KeyboardInterrupt) as e:
 		log.exception(e)
+		if args.pause_on_error:
+			raw_input("Press Enter to commence rollback")
 		log.error('Rolling back')
 		rollback_tasklist = TaskList()
 		provider.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest)
@@ -44,5 +50,5 @@ def run(args):
 			rollback_tasks = getattr(plugin, 'rollback_tasks', None)
 			if callable(rollback_tasks):
 				plugin.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest)
-		rollback_tasklist.run(bootstrap_info)
+		rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run)
 		log.info('Successfully completed rollback')

base/manifest-schema.json

@@ -9,24 +9,17 @@
 		"bootstrapper": {
 			"type": "object",
 			"properties": {
-				"mount_dir": { "type": "string" },
-				"mirror":   { "type": "string" },
-				"tarball": { "type": "boolean" },
-				"tarball_dir": { "type": "string" }
+				"workspace": { "$ref": "#/definitions/path" },
+				"mirror":    { "type": "string", "format": "uri" },
+				"tarball":   { "type": "boolean" }
 			},
-			"required": ["mount_dir"]
+			"required": ["workspace"]
 		},
 		"system": {
 			"type": "object",
 			"properties": {
-				"release": {
-					"type": "string",
-					"enum": ["wheezy"]
-				},
-				"architecture": {
-					"type": "string",
-					"enum": ["i386", "amd64"]
-				},
+				"release":      { "enum": ["wheezy"] },
+				"architecture": { "enum": ["i386", "amd64"] },
 				"timezone":     { "type": "string" },
 				"locale":       { "type": "string" },
 				"charmap":      { "type": "string" }
@@ -36,12 +29,16 @@
 		"volume": {
 			"type": "object",
 			"properties": {
-				"size": {
-					"type": "integer",
-					"minimum": 1
+				"backing": { "type": "string" },
+				"partitions": {
+					"type": "object",
+					"oneOf": [
+						{ "$ref": "#/definitions/no_partitions" },
+						{ "$ref": "#/definitions/partition_table" }
+					]
 				}
 			},
-			"required": ["size"]
+			"required": ["partitions"]
 		},
 		"plugins": {
 			"type": "object",
@@ -59,5 +56,44 @@
 			"additionalProperties": false
 		}
 	},
-	"required": ["provider", "bootstrapper", "volume", "system"]
+	"required": ["provider", "bootstrapper", "volume", "system"],
+	"definitions": {
+		"path": {
+			"type":    "string",
+			"pattern": "^[^\\0]+$"
+		},
+		"no_partitions": {
+			"type": "object",
+			"properties": {
+				"type": { "enum": ["none"] },
+				"root": { "$ref": "#/definitions/partition" }
+			},
+			"required": ["root"],
+			"additionalProperties": false
+		},
+		"partition_table": {
+			"type": "object",
+			"properties": {
+				"type": { "enum": ["mbr", "gpt"] },
+				"boot": { "$ref": "#/definitions/partition" },
+				"root": { "$ref": "#/definitions/partition" },
+				"swap": {
+					"type": "object",
+					"properties": { "size": { "type": "integer", "minimum": 1 } },
+					"required": ["size"]
+				}
+			},
+			"required": ["root"],
+			"additionalProperties": false
+		},
+		"partition": {
+			"type": "object",
+			"properties": {
+				"size": { "type": "integer", "minimum": 1 },
+				"filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] }
+			},
+			"required": ["size", "filesystem"]
+		}
+	},
+	"required": ["provider", "bootstrapper", "system", "volume"]
 }

base/manifest.py

@@ -49,10 +49,6 @@ class Manifest(object):
 		self.bootstrapper = data['bootstrapper']
 		if 'mirror' not in self.bootstrapper:
 			self.bootstrapper['mirror'] = 'http://http.debian.net/debian'
-		if 'tarball' not in self.bootstrapper:
-			self.bootstrapper['tarball'] = False
-		if 'tarball_dir' not in self.bootstrapper and self.bootstrapper['tarball']:
-			self.bootstrapper['tarball_dir'] = '/tmp'
 		self.volume = data['volume']
 		self.system = data['system']
 		self.plugins = data['plugins'] if 'plugins' in data else {}

base/task.py

@@ -1,4 +1,3 @@
-from common.exceptions import TaskListError
 
 
 class Task(object):
@@ -6,27 +5,8 @@ class Task(object):
 	before = []
 	after = []
 
-	def __init__(self):
-		self._check_ordering()
-
 	def __str__(self):
 		return '{module}.{task}'.format(module=self.__module__, task=self.__class__.__name__)
 
 	def __repr__(self):
 		return self.__str__()
-
-	def _check_ordering(self):
-		def name(ref):
-			return '{module}.{task}'.format(module=ref.__module__, task=ref.__class__.__name__)
-		for task in self.before:
-			if self.phase > task.phase:
-				msg = ("The task {self} is specified as running before {other}, "
-				       "but its phase '{phase}' lies after the phase '{other_phase}'"
-				       .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase))
-				raise TaskListError(msg)
-		for task in self.after:
-			if self.phase < task.phase:
-				msg = ("The task {self} is specified as running after {other}, "
-				       "but its phase '{phase}' lies before the phase '{other_phase}'"
-				       .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase))
-				raise TaskListError(msg)

base/tasklist.py

@@ -13,40 +13,34 @@ class TaskList(object):
 		self.tasks.update(args)
 
 	def remove(self, *args):
-		for task_type in args:
-			task = self.get(task_type)
-			if task is not None:
+		for task in args:
 			self.tasks.discard(task)
 
-	def replace(self, task, replacement):
-		self.remove(task)
-		self.add(replacement)
-
-	def get(self, ref):
-		return next((task for task in self.tasks if type(task) is ref), None)
-
-	def run(self, bootstrap_info):
+	def run(self, info={}, dry_run=False):
 		task_list = self.create_list(self.tasks)
 		log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list)))
 
-		for task in task_list:
+		for task_type in task_list:
+			task = task_type()
 			if hasattr(task, 'description'):
 				log.info(task.description)
 			else:
 				log.info('Running {task}'.format(task=task))
-			task.run(bootstrap_info)
+			if not dry_run:
+				task.run(info)
 			self.tasks_completed.append(task)
 
 	def create_list(self, tasks):
 		from common.phases import order
 		graph = {}
 		for task in tasks:
-			successors = []
-			successors.extend([self.get(succ) for succ in task.before])
-			successors.extend(filter(lambda succ: type(task) in succ.after, tasks))
+			self.check_ordering(task)
+			successors = set()
+			successors.update(task.before)
+			successors.update(filter(lambda succ: task in succ.after, tasks))
 			succeeding_phases = order[order.index(task.phase) + 1:]
-			successors.extend(filter(lambda succ: succ.phase in succeeding_phases, tasks))
-			graph[task] = filter(lambda succ: succ in self.tasks, successors)
+			successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks))
+			graph[task] = filter(lambda succ: succ in tasks, successors)
 
 		components = self.strongly_connected_components(graph)
 		cycles_found = 0
@@ -63,6 +57,20 @@ class TaskList(object):
 
 		return sorted_tasks
 
+	def check_ordering(self, task):
+		for successor in task.before:
+			if successor.phase > successor.phase:
+				msg = ("The task {task} is specified as running before {other}, "
+				       "but its phase '{phase}' lies after the phase '{other_phase}'"
+				       .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
+				raise TaskListError(msg)
+		for predecessor in task.after:
+			if task.phase < predecessor.phase:
+				msg = ("The task {task} is specified as running after {other}, "
+				       "but its phase '{phase}' lies before the phase '{other_phase}'"
+				       .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
+				raise TaskListError(msg)
+
 	def strongly_connected_components(self, graph):
 		# Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
 		# Find the strongly connected components in a graph using Tarjan's algorithm.

common/assets/init.d/expand-volume

@@ -13,7 +13,7 @@
 prog=$(basename $0)
 logger="logger -t $prog"
 
-device_path="/dev/xvda1"
+device_path="/dev/xvda"
 
 filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'`
 

common/fs/__init__.py

@@ -0,0 +1,39 @@
+
+
+def get_partitions():
+	import re
+	regexp = re.compile('^ *(?P<major>\d+) *(?P<minor>\d+) *(?P<num_blks>\d+) (?P<dev_name>\S+)$')
+	matches = {}
+	path = '/proc/partitions'
+	with open(path) as partitions:
+		next(partitions)
+		next(partitions)
+		for line in partitions:
+			match = regexp.match(line)
+			if match is None:
+				raise RuntimeError('Unable to parse {line} in {path}'.format(line=line, path=path))
+			matches[match.group('dev_name')] = match.groupdict()
+	return matches
+
+
+def remount(volume, fn):
+	from base.fs.partitionmaps.none import NoPartitions
+
+	p_map = volume.partition_map
+	volume.unmount_specials()
+	if hasattr(p_map, 'boot'):
+		boot_dir = p_map.boot.mount_dir
+		p_map.boot.unmount()
+	root_dir = p_map.root.mount_dir
+	p_map.root.unmount()
+	if not isinstance(p_map, NoPartitions):
+		p_map.unmap(volume)
+		result = fn()
+		p_map.map(volume)
+	else:
+		result = fn()
+	p_map.root.mount(root_dir)
+	if hasattr(p_map, 'boot'):
+		p_map.boot.mount(boot_dir)
+	volume.mount_specials()
+	return result

common/fs/loopbackvolume.py

@@ -0,0 +1,80 @@
+from base.fs.volume import Volume
+from common.tools import log_check_call
+from base.fs.exceptions import VolumeError
+
+
+# QEMU formats:
+# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop
+
+
+class LoopbackVolume(Volume):
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'},
+	          {'name': 'attach', 'src': 'detached', 'dst': 'attached'},
+	          {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'},
+            {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'attached'},
+	          {'name': 'detach', 'src': 'attached', 'dst': 'detached'},
+	          {'name': 'delete', 'src': 'detached', 'dst': 'deleted'},
+	          ]
+
+	extension = 'raw'
+
+	def can_mount_specials(self):
+		return self.fsm.current in ['attached', 'linked']
+
+	def create(self, image_path):
+		self.fsm.create(image_path=image_path)
+
+	def _before_create(self, e):
+		self.image_path = e.image_path
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M'])
+
+	def _before_attach(self, e):
+		[self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path])
+		self.device_path = self.loop_device_path
+
+	def _before_link_dm_node(self, e):
+		import os.path
+		from . import get_partitions
+		proc_partitions = get_partitions()
+		loop_device_name = os.path.basename(self.loop_device_path)
+		loop_device_partition = proc_partitions[loop_device_name]
+
+		sectors = self.size*1024*1024/512
+		table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}'
+		         .format(log_start_sec=0,
+		                 sectors=sectors,
+		                 major=loop_device_partition['major'],
+		                 minor=loop_device_partition['minor'],
+		                 start_sec=0))
+		import string
+		import os.path
+		for letter in string.ascii_lowercase:
+			dev_name = 'vd' + letter
+			dev_path = os.path.join('/dev/mapper', dev_name)
+			if not os.path.exists(dev_path):
+				self.dm_node_name = dev_name
+				self.dm_node_path = dev_path
+				break
+
+		if not hasattr(self, 'dm_node_name'):
+			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
+
+		log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table)
+		self.device_path = self.dm_node_path
+
+	def _before_unlink_dm_node(self, e):
+		log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name])
+		del self.dm_node_name
+		del self.dm_node_path
+		self.device_path = self.loop_device_path
+
+	def _before_detach(self, e):
+		log_check_call(['/sbin/losetup', '--detach', self.loop_device_path])
+		del self.loop_device_path
+		del self.device_path
+
+	def _before_delete(self, e):
+		from os import remove
+		remove(self.image_path)
+		del self.image_path

common/fsm_proxy.py

@@ -0,0 +1,42 @@
+
+
+class FSMProxy(object):
+
+	def __init__(self, cfg):
+		from fysom import Fysom
+		events = set([event['name'] for event in cfg['events']])
+		cfg['callbacks'] = self.collect_event_listeners(events, cfg['callbacks'])
+		self.fsm = Fysom(cfg)
+		self.attach_proxy_methods(self.fsm, events)
+
+	def collect_event_listeners(self, events, callbacks):
+		callbacks = callbacks.copy()
+		callback_names = []
+		for event in events:
+			callback_names.append(('_before_' + event, 'onbefore' + event))
+			callback_names.append(('_after_' + event, 'onafter' + event))
+		for fn_name, listener in callback_names:
+			fn = getattr(self, fn_name, None)
+			if callable(fn):
+				if listener in callbacks:
+					old_fn = callbacks[listener]
+
+					def wrapper(e, old_fn=old_fn, fn=fn):
+						old_fn(e)
+						fn(e)
+					callbacks[listener] = wrapper
+				else:
+					callbacks[listener] = fn
+		return callbacks
+
+	def attach_proxy_methods(self, fsm, events):
+		def make_proxy(fsm, event):
+			fn = getattr(fsm, event)
+
+			def proxy():
+				fn()
+			return proxy
+
+		for event in events:
+			if not hasattr(self, event):
+				setattr(self, event, make_proxy(fsm, event))

common/task_sets.py

@@ -0,0 +1,74 @@
+from common.tasks import workspace
+from common.tasks import packages
+from common.tasks import host
+from common.tasks import bootstrap
+from common.tasks import volume
+from common.tasks import filesystem
+from common.tasks import partitioning
+from common.tasks import cleanup
+from common.tasks import apt
+from common.tasks import security
+from common.tasks import locale
+
+base_set = [workspace.CreateWorkspace,
+            packages.HostPackages,
+            packages.ImagePackages,
+            host.CheckPackages,
+            bootstrap.Bootstrap,
+            workspace.DeleteWorkspace,
+            ]
+
+volume_set = [volume.Attach,
+              volume.Detach,
+              filesystem.Format,
+              filesystem.FStab,
+              ]
+
+partitioning_set = [partitioning.PartitionVolume,
+                    partitioning.MapPartitions,
+                    partitioning.UnmapPartitions,
+                    ]
+
+boot_partition_set = [filesystem.CreateBootMountDir,
+                      filesystem.MountBoot,
+                      filesystem.UnmountBoot,
+                      ]
+
+mounting_set = [filesystem.CreateMountDir,
+                filesystem.MountRoot,
+                filesystem.MountSpecials,
+                filesystem.UnmountSpecials,
+                filesystem.UnmountRoot,
+                filesystem.DeleteMountDir,
+                ]
+
+ssh_set = [security.DisableSSHPasswordAuthentication,
+           security.DisableSSHDNSLookup,
+           cleanup.ShredHostkeys,
+           ]
+
+apt_set = [apt.DisableDaemonAutostart,
+           apt.AptSources,
+           apt.AptUpgrade,
+           apt.PurgeUnusedPackages,
+           apt.AptClean,
+           apt.EnableDaemonAutostart,
+           ]
+
+locale_set = [locale.GenerateLocale,
+              locale.SetTimezone,
+              ]
+
+
+def get_fs_specific_set(partitions):
+	task_set = {'ext2': [filesystem.TuneVolumeFS],
+	            'ext3': [filesystem.TuneVolumeFS],
+	            'ext4': [filesystem.TuneVolumeFS],
+	            'xfs':  [filesystem.AddXFSProgs],
+	            }
+	tasks = set()
+	if 'boot' in partitions:
+		tasks.update(task_set.get(partitions['boot']['filesystem'], []))
+	if 'root' in partitions:
+		tasks.update(task_set.get(partitions['root']['filesystem'], []))
+	return tasks

common/tasks/bootstrap.py

@@ -29,7 +29,7 @@ class MakeTarball(Task):
 		hash_args = [arg for arg in arguments if arg != info.root]
 		tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8]
 		tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id)
-		info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename)
+		info.tarball = os.path.join(info.manifest.bootstrapper['workspace'], tarball_filename)
 		if os.path.isfile(info.tarball):
 			log.debug('Found matching tarball, skipping download')
 		else:

common/tasks/filesystem.py

@@ -1,29 +1,30 @@
 from base import Task
 from common import phases
-from common.exceptions import TaskError
 from common.tools import log_check_call
 from bootstrap import Bootstrap
+import volume
 
 
-class FormatVolume(Task):
+class Format(Task):
 	description = 'Formatting the volume'
 	phase = phases.volume_preparation
 
 	def run(self, info):
-		dev_path = info.bootstrap_device['path']
-		mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem'])
-		log_check_call([mkfs, dev_path])
-		info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']}
+		for partition in info.volume.partition_map.partitions:
+			partition.format()
 
 
 class TuneVolumeFS(Task):
 	description = 'Tuning the bootstrap volume filesystem'
 	phase = phases.volume_preparation
-	after = [FormatVolume]
+	after = [Format]
 
 	def run(self, info):
+		import re
 		# Disable the time based filesystem check
-		log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']])
+		for partition in info.volume.partition_map.partitions:
+			if re.match('^ext[2-4]$', partition.filesystem) is not None:
+				log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path])
 
 
 class AddXFSProgs(Task):
@@ -36,33 +37,43 @@ class AddXFSProgs(Task):
 
 
 class CreateMountDir(Task):
-	description = 'Creating mountpoint for the bootstrap volume'
+	description = 'Creating mountpoint for the root partition'
 	phase = phases.volume_mounting
 
 	def run(self, info):
 		import os
-		mount_dir = info.manifest.bootstrapper['mount_dir']
-		info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id)
-		# Works recursively, fails if last part exists, which is exaclty what we want.
+		info.root = os.path.join(info.workspace, 'root')
 		os.makedirs(info.root)
 
 
-class MountVolume(Task):
-	description = 'Mounting the bootstrap volume'
+class MountRoot(Task):
+	description = 'Mounting the root partition'
 	phase = phases.volume_mounting
 	after = [CreateMountDir]
 
 	def run(self, info):
-		with open('/proc/mounts') as mounts:
-			for mount in mounts:
-				if info.root in mount:
-					msg = 'Something is already mounted at {root}'.format(root=info.root)
-					raise TaskError(msg)
+		info.volume.partition_map.root.mount(info.root)
 
-		log_check_call(['/bin/mount',
-		                '--types', info.manifest.volume['filesystem'],
-		                info.bootstrap_device['partitions']['root_path'],
-		                info.root])
+
+class MountBoot(Task):
+	description = 'Mounting the boot partition'
+	phase = phases.volume_mounting
+	after = [MountRoot]
+
+	def run(self, info):
+		info.volume.partition_map.boot.mount(info.boot_dir)
+
+
+class CreateBootMountDir(Task):
+	description = 'Creating mountpoint for the boot partition'
+	phase = phases.volume_mounting
+	after = [MountRoot]
+	before = [MountBoot]
+
+	def run(self, info):
+		import os
+		info.boot_dir = os.path.join(info.root, 'boot')
+		os.makedirs(info.boot_dir)
 
 
 class MountSpecials(Task):
@@ -71,36 +82,40 @@ class MountSpecials(Task):
 	after = [Bootstrap]
 
 	def run(self, info):
-		log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts'])
+		info.volume.mount_specials()
+
+
+class UnmountRoot(Task):
+	description = 'Unmounting the bootstrap volume'
+	phase = phases.volume_unmounting
+	before = [volume.Detach]
+
+	def run(self, info):
+		info.volume.partition_map.root.unmount()
+
+
+class UnmountBoot(Task):
+	description = 'Unmounting the boot partition'
+	phase = phases.volume_unmounting
+	before = [UnmountRoot]
+
+	def run(self, info):
+		info.volume.partition_map.boot.unmount()
 
 
 class UnmountSpecials(Task):
 	description = 'Unmunting special block devices'
 	phase = phases.volume_unmounting
+	before = [UnmountRoot]
 
 	def run(self, info):
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc'])
-		log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)])
-
-
-class UnmountVolume(Task):
-	description = 'Unmounting the bootstrap volume'
-	phase = phases.volume_unmounting
-	after = [UnmountSpecials]
-
-	def run(self, info):
-		log_check_call(['/bin/umount', info.root])
+		info.volume.unmount_specials()
 
 
 class DeleteMountDir(Task):
 	description = 'Deleting mountpoint for the bootstrap volume'
 	phase = phases.volume_unmounting
-	after = [UnmountVolume]
+	after = [UnmountRoot]
 
 	def run(self, info):
 		import os
@@ -108,24 +123,44 @@ class DeleteMountDir(Task):
 		del info.root
 
 
-class ModifyFstab(Task):
-	description = 'Adding root volume to the fstab'
+class FStab(Task):
+	description = 'Adding partitions to the fstab'
 	phase = phases.system_modification
 
 	def run(self, info):
 		import os.path
-		mount_opts = ['defaults']
-		if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
-			mount_opts.append('barrier=0')
-		if info.manifest.volume['filesystem'].lower() == 'xfs':
-			mount_opts.append('nobarrier')
-		fstab_path = os.path.join(info.root, 'etc/fstab')
+		p_map = info.volume.partition_map
+		mount_points = [{'path': '/',
+			               'partition': p_map.root,
+			               'dump': '1',
+			               'pass_num': '1',
+			                }]
+		if hasattr(p_map, 'boot'):
+			mount_points.append({'path': '/boot',
+			                     'partition': p_map.boot,
+			                     'dump': '1',
+			                     'pass_num': '2',
+			                     })
+		if hasattr(p_map, 'swap'):
+			mount_points.append({'path': 'none',
+			                     'partition': p_map.swap,
+			                     'dump': '1',
+			                     'pass_num': '0',
+			                     })
 
-		device = '/dev/sda1'
-		if info.manifest.virtualization == 'pvm':
-			device = '/dev/xvda1'
-		with open(fstab_path, 'a') as fstab:
-			fstab.write('{device} /     {filesystem}    {mount_opts} 1 1\n'
-			            .format(device=device,
-			                    filesystem=info.manifest.volume['filesystem'].lower(),
-			                    mount_opts=','.join(mount_opts)))
+		fstab_lines = []
+		for mount_point in mount_points:
+			partition = mount_point['partition']
+			mount_opts = ['defaults']
+			fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}'
+			                   .format(uuid=partition.get_uuid(),
+			                           mountpoint=mount_point['path'],
+			                           filesystem=partition.filesystem,
+			                           mount_opts=','.join(mount_opts),
+			                           dump=mount_point['dump'],
+			                           pass_num=mount_point['pass_num']))
+
+		fstab_path = os.path.join(info.root, 'etc/fstab')
+		with open(fstab_path, 'w') as fstab:
+			fstab.write('\n'.join(fstab_lines))
+			fstab.write('\n')

common/tasks/host.py

@@ -14,7 +14,7 @@ class CheckPackages(Task):
 		from subprocess import CalledProcessError
 		for package in info.host_packages:
 			try:
-				log_check_call(['/usr/bin/dpkg', '--status', package])
+				log_check_call(['/usr/bin/dpkg-query', '-s', package])
 			except CalledProcessError:
 				msg = "The package ``{0}\'\' is not installed".format(package)
 				raise TaskError(msg)

common/tasks/initd.py

@@ -1,5 +1,6 @@
 from base import Task
 from common import phases
+from common.tools import log_check_call
 import os.path
 
 
@@ -8,11 +9,18 @@ class ResolveInitScripts(Task):
 	phase = phases.system_modification
 
 	def run(self, info):
-		init_scripts = {'expand-volume': 'expand-volume'}
+		init_scripts = {}
+		init_scripts['expand-volume'] = 'expand-volume'
 
+		from subprocess import CalledProcessError
+		try:
+			log_check_call(['/usr/sbin/chroot', info.root,
+			                '/usr/bin/dpkg-query', '-W', 'openssh-server'])
 			init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys'
 			if info.manifest.system['release'] == 'squeeze':
 				init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys'
+		except CalledProcessError:
+			pass
 
 		disable_scripts = ['hwclock.sh']
 		if info.manifest.system['release'] == 'squeeze':
@@ -37,7 +45,6 @@ class InstallInitScripts(Task):
 		             stat.S_IRGRP                | stat.S_IXGRP |
 		             stat.S_IROTH                | stat.S_IXOTH)
 		from shutil import copy
-		from common.tools import log_check_call
 		for name, src in info.initd['install'].iteritems():
 			dst = os.path.join(info.root, 'etc/init.d', name)
 			copy(src, dst)

common/tasks/loopback.py

@@ -1,60 +1,29 @@
 from base import Task
 from common import phases
-from filesystem import UnmountVolume
-from common.tools import log_check_call
+import volume
 
 
 class Create(Task):
 	description = 'Creating a loopback volume'
 	phase = phases.volume_creation
+	before = [volume.Attach]
 
 	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
 		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
-		log_check_call(['/bin/dd',
-		                'if=/dev/zero', 'of=' + info.loopback_file,
-		                'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0'])
+		image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension))
+		info.volume.create(image_path)
 
 
-class CreateQemuImg(Task):
-	description = 'Creating a loopback volume with qemu'
-	phase = phases.volume_creation
+class MoveImage(Task):
+	description = 'Moving volume image'
+	phase = phases.image_registration
 
 	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
 		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
-		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw',
-		                info.loopback_file, str(info.manifest.volume['size']) + 'M'])
-
-
-class Attach(Task):
-	description = 'Attaching the loopback volume'
-	phase = phases.volume_creation
-	after = [Create, CreateQemuImg]
-
-	def run(self, info):
-		info.bootstrap_device = {}
-		command = ['/sbin/losetup', '--show', '--find', info.loopback_file]
-		[info.bootstrap_device['path']] = log_check_call(command)
-
-
-class Detach(Task):
-	description = 'Detaching the loopback volume'
-	phase = phases.volume_unmounting
-	after = [UnmountVolume]
-
-	def run(self, info):
-		log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']])
-		del info.bootstrap_device
-
-
-class Delete(Task):
-	description = 'Deleting the loopback volume'
-	phase = phases.cleaning
-
-	def run(self, info):
-		from os import remove
-		remove(info.loopback_file)
-		del info.loopback_file
+		filename = 'volume-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension)
+		destination = os.path.join(info.manifest.bootstrapper['workspace'], filename)
+		import shutil
+		shutil.move(info.volume.image_path, destination)
+		import logging
+		log = logging.getLogger(__name__)
+		log.info('The volume image has been moved to {image_path}'.format(image_path=destination))

common/tasks/network.py

@@ -12,6 +12,15 @@ class RemoveDNSInfo(Task):
 		remove(os.path.join(info.root, 'etc/resolv.conf'))
 
 
+class RemoveHostname(Task):
+	description = 'Removing the hostname file'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from os import remove
+		remove(os.path.join(info.root, 'etc/hostname'))
+
+
 class ConfigureNetworkIF(Task):
 	description = 'Configuring network interfaces'
 	phase = phases.system_modification
@@ -26,13 +35,3 @@ class ConfigureNetworkIF(Task):
 		                        'iface eth0 inet dhcp\n'}
 		with open(interfaces_path, 'a') as interfaces:
 			interfaces.write(if_config.get(info.manifest.system['release']))
-
-
-class ConfigureDHCP(Task):
-	description = 'Configuring the DHCP client'
-	phase = phases.system_modification
-
-	def run(self, info):
-		from common.tools import sed_i
-		dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
-		sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')

common/tasks/packages.py

@@ -7,8 +7,19 @@ class HostPackages(Task):
 	phase = phases.preparation
 
 	def run(self, info):
-		packages = set(['debootstrap'])
-		info.host_packages = packages
+		info.host_packages = set()
+		info.host_packages.add('debootstrap')
+
+		from common.fs.loopbackvolume import LoopbackVolume
+		if isinstance(info.volume, LoopbackVolume):
+			info.host_packages.add('qemu-utils')
+
+		if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions):
+			info.host_packages.add('xfsprogs')
+
+		from base.fs.partitionmaps.none import NoPartitions
+		if not isinstance(info.volume.partition_map, NoPartitions):
+			info.host_packages.update(['parted', 'kpartx'])
 
 
 class ImagePackages(Task):
@@ -16,12 +27,7 @@ class ImagePackages(Task):
 	phase = phases.preparation
 
 	def run(self, info):
-		# Add some basic packages we are going to need
-		include = set(['udev',
-		               'openssh-server',
+		info.img_packages = set(), set()
+		include, exclude = info.img_packages
 		# We could bootstrap without locales, but things just suck without them, error messages etc.
-		               'locales',
-		               ])
-		exclude = set()
-
-		info.img_packages = include, exclude
+		include.add('locales')

common/tasks/parted.py

@@ -1,53 +0,0 @@
-from base import Task
-from common import phases
-from common.tools import log_check_call
-import filesystem
-import loopback
-
-
-class PartitionVolume(Task):
-	description = 'Partitioning the volume'
-	phase = phases.volume_preparation
-
-	def run(self, info):
-		# parted
-		log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'],
-		                '--', 'mklabel', 'msdos'])
-		log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'],
-		                '--', 'mkpart', 'primary', 'ext4', '32k', '-1'])
-		log_check_call(['parted', '--script', info.bootstrap_device['path'],
-		                '--', 'set', '1', 'boot', 'on'])
-
-
-class MapPartitions(Task):
-	description = 'Mapping volume partitions'
-	phase = phases.volume_preparation
-	after = [PartitionVolume]
-
-	def run(self, info):
-		root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1'
-		log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']])
-		info.bootstrap_device['partitions'] = {'root_path': root_partition_path}
-
-
-class FormatPartitions(Task):
-	description = 'Formatting the partitions'
-	phase = phases.volume_preparation
-	before = [filesystem.TuneVolumeFS]
-	after = [MapPartitions]
-
-	def run(self, info):
-		# These params will fail for mkfs.xfs
-		log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']),
-		                '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']])
-
-
-class UnmapPartitions(Task):
-	description = 'Removing volume partitions mapping'
-	phase = phases.volume_unmounting
-	before = [loopback.Detach]
-	after = [filesystem.UnmountVolume]
-
-	def run(self, info):
-		log_check_call(['kpartx', '-d', info.bootstrap_device['path']])
-		del info.bootstrap_device['partitions']['root_path']

common/tasks/partitioning.py

@@ -0,0 +1,32 @@
+from base import Task
+from common import phases
+import filesystem
+import volume
+
+
+class PartitionVolume(Task):
+	description = 'Partitioning the volume'
+	phase = phases.volume_preparation
+
+	def run(self, info):
+		info.volume.partition_map.create(info.volume)
+
+
+class MapPartitions(Task):
+	description = 'Mapping volume partitions'
+	phase = phases.volume_preparation
+	before = [filesystem.Format]
+	after = [PartitionVolume]
+
+	def run(self, info):
+		info.volume.partition_map.map(info.volume)
+
+
+class UnmapPartitions(Task):
+	description = 'Removing volume partitions mapping'
+	phase = phases.volume_unmounting
+	before = [volume.Detach]
+	after = [filesystem.UnmountRoot]
+
+	def run(self, info):
+		info.volume.partition_map.unmap(info.volume)

common/tasks/volume.py

@@ -0,0 +1,28 @@
+from base import Task
+from common import phases
+from common.tasks import workspace
+
+
+class Attach(Task):
+	description = 'Attaching the volume'
+	phase = phases.volume_creation
+
+	def run(self, info):
+		info.volume.attach()
+
+
+class Detach(Task):
+	description = 'Detaching the volume'
+	phase = phases.volume_unmounting
+
+	def run(self, info):
+		info.volume.detach()
+
+
+class Delete(Task):
+	description = 'Deleting the volume'
+	phase = phases.cleaning
+	before = [workspace.DeleteWorkspace]
+
+	def run(self, info):
+		info.volume.delete()

common/tasks/workspace.py

@@ -0,0 +1,20 @@
+from base import Task
+from common import phases
+
+
+class CreateWorkspace(Task):
+	description = 'Creating workspace'
+	phase = phases.preparation
+
+	def run(self, info):
+		import os
+		os.makedirs(info.workspace)
+
+
+class DeleteWorkspace(Task):
+	description = 'Deleting workspace'
+	phase = phases.cleaning
+
+	def run(self, info):
+		import os
+		os.rmdir(info.workspace)

common/tools.py

@@ -16,6 +16,7 @@ def log_call(command, stdin=None):
 	from os.path import realpath
 	command_log = realpath(command[0]).replace('/', '.')
 	log = logging.getLogger(__name__ + command_log)
+	log.debug('Executing: {command}'.format(command=' '.join(command)))
 
 	if stdin is not None:
 		process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -31,13 +32,11 @@ def log_call(command, stdin=None):
 		ret = select.select(reads, [], [])
 		for fd in ret[0]:
 			if fd == process.stdout.fileno():
-				line = process.stdout.readline()
-				if line != '':
+				for line in iter(process.stdout.readline, ''):
 					log.debug(line.strip())
 					stdout.append(line.strip())
 			if fd == process.stderr.fileno():
-				line = process.stderr.readline()
-				if line != '':
+				for line in iter(process.stderr.readline, ''):
 					log.error(line.strip())
 					stderr.append(line.strip())
 		if process.poll() is not None:

manifests/ec2-ebs-debian-official.manifest.json

@@ -2,21 +2,16 @@
 	"provider":       "ec2",
 	"virtualization": "pvm",
 	"credentials":    {
-		"access-key":  null,
-		"secret-key":  null,
-		"certificate": null,
-		"private-key": null,
-		"user-id":     null
+		// "access-key": null,
+		// "secret-key": null
 	},
 
 	"bootstrapper": {
-		"mount_dir": "/target",
-		"mirror": "http://http.debian.net/debian"
+		"workspace": "/target"
 	},
 	"image": {
-		"name":        "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}",
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
 		"description": "Debian {release} {architecture} AMI ({virtualization})"
-		"bucket":      ""
 	},
 	"system": {
 		"release":      "wheezy",
@@ -26,18 +21,19 @@
 		"charmap":      "UTF-8"
 	},
 	"volume": {
-		"backing":      "s3",
-		"filesystem":   "ext4",
-		"size":         1024
+		"backing": "ebs",
+		"partitions": {
+			"type": "none",
+			"root": {
+				"size":       1024,
+				"filesystem": "ext4"
+			}
+		}
 	},
 	"plugins": {
 		"admin_user": {
 			"enabled": true,
 			"username": "admin"
-		},
-		"prebootstrapped": {
-			"enabled":  false,
-			"image": null
 		}
 	}
 }

manifests/ec2-ebs-partitioned.manifest.json

@@ -0,0 +1,33 @@
+{
+	"provider":       "ec2",
+	"virtualization": "pvm",
+	"credentials":    {
+		// "access-key": null,
+		// "secret-key": null
+	},
+
+	"bootstrapper": {
+		"workspace": "/target"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} AMI ({virtualization})"
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing": "ebs",
+		"partitions": {
+			"type": "mbr",
+			"root": {
+				"size":       1023,
+				"filesystem": "ext4"
+			}
+		}
+	}
+}

manifests/ec2-ebs-pvm.manifest.json

@@ -1,47 +0,0 @@
-{
-	"provider":       "ec2",
-	"virtualization": "pvm",
-	"credentials":    {
-		"access-key": null,
-		"secret-key": null
-	},
-
-	"bootstrapper": {
-		"mount_dir": "/target",
-		"mirror": "http://http.debian.net/debian"
-	},
-	"image": {
-		"name":        "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}",
-		"description": "Debian {release} {architecture} AMI ({virtualization})"
-	},
-	"system": {
-		"release":      "wheezy",
-		"architecture": "amd64",
-		"timezone":     "UTC",
-		"locale":       "en_US",
-		"charmap":      "UTF-8"
-	},
-	"volume": {
-		"backing":    "ebs",
-		"filesystem": "ext4",
-		"size":       8192
-	},
-	"plugins": {
-		"admin_user": {
-			"enabled": true,
-			"username": "admin"
-		},
-		"backports": {
-			"enabled": true,
-			"packages": [ "cloud-init" ]
-		},
-		"build_metadata": {
-			"enabled": false,
-			"path":    "/root/build-metadata-{ami_name}"
-		},
-		"prebootstrapped": {
-			"enabled":  false,
-			"snapshot": ""
-		}
-	}
-}

manifests/ec2-ebs.manifest.json

@@ -0,0 +1,33 @@
+{
+	"provider":       "ec2",
+	"virtualization": "pvm",
+	"credentials":    {
+		// "access-key": null,
+		// "secret-key": null
+	},
+
+	"bootstrapper": {
+		"workspace": "/target"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} AMI ({virtualization})"
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing": "ebs",
+		"partitions": {
+			"type": "none",
+			"root": {
+				"size":       1024,
+				"filesystem": "ext4"
+			}
+		}
+	}
+}

manifests/ec2-s3.manifest.json

@@ -0,0 +1,37 @@
+{
+	"provider":       "ec2",
+	"virtualization": "pvm",
+	"credentials":    {
+		// "access-key":  null,
+		// "secret-key":  null,
+		// "certificate": null,
+		// "private-key": null,
+		// "user-id":     null
+	},
+
+	"bootstrapper": {
+		"workspace": "/target"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} AMI",
+		"bucket":      "debian-amis"
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing": "s3",
+		"partitions": {
+			"type": "none",
+			"root": {
+				"size":       1024,
+				"filesystem": "ext4"
+			}
+		}
+	}
+}

manifests/kvm-virtio.manifest.json

@@ -1,40 +0,0 @@
-{
-	"provider"      : "kvm",
-	"virtualization": "virtio",
-
-	"bootstrapper": {
-		"mount_dir": "/mnt/target",
-		"mirror"      : "http://ftp.fr.debian.org/debian/"
-	},
-	"image": {
-		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {release} {architecture} ({virtualization})"
-	},
-	"system": {
-		"release"     : "wheezy",
-		"architecture": "amd64",
-		"timezone"    : "UTC",
-		"locale"      : "en_US",
-		"charmap"     : "UTF-8"
-	},
-	"volume": {
-		"backing"   : "raw",
-		"filesystem": "ext4",
-		"size"      : 1024,
-		"loopback_dir" : "/tmp"
-	},
-	"plugins": {
-		"user_packages": {
-			"enabled": true,
-			"repo": [ "apache2" ],
-			"local": []
-		},
-		"root_password": {
-			"enabled": true,
-			"password": "test"
-                },
-		"opennebula": {
-			"enabled": true
-		}
-	}
-}

manifests/one-ide.manifest.json

@@ -1,31 +0,0 @@
-{
-	"provider"      : "one",
-	"virtualization": "ide",
-
-	"bootstrapper": {
-		"mount_dir": "/target"
-	},
-	"image": {
-		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {release} {architecture} ({virtualization})"
-	},
-	"system": {
-		"release"     : "wheezy",
-		"architecture": "amd64",
-		"timezone"    : "UTC",
-		"locale"      : "en_US",
-		"charmap"     : "UTF-8"
-	},
-	"volume": {
-		"backing"   : "raw",
-		"filesystem": "ext4",
-		"size"      : 1024
-	},
-	"plugins": {
-		"user_packages": {
-			"enabled": true,
-			"repo": [ "apache2" ],
-			"local": []
-		}
-	}
-}

manifests/one-virtio.manifest.json

@@ -1,31 +0,0 @@
-{
-	"provider"      : "one",
-	"virtualization": "virtio",
-
-	"bootstrapper": {
-		"mount_dir": "/target"
-	},
-	"image": {
-		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {release} {architecture} ({virtualization})"
-	},
-	"system": {
-		"release"     : "wheezy",
-		"architecture": "amd64",
-		"timezone"    : "UTC",
-		"locale"      : "en_US",
-		"charmap"     : "UTF-8"
-	},
-	"volume": {
-		"backing"   : "raw",
-		"filesystem": "ext4",
-		"size"      : 1024
-	},
-	"plugins": {
-		"user_packages": {
-			"enabled": true,
-			"repo": [ "apache2" ],
-			"local": []
-		}
-	}
-}

manifests/virtualbox.json

@@ -1,41 +0,0 @@
-{
-	"provider"      : "virtualbox",
-	"virtualization": "ide",
-
-	"bootstrapper": {
-		"mount_dir": "/mnt/target",
-		"mirror"      : "http://ftp.fr.debian.org/debian/"
-	},
-	"image": {
-		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {release} {architecture} ({virtualization})"
-	},
-	"system": {
-		"release"     : "wheezy",
-		"architecture": "amd64",
-		"timezone"    : "UTC",
-		"locale"      : "en_US",
-		"charmap"     : "UTF-8"
-	},
-	"volume": {
-		"backing"   : "raw",
-		"filesystem": "ext4",
-		"size"      : 1024,
-		"loopback_dir" : "/tmp"
-	},
-	"plugins": {
-		"user_packages": {
-			"enabled": true,
-			"repo": [ "apache2" ],
-			"local": []
-		},
-		"root_password": {
-			"enabled": true,
-			"password": "test"
-                },
-		"convert_image": {
-			"enabled": true,
-			"format": "vdi"
-		}
-	}
-}

manifests/virtualbox.manifest.json

@@ -0,0 +1,33 @@
+{
+	"provider"      : "virtualbox",
+	"bootstrapper": {
+		"workspace": "/target",
+		"tarball":   true
+	},
+	"image": {
+		"name"       : "debian-{release}-{architecture}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture}"
+	},
+	"system": {
+		"release"     : "wheezy",
+		"architecture": "amd64",
+		"timezone"    : "UTC",
+		"locale"      : "en_US",
+		"charmap"     : "UTF-8"
+	},
+	"volume": {
+		"backing": "vdi",
+		"partitions": {
+			"type": "mbr",
+			"boot": {
+				"size":       32,
+				"filesystem": "ext2"
+			},
+			"root": {
+				"size":       991,
+				"filesystem": "ext4"
+			},
+			"swap": {"size": 128}
+		}
+	}
+}

plugins/admin_user/__init__.py

@@ -2,11 +2,11 @@
 
 def tasks(tasklist, manifest):
 	import tasks
-	tasklist.add(tasks.AddSudoPackage())
-	tasklist.add(tasks.CreateAdminUser())
-	tasklist.add(tasks.PasswordlessSudo())
-	tasklist.add(tasks.AdminUserCredentials())
-	tasklist.add(tasks.DisableRootLogin())
+	tasklist.add(tasks.AddSudoPackage.
+	             tasks.CreateAdminUser,
+	             tasks.PasswordlessSudo,
+	             tasks.AdminUserCredentials,
+	             tasks.DisableRootLogin)
 
 
 def validate_manifest(data, schema_validate):

plugins/backports/__init__.py

@@ -2,8 +2,8 @@
 
 def tasks(tasklist, manifest):
 	import tasks
-	tasklist.add(tasks.AptSourcesBackports())
-	tasklist.add(tasks.AddBackportsPackages())
+	tasklist.add(tasks.AptSourcesBackports,
+	             tasks.AddBackportsPackages)
 
 
 def validate_manifest(data, schema_validate):

plugins/build_metadata/__init__.py

@@ -2,4 +2,4 @@
 
 def tasks(tasklist, manifest):
 	from tasks import WriteMetadata
-	tasklist.add(WriteMetadata())
+	tasklist.add(WriteMetadata

plugins/convert_image/__init__.py

@@ -2,7 +2,7 @@
 
 def tasks(tasklist, manifest):
 	from tasks import ConvertImage
-	tasklist.add(ConvertImage())
+	tasklist.add(ConvertImage)
 
 
 def validate_manifest(data, schema_validate):

plugins/convert_image/tasks.py

@@ -1,10 +1,12 @@
 from base import Task
 from common import phases
+from common.tasks import loopback
 
 
 class ConvertImage(Task):
 	description = 'Converting raw image'
 	phase = phases.image_registration
+	before = [loopback.MoveImage]
 
 	def run(self, info):
 		from common.tools import log_check_call

plugins/opennebula/__init__.py

@@ -2,4 +2,4 @@
 
 def tasks(tasklist, manifest):
 	import tasks
-	tasklist.add(tasks.OpenNebulaContext())
+	tasklist.add(tasks.OpenNebulaContext)

plugins/prebootstrapped/__init__.py

@@ -2,39 +2,43 @@ from tasks import Snapshot
 from tasks import CopyImage
 from tasks import CreateFromSnapshot
 from tasks import CreateFromImage
+from tasks import SetBootMountDir
 from providers.ec2.tasks import ebs
 from common.tasks import loopback
+from common.tasks import volume
 from common.tasks import bootstrap
 from common.tasks import filesystem
-from common.tasks import parted
+from common.tasks import partitioning
 
 
 def tasks(tasklist, manifest):
 	settings = manifest.plugins['prebootstrapped']
+	skip_tasks = [ebs.Create,
+	              loopback.Create,
+
+	              filesystem.Format,
+	              partitioning.PartitionVolume,
+	              filesystem.TuneVolumeFS,
+	              filesystem.AddXFSProgs,
+	              filesystem.CreateBootMountDir,
+	              bootstrap.MakeTarball,
+	              bootstrap.Bootstrap]
 	if manifest.volume['backing'] == 'ebs':
 		if 'snapshot' in settings and settings['snapshot'] is not None:
-			tasklist.replace(ebs.Create, CreateFromSnapshot())
-			tasklist.remove(filesystem.FormatVolume,
-			                filesystem.TuneVolumeFS,
-			                filesystem.AddXFSProgs,
-			                bootstrap.MakeTarball,
-			                bootstrap.Bootstrap)
+			tasklist.add(CreateFromSnapshot)
+			tasklist.remove(*skip_tasks)
+			if 'boot' in manifest.volume['partitions']:
+				tasklist.add(SetBootMountDir)
 		else:
-			tasklist.add(Snapshot())
+			tasklist.add(Snapshot)
 	else:
 		if 'image' in settings and settings['image'] is not None:
-			tasklist.add(CreateFromImage())
-			tasklist.remove(loopback.Create,
-			                loopback.CreateQemuImg,
-			                parted.PartitionVolume,
-			                parted.FormatPartitions,
-			                filesystem.FormatVolume,
-			                filesystem.TuneVolumeFS,
-			                filesystem.AddXFSProgs,
-			                bootstrap.MakeTarball,
-			                bootstrap.Bootstrap)
+			tasklist.add(CreateFromImage)
+			tasklist.remove(*skip_tasks)
+			if 'boot' in manifest.volume['partitions']:
+				tasklist.add(SetBootMountDir)
 		else:
-			tasklist.add(CopyImage())
+			tasklist.add(CopyImage)
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -42,12 +46,12 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 
 	def counter_task(task, counter):
 		if task in completed and counter not in completed:
-			tasklist.add(counter())
+			tasklist.add(counter)
 
 	if manifest.volume['backing'] == 'ebs':
-		counter_task(CreateFromSnapshot, ebs.Delete)
+		counter_task(CreateFromSnapshot, volume.Delete)
 	else:
-		counter_task(CreateFromImage, loopback.Delete)
+		counter_task(CreateFromImage, volume.Delete)
 
 
 def validate_manifest(data, schema_validate):

plugins/prebootstrapped/tasks.py

@@ -1,64 +1,96 @@
 from base import Task
 from common import phases
-from providers.ec2.tasks import ebs
-from common.tasks import loopback
+from common.tasks import volume
 from common.tasks import bootstrap
+from common.tasks import filesystem
+from common.fs import remount
+from shutil import copyfile
+import os.path
 import time
 import logging
 log = logging.getLogger(__name__)
 
 
-class Snapshot(ebs.Snapshot):
+class Snapshot(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
 	phase = phases.os_installation
-	after = [bootstrap.Bootstrap]
+	after = [bootstrap.Bootstrap, filesystem.MountSpecials]
 
 	def run(self, info):
-		super(Snapshot, self).run(info)
-		msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id)
+		def mk_snapshot():
+			return info.volume.snapshot()
+		snapshot = remount(info.volume, mk_snapshot)
+		msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=snapshot.id)
 		log.info(msg)
 
 
 class CreateFromSnapshot(Task):
 	description = 'Creating EBS volume from a snapshot'
 	phase = phases.volume_creation
-	before = [ebs.Attach]
+	before = [volume.Attach]
 
 	def run(self, info):
 		volume_size = int(info.manifest.volume['size'] / 1024)
 		snapshot = info.manifest.plugins['prebootstrapped']['snapshot']
-		info.volume = info.connection.create_volume(volume_size,
+		info.volume.volume = info.connection.create_volume(volume_size,
 		                                                   info.host['availabilityZone'],
 		                                                   snapshot=snapshot)
 		while info.volume.volume_state() != 'available':
 			time.sleep(5)
 			info.volume.update()
 
+		set_fs_states(info.volume)
+
 
 class CopyImage(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
 	phase = phases.os_installation
-	after = [bootstrap.Bootstrap]
+	after = [bootstrap.Bootstrap, filesystem.MountSpecials]
 
 	def run(self, info):
-		import os.path
-		from shutil import copyfile
-		loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id)
-		image_copy_path = os.path.join('/tmp', loopback_backup_name)
-		copyfile(info.loopback_file, image_copy_path)
-		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path)
+		loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension)
+		destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name)
+
+		def mk_snapshot():
+			copyfile(info.volume.image_path, destination)
+		remount(info.volume, mk_snapshot)
+		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination)
 		log.info(msg)
 
 
 class CreateFromImage(Task):
 	description = 'Creating loopback image from a copy'
 	phase = phases.volume_creation
-	before = [loopback.Attach]
+	before = [volume.Attach]
 
 	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
-		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension))
 		loopback_backup_path = info.manifest.plugins['prebootstrapped']['image']
-		from shutil import copyfile
-		copyfile(loopback_backup_path, info.loopback_file)
+		copyfile(loopback_backup_path, info.volume.image_path)
+
+		set_fs_states(info.volume)
+
+
+class SetBootMountDir(Task):
+	description = 'Setting mountpoint for the boot partition'
+	phase = phases.volume_mounting
+	after = [filesystem.MountRoot]
+	before = [filesystem.MountBoot]
+
+	def run(self, info):
+		info.boot_dir = os.path.join(info.root, 'boot')
+
+
+def set_fs_states(volume):
+		volume.fsm.current = 'detached'
+
+		p_map = volume.partition_map
+		partitions_state = 'attached'
+		from base.fs.partitionmaps.none import NoPartitions
+		if isinstance(p_map, NoPartitions):
+			partitions_state = 'formatted'
+		else:
+			p_map.fsm.current = 'unmapped'
+			partitions_state = 'unmapped_fmt'
+		for partition in p_map.partitions:
+			partition.fsm.current = partitions_state

plugins/root_password/__init__.py

@@ -3,7 +3,8 @@
 def tasks(tasklist, manifest):
 	from common.tasks.security import DisableSSHPasswordAuthentication
 	from tasks import SetRootPassword
-	tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword())
+	tasklist.remove(DisableSSHPasswordAuthentication)
+	tasklist.add(SetRootPassword)
 
 
 def validate_manifest(data, schema_validate):

plugins/unattended_upgrades/__init__.py

@@ -2,8 +2,8 @@
 
 def tasks(tasklist, manifest):
 	import tasks
-	tasklist.add(tasks.AddUnattendedUpgradesPackage())
-	tasklist.add(tasks.EnablePeriodicUpgrades())
+	tasklist.add(tasks.AddUnattendedUpgradesPackage,
+	             tasks.EnablePeriodicUpgrades)
 
 
 def validate_manifest(data, schema_validate):

plugins/user_packages/__init__.py

@@ -2,5 +2,5 @@
 
 def tasks(tasklist, manifest):
 	from user_packages import AddUserPackages, AddLocalUserPackages
-	tasklist.add(AddUserPackages())
-	tasklist.add(AddLocalUserPackages())
+	tasklist.add(AddUserPackages,
+	             AddLocalUserPackages)

providers/ec2/__init__.py

@@ -1,24 +1,25 @@
 from manifest import Manifest
 import logging
 from tasks import packages
-from common.tasks import packages as common_packages
 from tasks import connection
 from tasks import host
-from common.tasks import host as common_host
 from tasks import ami
+from common.tasks import volume as volume_tasks
 from tasks import ebs
+from common.tasks import partitioning
 from common.tasks import loopback
-from common.tasks import filesystem
+from common.tasks import filesystem as common_filesystem
+from tasks import filesystem
 from common.tasks import bootstrap
-from common.tasks import locale
-from common.tasks import apt
 from tasks import boot
 from common.tasks import boot as common_boot
 from common.tasks import security
-from common.tasks import network
+from tasks import network
+from common.tasks import network as common_network
 from tasks import initd
 from common.tasks import initd as common_initd
 from common.tasks import cleanup
+from common.tasks import workspace
 
 
 def initialize():
@@ -27,74 +28,68 @@ def initialize():
 
 
 def tasks(tasklist, manifest):
-	tasklist.add(packages.HostPackages(),
-	             common_packages.HostPackages(),
-	             packages.ImagePackages(),
-	             common_packages.ImagePackages(),
-	             common_host.CheckPackages(),
-	             connection.GetCredentials(),
-	             host.GetInfo(),
-	             ami.AMIName(),
-	             connection.Connect(),
+	from common.task_sets import base_set
+	from common.task_sets import mounting_set
+	from common.task_sets import apt_set
+	from common.task_sets import locale_set
+	from common.task_sets import ssh_set
+	tasklist.add(*base_set)
+	tasklist.add(*mounting_set)
+	tasklist.add(*apt_set)
+	tasklist.add(*locale_set)
+	tasklist.add(*ssh_set)
 
-	             filesystem.FormatVolume(),
-	             filesystem.CreateMountDir(),
-	             filesystem.MountVolume(),
+	if manifest.volume['partitions']['type'] != 'none':
+		from common.task_sets import partitioning_set
+		tasklist.add(*partitioning_set)
 
-	             bootstrap.Bootstrap(),
-	             filesystem.MountSpecials(),
-	             locale.GenerateLocale(),
-	             locale.SetTimezone(),
-	             apt.DisableDaemonAutostart(),
-	             apt.AptSources(),
-	             apt.AptUpgrade(),
-	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
-	             common_boot.BlackListModules(),
-	             common_boot.DisableGetTTYs(),
-	             security.EnableShadowConfig(),
-	             security.DisableSSHPasswordAuthentication(),
-	             security.DisableSSHDNSLookup(),
-	             network.RemoveDNSInfo(),
-	             network.ConfigureNetworkIF(),
-	             network.ConfigureDHCP(),
-	             common_initd.ResolveInitScripts(),
-	             initd.AddEC2InitScripts(),
-	             common_initd.InstallInitScripts(),
-	             cleanup.ClearMOTD(),
-	             cleanup.ShredHostkeys(),
-	             cleanup.CleanTMP(),
-	             apt.PurgeUnusedPackages(),
-	             apt.AptClean(),
-	             apt.EnableDaemonAutostart(),
-	             filesystem.UnmountSpecials(),
+	tasklist.add(packages.HostPackages,
+	             packages.ImagePackages,
+	             connection.GetCredentials,
+	             host.GetInfo,
+	             ami.AMIName,
+	             connection.Connect,
 
-	             filesystem.UnmountVolume(),
-	             filesystem.DeleteMountDir(),
-	             ami.RegisterAMI())
+	             boot.ConfigureGrub,
+	             common_boot.BlackListModules,
+	             common_boot.DisableGetTTYs,
+	             security.EnableShadowConfig,
+	             common_network.RemoveDNSInfo,
+	             common_network.ConfigureNetworkIF,
+	             network.EnableDHCPCDDNS,
+	             common_initd.ResolveInitScripts,
+	             initd.AddEC2InitScripts,
+	             common_initd.InstallInitScripts,
+	             initd.AdjustExpandVolumeScript,
+	             cleanup.ClearMOTD,
+	             cleanup.CleanTMP,
 
-	if manifest.bootstrapper['tarball']:
-		tasklist.add(bootstrap.MakeTarball())
+	             ami.RegisterAMI)
 
-	backing_specific_tasks = {'ebs': [ebs.Create(),
-	                                  ebs.Attach(),
-	                                  ebs.Detach(),
-	                                  ebs.Snapshot(),
-	                                  ebs.Delete()],
-	                          's3': [loopback.Create(),
-	                                 loopback.Attach(),
-	                                 loopback.Detach(),
-	                                 ami.BundleImage(),
-	                                 ami.UploadImage(),
-	                                 loopback.Delete(),
-	                                 ami.RemoveBundle()]}
+	backing_specific_tasks = {'ebs': [ebs.Create,
+	                                  ebs.Attach,
+	                                  common_filesystem.FStab,
+	                                  ebs.Snapshot],
+	                          's3': [loopback.Create,
+	                                 volume_tasks.Attach,
+	                                 filesystem.S3FStab,
+	                                 ami.BundleImage,
+	                                 ami.UploadImage,
+	                                 ami.RemoveBundle]}
 	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
+	tasklist.add(common_filesystem.Format,
+	             volume_tasks.Detach,
+	             volume_tasks.Delete)
 
-	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
-	                             'ext2': [filesystem.TuneVolumeFS()],
-	                             'ext3': [filesystem.TuneVolumeFS()],
-	                             'ext4': [filesystem.TuneVolumeFS()]}
-	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
+	if manifest.bootstrapper.get('tarball', False):
+		tasklist.add(bootstrap.MakeTarball)
+
+	from common.task_sets import get_fs_specific_set
+	tasklist.add(*get_fs_specific_set(manifest.volume['partitions']))
+
+	if 'boot' in manifest.volume['partitions']:
+		from common.task_sets import boot_partition_set
+		tasklist.add(*boot_partition_set)
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -102,14 +97,20 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 
 	def counter_task(task, counter):
 		if task in completed and counter not in completed:
-			tasklist.add(counter())
+			tasklist.add(counter)
 
-	if manifest.volume['backing'].lower() == 'ebs':
-		counter_task(ebs.Create, ebs.Delete)
-		counter_task(ebs.Attach, ebs.Detach)
-	if manifest.volume['backing'].lower() == 's3':
-		counter_task(loopback.Create, loopback.Delete)
-		counter_task(loopback.Attach, loopback.Detach)
-	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
-	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
-	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
+	counter_task(ebs.Create, volume_tasks.Delete)
+	counter_task(ebs.Attach, volume_tasks.Detach)
+
+	counter_task(loopback.Create, volume_tasks.Delete)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)
+
+	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
+	counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir)
+	counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials)
+
+	counter_task(common_filesystem.MountRoot, common_filesystem.UnmountRoot)
+	counter_task(common_filesystem.MountBoot, common_filesystem.UnmountBoot)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)
+	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)
+	counter_task(ami.BundleImage, ami.RemoveBundle)

providers/ec2/assets/grub.d/40_custom

@@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib
 export TEXTDOMAIN=grub
 export TEXTDOMAINDIR=${prefix}/share/locale
 
-GRUB_DEVICE=/dev/xvda1
+GRUB_DEVICE=/dev/xvda
 
 
 cat << EOF

providers/ec2/ebsvolume.py

@@ -0,0 +1,57 @@
+from base.fs.volume import Volume
+from base.fs.exceptions import VolumeError
+import time
+
+
+class EBSVolume(Volume):
+
+	def create(self, conn, zone):
+		self.fsm.create(connection=conn, zone=zone)
+
+	def _before_create(self, e):
+		conn = e.connection
+		zone = e.zone
+		import math
+		size = int(math.ceil(self.partition_map.get_total_size() / 1024))
+		self.volume = conn.create_volume(size, zone)
+		while self.volume.volume_state() != 'available':
+			time.sleep(5)
+			self.volume.update()
+
+	def attach(self, instance_id):
+		self.fsm.attach(instance_id=instance_id)
+
+	def _before_attach(self, e):
+		instance_id = e.instance_id
+		import os.path
+		import string
+		for letter in string.ascii_lowercase[5:]:
+			dev_path = os.path.join('/dev', 'xvd' + letter)
+			if not os.path.exists(dev_path):
+				self.device_path = dev_path
+				self.ec2_device_path = os.path.join('/dev', 'sd' + letter)
+				break
+
+		if self.device_path is None:
+			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
+
+		self.volume.attach(instance_id, self.ec2_device_path)
+		while self.volume.attachment_state() != 'attached':
+			time.sleep(2)
+			self.volume.update()
+
+	def _before_detach(self, e):
+		self.volume.detach()
+		while self.volume.attachment_state() is not None:
+			time.sleep(2)
+			self.volume.update()
+
+	def _before_delete(self, e):
+		self.volume.delete()
+
+	def snapshot(self):
+		snapshot = self.volume.create_snapshot()
+		while snapshot.status != 'completed':
+			time.sleep(2)
+			snapshot.update()
+		return snapshot

providers/ec2/manifest-schema-s3.json

@@ -25,6 +25,17 @@
 				}
 			},
 			"required": ["bucket"]
+		},
+		"volume": {
+			"type": "object",
+			"properties": {
+				"partitions": {
+					"type": "object",
+					"properties": {
+						"type": { "enum": ["none"] }
+					}
+				}
+			}
 		}
 	},
 	"required": ["image"]

providers/ec2/manifest-schema.json

@@ -3,6 +3,17 @@
 	"title": "EC2 manifest",
 	"type": "object",
 	"properties": {
+		"image": {
+			"type": "object",
+			"properties": {
+				"name": {
+					"type": "string"
+				},
+				"description": {
+					"type": "string"
+				}
+			}
+		},
 		"credentials": {
 			"type": "object",
 			"properties": {
@@ -17,17 +28,16 @@
 		"volume": {
 			"type": "object",
 			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["ebs", "s3"]
-				},
-				"filesystem": {
-					"type": "string",
-					"enum": ["ext2", "ext3", "ext4", "xfs"]
+				"backing": { "enum": ["ebs", "s3"] },
+				"partitions": {
+					"type": "object",
+					"properties": {
+						"type": { "enum": ["none", "mbr"] }
+					}
 				}
 			},
-			"required": ["backing", "filesystem"]
+			"required": ["backing"]
 		}
 	},
-	"required": ["volume"]
+	"required": ["image"]
 }

providers/ec2/manifest.py

@@ -9,8 +9,10 @@ class Manifest(base.Manifest):
 		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
 		self.schema_validate(data, schema_path)
 		if data['volume']['backing'] == 'ebs':
-			if data['volume']['size'] % 1024 != 0:
-				msg = 'The volume size must be a multiple of 1024 when using EBS backing'
+			volume_size = self._calculate_volume_size(data['volume']['partitions'])
+			if volume_size % 1024 != 0:
+				msg = ('The volume size must be a multiple of 1024 when using EBS backing '
+				       '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)')
 				raise ManifestError(msg, self)
 		else:
 			schema_path = path.join(path.dirname(__file__), 'manifest-schema-s3.json')
@@ -21,9 +23,15 @@ class Manifest(base.Manifest):
 		self.credentials    = data['credentials']
 		self.virtualization = data['virtualization']
 		self.image          = data['image']
-		if data['volume']['backing'] == 'ebs':
-			self.ebs_volume_size = data['volume']['size'] / 1024
-		if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3':
-			self.volume['loopback_dir'] = '/tmp'
-		if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3':
-			self.image['bundle_dir'] = '/tmp'
+
+	def _calculate_volume_size(self, partitions):
+		if partitions['type'] == 'mbr':
+			size = 1
+		else:
+			size = 0
+		if 'boot' in partitions:
+			size += partitions['boot']['size']
+		size += partitions['root']['size']
+		if 'swap' in partitions:
+			size += partitions['swap']['size']
+		return size

providers/ec2/tasks/ami.py

@@ -3,6 +3,7 @@ from common import phases
 from common.exceptions import TaskError
 from common.tools import log_check_call
 from ebs import Snapshot
+from common.tasks import workspace
 from connection import Connect
 import os.path
 
@@ -45,9 +46,9 @@ class BundleImage(Task):
 
 	def run(self, info):
 		bundle_name = 'bundle-{id:x}'.format(id=info.run_id)
-		info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name)
+		info.bundle_path = os.path.join(info.workspace, bundle_name)
 		log_check_call(['/usr/bin/euca-bundle-image',
-		                '--image', info.loopback_file,
+		                '--image', info.volume.image_path,
 		                '--user', info.credentials['user-id'],
 		                '--privatekey', info.credentials['private-key'],
 		                '--cert', info.credentials['certificate'],
@@ -80,6 +81,7 @@ class UploadImage(Task):
 class RemoveBundle(Task):
 	description = 'Removing the bundle files'
 	phase = phases.cleaning
+	before = [workspace.DeleteWorkspace]
 
 	def run(self, info):
 		from shutil import rmtree
@@ -92,46 +94,109 @@ class RegisterAMI(Task):
 	phase = phases.image_registration
 	after = [Snapshot, UploadImage]
 
-	kernel_mapping = {'us-east-1':      {'amd64': 'aki-88aa75e1',
-	                                     'i386':  'aki-b6aa75df'},
-	                  'us-west-1':      {'amd64': 'aki-f77e26b2',
-	                                     'i386':  'aki-f57e26b0'},
-	                  'us-west-2':      {'amd64': 'aki-fc37bacc',
-	                                     'i386':  'aki-fa37baca'},
-	                  'eu-west-1':      {'amd64': 'aki-71665e05',
-	                                     'i386':  'aki-75665e01'},
-	                  'ap-southeast-1': {'amd64': 'aki-fe1354ac',
-	                                     'i386':  'aki-f81354aa'},
-	                  'ap-southeast-2': {'amd64': 'aki-31990e0b',
-	                                     'i386':  'aki-33990e09'},
-	                  'ap-northeast-1': {'amd64': 'aki-44992845',
-	                                     'i386':  'aki-42992843'},
-	                  'sa-east-1':      {'amd64': 'aki-c48f51d9',
-	                                     'i386':  'aki-ca8f51d7'},
-	                  'us-gov-west-1':  {'amd64': 'aki-79a4c05a',
-	                                     'i386':  'aki-7ba4c058'}}
+	# Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs
+	kernel_mapping = {'ap-northeast-1': {  # Asia Pacific (Tokyo) Region
+	                  'hd0':  {'i386':  'aki-136bf512',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-176bf516'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-196bf518',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-1f6bf51e'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'ap-southeast-1': {  # Asia Pacific (Singapore) Region
+	                  'hd0':  {'i386':  'aki-ae3973fc',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-503e7402'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-563e7404',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-5e3e740c'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'ap-southeast-2': {  # Asia Pacific (Sydney) Region
+	                  'hd0':  {'i386':  'aki-cd62fff7',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-c362fff9'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-c162fffb',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-3b1d8001'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'eu-west-1': {  # EU (Ireland) Region
+	                  'hd0':  {'i386':  'aki-68a3451f',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-52a34525'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-5ea34529',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-58a3452f'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'sa-east-1': {  # South America (Sao Paulo) Region
+	                  'hd0':  {'i386':  'aki-5b53f446',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-5553f448'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-5753f44a',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-5153f44c'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-east-1': {  # US East (Northern Virginia) Region
+	                  'hd0':  {'i386':  'aki-8f9dcae6',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-919dcaf8'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-659ccb0c',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-499ccb20'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-gov-west-1': {  # AWS GovCloud (US)
+	                  'hd0':  {'i386':  'aki-1fe98d3c',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-1de98d3e'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-63e98d40',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-61e98d42'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-west-1': {  # US West (Northern California) Region
+	                  'hd0':  {'i386':  'aki-8e0531cb',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-880531cd'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-960531d3',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-920531d7'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  },
+	                  'us-west-2': {  # US West (Oregon) Region
+	                  'hd0':  {'i386':  'aki-f08f11c0',   # pv-grub-hd0_1.04-i386.gz
+	                           'amd64': 'aki-fc8f11cc'},  # pv-grub-hd0_1.04-x86_64.gz
+	                  'hd00': {'i386':  'aki-e28f11d2',   # pv-grub-hd00_1.04-i386.gz
+	                           'amd64': 'aki-e68f11d6'}   # pv-grub-hd00_1.04-x86_64.gz
+	                  }}
 
 	def run(self, info):
-		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
-		kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
-
 		if info.manifest.volume['backing'] == 'ebs':
+			self.run_ebs(info)
+		if info.manifest.volume['backing'] == 's3':
+			self.run_s3(info)
+
+	def run_ebs(self, info):
+		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
+
+		from base.fs.partitionmaps.none import NoPartitions
+		if isinstance(info.volume.partition_map, NoPartitions):
+			grub_boot_device = 'hd0'
+			root_device_name = '/dev/sda'
+		else:
+			grub_boot_device = 'hd00'
+			root_idx = info.volume.partition_map.root.get_index()
+			root_device_name = '/dev/sda{idx}'.format(idx=root_idx)
+
+		kernel_id = (self.kernel_mapping
+		             .get(info.host['region'])
+		             .get(grub_boot_device)
+		             .get(info.manifest.system['architecture']))
+
 		from boto.ec2.blockdevicemapping import BlockDeviceType
 		from boto.ec2.blockdevicemapping import BlockDeviceMapping
 		block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
-			                               size=info.manifest.ebs_volume_size)
+		                               size=info.volume.partition_map.get_total_size()/1024)
 		block_device_map = BlockDeviceMapping()
-			block_device_map['/dev/sda1'] = block_device
+		block_device_map['/dev/sda'] = block_device
 
 		info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
 		                                            architecture=arch, kernel_id=kernel_id,
-			                                            root_device_name='/dev/sda1',
+		                                            root_device_name=root_device_name,
 		                                            block_device_map=block_device_map)
-		if info.manifest.volume['backing'] == 's3':
-			image_location = ('{bucket}/{ami_name}.manifest.xml'
+
+	def run_s3(self, info):
+		arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
+
+		kernel_id = (self.kernel_mapping
+		             .get(info.host['region'])
+		             .get('hd0')
+		             .get(info.manifest.system['architecture']))
+
+		image_manifest = ('{bucket}/{ami_name}.manifest.xml'
 		                  .format(bucket=info.manifest.image['bucket'],
 		                          ami_name=info.ami_name))
 		info.image = info.connection.register_image(description=info.ami_description,
 		                                            architecture=arch, kernel_id=kernel_id,
-			                                            root_device_name='/dev/sda1',
-			                                            image_location=image_location)
+		                                            root_device_name='dev/sda1',
+		                                            image_location=image_manifest)

providers/ec2/tasks/boot.py

@@ -24,6 +24,19 @@ class ConfigureGrub(Task):
 		copy(script_src, script_dst)
 		os.chmod(script_dst, rwxr_xr_x)
 
+		from base.fs.partitionmaps.none import NoPartitions
+		if not isinstance(info.volume.partition_map, NoPartitions):
+			from common.tools import sed_i
+			root_idx = info.volume.partition_map.root.get_index()
+			grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx)
+			sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device)
+			grub_root = '\troot (hd0,{idx})'.format(idx=root_idx-1)
+			sed_i(script_dst, '^\troot \(hd0\)$', grub_root)
+
+		if info.manifest.volume['backing'] == 's3':
+			from common.tools import sed_i
+			sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1')
+
 		from common.tools import sed_i
 		grub_def = os.path.join(info.root, 'etc/default/grub')
 		sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n'

providers/ec2/tasks/connection.py

@@ -20,9 +20,12 @@ class GetCredentials(Task):
 			for key in keys:
 				creds[key] = manifest.credentials[key]
 			return creds
-		if all(getenv(key) is not None for key in keys):
+
+		def env_key(key):
+			return ('aws-'+key).upper().replace('-', '_')
+		if all(getenv(env_key(key)) is not None for key in keys):
 			for key in keys:
-				creds[key] = getenv(key)
+				creds[key] = getenv(env_key(key))
 			return creds
 		raise RuntimeError(('No ec2 credentials found, they must all be specified '
 		                    'exclusively via environment variables or through the manifest.'))

providers/ec2/tasks/ebs.py

@@ -1,60 +1,22 @@
 from base import Task
 from common import phases
-from common.exceptions import TaskError
-from common.tasks.filesystem import UnmountVolume
-import time
 
 
 class Create(Task):
-	description = 'Creating an EBS volume for bootstrapping'
+	description = 'Creating the EBS volume'
 	phase = phases.volume_creation
 
 	def run(self, info):
-		info.volume = info.connection.create_volume(info.manifest.ebs_volume_size,
-		                                            info.host['availabilityZone'])
-		while info.volume.volume_state() != 'available':
-			time.sleep(5)
-			info.volume.update()
+		info.volume.create(info.connection, info.host['availabilityZone'])
 
 
 class Attach(Task):
-	description = 'Attaching the EBS volume'
+	description = 'Attaching the volume'
 	phase = phases.volume_creation
 	after = [Create]
 
 	def run(self, info):
-		def char_range(c1, c2):
-			"""Generates the characters from `c1` to `c2`, inclusive."""
-			for c in xrange(ord(c1), ord(c2) + 1):
-				yield chr(c)
-
-		import os.path
-		info.bootstrap_device = {}
-		for letter in char_range('f', 'z'):
-			dev_path = os.path.join('/dev', 'xvd' + letter)
-			if not os.path.exists(dev_path):
-				info.bootstrap_device['path'] = dev_path
-				info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter)
-				break
-		if 'path' not in info.bootstrap_device:
-			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
-
-		info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path'])
-		while info.volume.attachment_state() != 'attached':
-			time.sleep(2)
-			info.volume.update()
-
-
-class Detach(Task):
-	description = 'Detaching the EBS volume'
-	phase = phases.volume_unmounting
-	after = [UnmountVolume]
-
-	def run(self, info):
-		info.volume.detach()
-		while info.volume.attachment_state() is not None:
-			time.sleep(2)
-			info.volume.update()
+		info.volume.attach(info.host['instanceId'])
 
 
 class Snapshot(Task):
@@ -62,20 +24,4 @@ class Snapshot(Task):
 	phase = phases.image_registration
 
 	def run(self, info):
-		info.snapshot = info.volume.create_snapshot()
-		while info.snapshot.status != 'completed':
-			time.sleep(2)
-			info.snapshot.update()
-
-
-class Delete(Task):
-	description = 'Deleting the EBS volume'
-	phase = phases.cleaning
-
-	def run(self, info):
-		info.volume.delete()
-		del info.volume
-
-
-class VolumeError(TaskError):
-	pass
+		info.snapshot = info.volume.snapshot()

providers/ec2/tasks/filesystem.py

@@ -0,0 +1,27 @@
+from base import Task
+from common import phases
+
+
+class S3FStab(Task):
+	description = 'Adding the S3 root partition to the fstab'
+	phase = phases.system_modification
+
+	def run(self, info):
+		import os.path
+		root = info.volume.partition_map.root
+
+		fstab_lines = []
+		mount_opts = ['defaults']
+		fstab_lines.append('{device_path}{idx} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}'
+		                   .format(device_path='/dev/xvda',
+		                   	       idx=1,
+		                           mountpoint='/',
+		                           filesystem=root.filesystem,
+		                           mount_opts=','.join(mount_opts),
+		                           dump='1',
+		                           pass_num='1'))
+
+		fstab_path = os.path.join(info.root, 'etc/fstab')
+		with open(fstab_path, 'w') as fstab:
+			fstab.write('\n'.join(fstab_lines))
+			fstab.write('\n')

providers/ec2/tasks/initd.py

@@ -1,5 +1,6 @@
 from base import Task
 from common import phases
+from common.exceptions import TaskError
 from common.tasks import initd
 import os.path
 
@@ -17,3 +18,22 @@ class AddEC2InitScripts(Task):
 		init_scripts_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d'))
 		for name, path in init_scripts.iteritems():
 			info.initd['install'][name] = os.path.join(init_scripts_dir, path)
+
+
+class AdjustExpandVolumeScript(Task):
+	description = 'Adjusting the expand-volume script'
+	phase = phases.system_modification
+	after = [initd.InstallInitScripts]
+
+	def run(self, info):
+		if 'expand-volume' not in info.initd['install']:
+			raise TaskError('The expand-volume script was not installed')
+
+		from base.fs.partitionmaps.none import NoPartitions
+		if not isinstance(info.volume.partition_map, NoPartitions):
+			import os.path
+			from common.tools import sed_i
+			script = os.path.join(info.root, 'etc/init.d.expand-volume')
+			root_idx = info.volume.partition_map.root.get_index()
+			device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx)
+			sed_i(script, '^device_path="/dev/xvda$', device_path)

providers/ec2/tasks/network.py

@@ -0,0 +1,15 @@
+from base import Task
+from common import phases
+import os.path
+
+
+class EnableDHCPCDDNS(Task):
+	description = 'Configuring the DHCP client to set the nameservers'
+	phase = phases.system_modification
+
+	def run(self, info):
+		# The dhcp client that ships with debian sets the DNS servers per default.
+		# For dhcpcd we need to configure it to do that.
+		from common.tools import sed_i
+		dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
+		sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')

providers/ec2/tasks/packages.py

@@ -11,8 +11,6 @@ class HostPackages(Task):
 	after = [packages.HostPackages]
 
 	def run(self, info):
-		if info.manifest.volume['filesystem'] == 'xfs':
-			info.host_packages.add('xfsprogs')
 		if info.manifest.volume['backing'] == 's3':
 			info.host_packages.add('euca2ools')
 
@@ -25,6 +23,7 @@ class ImagePackages(Task):
 	def run(self, info):
 		manifest = info.manifest
 		include, exclude = info.img_packages
+		include.add('openssh-server')
 		include.add('file')  # Needed for the init scripts
 		include.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
 		if manifest.virtualization == 'pvm':

providers/kvm/README.md

@@ -1,9 +0,0 @@
-this provider creates images for KVM.
-
-It is possible to add opennebula plugin for OpenNebula images.
-
-Disk virtuzalition is specified by the virtualization field in the manifest.
-Allowed values are:
-
-* ide: basic disk emulation on /dev/sda1
-* virtio: enhanced performance on /dev/vda1. It adds virtio modules in the kernel and needs configuration update in VM template: <target dev='vda' bus='virtio'/>

providers/kvm/__init__.py

@@ -1,92 +0,0 @@
-from manifest import Manifest
-from tasks import packages
-from common.tasks import packages as common_packages
-from common.tasks import host
-from common.tasks import loopback
-from common.tasks import parted
-from common.tasks import filesystem
-from common.tasks import bootstrap
-from common.tasks import locale
-from common.tasks import apt
-from tasks import boot
-from common.tasks import boot as common_boot
-from common.tasks import security
-from common.tasks import network
-from common.tasks import initd
-from common.tasks import cleanup
-from common.tasks import loopback
-
-
-def initialize():
-	pass
-
-
-def tasks(tasklist, manifest):
-	tasklist.add(packages.HostPackages(),
-	             common_packages.HostPackages(),
-	             packages.ImagePackages(),
-	             common_packages.ImagePackages(),
-	             host.CheckPackages(),
-
-	             loopback.CreateQemuImg(),
-	             loopback.Attach(),
-	             parted.PartitionVolume(),
-	             parted.MapPartitions(),
-	             parted.FormatPartitions(),
-	             filesystem.CreateMountDir(),
-	             filesystem.MountVolume(),
-
-	             bootstrap.Bootstrap(),
-	             filesystem.MountSpecials(),
-	             locale.GenerateLocale(),
-	             locale.SetTimezone(),
-	             apt.DisableDaemonAutostart(),
-	             apt.AptSources(),
-	             apt.AptUpgrade(),
-	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
-	             common_boot.BlackListModules(),
-	             common_boot.DisableGetTTYs(),
-	             security.EnableShadowConfig(),
-	             security.DisableSSHPasswordAuthentication(),
-	             security.DisableSSHDNSLookup(),
-	             network.RemoveDNSInfo(),
-	             network.ConfigureNetworkIF(),
-	             network.ConfigureDHCP(),
-	             initd.ResolveInitScripts(),
-	             initd.InstallInitScripts(),
-	             cleanup.ClearMOTD(),
-	             cleanup.ShredHostkeys(),
-	             cleanup.CleanTMP(),
-	             apt.PurgeUnusedPackages(),
-	             apt.AptClean(),
-	             apt.EnableDaemonAutostart(),
-	             filesystem.UnmountSpecials(),
-
-	             filesystem.UnmountVolume(),
-	             parted.UnmapPartitions(),
-	             loopback.Detach(),
-	             filesystem.DeleteMountDir())
-
-	if manifest.bootstrapper['tarball']:
-		tasklist.add(bootstrap.MakeTarball())
-
-	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
-	                             'ext2': [filesystem.TuneVolumeFS()],
-	                             'ext3': [filesystem.TuneVolumeFS()],
-	                             'ext4': [filesystem.TuneVolumeFS()]}
-	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
-
-
-def rollback_tasks(tasklist, tasks_completed, manifest):
-	completed = [type(task) for task in tasks_completed]
-
-	def counter_task(task, counter):
-		if task in completed and counter not in completed:
-			tasklist.add(counter())
-
-	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
-	counter_task(parted.MapPartitions, parted.UnmapPartitions)
-	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
-	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
-	counter_task(loopback.Attach, loopback.Detach)

providers/kvm/assets/grub.d/00_header

@@ -1,4 +0,0 @@
-#! /bin/sh
-set -e
-
-# nothing to do, skip grub mkconfig for this

providers/kvm/assets/grub.d/10_linux

@@ -1,93 +0,0 @@
-#!/bin/sh
-
-# This file generates the old menu.lst configuration with grub2
-# It was copied from tomheadys github repo:
-# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
-
-prefix=/usr
-exec_prefix=${prefix}
-bindir=${exec_prefix}/bin
-libdir=${exec_prefix}/lib
-. ${libdir}/grub/grub-mkconfig_lib
-
-export TEXTDOMAIN=grub
-export TEXTDOMAINDIR=${prefix}/share/locale
-
-GRUB_DEVICE=/dev/sda1
-
-
-cat << EOF
-set default=${GRUB_DEFAULT}
-set timeout=${GRUB_TIMEOUT}
-insmod part_msdos
-insmod ext2
-insmod gettext
-set menu_color_normal=cyan/blue
-set menu_color_highlight=white/blue
-set root='(hd0,msdos1)'
-EOF
-
-if ${GRUB_HIDDEN_TIMEOUT:-false}; then
-	printf "hiddenmenu\n"
-fi
-
-linux_entry ()
-{
-	os="$1"
-	version="$2"
-	args="$4"
-
-	title="$(gettext_quoted "%s, with Linux %s")"
-
-	cat << EOF
-menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os {
-        insmod part_msdos
-        insmod ext2
-	set timeout=${GRUB_TIMEOUT}
-	set root='(hd0,msdos1)'
-        echo 'Loading Linux ${version}'
-	linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
-        echo 'Loading initial ramdisk ...'
-	initrd ${rel_dirname}/${initrd}
-}
-EOF
-}
-
-list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
-		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
-		done`
-prepare_boot_cache=
-
-while [ "x$list" != "x" ] ; do
-	linux=`version_find_latest $list`
-	basename=`basename $linux`
-	dirname=`dirname $linux`
-	rel_dirname=`make_system_path_relative_to_its_root $dirname`
-	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
-	alt_version=`echo $version | sed -e "s,\.old$,,g"`
-	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
-
-	initrd=
-	for i in "initrd.img-${version}" "initrd-${version}.img" \
-		"initrd-${version}" "initramfs-${version}.img" \
-		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
-		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
-	if test -e "${dirname}/${i}" ; then
-		initrd="$i"
-		break
-	fi
-	done
-
-	initramfs=
-	for i in "config-${version}" "config-${alt_version}"; do
-	if test -e "${dirname}/${i}" ; then
-		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut   -f2 -d= | tr -d \"`
-		break
-	fi
-	done
-
-	linux_entry "${OS}" "${version}" \
-		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
-
-	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
-done

providers/kvm/manifest-schema.json

@@ -1,22 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Kvm manifest",
-	"type": "object",
-	"properties": {
-		"volume": {
-			"type": "object",
-			"properties": {
-				"backing": {
-					"type": "string",
-					"enum": ["raw", "qcow2"]
-				},
-				"filesystem": {
-					"type": "string",
-					"enum": ["ext2", "ext3", "ext4", "xfs"]
-				}
-			},
-			"required": ["backing", "filesystem"]
-		}
-	},
-	"required": ["volume"]
-}

providers/kvm/manifest.py

@@ -1,16 +0,0 @@
-import base
-
-
-class Manifest(base.Manifest):
-	def validate(self, data):
-		super(Manifest, self).validate(data)
-		from os import path
-		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
-		self.schema_validate(data, schema_path)
-
-	def parse(self, data):
-		super(Manifest, self).parse(data)
-		self.image          = data['image']
-		self.virtualization = data['virtualization']
-		if 'loopback_dir' not in self.volume:
-			self.volume['loopback_dir'] = '/tmp'

providers/kvm/tasks/boot.py

@@ -1,55 +0,0 @@
-from base import Task
-from common import phases
-
-
-class ConfigureGrub(Task):
-	description = 'Configuring grub for KVM'
-	phase = phases.system_modification
-
-	def run(self, info):
-		import stat
-	        rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
-                     stat.S_IRGRP                | stat.S_IXGRP |
-                     stat.S_IROTH                | stat.S_IXOTH)
-        	x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
-		import os.path
-		device_map_path = os.path.join(info.root, 'boot/grub/device.map')
-		with open(device_map_path, 'w') as device_map:
-			device_map.write('(hd0)   /dev/sda\n')
-
-		from common.tools import log_check_call
-
-		from shutil import copy
-        	script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux'))
-        	script_dst = os.path.join(info.root, 'etc/grub.d/10_linux')
-        	copy(script_src, script_dst)
-        	os.chmod(script_dst, rwxr_xr_x)
-                script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header'))
-                script_dst = os.path.join(info.root, 'etc/grub.d/00_header')
-                copy(script_src, script_dst)
-                os.chmod(script_dst, rwxr_xr_x)
-
-                if info.manifest.virtualization == 'virtio':
-			print "Using virtio"
-                        modules_path = os.path.join(info.root,
-                                        'etc/initramfs-tools/modules')
-                        with open(modules_path, 'a') as modules:
-                                modules.write("\nvirtio_pci\nvirtio_blk\n")
-
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u'])
-		# Install grub in mbr
-		log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']])
-
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
-
-                if info.manifest.virtualization == 'virtio':
-			from common.tools import sed_i
-                        grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg')
-                        sed_i(grub_cfg, 'sda', 'vda')
-                        device_map = os.path.join(info.root,
-                                        'boot/grub/device.map')
-                        sed_i(device_map, 'sda', 'vda')
-			fstab_file =  os.path.join(info.root,
-                                        'etc/fstab')
-			sed_i(fstab_file, 'sda', 'vda')
-

providers/kvm/tasks/packages.py

@@ -1,48 +0,0 @@
-from base import Task
-from common import phases
-from common.tasks import packages
-from common.tasks.host import CheckPackages
-
-
-class HostPackages(Task):
-	description = 'Determining required host packages'
-	phase = phases.preparation
-	before = [CheckPackages]
-	after = [packages.HostPackages]
-
-	def run(self, info):
-		info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx'])
-		if info.manifest.volume['filesystem'] == 'xfs':
-			info.host_packages.add('xfsprogs')
-
-
-class ImagePackages(Task):
-	description = 'Determining required image packages'
-	phase = phases.preparation
-	after = [packages.ImagePackages]
-
-	def run(self, info):
-		manifest = info.manifest
-		include, exclude = info.img_packages
-		# Add some basic packages we are going to need
-		include.update(['parted',
-		                'kpartx',
-		               # Needed for the init scripts
-		               'file',
-		               # isc-dhcp-client doesn't work properly with ec2
-		               'dhcpcd',
-		               'chkconfig',
-		               'openssh-client',
-			       'grub2'
-		                ])
-
-		exclude.update(['isc-dhcp-client',
-		               'isc-dhcp-common',
-		                ])
-
-		# In squeeze, we need a special kernel flavor for xen
-		kernels = {'squeeze': {'amd64': 'linux-image-amd64',
-		                       'i386':  'linux-image-686', },
-		           'wheezy':  {'amd64': 'linux-image-amd64',
-		                       'i386':  'linux-image-686', }, }
-		include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))

providers/virtualbox/__init__.py

@@ -1,20 +1,17 @@
 from manifest import Manifest
 from tasks import packages
-from common.tasks import packages as common_packages
-from common.tasks import host
+from common.tasks import volume as volume_tasks
 from common.tasks import loopback
-from common.tasks import parted
+from common.tasks import partitioning
 from common.tasks import filesystem
 from common.tasks import bootstrap
-from common.tasks import locale
-from common.tasks import apt
 from tasks import boot
 from common.tasks import boot as common_boot
 from common.tasks import security
 from common.tasks import network
 from common.tasks import initd
 from common.tasks import cleanup
-from common.tasks import loopback
+from common.tasks import workspace
 
 
 def initialize():
@@ -22,60 +19,48 @@ def initialize():
 
 
 def tasks(tasklist, manifest):
-	tasklist.add(packages.HostPackages(),
-	             common_packages.HostPackages(),
-	             packages.ImagePackages(),
-	             common_packages.ImagePackages(),
-	             host.CheckPackages(),
+	from common.task_sets import base_set
+	from common.task_sets import volume_set
+	from common.task_sets import mounting_set
+	from common.task_sets import apt_set
+	from common.task_sets import locale_set
+	tasklist.add(*base_set)
+	tasklist.add(*volume_set)
+	tasklist.add(*mounting_set)
+	tasklist.add(*apt_set)
+	tasklist.add(*locale_set)
 
-	             loopback.CreateQemuImg(),
-	             loopback.Attach(),
-	             parted.PartitionVolume(),
-	             parted.MapPartitions(),
-	             parted.FormatPartitions(),
-	             filesystem.CreateMountDir(),
-	             filesystem.MountVolume(),
+	if manifest.volume['partitions']['type'] != 'none':
+		from common.task_sets import partitioning_set
+		tasklist.add(*partitioning_set)
 
-	             bootstrap.Bootstrap(),
-	             filesystem.MountSpecials(),
-	             locale.GenerateLocale(),
-	             locale.SetTimezone(),
-	             apt.DisableDaemonAutostart(),
-	             apt.AptSources(),
-	             apt.AptUpgrade(),
-	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
-	             common_boot.BlackListModules(),
-	             common_boot.DisableGetTTYs(),
-	             security.EnableShadowConfig(),
-	             security.DisableSSHPasswordAuthentication(),
-	             security.DisableSSHDNSLookup(),
-	             network.RemoveDNSInfo(),
-	             network.ConfigureNetworkIF(),
-	             network.ConfigureDHCP(),
-	             initd.ResolveInitScripts(),
-	             initd.InstallInitScripts(),
-	             cleanup.ClearMOTD(),
-	             cleanup.ShredHostkeys(),
-	             cleanup.CleanTMP(),
-	             apt.PurgeUnusedPackages(),
-	             apt.AptClean(),
-	             apt.EnableDaemonAutostart(),
-	             filesystem.UnmountSpecials(),
+	tasklist.add(packages.ImagePackages,
 
-	             filesystem.UnmountVolume(),
-	             parted.UnmapPartitions(),
-	             loopback.Detach(),
-	             filesystem.DeleteMountDir())
+	             loopback.Create,
 
-	if manifest.bootstrapper['tarball']:
-		tasklist.add(bootstrap.MakeTarball())
+	             boot.ConfigureGrub,
+	             common_boot.BlackListModules,
+	             common_boot.DisableGetTTYs,
+	             security.EnableShadowConfig,
+	             network.RemoveDNSInfo,
+	             network.ConfigureNetworkIF,
+	             network.RemoveHostname,
+	             initd.ResolveInitScripts,
+	             initd.InstallInitScripts,
+	             cleanup.ClearMOTD,
+	             cleanup.CleanTMP,
 
-	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
-	                             'ext2': [filesystem.TuneVolumeFS()],
-	                             'ext3': [filesystem.TuneVolumeFS()],
-	                             'ext4': [filesystem.TuneVolumeFS()]}
-	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
+	             loopback.MoveImage)
+
+	if manifest.bootstrapper.get('tarball', False):
+		tasklist.add(bootstrap.MakeTarball)
+
+	from common.task_sets import get_fs_specific_set
+	tasklist.add(*get_fs_specific_set(manifest.volume['partitions']))
+
+	if 'boot' in manifest.volume['partitions']:
+		from common.task_sets import boot_partition_set
+		tasklist.add(*boot_partition_set)
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -83,10 +68,13 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 
 	def counter_task(task, counter):
 		if task in completed and counter not in completed:
-			tasklist.add(counter())
+			tasklist.add(counter)
 
+	counter_task(loopback.Create, volume_tasks.Delete)
 	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
-	counter_task(parted.MapPartitions, parted.UnmapPartitions)
-	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
+	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
+	counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
+	counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
-	counter_task(loopback.Attach, loopback.Detach)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)
+	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)

providers/virtualbox/assets/grub.d/00_header

@@ -1,4 +0,0 @@
-#! /bin/sh
-set -e
-
-# nothing to do, skip grub mkconfig for this

providers/virtualbox/assets/grub.d/10_linux

@@ -1,93 +0,0 @@
-#!/bin/sh
-
-# This file generates the old menu.lst configuration with grub2
-# It was copied from tomheadys github repo:
-# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
-
-prefix=/usr
-exec_prefix=${prefix}
-bindir=${exec_prefix}/bin
-libdir=${exec_prefix}/lib
-. ${libdir}/grub/grub-mkconfig_lib
-
-export TEXTDOMAIN=grub
-export TEXTDOMAINDIR=${prefix}/share/locale
-
-GRUB_DEVICE=/dev/sda1
-
-
-cat << EOF
-set default=${GRUB_DEFAULT}
-set timeout=${GRUB_TIMEOUT}
-insmod part_msdos
-insmod ext2
-insmod gettext
-set menu_color_normal=cyan/blue
-set menu_color_highlight=white/blue
-set root='(hd0,msdos1)'
-EOF
-
-if ${GRUB_HIDDEN_TIMEOUT:-false}; then
-	printf "hiddenmenu\n"
-fi
-
-linux_entry ()
-{
-	os="$1"
-	version="$2"
-	args="$4"
-
-	title="$(gettext_quoted "%s, with Linux %s")"
-
-	cat << EOF
-menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os {
-        insmod part_msdos
-        insmod ext2
-	set timeout=${GRUB_TIMEOUT}
-	set root='(hd0,msdos1)'
-        echo 'Loading Linux ${version}'
-	linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
-        echo 'Loading initial ramdisk ...'
-	initrd ${rel_dirname}/${initrd}
-}
-EOF
-}
-
-list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
-		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
-		done`
-prepare_boot_cache=
-
-while [ "x$list" != "x" ] ; do
-	linux=`version_find_latest $list`
-	basename=`basename $linux`
-	dirname=`dirname $linux`
-	rel_dirname=`make_system_path_relative_to_its_root $dirname`
-	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
-	alt_version=`echo $version | sed -e "s,\.old$,,g"`
-	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
-
-	initrd=
-	for i in "initrd.img-${version}" "initrd-${version}.img" \
-		"initrd-${version}" "initramfs-${version}.img" \
-		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
-		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
-	if test -e "${dirname}/${i}" ; then
-		initrd="$i"
-		break
-	fi
-	done
-
-	initramfs=
-	for i in "config-${version}" "config-${alt_version}"; do
-	if test -e "${dirname}/${i}" ; then
-		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut   -f2 -d= | tr -d \"`
-		break
-	fi
-	done
-
-	linux_entry "${OS}" "${version}" \
-		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
-
-	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
-done

providers/virtualbox/manifest-schema.json

@@ -8,15 +8,16 @@
 			"properties": {
 				"backing": {
 					"type": "string",
-					"enum": ["raw", "qcow2"]
+					"enum": ["raw", "vdi", "qcow2"]
 				},
-				"filesystem": {
-					"type": "string",
-					"enum": ["ext2", "ext3", "ext4", "xfs"]
+				"partitions": {
+					"type": "object",
+					"properties": {
+						"type": { "enum": ["none", "mbr"] }
+					}
 				}
 			},
-			"required": ["backing", "filesystem"]
+			"required": ["backing"]
+		}
 	}
-	},
-	"required": ["volume"]
 }

providers/virtualbox/manifest.py

@@ -12,5 +12,3 @@ class Manifest(base.Manifest):
 		super(Manifest, self).parse(data)
 		self.virtualization = None
 		self.image          = data['image']
-		if 'loopback_dir' not in self.volume:
-			self.volume['loopback_dir'] = '/tmp'

providers/virtualbox/tasks/boot.py

@@ -1,34 +1,65 @@
 from base import Task
 from common import phases
+from common.tasks import apt
+from common.fs.loopbackvolume import LoopbackVolume
 
 
 class ConfigureGrub(Task):
 	description = 'Configuring grub'
 	phase = phases.system_modification
+	after = [apt.AptUpgrade]
 
 	def run(self, info):
-		import stat
-		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
-		             stat.S_IRGRP                | stat.S_IXGRP |
-		             stat.S_IROTH                | stat.S_IXOTH)
-		import os.path
-		device_map_path = os.path.join(info.root, 'boot/grub/device.map')
-		with open(device_map_path, 'w') as device_map:
-			device_map.write('(hd0)   /dev/sda\n')
-
+		import os
 		from common.tools import log_check_call
 
-		from shutil import copy
-		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux'))
-		script_dst = os.path.join(info.root, 'etc/grub.d/10_linux')
-		copy(script_src, script_dst)
-		os.chmod(script_dst, rwxr_xr_x)
-		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header'))
-		script_dst = os.path.join(info.root, 'etc/grub.d/00_header')
-		copy(script_src, script_dst)
-		os.chmod(script_dst, rwxr_xr_x)
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u'])
-		# Install grub in mbr
-		log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/",
-		                info.bootstrap_device['path']])
+		boot_dir = os.path.join(info.root, 'boot')
+		grub_dir = os.path.join(boot_dir, 'grub')
+
+		from base.fs.partitionmaps.none import NoPartitions
+		from base.fs.partitionmaps.gpt import GPTPartitionMap
+		from common.fs import remount
+		p_map = info.volume.partition_map
+
+		def mk_remount_fn(fn):
+			def set_device_path():
+				fn()
+				if isinstance(p_map, NoPartitions):
+					p_map.root.device_path = info.volume.device_path
+			return set_device_path
+		link_fn = mk_remount_fn(info.volume.link_dm_node)
+		unlink_fn = mk_remount_fn(info.volume.unlink_dm_node)
+
+		# GRUB cannot deal with installing to loopback devices
+		# so we fake a real harddisk with dmsetup.
+		# Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, link_fn)
+		try:
+			[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
+			device_map_path = os.path.join(grub_dir, 'device.map')
+			partition_prefix = 'msdos'
+			if isinstance(p_map, GPTPartitionMap):
+				partition_prefix = 'gpt'
+			with open(device_map_path, 'w') as device_map:
+				device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
+				if not isinstance(p_map, NoPartitions):
+					for idx, partition in enumerate(info.volume.partition_map.partitions):
+						[partition_path] = log_check_call(['readlink', '-f', partition.device_path])
+						device_map.write('(hd0,{prefix}{idx}) {device_path}\n'
+						                 .format(device_path=partition_path, prefix=partition_prefix, idx=idx+1))
+
+			# Install grub
+			log_check_call(['/usr/sbin/chroot', info.root,
+			                '/usr/sbin/grub-install',
+			                # '--root-directory=' + info.root,
+			                # '--boot-directory=' + boot_dir,
+			                device_path])
 			log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+		except Exception as e:
+			if isinstance(info.volume, LoopbackVolume):
+				remount(info.volume, unlink_fn)
+			raise e
+
+		if isinstance(info.volume, LoopbackVolume):
+			remount(info.volume, unlink_fn)

providers/virtualbox/tasks/packages.py

@@ -1,19 +1,6 @@
 from base import Task
 from common import phases
 from common.tasks import packages
-from common.tasks.host import CheckPackages
-
-
-class HostPackages(Task):
-	description = 'Determining required host packages'
-	phase = phases.preparation
-	before = [CheckPackages]
-	after = [packages.HostPackages]
-
-	def run(self, info):
-		info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx'])
-		if info.manifest.volume['filesystem'] == 'xfs':
-			info.host_packages.add('xfsprogs')
 
 
 class ImagePackages(Task):
@@ -25,20 +12,7 @@ class ImagePackages(Task):
 		manifest = info.manifest
 		include, exclude = info.img_packages
 		# Add some basic packages we are going to need
-		include.update(['parted',
-		                'kpartx',
-		               # Needed for the init scripts
-		               'file',
-		               # isc-dhcp-client doesn't work properly with ec2
-		               'dhcpcd',
-		               'chkconfig',
-		               'openssh-client',
-		               'grub2'
-		                ])
-
-		exclude.update(['isc-dhcp-client',
-		               'isc-dhcp-common',
-		                ])
+		include.add('grub2')
 
 		# In squeeze, we need a special kernel flavor for xen
 		kernels = {'squeeze': {'amd64': 'linux-image-amd64',

providers/virtualbox/volume.py

@@ -0,0 +1,77 @@
+from common.fs.loopbackvolume import LoopbackVolume
+from base.fs.exceptions import VolumeError
+from common.tools import log_check_call
+from common.fs import get_partitions
+
+
+class VirtualBoxVolume(LoopbackVolume):
+
+	extension = 'vdi'
+
+	def _before_create(self, e):
+		self.image_path = e.image_path
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M'])
+
+	def _check_nbd_module(self):
+		from base.fs.partitionmaps.none import NoPartitions
+		if isinstance(self.partition_map, NoPartitions):
+			if not self._module_loaded('nbd'):
+				raise VolumeError('The kernel module `nbd\' must be loaded '
+				                  '(`modprobe nbd\') to attach .vdi images')
+		else:
+			num_partitions = len(self.partition_map.partitions)
+			if not self._module_loaded('nbd'):
+				msg = ('The kernel module `nbd\' must be loaded '
+				       '(`modprobe nbd max_part={num_partitions}\') to attach .vdi images'
+				       .format(num_partitions=num_partitions))
+				raise VolumeError(msg)
+			nbd_max_part = int(self._module_param('nbd', 'max_part'))
+			if nbd_max_part < num_partitions:
+				# Found here: http://bethesignal.org/blog/2011/01/05/how-to-mount-virtualbox-vdi-image/
+				msg = ('The kernel module `nbd\' was loaded with the max_part '
+				       'parameter set to {max_part}, which is below '
+				       'the amount of partitions for this volume ({num_partitions}). '
+				       'Reload the nbd kernel module with max_part set to at least {num_partitions} '
+				       '(`rmmod nbd; modprobe nbd max_part={num_partitions}\').'
+				       .format(max_part=nbd_max_part, num_partitions=num_partitions))
+				raise VolumeError(msg)
+
+	def _before_attach(self, e):
+		self._check_nbd_module()
+		self.loop_device_path = self._find_free_nbd_device()
+		log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path])
+		self.device_path = self.loop_device_path
+
+	def _before_detach(self, e):
+		log_check_call(['/usr/bin/qemu-nbd', '--disconnect', self.loop_device_path])
+		del self.loop_device_path
+		del self.device_path
+
+	def _module_loaded(self, module):
+		import re
+		regexp = re.compile('^{module} +'.format(module=module))
+		with open('/proc/modules') as loaded_modules:
+			for line in loaded_modules:
+				match = regexp.match(line)
+				if match is not None:
+					return True
+		return False
+
+	def _module_param(self, module, param):
+		import os.path
+		param_path = os.path.join('/sys/module', module, 'parameters', param)
+		with open(param_path) as param:
+			return param.read().strip()
+
+	# From http://lists.gnu.org/archive/html/qemu-devel/2011-11/msg02201.html
+	# Apparently it's not in the current qemu-nbd shipped with wheezy
+	def _is_nbd_used(self, device_name):
+		return device_name in get_partitions()
+
+	def _find_free_nbd_device(self):
+		import os.path
+		for i in xrange(0, 15):
+			device_name = 'nbd' + str(i)
+			if not self._is_nbd_used(device_name):
+				return os.path.join('/dev', device_name)
+		raise VolumeError('Unable to find free nbd device.')