Support for partitions

MAJOR refactor. The volume is now abstracted into a model along with a partitionmap and partitions.
Volumes and partitions are now controlled via an FSM to ensure that commands are called in the proper sequence.
GRUB can now be installed properly onto loop devices by using dmsetup to fake a proper harddisk.

base/bootstrapinfo.py

@@ -3,6 +3,8 @@
 class BootstrapInformation(object):
 	def __init__(self, manifest=None, debug=False):
 		self.manifest = manifest
+		from fs import load_volume
+		self.volume = load_volume(self.manifest.volume)
 		self.debug = debug
 		import random
 		self.run_id = random.randrange(16 ** 8)

base/fs/__init__.py

@@ -0,0 +1,17 @@
+
+
+def load_volume(data):
+	from common.fs.loopbackvolume import LoopbackVolume
+	from providers.ec2.volume import EBSVolume
+	from providers.virtualbox.volume import VirtualBoxVolume
+	from partitionmap import PartitionMap
+	from nopartitions import NoPartitions
+	partition_maps = {'none': NoPartitions,
+	                  'gpt': PartitionMap,
+	                  }
+	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'])
+	volume_backings = {'raw': LoopbackVolume,
+	                   'vdi': VirtualBoxVolume,
+	                   'ebs': EBSVolume
+	                   }
+	return volume_backings.get(data['backing'])(partition_map)

base/fs/exceptions.py

@@ -0,0 +1,8 @@
+
+
+class VolumeError(Exception):
+	pass
+
+
+class PartitionError(Exception):
+	pass

base/fs/nopartitions.py

@@ -0,0 +1,30 @@
+from partitions.singlepartition import SinglePartition
+
+
+class NoPartitions(object):
+
+	def __init__(self, data):
+		root = data['root']
+		self.root = SinglePartition(root['size'], root['filesystem'])
+		self.mount_points = [('/', self.root)]
+
+	def get_total_size(self):
+		return self.root.size
+
+	def create(self, volume):
+		pass
+
+	def map(self, volume):
+		pass
+
+	def unmap(self, volume):
+		pass
+
+	def format(self):
+		self.root.format()
+
+	def mount_root(self, destination):
+		self.root.mount(destination)
+
+	def unmount_root(self):
+		self.root.unmount()

base/fs/partitionmap.py

@@ -0,0 +1,88 @@
+from common.tools import log_check_call
+from partitions.partition import Partition
+from partitions.swap import Swap
+from exceptions import PartitionError
+
+
+class PartitionMap(object):
+
+	def __init__(self, data):
+		self.boot = None
+		self.swap = None
+		self.mount_points = []
+		if 'boot' in data:
+			self.boot = Partition(data['boot']['size'], data['boot']['filesystem'], None)
+			self.mount_points.append(('/boot', self.boot))
+		self.root = Partition(data['root']['size'], data['root']['filesystem'], self.boot)
+		self.mount_points.append(('/', self.root))
+		if 'swap' in data:
+			self.swap = Swap(data['swap']['size'], self.root)
+			self.mount_points.append(('none', self.root))
+		self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap])
+
+	def get_total_size(self):
+		return sum(p.size for p in self.partitions)
+
+	def create(self, volume):
+		log_check_call(['/sbin/parted', '--script', '--align', 'optimal', volume.device_path,
+		                '--', 'mklabel', 'gpt'])
+		for partition in self.partitions:
+			partition.create(volume)
+
+		boot_idx = self.root.get_index()
+		if self.boot is not None:
+			boot_idx = self.boot.get_index()
+		log_check_call(['/sbin/parted', '--script', volume.device_path,
+		                '--', 'set', str(boot_idx), 'bios_grub', 'on'])
+
+	def map(self, volume):
+		try:
+			mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path])
+			import re
+			regexp = re.compile('^(?P<name>.+[^\d](?P<p_idx>\d+)) : '
+			                    '(?P<start_blk>\d) (?P<num_blks>\d+) '
+			                    '{device_path} (?P<blk_offset>\d+)$'
+			                    .format(device_path=volume.device_path))
+			log_check_call(['/sbin/kpartx', '-a', volume.device_path])
+			import os.path
+			for mapping in mappings:
+				match = regexp.match(mapping)
+				if match is None:
+					raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping))
+				partition_path = os.path.join('/dev/mapper', match.group('name'))
+				p_idx = int(match.group('p_idx'))-1
+				self.partitions[p_idx].map(partition_path)
+
+			for idx, partition in enumerate(self.partitions):
+				if not partition.state() in ['mapped', 'formatted']:
+					raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1))
+
+		except PartitionError as e:
+			for partition in self.partitions:
+				if partition.state() in ['mapped', 'formatted']:
+					partition.unmap()
+			log_check_call(['/sbin/kpartx', '-d', volume.device_path])
+			raise e
+
+	def unmap(self, volume):
+		for partition in self.partitions:
+			partition.unmap()
+		log_check_call(['/sbin/kpartx', '-d', volume.device_path])
+
+	def format(self):
+		for partition in self.partitions:
+			partition.format()
+
+	def mount_root(self, destination):
+		self.root.mount(destination)
+
+	def unmount_root(self):
+		self.root.unmount()
+
+	def mount_boot(self):
+		import os.path
+		destination = os.path.join(self.root.mount_dir, 'boot')
+		self.boot.mount(destination)
+
+	def unmount_boot(self):
+		self.boot.unmount()

base/fs/partitions/abstractpartition.py

@@ -0,0 +1,56 @@
+from common.tools import log_check_call
+from abc import ABCMeta
+from fysom import Fysom
+
+
+class AbstractPartition(object):
+
+	__metaclass__ = ABCMeta
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'},
+	          {'name': 'format', 'src': 'created', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          ]
+
+	def __init__(self, size, filesystem, callbacks={}):
+		self.size          = size
+		self.filesystem    = filesystem
+		self.device_path   = None
+		self.initial_state = 'nonexistent'
+
+		callbacks.update({'onbeforeformat': self._format,
+		                  'onbeforemount': self._mount,
+		                  'onbeforeunmount': self._unmount,
+		                  })
+
+		self.fsm = Fysom({'initial': 'nonexistent',
+		                  'events': self.events,
+		                  'callbacks': callbacks})
+		from common.fsm import attach_proxy_methods
+		attach_proxy_methods(self, self.events, self.fsm)
+
+	def state(self):
+		return self.fsm.current
+
+	def force_state(self, state):
+		self.fsm.current = state
+
+	def get_uuid(self):
+		[uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path])
+		return uuid
+
+	def _format(self, e):
+		mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem)
+		log_check_call([mkfs, self.device_path])
+
+	def mount(self, destination):
+		self.fsm.mount(destination=destination)
+
+	def _mount(self, e):
+		log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination])
+		self.mount_dir = e.destination
+
+	def _unmount(self, e):
+		log_check_call(['/bin/umount', self.mount_dir])
+		del self.mount_dir

base/fs/partitions/partition.py

@@ -0,0 +1,60 @@
+from common.tools import log_check_call
+from abstractpartition import AbstractPartition
+
+
+class Partition(AbstractPartition):
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'},
+	          {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'},
+	          {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'},
+	          {'name': 'format', 'src': 'mapped', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'},
+	          {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'},
+	          ]
+
+	def __init__(self, size, filesystem, previous, callbacks={}):
+		self.previous = previous
+		callbacks.update({'onbeforecreate': self._create,
+		                  'onbeforemap': self._map,
+		                  'onbeforeunmap': self._unmap,
+		                  })
+		super(Partition, self).__init__(size, filesystem, callbacks=callbacks)
+
+	def get_index(self):
+		if self.previous is None:
+			return 1
+		else:
+			return self.previous.get_index()+1
+
+	def get_start(self):
+		if self.previous is None:
+			return 0
+		else:
+			return self.previous.get_start() + self.previous.size
+
+	def create(self, volume):
+		self.fsm.create(volume=volume)
+
+	def _create(self, e):
+		start = self.get_start()
+		# maybe use `parted -- name` to set partition name
+		log_check_call(['/sbin/parted', '--script', '--align', 'optimal', e.volume.device_path,
+		                '--', 'mkpart', 'primary', str(start), str(start + self.size)])
+
+	def map(self, device_path):
+		self.fsm.map(device_path=device_path)
+
+	def _map(self, e):
+		self.device_path = e.device_path
+
+	def _unmap(self, e):
+		self.device_path = None
+
+# Partition flags:
+# boot, root, swap, hidden, raid, lvm, lba, legacy_boot, palo
+
+
+# Partition tables
+# bsd, dvh, gpt, loop, mac, msdos, pc98, sun

base/fs/partitions/singlepartition.py

@@ -0,0 +1,20 @@
+from abstractpartition import AbstractPartition
+
+
+class SinglePartition(AbstractPartition):
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'},
+	          {'name': 'format', 'src': 'created', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          ]
+
+	def __init__(self, size, filesystem, callbacks={}):
+		callbacks['oncreate'] = self._create
+		super(SinglePartition, self).__init__(size, filesystem, callbacks=callbacks)
+
+	def create(self, volume):
+		self.fsm.create(volume=volume)
+
+	def _create(self, e):
+		self.device_path = e.volume.device_path

base/fs/partitions/swap.py

@@ -0,0 +1,11 @@
+from common.tools import log_check_call
+from partition import Partition
+
+
+class Swap(Partition):
+
+	def __init__(self, size, previous):
+		super(Swap, self).__init__(size, 'swap', previous)
+
+	def _format(self, e):
+		log_check_call(['/sbin/mkswap', self.device_path])

base/fs/volume.py

@@ -0,0 +1,138 @@
+from abc import ABCMeta
+from fysom import Fysom
+
+
+class Volume(object):
+
+	__metaclass__ = ABCMeta
+
+	events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'},
+	          {'name': 'attach', 'src': 'detached', 'dst': 'attached'},
+	          {'name': 'partition', 'src': 'attached', 'dst': 'partitioned'},
+	          {'name': 'map', 'src': 'partitioned', 'dst': 'mapped'},
+	          {'name': 'format', 'src': 'mapped', 'dst': 'formatted'},
+	          {'name': 'mount', 'src': ['formatted', 'mounted'], 'dst': 'mounted'},
+	          {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'},
+	          {'name': 'unmap', 'src': 'formatted', 'dst': 'partitioned_fmt'},
+	          {'name': 'detach', 'src': 'partitioned_fmt', 'dst': 'detached_fmt'},
+	          {'name': 'delete', 'src': ['detached', 'detached_prt', 'detached_fmt'], 'dst': 'deleted'},
+
+	          {'name': 'attach', 'src': 'detached_fmt', 'dst': 'partitioned_fmt'},
+	          {'name': 'map', 'src': 'partitioned_fmt', 'dst': 'formatted'},
+
+	          {'name': 'attach', 'src': 'detached_prt', 'dst': 'partitioned'},
+	          {'name': 'detach', 'src': 'partitioned', 'dst': 'detached_prt'},
+
+	          {'name': 'detach', 'src': 'attached', 'dst': 'detached'},
+	          {'name': 'unmap', 'src': 'mapped', 'dst': 'partitioned'},
+	          ]
+
+	mount_events = [{'name': 'mount_root', 'src': 'unmounted', 'dst': 'root_mounted'},
+	                {'name': 'mount_boot', 'src': 'root_mounted', 'dst': 'boot_mounted'},
+	                {'name': 'mount_specials', 'src': 'boot_mounted', 'dst': 'specials_mounted'},
+	                {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'boot_mounted'},
+	                {'name': 'unmount_boot', 'src': 'boot_mounted', 'dst': 'root_mounted'},
+	                {'name': 'unmount_root', 'src': 'root_mounted', 'dst': 'unmounted'},
+
+	                {'name': 'mount_specials', 'src': 'root_mounted', 'dst': 'specials_mounted_no_boot'},
+	                {'name': 'mount_boot', 'src': 'specials_mounted_no_boot', 'dst': 'specials_mounted'},
+	                {'name': 'unmount_specials', 'src': 'specials_mounted_no_boot', 'dst': 'root_mounted'},
+	                {'name': 'unmount_boot', 'src': 'specials_mounted', 'dst': 'specials_mounted_no_boot'},
+	                ]
+
+	def __init__(self, partition_map, callbacks={}):
+		self.device_path = None
+		self.partition_map = partition_map
+		self.size = self.partition_map.get_total_size()
+
+		callbacks.update({'onbeforepartition': self._partition,
+		                  'onbeforemap': self._map,
+		                  'onbeforeunmap': self._unmap,
+		                  'onbeforeformat': self._format,
+		                  'onbeforeunmount': self._unmount,
+		                  })
+
+		mount_callbacks = {'onbeforemount_root': self._mount_root,
+		                   'onbeforemount_boot': self._mount_boot,
+		                   'onbeforemount_specials': self._mount_specials,
+		                   'onbeforeunmount_root': self._unmount_root,
+		                   'onbeforeunmount_boot': self._unmount_boot,
+		                   'onbeforeunmount_specials': self._unmount_specials,
+		                   }
+		self.fsm = Fysom({'initial': 'nonexistent',
+		                  'events': self.events,
+		                  'callbacks': callbacks})
+
+		self.mount_fsm = Fysom({'initial': 'unmounted',
+		                        'events': self.mount_events,
+		                        'callbacks': mount_callbacks})
+
+		from common.fsm import attach_proxy_methods
+		attach_proxy_methods(self, self.events, self.fsm)
+		attach_proxy_methods(self, self.mount_events, self.mount_fsm)
+
+	def state(self):
+		return self.fsm.current
+
+	def force_state(self, state):
+		self.fsm.current = state
+
+	def _partition(self, e):
+		self.partition_map.create(self)
+
+	def _map(self, e):
+		self.partition_map.map(self)
+
+	def _unmap(self, e):
+		self.partition_map.unmap(self)
+
+	def _format(self, e):
+		self.partition_map.format()
+
+	def _unmount(self, e):
+		if self.mount_fsm.current is 'specials_mounted':
+			self.unmount_specials()
+		if self.mount_fsm.current is 'specials_mounted_no_boot':
+			self.unmount_specials()
+		if self.mount_fsm.current is 'boot_mounted':
+			self.unmount_boot()
+		if self.mount_fsm.current is 'root_mounted':
+			self.unmount_root()
+
+	def mount_root(self, destination):
+		self.mount_fsm.mount_root(destination=destination)
+
+	def unmount_root(self):
+		self.mount_fsm.unmount_root()
+		self.fsm.unmount()
+
+	def _mount_root(self, e):
+		self.mount()
+		self.partition_map.mount_root(e.destination)
+
+	def _unmount_root(self, e):
+		self.partition_map.unmount_root()
+
+	def _mount_boot(self, e):
+		self.mount()
+		self.partition_map.mount_boot()
+
+	def _unmount_boot(self, e):
+		self.partition_map.unmount_boot()
+
+	def _mount_specials(self, e):
+		self.mount()
+		from common.tools import log_check_call
+		root = self.partition_map.root.mount_dir
+		log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts'])
+
+	def _unmount_specials(self, e):
+		from common.tools import log_check_call
+		root = self.partition_map.root.mount_dir
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys'])
+		log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc'])
+		log_check_call(['/bin/umount', '{root}/dev'.format(root=root)])

base/manifest-schema.json

@@ -34,14 +34,13 @@
 			"required": ["release", "architecture", "timezone", "locale", "charmap"]
 		},
 		"volume": {
-			"type": "object",
+			"type": "object"
-			"properties": {
+			// "properties": {
-				"size": {
+			// 	"backing": {
-					"type": "integer",
+			// 		"type": "string"
-					"minimum": 1
+			// 	}
-				}
+			// },
-			},
+			// "required": ["backing"]
-			"required": ["size"]
 		},
 		"plugins": {
 			"type": "object",

common/fs/__init__.py

@@ -0,0 +1,11 @@
+
+
+def get_major_minor_dev_num(device_name):
+	import re
+	regexp = re.compile('^ *(?P<major>\d+) *(?P<minor>\d+) *(?P<num_blks>\d+) {device_name}$'
+	                    .format(device_name=device_name))
+	with open('/proc/partitions') as partitions:
+		for line in partitions:
+			match = regexp.match(line)
+			if match is not None:
+				return match.group('major'), match.group('minor')

common/fs/loopbackvolume.py

@@ -0,0 +1,90 @@
+from base.fs.volume import Volume
+from common.tools import log_check_call
+from base.fs.exceptions import VolumeError
+
+
+# QEMU formats:
+# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop
+
+
+class LoopbackVolume(Volume):
+
+	link_dm_events = [{'name': 'link_dm_node', 'src': 'partitioned', 'dst': 'linked'},
+	                  {'name': 'map', 'src': 'linked', 'dst': 'mapped_lnk'},
+	                  {'name': 'format', 'src': 'mapped_lnk', 'dst': 'formatted_lnk'},
+	                  {'name': 'mount', 'src': ['formatted_lnk', 'mounted_lnk'], 'dst': 'mounted_lnk'},
+	                  {'name': 'unmount', 'src': 'mounted_lnk', 'dst': 'formatted_lnk'},
+	                  {'name': 'unmap', 'src': 'formatted_lnk', 'dst': 'partitioned_fmt_lnk'},
+	                  {'name': 'unlink_dm_node', 'src': 'partitioned_fmt_lnk', 'dst': 'partitioned_fmt'},
+
+	                  {'name': 'link_dm_node', 'src': 'partitioned_fmt', 'dst': 'partitioned_fmt_lnk'},
+	                  {'name': 'map', 'src': 'partitioned_fmt_lnk', 'dst': 'formatted_lnk'},
+	                  {'name': 'unmap', 'src': 'mapped_lnk', 'dst': 'linked'},
+	                  {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'},
+	                  ]
+
+	def __init__(self, partition_map, callbacks={}):
+		callbacks.update({'onbeforecreate': self._create,
+		                  'onbeforeattach': self._attach,
+		                  'onbeforedetach': self._detach,
+		                  'onbeforedelete': self._delete,
+		                  'onbeforelink_dm_node': self._link_dm_node,
+		                  'onbeforeunlink_dm_node': self._unlink_dm_node,
+		                  })
+		self.events.extend(self.link_dm_events)
+		super(LoopbackVolume, self).__init__(partition_map, callbacks=callbacks)
+
+	def create(self, image_path):
+		self.fsm.create(image_path=image_path)
+
+	def _create(self, e):
+		self.image_path = e.image_path
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M'])
+
+	def _attach(self, e):
+		[self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path])
+		self.device_path = self.loop_device_path
+
+	def _link_dm_node(self, e):
+		import re
+		loop_device_name = re.match('^/dev/(?P<name>.*)$', self.loop_device_path).group('name')
+		from . import get_major_minor_dev_num
+		major, minor = get_major_minor_dev_num(loop_device_name)
+		sectors = self.size*1024*1024/512
+		table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}'
+		         .format(log_start_sec=0,
+		                 sectors=sectors,
+		                 major=major,
+		                 minor=minor,
+		                 start_sec=0))
+		import string
+		import os.path
+		for letter in string.ascii_lowercase:
+			dev_name = 'vd' + letter
+			dev_path = os.path.join('/dev/mapper', dev_name)
+			if not os.path.exists(dev_path):
+				self.dm_node_name = dev_name
+				self.dm_node_path = dev_path
+				break
+
+		if not hasattr(self, 'dm_node_name'):
+			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
+
+		log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table)
+		self.device_path = self.dm_node_path
+
+	def _unlink_dm_node(self, e):
+		log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name])
+		del self.dm_node_name
+		del self.dm_node_path
+		self.device_path = self.loop_device_path
+
+	def _detach(self, e):
+		log_check_call(['/sbin/losetup', '--detach', self.loop_device_path])
+		del self.loop_device_path
+		del self.device_path
+
+	def _delete(self, e):
+		from os import remove
+		remove(self.image_path)
+		del self.image_path

common/fsm.py

@@ -0,0 +1,7 @@
+
+
+def attach_proxy_methods(obj, events, fsm):
+	methods = set([e['name'] for e in events])
+	for event in methods:
+		if not hasattr(obj, event):
+			setattr(obj, event, lambda e=event: getattr(fsm, e)())

common/tasks/filesystem.py

@@ -1,29 +1,29 @@
 from base import Task
 from common import phases
-from common.exceptions import TaskError
 from common.tools import log_check_call
 from bootstrap import Bootstrap
+import volume
 
 
-class FormatVolume(Task):
+class Format(Task):
 	description = 'Formatting the volume'
 	phase = phases.volume_preparation
 
 	def run(self, info):
-		dev_path = info.bootstrap_device['path']
+		info.volume.format()
-		mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem'])
-		log_check_call([mkfs, dev_path])
-		info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']}
 
 
 class TuneVolumeFS(Task):
 	description = 'Tuning the bootstrap volume filesystem'
 	phase = phases.volume_preparation
-	after = [FormatVolume]
+	after = [Format]
 
 	def run(self, info):
+		import re
 		# Disable the time based filesystem check
-		log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']])
+		for partition in info.volume.partition_map.partitions:
+			if re.match('^ext[2-4]$', partition.filesystem) is not None:
+				log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path])
 
 
 class AddXFSProgs(Task):
@@ -36,33 +36,45 @@ class AddXFSProgs(Task):
 
 
 class CreateMountDir(Task):
-	description = 'Creating mountpoint for the bootstrap volume'
+	description = 'Creating mountpoint for the root partition'
 	phase = phases.volume_mounting
 
 	def run(self, info):
 		import os
 		mount_dir = info.manifest.bootstrapper['mount_dir']
 		info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id)
-		# Works recursively, fails if last part exists, which is exaclty what we want.
+		# Works recursively, fails if last part exists, which is exactly what we want.
 		os.makedirs(info.root)
 
 
-class MountVolume(Task):
+class MountRoot(Task):
-	description = 'Mounting the bootstrap volume'
+	description = 'Mounting the root partition'
 	phase = phases.volume_mounting
 	after = [CreateMountDir]
 
 	def run(self, info):
-		with open('/proc/mounts') as mounts:
+		info.volume.mount_root(info.root)
-			for mount in mounts:
-				if info.root in mount:
-					msg = 'Something is already mounted at {root}'.format(root=info.root)
-					raise TaskError(msg)
 
-		log_check_call(['/bin/mount',
+
-		                '--types', info.manifest.volume['filesystem'],
+class MountBoot(Task):
-		                info.bootstrap_device['partitions']['root_path'],
+	description = 'Mounting the boot partition'
-		                info.root])
+	phase = phases.volume_mounting
+	after = [MountRoot]
+
+	def run(self, info):
+		info.volume.mount_boot()
+
+
+class CreateBootMountDir(Task):
+	description = 'Creating mountpoint boot partition'
+	phase = phases.volume_mounting
+	after = [MountRoot]
+	before = [MountBoot]
+
+	def run(self, info):
+		import os
+		info.boot = os.path.join(info.root, 'boot')
+		os.makedirs()
 
 
 class MountSpecials(Task):
@@ -71,36 +83,40 @@ class MountSpecials(Task):
 	after = [Bootstrap]
 
 	def run(self, info):
-		log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)])
+		info.volume.mount_specials()
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc'])
+
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys'])
+
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts'])
+class UnmountRoot(Task):
+	description = 'Unmounting the bootstrap volume'
+	phase = phases.volume_unmounting
+	before = [volume.Detach]
+
+	def run(self, info):
+		info.volume.unmount_root()
+
+
+class UnmountBoot(Task):
+	description = 'Unmounting the boot partition'
+	phase = phases.volume_unmounting
+	before = [UnmountRoot]
+
+	def run(self, info):
+		info.volume.unmount_boot()
 
 
 class UnmountSpecials(Task):
 	description = 'Unmunting special block devices'
 	phase = phases.volume_unmounting
+	before = [UnmountRoot]
 
 	def run(self, info):
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts'])
+		info.volume.unmount_specials()
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc'])
-		log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)])
-
-
-class UnmountVolume(Task):
-	description = 'Unmounting the bootstrap volume'
-	phase = phases.volume_unmounting
-	after = [UnmountSpecials]
-
-	def run(self, info):
-		log_check_call(['/bin/umount', info.root])
 
 
 class DeleteMountDir(Task):
 	description = 'Deleting mountpoint for the bootstrap volume'
 	phase = phases.volume_unmounting
-	after = [UnmountVolume]
+	after = [UnmountRoot]
 
 	def run(self, info):
 		import os
@@ -108,24 +124,28 @@ class DeleteMountDir(Task):
 		del info.root
 
 
-class ModifyFstab(Task):
+class FStab(Task):
-	description = 'Adding root volume to the fstab'
+	description = 'Adding partitions to the fstab'
 	phase = phases.system_modification
 
 	def run(self, info):
 		import os.path
-		mount_opts = ['defaults']
+		# device = '/dev/sda'
-		if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
+		# if info.manifest.virtualization == 'pvm':
-			mount_opts.append('barrier=0')
+		# 	device = '/dev/xvda'
-		if info.manifest.volume['filesystem'].lower() == 'xfs':
+		fstab_lines = []
-			mount_opts.append('nobarrier')
+		for mount_point, partition in info.volume.partition_map.mount_points:
-		fstab_path = os.path.join(info.root, 'etc/fstab')
+			mount_opts = ['defaults']
+			if partition.filesystem in ['ext2', 'ext3', 'ext4']:
+				mount_opts.append('barrier=0')
+			if partition.filesystem == 'xfs':
+				mount_opts.append('nobarrier')
+			fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} 1 1'
+			                   .format(uuid=partition.get_uuid(),
+			                           mountpoint=mount_point,
+			                           filesystem=partition.filesystem,
+			                           mount_opts=','.join(mount_opts)))
 
-		device = '/dev/sda1'
+		fstab_path = os.path.join(info.root, 'etc/fstab')
-		if info.manifest.virtualization == 'pvm':
+		with open(fstab_path, 'w') as fstab:
-			device = '/dev/xvda1'
+			fstab.write('\n'.join(fstab_lines))
-		with open(fstab_path, 'a') as fstab:
-			fstab.write('{device} /     {filesystem}    {mount_opts} 1 1\n'
-			            .format(device=device,
-			                    filesystem=info.manifest.volume['filesystem'].lower(),
-			                    mount_opts=','.join(mount_opts)))

common/tasks/loopback.py

@@ -1,60 +1,15 @@
 from base import Task
 from common import phases
-from filesystem import UnmountVolume
+import volume
-from common.tools import log_check_call
 
 
 class Create(Task):
 	description = 'Creating a loopback volume'
 	phase = phases.volume_creation
+	before = [volume.Attach]
 
 	def run(self, info):
 		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
 		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
-		log_check_call(['/bin/dd',
+		info.volume.create(image_path)
-		                'if=/dev/zero', 'of=' + info.loopback_file,
-		                'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0'])
-
-
-class CreateQemuImg(Task):
-	description = 'Creating a loopback volume with qemu'
-	phase = phases.volume_creation
-
-	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
-		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
-		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw',
-		                info.loopback_file, str(info.manifest.volume['size']) + 'M'])
-
-
-class Attach(Task):
-	description = 'Attaching the loopback volume'
-	phase = phases.volume_creation
-	after = [Create, CreateQemuImg]
-
-	def run(self, info):
-		info.bootstrap_device = {}
-		command = ['/sbin/losetup', '--show', '--find', info.loopback_file]
-		[info.bootstrap_device['path']] = log_check_call(command)
-
-
-class Detach(Task):
-	description = 'Detaching the loopback volume'
-	phase = phases.volume_unmounting
-	after = [UnmountVolume]
-
-	def run(self, info):
-		log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']])
-		del info.bootstrap_device
-
-
-class Delete(Task):
-	description = 'Deleting the loopback volume'
-	phase = phases.cleaning
-
-	def run(self, info):
-		from os import remove
-		remove(info.loopback_file)
-		del info.loopback_file

common/tasks/parted.py

@@ -1,53 +0,0 @@
-from base import Task
-from common import phases
-from common.tools import log_check_call
-import filesystem
-import loopback
-
-
-class PartitionVolume(Task):
-	description = 'Partitioning the volume'
-	phase = phases.volume_preparation
-
-	def run(self, info):
-		# parted
-		log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'],
-		                '--', 'mklabel', 'msdos'])
-		log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'],
-		                '--', 'mkpart', 'primary', 'ext4', '32k', '-1'])
-		log_check_call(['parted', '--script', info.bootstrap_device['path'],
-		                '--', 'set', '1', 'boot', 'on'])
-
-
-class MapPartitions(Task):
-	description = 'Mapping volume partitions'
-	phase = phases.volume_preparation
-	after = [PartitionVolume]
-
-	def run(self, info):
-		root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1'
-		log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']])
-		info.bootstrap_device['partitions'] = {'root_path': root_partition_path}
-
-
-class FormatPartitions(Task):
-	description = 'Formatting the partitions'
-	phase = phases.volume_preparation
-	before = [filesystem.TuneVolumeFS]
-	after = [MapPartitions]
-
-	def run(self, info):
-		# These params will fail for mkfs.xfs
-		log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']),
-		                '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']])
-
-
-class UnmapPartitions(Task):
-	description = 'Removing volume partitions mapping'
-	phase = phases.volume_unmounting
-	before = [loopback.Detach]
-	after = [filesystem.UnmountVolume]
-
-	def run(self, info):
-		log_check_call(['kpartx', '-d', info.bootstrap_device['path']])
-		del info.bootstrap_device['partitions']['root_path']

common/tasks/partitioning.py

@@ -0,0 +1,32 @@
+from base import Task
+from common import phases
+import filesystem
+import volume
+
+
+class PartitionVolume(Task):
+	description = 'Partitioning the volume'
+	phase = phases.volume_preparation
+
+	def run(self, info):
+		info.volume.partition()
+
+
+class MapPartitions(Task):
+	description = 'Mapping volume partitions'
+	phase = phases.volume_preparation
+	before = [filesystem.Format]
+	after = [PartitionVolume]
+
+	def run(self, info):
+		info.volume.map()
+
+
+class UnmapPartitions(Task):
+	description = 'Removing volume partitions mapping'
+	phase = phases.volume_unmounting
+	before = [volume.Detach]
+	after = [filesystem.UnmountRoot]
+
+	def run(self, info):
+		info.volume.unmap()

common/tasks/volume.py

@@ -0,0 +1,26 @@
+from base import Task
+from common import phases
+
+
+class Attach(Task):
+	description = 'Attaching the volume'
+	phase = phases.volume_creation
+
+	def run(self, info):
+		info.volume.attach()
+
+
+class Detach(Task):
+	description = 'Detaching the volume'
+	phase = phases.volume_unmounting
+
+	def run(self, info):
+		info.volume.detach()
+
+
+class Delete(Task):
+	description = 'Deleting the volume'
+	phase = phases.cleaning
+
+	def run(self, info):
+		info.volume.delete()

manifests/ec2-ebs-pvm.manifest.json

@@ -23,8 +23,14 @@
 	},
 	"volume": {
 		"backing":    "ebs",
-		"filesystem": "ext4",
+		"partition_table": {
-		"size":       8192
+			"type": null,
+			"partitions": [
+				{"size":       8192,
+				 "filesystem": "ext4",
+				 "flags":      ["root"]}
+			]
+		}
 	},
 	"plugins": {
 		"admin_user": {

manifests/virtualbox.json

@@ -1,41 +0,0 @@
-{
-	"provider"      : "virtualbox",
-	"virtualization": "ide",
-
-	"bootstrapper": {
-		"mount_dir": "/mnt/target",
-		"mirror"      : "http://ftp.fr.debian.org/debian/"
-	},
-	"image": {
-		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
-		"description": "Debian {release} {architecture} ({virtualization})"
-	},
-	"system": {
-		"release"     : "wheezy",
-		"architecture": "amd64",
-		"timezone"    : "UTC",
-		"locale"      : "en_US",
-		"charmap"     : "UTF-8"
-	},
-	"volume": {
-		"backing"   : "raw",
-		"filesystem": "ext4",
-		"size"      : 1024,
-		"loopback_dir" : "/tmp"
-	},
-	"plugins": {
-		"user_packages": {
-			"enabled": true,
-			"repo": [ "apache2" ],
-			"local": []
-		},
-		"root_password": {
-			"enabled": true,
-			"password": "test"
-                },
-		"convert_image": {
-			"enabled": true,
-			"format": "vdi"
-		}
-	}
-}

manifests/virtualbox.manifest.json

@@ -0,0 +1,50 @@
+{
+	"provider":       "virtualbox",
+	"virtualization": "ide",
+
+	"bootstrapper": {
+		"mount_dir": "/mnt/target",
+		"mirror":    "http://http.debian.net/debian/"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} ({virtualization})"
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing":         "raw",
+		"loopback_dir":    "/tmp",
+		"partitions": {
+			"boot": {
+				"size":       12,
+				"filesystem": "ext2"
+			},
+			"root": {
+				"size":       812,
+				"filesystem": "ext4"
+			},
+			"swap": {"size": 200}
+		}
+	},
+	"plugins": {
+		"user_packages": {
+			"enabled": true,
+			"repo":    [ "apache2" ],
+			"local":   []
+		},
+		"root_password": {
+			"enabled":  true,
+			"password": "test"
+		},
+		"convert_image": {
+			"enabled": true,
+			"format":  "vdi"
+		}
+	}
+}

plugins/prebootstrapped/__init__.py

@@ -4,35 +4,31 @@ from tasks import CreateFromSnapshot
 from tasks import CreateFromImage
 from providers.ec2.tasks import ebs
 from common.tasks import loopback
+from common.tasks import volume
 from common.tasks import bootstrap
 from common.tasks import filesystem
-from common.tasks import parted
+from common.tasks import partitioning
 
 
 def tasks(tasklist, manifest):
 	settings = manifest.plugins['prebootstrapped']
+	skip_tasks = [filesystem.Format,
+	              partitioning.PartitionVolume,
+	              filesystem.TuneVolumeFS,
+	              filesystem.AddXFSProgs,
+	              filesystem.CreateBootMountDir,
+	              bootstrap.MakeTarball,
+	              bootstrap.Bootstrap]
 	if manifest.volume['backing'] == 'ebs':
 		if 'snapshot' in settings and settings['snapshot'] is not None:
 			tasklist.replace(ebs.Create, CreateFromSnapshot())
-			tasklist.remove(filesystem.FormatVolume,
+			tasklist.remove(*skip_tasks)
-			                filesystem.TuneVolumeFS,
-			                filesystem.AddXFSProgs,
-			                bootstrap.MakeTarball,
-			                bootstrap.Bootstrap)
 		else:
 			tasklist.add(Snapshot())
 	else:
 		if 'image' in settings and settings['image'] is not None:
-			tasklist.add(CreateFromImage())
+			tasklist.replace(loopback.Create, CreateFromImage())
-			tasklist.remove(loopback.Create,
+			tasklist.remove(*skip_tasks)
-			                loopback.CreateQemuImg,
-			                parted.PartitionVolume,
-			                parted.FormatPartitions,
-			                filesystem.FormatVolume,
-			                filesystem.TuneVolumeFS,
-			                filesystem.AddXFSProgs,
-			                bootstrap.MakeTarball,
-			                bootstrap.Bootstrap)
 		else:
 			tasklist.add(CopyImage())
 
@@ -45,9 +41,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 			tasklist.add(counter())
 
 	if manifest.volume['backing'] == 'ebs':
-		counter_task(CreateFromSnapshot, ebs.Delete)
+		counter_task(CreateFromSnapshot, volume.Delete)
 	else:
-		counter_task(CreateFromImage, loopback.Delete)
+		counter_task(CreateFromImage, volume.Delete)
 
 
 def validate_manifest(data, schema_validate):

plugins/prebootstrapped/tasks.py

@@ -1,7 +1,7 @@
 from base import Task
 from common import phases
 from providers.ec2.tasks import ebs
-from common.tasks import loopback
+from common.tasks import volume
 from common.tasks import bootstrap
 import time
 import logging
@@ -22,18 +22,25 @@ class Snapshot(ebs.Snapshot):
 class CreateFromSnapshot(Task):
 	description = 'Creating EBS volume from a snapshot'
 	phase = phases.volume_creation
-	before = [ebs.Attach]
+	before = [volume.Attach]
 
 	def run(self, info):
 		volume_size = int(info.manifest.volume['size'] / 1024)
 		snapshot = info.manifest.plugins['prebootstrapped']['snapshot']
-		info.volume = info.connection.create_volume(volume_size,
+		info.volume.volume = info.connection.create_volume(volume_size,
-		                                            info.host['availabilityZone'],
+		                                                   info.host['availabilityZone'],
-		                                            snapshot=snapshot)
+		                                                   snapshot=snapshot)
 		while info.volume.volume_state() != 'available':
 			time.sleep(5)
 			info.volume.update()
 
+		info.volume.force_state('detached_fmt')
+		partitions_state = 'formatted'
+		if 'partitions' in info.manifest.volume:
+			partitions_state = 'unmapped_fmt'
+		for partition in info.volume.partition_map.partitions:
+			partition.force_state(partitions_state)
+
 
 class CopyImage(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
@@ -45,7 +52,7 @@ class CopyImage(Task):
 		from shutil import copyfile
 		loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id)
 		image_copy_path = os.path.join('/tmp', loopback_backup_name)
-		copyfile(info.loopback_file, image_copy_path)
+		copyfile(info.volume.image_path, image_copy_path)
 		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path)
 		log.info(msg)
 
@@ -53,12 +60,19 @@ class CopyImage(Task):
 class CreateFromImage(Task):
 	description = 'Creating loopback image from a copy'
 	phase = phases.volume_creation
-	before = [loopback.Attach]
+	before = [volume.Attach]
 
 	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
 		import os.path
-		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
-		loopback_backup_path = info.manifest.plugins['prebootstrapped']['image']
 		from shutil import copyfile
-		copyfile(loopback_backup_path, info.loopback_file)
+		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
+		info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		loopback_backup_path = info.manifest.plugins['prebootstrapped']['image']
+		copyfile(loopback_backup_path, info.volume.image_path)
+
+		info.volume.force_state('detached_fmt')
+		partitions_state = 'formatted'
+		if 'partitions' in info.manifest.volume:
+			partitions_state = 'unmapped_fmt'
+		for partition in info.volume.partition_map.partitions:
+			partition.force_state(partitions_state)

providers/ec2/__init__.py

@@ -6,6 +6,7 @@ from tasks import connection
 from tasks import host
 from common.tasks import host as common_host
 from tasks import ami
+from common.tasks import volume
 from tasks import ebs
 from common.tasks import loopback
 from common.tasks import filesystem
@@ -49,7 +50,7 @@ def tasks(tasklist, manifest):
 	             apt.AptSources(),
 	             apt.AptUpgrade(),
 	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
+	             filesystem.FStab(),
 	             common_boot.BlackListModules(),
 	             common_boot.DisableGetTTYs(),
 	             security.EnableShadowConfig(),
@@ -77,16 +78,16 @@ def tasks(tasklist, manifest):
 		tasklist.add(bootstrap.MakeTarball())
 
 	backing_specific_tasks = {'ebs': [ebs.Create(),
-	                                  ebs.Attach(),
+	                                  volume.Attach(),
-	                                  ebs.Detach(),
+	                                  volume.Detach(),
 	                                  ebs.Snapshot(),
-	                                  ebs.Delete()],
+	                                  volume.Delete()],
 	                          's3': [loopback.Create(),
-	                                 loopback.Attach(),
+	                                 volume.Attach(),
-	                                 loopback.Detach(),
+	                                 volume.Detach(),
 	                                 ami.BundleImage(),
 	                                 ami.UploadImage(),
-	                                 loopback.Delete(),
+	                                 volume.Delete(),
 	                                 ami.RemoveBundle()]}
 	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
 
@@ -105,11 +106,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 			tasklist.add(counter())
 
 	if manifest.volume['backing'].lower() == 'ebs':
-		counter_task(ebs.Create, ebs.Delete)
+		counter_task(ebs.Create, volume.Delete)
-		counter_task(ebs.Attach, ebs.Detach)
+		counter_task(volume.Attach, volume.Detach)
 	if manifest.volume['backing'].lower() == 's3':
-		counter_task(loopback.Create, loopback.Delete)
+		counter_task(loopback.Create, volume.Delete)
-		counter_task(loopback.Attach, loopback.Detach)
+		counter_task(volume.Attach, volume.Detach)
 	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
 	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)

providers/ec2/tasks/ebs.py

@@ -1,60 +1,15 @@
 from base import Task
 from common import phases
-from common.exceptions import TaskError
+from common.tasks import volume
-from common.tasks.filesystem import UnmountVolume
-import time
 
 
 class Create(Task):
-	description = 'Creating an EBS volume for bootstrapping'
+	description = 'Creating the EBS volume'
 	phase = phases.volume_creation
+	before = [volume.Attach]
 
 	def run(self, info):
-		info.volume = info.connection.create_volume(info.manifest.ebs_volume_size,
+		info.volume.create(info.connection, info.host['availabilityZone'])
-		                                            info.host['availabilityZone'])
-		while info.volume.volume_state() != 'available':
-			time.sleep(5)
-			info.volume.update()
-
-
-class Attach(Task):
-	description = 'Attaching the EBS volume'
-	phase = phases.volume_creation
-	after = [Create]
-
-	def run(self, info):
-		def char_range(c1, c2):
-			"""Generates the characters from `c1` to `c2`, inclusive."""
-			for c in xrange(ord(c1), ord(c2) + 1):
-				yield chr(c)
-
-		import os.path
-		info.bootstrap_device = {}
-		for letter in char_range('f', 'z'):
-			dev_path = os.path.join('/dev', 'xvd' + letter)
-			if not os.path.exists(dev_path):
-				info.bootstrap_device['path'] = dev_path
-				info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter)
-				break
-		if 'path' not in info.bootstrap_device:
-			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
-
-		info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path'])
-		while info.volume.attachment_state() != 'attached':
-			time.sleep(2)
-			info.volume.update()
-
-
-class Detach(Task):
-	description = 'Detaching the EBS volume'
-	phase = phases.volume_unmounting
-	after = [UnmountVolume]
-
-	def run(self, info):
-		info.volume.detach()
-		while info.volume.attachment_state() is not None:
-			time.sleep(2)
-			info.volume.update()
 
 
 class Snapshot(Task):
@@ -62,20 +17,4 @@ class Snapshot(Task):
 	phase = phases.image_registration
 
 	def run(self, info):
-		info.snapshot = info.volume.create_snapshot()
+		info.snapshot = info.volume.snapshot()
-		while info.snapshot.status != 'completed':
-			time.sleep(2)
-			info.snapshot.update()
-
-
-class Delete(Task):
-	description = 'Deleting the EBS volume'
-	phase = phases.cleaning
-
-	def run(self, info):
-		info.volume.delete()
-		del info.volume
-
-
-class VolumeError(TaskError):
-	pass

providers/ec2/volume.py

@@ -0,0 +1,56 @@
+from base.fs.volume import Volume
+from base.fs.exceptions import VolumeError
+import time
+
+
+class EBSVolume(Volume):
+
+	volume = None
+
+	def create(self, conn, zone):
+		super(EBSVolume, self).create(self)
+		import math
+		# TODO: Warn if volume size is not a multiple of 1024
+		size = int(math.ceil(self.partition_map.get_volume_size() / 1024))
+		self.volume = conn.create_volume(size, zone)
+		while self.volume.volume_state() != 'available':
+			time.sleep(5)
+			self.volume.update()
+		self.created = True
+
+	def attach(self, instance_id):
+		super(EBSVolume, self).attach(self)
+		import os.path
+		import string
+		for letter in string.ascii_lowercase:
+			dev_path = os.path.join('/dev', 'xvd' + letter)
+			if not os.path.exists(dev_path):
+				self.device_path = dev_path
+				self.ec2_device_path = os.path.join('/dev', 'sd' + letter)
+				break
+
+		if self.device_path is None:
+			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
+
+		self.volume.attach(instance_id, self.ec2_device_path)
+		while self.volume.attachment_state() != 'attached':
+			time.sleep(2)
+			self.volume.update()
+
+	def detach(self):
+		super(EBSVolume, self).detach(self)
+		self.volume.detach()
+		while self.volume.attachment_state() is not None:
+			time.sleep(2)
+			self.volume.update()
+
+	def delete(self):
+		super(EBSVolume, self).delete(self)
+		self.volume.delete()
+
+	def snapshot(self):
+		snapshot = self.volume.create_snapshot()
+		while snapshot.status != 'completed':
+			time.sleep(2)
+			snapshot.update()
+		return snapshot

providers/kvm/__init__.py

@@ -44,7 +44,7 @@ def tasks(tasklist, manifest):
 	             apt.AptSources(),
 	             apt.AptUpgrade(),
 	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
+	             filesystem.FStab(),
 	             common_boot.BlackListModules(),
 	             common_boot.DisableGetTTYs(),
 	             security.EnableShadowConfig(),

providers/virtualbox/__init__.py

@@ -2,8 +2,9 @@ from manifest import Manifest
 from tasks import packages
 from common.tasks import packages as common_packages
 from common.tasks import host
+from common.tasks import volume as volume_tasks
 from common.tasks import loopback
-from common.tasks import parted
+from common.tasks import partitioning
 from common.tasks import filesystem
 from common.tasks import bootstrap
 from common.tasks import locale
@@ -14,7 +15,6 @@ from common.tasks import security
 from common.tasks import network
 from common.tasks import initd
 from common.tasks import cleanup
-from common.tasks import loopback
 
 
 def initialize():
@@ -28,13 +28,13 @@ def tasks(tasklist, manifest):
 	             common_packages.ImagePackages(),
 	             host.CheckPackages(),
 
-	             loopback.CreateQemuImg(),
+	             loopback.Create(),
-	             loopback.Attach(),
+	             volume_tasks.Attach(),
-	             parted.PartitionVolume(),
+	             partitioning.PartitionVolume(),
-	             parted.MapPartitions(),
+	             partitioning.MapPartitions(),
-	             parted.FormatPartitions(),
+	             filesystem.Format(),
 	             filesystem.CreateMountDir(),
-	             filesystem.MountVolume(),
+	             filesystem.MountRoot(),
 
 	             bootstrap.Bootstrap(),
 	             filesystem.MountSpecials(),
@@ -44,7 +44,7 @@ def tasks(tasklist, manifest):
 	             apt.AptSources(),
 	             apt.AptUpgrade(),
 	             boot.ConfigureGrub(),
-	             filesystem.ModifyFstab(),
+	             filesystem.FStab(),
 	             common_boot.BlackListModules(),
 	             common_boot.DisableGetTTYs(),
 	             security.EnableShadowConfig(),
@@ -63,19 +63,32 @@ def tasks(tasklist, manifest):
 	             apt.EnableDaemonAutostart(),
 	             filesystem.UnmountSpecials(),
 
-	             filesystem.UnmountVolume(),
+	             filesystem.UnmountRoot(),
-	             parted.UnmapPartitions(),
+	             partitioning.UnmapPartitions(),
-	             loopback.Detach(),
+	             volume_tasks.Detach(),
-	             filesystem.DeleteMountDir())
+	             filesystem.DeleteMountDir(),
+	             volume_tasks.Delete())
 
 	if manifest.bootstrapper['tarball']:
 		tasklist.add(bootstrap.MakeTarball())
 
-	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
+	partitions = manifest.volume['partitions']
-	                             'ext2': [filesystem.TuneVolumeFS()],
+	import re
-	                             'ext3': [filesystem.TuneVolumeFS()],
+	for key in ['boot', 'root']:
-	                             'ext4': [filesystem.TuneVolumeFS()]}
+		if key not in partitions:
-	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
+			continue
+		if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None:
+			tasklist.add(filesystem.TuneVolumeFS())
+			break
+	for key in ['boot', 'root']:
+		if key not in partitions:
+			continue
+		if partitions[key]['filesystem'] == 'xfs':
+			tasklist.add(filesystem.AddXFSProgs())
+			break
+
+	if 'boot' in manifest.volume['partitions']:
+		tasklist.add(filesystem.MountBoot(), filesystem.UnmountBoot())
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):
@@ -85,8 +98,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 		if task in completed and counter not in completed:
 			tasklist.add(counter())
 
+	counter_task(loopback.Create, volume_tasks.Delete)
 	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
-	counter_task(parted.MapPartitions, parted.UnmapPartitions)
+	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
-	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
+	counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
-	counter_task(loopback.Attach, loopback.Detach)
+	counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
+	counter_task(volume_tasks.Attach, volume_tasks.Detach)

providers/virtualbox/manifest-schema.json

@@ -9,13 +9,13 @@
 				"backing": {
 					"type": "string",
 					"enum": ["raw", "qcow2"]
-				},
-				"filesystem": {
-					"type": "string",
-					"enum": ["ext2", "ext3", "ext4", "xfs"]
 				}
+				// "filesystem": {
+				// 	"type": "string",
+				// 	"enum": ["ext2", "ext3", "ext4", "xfs"]
+				// }
 			},
-			"required": ["backing", "filesystem"]
+			"required": ["backing"]
 		}
 	},
 	"required": ["volume"]

providers/virtualbox/tasks/boot.py

@@ -1,34 +1,47 @@
 from base import Task
 from common import phases
+from common.tasks import apt
+from common.fs.loopbackvolume import LoopbackVolume
 
 
 class ConfigureGrub(Task):
 	description = 'Configuring grub'
 	phase = phases.system_modification
+	after = [apt.AptUpgrade]
 
 	def run(self, info):
-		import stat
+		import os
-		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
-		             stat.S_IRGRP                | stat.S_IXGRP |
-		             stat.S_IROTH                | stat.S_IXOTH)
-		import os.path
-		device_map_path = os.path.join(info.root, 'boot/grub/device.map')
-		with open(device_map_path, 'w') as device_map:
-			device_map.write('(hd0)   /dev/sda\n')
-
 		from common.tools import log_check_call
 
-		from shutil import copy
+		boot_dir = os.path.join(info.root, 'boot')
-		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux'))
+		grub_dir = os.path.join(boot_dir, 'grub')
-		script_dst = os.path.join(info.root, 'etc/grub.d/10_linux')
+
-		copy(script_src, script_dst)
+		if isinstance(info.volume, LoopbackVolume):
-		os.chmod(script_dst, rwxr_xr_x)
+			info.volume.unmount()
-		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header'))
+			info.volume.unmap()
-		script_dst = os.path.join(info.root, 'etc/grub.d/00_header')
+			info.volume.link_dm_node()
-		copy(script_src, script_dst)
+			info.volume.map()
-		os.chmod(script_dst, rwxr_xr_x)
+			info.volume.mount_root(info.root)
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u'])
+			info.volume.mount_boot()
-		# Install grub in mbr
+			info.volume.mount_specials()
-		log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/",
+		[device_path] = log_check_call(['readlink', '-f', info.volume.device_path])
-		                info.bootstrap_device['path']])
+		device_map_path = os.path.join(grub_dir, 'device.map')
+		with open(device_map_path, 'w') as device_map:
+			device_map.write('(hd0) {device_path}\n'.format(device_path=device_path))
+
+		# Install grub
+		log_check_call(['/usr/sbin/grub-install',
+		                '--root-directory=' + info.root,
+		                '--boot-directory=' + boot_dir,
+		                device_path])
 		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+		# log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u'])
+
+		if isinstance(info.volume, LoopbackVolume):
+			info.volume.unmount()
+			info.volume.unmap()
+			info.volume.unlink_dm_node()
+			info.volume.map()
+			info.volume.mount_root(info.root)
+			info.volume.mount_boot()
+			info.volume.mount_specials()

providers/virtualbox/tasks/packages.py

@@ -12,7 +12,7 @@ class HostPackages(Task):
 
 	def run(self, info):
 		info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx'])
-		if info.manifest.volume['filesystem'] == 'xfs':
+		if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions):
 			info.host_packages.add('xfsprogs')
 
 

providers/virtualbox/volume.py

@@ -0,0 +1,11 @@
+from common.fs.loopbackvolume import LoopbackVolume
+from common.tools import log_check_call
+
+
+class VirtualBoxVolume(LoopbackVolume):
+
+	def create(self, image_path):
+		super(VirtualBoxVolume, self).create(self)
+		self.image_path = image_path
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M'])
+		self.created = True