Partition volumes by sectors instead of bytes

This allows for finer grained control over the partition sizes and gaps

bootstrapvz/base/fs/__init__.py

@@ -9,27 +9,33 @@ def load_volume(data, bootloader):
 	:return: The volume that represents all information pertaining to the volume we bootstrap on.
 	:rtype: Volume
 	"""
-	# Create a mapping between valid partition maps in the manifest and their corresponding classes
+	# Map valid partition maps in the manifest and their corresponding classes
 	from partitionmaps.gpt import GPTPartitionMap
 	from partitionmaps.msdos import MSDOSPartitionMap
 	from partitionmaps.none import NoPartitions
-	partition_maps = {'none': NoPartitions,
+	partition_map = {'none': NoPartitions,
-	                  'gpt': GPTPartitionMap,
+	                 'gpt': GPTPartitionMap,
-	                  'msdos': MSDOSPartitionMap,
+	                 'msdos': MSDOSPartitionMap,
-	                  }
+	                 }.get(data['partitions']['type'])
-	# Instantiate the partition map
-	partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader)
 
-	# Create a mapping between valid volume backings in the manifest and their corresponding classes
+	# Map valid volume backings in the manifest and their corresponding classes
 	from bootstrapvz.common.fs.loopbackvolume import LoopbackVolume
 	from bootstrapvz.providers.ec2.ebsvolume import EBSVolume
 	from bootstrapvz.common.fs.virtualdiskimage import VirtualDiskImage
 	from bootstrapvz.common.fs.virtualmachinedisk import VirtualMachineDisk
-	volume_backings = {'raw': LoopbackVolume,
+	volume_backing = {'raw': LoopbackVolume,
-	                   's3':  LoopbackVolume,
+	                  's3':  LoopbackVolume,
-	                   'vdi': VirtualDiskImage,
+	                  'vdi': VirtualDiskImage,
-	                   'vmdk': VirtualMachineDisk,
+	                  'vmdk': VirtualMachineDisk,
-	                   'ebs': EBSVolume
+	                  'ebs': EBSVolume
-	                   }
+	                  }.get(data['backing'])
+
+	# Instantiate the partition map
+	from bootstrapvz.common.bytes import Bytes
+	# Only operate with a physical sector size of 512 bytes for now,
+	# not sure if we can change that for some of the virtual disks
+	sector_size = Bytes('512B')
+	partition_map = partition_map(data['partitions'], sector_size, bootloader)
+
 	# Create the volume with the partition map as an argument
-	return volume_backings.get(data['backing'])(partition_map)
+	return volume_backing(partition_map)

bootstrapvz/base/fs/partitionmaps/abstract.py

@@ -1,5 +1,6 @@
 from abc import ABCMeta
 from abc import abstractmethod
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 from bootstrapvz.common.fsm_proxy import FSMProxy
 from ..exceptions import PartitionError
@@ -86,6 +87,8 @@ class AbstractPartitionMap(FSMProxy):
 
 			# Check if any partition was not mapped
 			for idx, partition in enumerate(self.partitions):
+				if isinstance(partition, PartitionGap):
+					continue
 				if partition.fsm.current not in ['mapped', 'formatted']:
 					raise PartitionError('kpartx did not map partition #' + str(idx + 1))
 
@@ -111,6 +114,8 @@ class AbstractPartitionMap(FSMProxy):
 		volume = event.volume
 		# Run through all partitions before unmapping and make sure they can all be unmapped
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			if partition.fsm.cannot('unmap'):
 				msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
 				raise PartitionError(msg)
@@ -118,4 +123,6 @@ class AbstractPartitionMap(FSMProxy):
 		log_check_call(['kpartx', '-ds', volume.device_path])
 		# Call unmap on all partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.unmap()

bootstrapvz/base/fs/partitionmaps/gpt.py

@@ -1,6 +1,7 @@
 from abstract import AbstractPartitionMap
 from ..partitions.gpt import GPTPartition
 from ..partitions.gpt_swap import GPTSwapPartition
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -8,12 +9,14 @@ class GPTPartitionMap(AbstractPartitionMap):
 	"""Represents a GPT partition map
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# List of partitions
 		self.partitions = []
 
@@ -26,27 +29,27 @@ class GPTPartitionMap(AbstractPartitionMap):
 		# next partition.
 		if bootloader == 'grub':
 			from ..partitions.unformatted import UnformattedPartition
-			self.grub_boot = UnformattedPartition(Bytes('1007KiB'), last_partition())
+			self.grub_boot = UnformattedPartition(Sectors('1007KiB', sector_size), last_partition())
 			# Mark the partition as a bios_grub partition
 			self.grub_boot.flags.append('bios_grub')
 			self.partitions.append(self.grub_boot)
 
 		# The boot and swap partitions are optional
 		if 'boot' in data:
-			self.boot = GPTPartition(Bytes(data['boot']['size']),
+			self.boot = GPTPartition(Sectors(data['boot']['size'], sector_size),
 			                         data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                         'boot', last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition())
+			self.swap = GPTSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
 			self.partitions.append(self.swap)
-		self.root = GPTPartition(Bytes(data['root']['size']),
+		self.root = GPTPartition(Sectors(data['root']['size'], sector_size),
 		                         data['root']['filesystem'], data['root'].get('format_command', None),
 		                         'root', last_partition())
 		self.partitions.append(self.root)
 
 		# We need to move the first partition to make space for the gpt offset
-		gpt_offset = Bytes('17KiB')
+		gpt_offset = Sectors('17KiB', sector_size)
 		self.partitions[0].offset += gpt_offset
 
 		if hasattr(self, 'grub_boot'):
@@ -58,6 +61,10 @@ class GPTPartitionMap(AbstractPartitionMap):
 			# Avoid increasing the volume size because of gpt_offset
 			self.partitions[0].size -= gpt_offset
 
+		# Leave the last sector unformatted
+		self.partitions[-1].size -= 1
+		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
+
 		super(GPTPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
@@ -70,4 +77,6 @@ class GPTPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'gpt'])
 		# Create the partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.create(volume)

bootstrapvz/base/fs/partitionmaps/msdos.py

@@ -1,6 +1,7 @@
 from abstract import AbstractPartitionMap
 from ..partitions.msdos import MSDOSPartition
 from ..partitions.msdos_swap import MSDOSSwapPartition
+from ..partitions.gap import PartitionGap
 from bootstrapvz.common.tools import log_check_call
 
 
@@ -9,12 +10,14 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 	Sometimes also called MBR (but that confuses the hell out of me, so ms-dos it is)
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# List of partitions
 		self.partitions = []
 
@@ -24,14 +27,14 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 
 		# The boot and swap partitions are optional
 		if 'boot' in data:
-			self.boot = MSDOSPartition(Bytes(data['boot']['size']),
+			self.boot = MSDOSPartition(Sectors(data['boot']['size'], sector_size),
 			                           data['boot']['filesystem'], data['boot'].get('format_command', None),
 			                           last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition())
+			self.swap = MSDOSSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition())
 			self.partitions.append(self.swap)
-		self.root = MSDOSPartition(Bytes(data['root']['size']),
+		self.root = MSDOSPartition(Sectors(data['root']['size'], sector_size),
 		                           data['root']['filesystem'], data['root'].get('format_command', None),
 		                           last_partition())
 		self.partitions.append(self.root)
@@ -44,13 +47,17 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		# The MBR offset is included in the grub offset, so if we don't use grub
 		# we should reduce the size of the first partition and move it by only 512 bytes.
 		if bootloader == 'grub':
-			offset = Bytes('2MiB')
+			offset = Sectors('2MiB', sector_size)
 		else:
-			offset = Bytes('512B')
+			offset = Sectors('512B', sector_size)
 
 		self.partitions[0].offset += offset
 		self.partitions[0].size -= offset
 
+		# Leave the last sector unformatted
+		self.partitions[-1].size -= 1
+		self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition()))
+
 		super(MSDOSPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):
@@ -61,4 +68,6 @@ class MSDOSPartitionMap(AbstractPartitionMap):
 		                '--', 'mklabel', 'msdos'])
 		# Create the partitions
 		for partition in self.partitions:
+			if isinstance(partition, PartitionGap):
+				continue
 			partition.create(volume)

bootstrapvz/base/fs/partitionmaps/none.py

@@ -7,14 +7,16 @@ class NoPartitions(object):
 	simply always deal with partition maps and then let the base abstract that away.
 	"""
 
-	def __init__(self, data, bootloader):
+	def __init__(self, data, sector_size, bootloader):
 		"""
 		:param dict data: volume.partitions part of the manifest
+		:param int sector_size: Sectorsize of the volume
 		:param str bootloader: Name of the bootloader we will use for bootstrapping
 		"""
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
+
 		# In the NoPartitions partitions map we only have a single 'partition'
-		self.root = SinglePartition(Bytes(data['root']['size']),
+		self.root = SinglePartition(Sectors(data['root']['size'], sector_size),
 		                            data['root']['filesystem'], data['root'].get('format_command', None))
 		self.partitions = [self.root]
 
@@ -29,7 +31,7 @@ class NoPartitions(object):
 		"""Returns the total size the partitions occupy
 
 		:return: The size of all the partitions
-		:rtype: Bytes
+		:rtype: Sectors
 		"""
 		return self.root.get_end()
 

bootstrapvz/base/fs/partitions/base.py

@@ -28,9 +28,9 @@ class BasePartition(AbstractPartition):
 		# By saving the previous partition we have
 		# a linked list that partitions can go backwards in to find the first partition.
 		self.previous = previous
-		from bootstrapvz.common.bytes import Bytes
+		from bootstrapvz.common.sectors import Sectors
-		# Initialize the offset to 0 bytes, may be changed later
+		# Initialize the offset to 0 sectors, may be changed later
-		self.offset = Bytes(0)
+		self.offset = Sectors(0, size.sector_size)
 		# List of flags that parted should put on the partition
 		self.flags = []
 		super(BasePartition, self).__init__(size, filesystem, format_command)

bootstrapvz/base/fs/partitions/gap.py

@@ -0,0 +1,17 @@
+from base import BasePartition
+
+
+class PartitionGap(BasePartition):
+	"""Represents a non-existent partition
+	A gap in the partitionmap
+	"""
+
+	# The states for our state machine. It can neither be create nor mapped.
+	events = []
+
+	def __init__(self, size, previous):
+		"""
+		:param Bytes size: Size of the partition
+		:param BasePartition previous: The partition that preceeds this one
+		"""
+		super(PartitionGap, self).__init__(size, None, None, previous)

bootstrapvz/common/exceptions.py

@@ -34,3 +34,7 @@ class NoMatchesError(Exception):
 
 class TooManyMatchesError(Exception):
 	pass
+
+
+class UnitError(Exception):
+	pass

bootstrapvz/common/fs/loopbackvolume.py

@@ -11,7 +11,7 @@ class LoopbackVolume(Volume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		size_opt = '--size={mib}M'.format(mib=self.size.get_qty_in('MiB'))
+		size_opt = '--size={mib}M'.format(mib=self.size.bytes.get_qty_in('MiB'))
 		log_check_call(['truncate', 'create', size_opt, self.image_path])
 
 	def _before_attach(self, e):

bootstrapvz/common/fs/qemuvolume.py

@@ -8,7 +8,7 @@ class QEMUVolume(LoopbackVolume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		vol_size = str(self.size.get_qty_in('MiB')) + 'M'
+		vol_size = str(self.size.bytes.get_qty_in('MiB')) + 'M'
 		log_check_call(['qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size])
 
 	def _check_nbd_module(self):

bootstrapvz/common/tasks/filesystem.py

@@ -25,9 +25,11 @@ class Format(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
+		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		for partition in info.volume.partition_map.partitions:
-			if not isinstance(partition, UnformattedPartition):
+			if isinstance(partition, (UnformattedPartition, PartitionGap)):
-				partition.format()
+				continue
+			partition.format()
 
 
 class TuneVolumeFS(Task):
@@ -38,12 +40,14 @@ class TuneVolumeFS(Task):
 	@classmethod
 	def run(cls, info):
 		from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition
+		from bootstrapvz.base.fs.partitions.gap import PartitionGap
 		import re
 		# Disable the time based filesystem check
 		for partition in info.volume.partition_map.partitions:
-			if not isinstance(partition, UnformattedPartition):
+			if isinstance(partition, (UnformattedPartition, PartitionGap)):
-				if re.match('^ext[2-4]$', partition.filesystem) is not None:
+				continue
-					log_check_call(['tune2fs', '-i', '0', partition.device_path])
+			if re.match('^ext[2-4]$', partition.filesystem) is not None:
+				log_check_call(['tune2fs', '-i', '0', partition.device_path])
 
 
 class AddXFSProgs(Task):

bootstrapvz/plugins/prebootstrapped/tasks.py

@@ -34,7 +34,7 @@ class CreateFromSnapshot(Task):
 	@classmethod
 	def run(cls, info):
 		snapshot = info.manifest.plugins['prebootstrapped']['snapshot']
-		ebs_volume = info._ec2['connection'].create_volume(info.volume.size.get_qty_in('GiB'),
+		ebs_volume = info._ec2['connection'].create_volume(info.volume.size.bytes.get_qty_in('GiB'),
 		                                                   info._ec2['host']['availabilityZone'],
 		                                                   snapshot=snapshot)
 		while ebs_volume.volume_state() != 'available':

bootstrapvz/plugins/vagrant/tasks.py

@@ -184,7 +184,7 @@ class PackageBox(Task):
 		# VHDURI = "http://go.microsoft.com/fwlink/?LinkId=137171"
 		volume_uuid = info.volume.get_uuid()
 		[disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces)
-		attr(disk, 'ovf:capacity', info.volume.size.get_qty_in('B'))
+		attr(disk, 'ovf:capacity', info.volume.size.bytes.get_qty_in('B'))
 		attr(disk, 'ovf:format', info.volume.ovf_uri)
 		attr(disk, 'ovf:uuid', volume_uuid)
 

bootstrapvz/providers/ec2/ebsvolume.py

@@ -11,7 +11,7 @@ class EBSVolume(Volume):
 	def _before_create(self, e):
 		conn = e.connection
 		zone = e.zone
-		size = self.size.get_qty_in('GiB')
+		size = self.size.bytes.get_qty_in('GiB')
 		self.volume = conn.create_volume(size, zone)
 		while self.volume.volume_state() != 'available':
 			time.sleep(5)

bootstrapvz/providers/ec2/tasks/ami.py

@@ -109,7 +109,7 @@ class RegisterAMI(Task):
 			from boto.ec2.blockdevicemapping import BlockDeviceType
 			from boto.ec2.blockdevicemapping import BlockDeviceMapping
 			block_device = BlockDeviceType(snapshot_id=info._ec2['snapshot'].id, delete_on_termination=True,
-			                               size=info.volume.size.get_qty_in('GiB'))
+			                               size=info.volume.size.bytes.get_qty_in('GiB'))
 			registration_params['block_device_map'] = BlockDeviceMapping()
 			registration_params['block_device_map'][root_dev_name] = block_device