MBR-gap, grub and GPT offset are now subtracted

automatically from the first available partition.
There is no need to fiddle with sizes while keeping the offsets in mind any longer.
Introduced Bytes() class which makes it a lot easier to handle size units.

base/fs/partitionmaps/gpt.py

@@ -7,28 +7,36 @@ from common.tools import log_check_call
 class GPTPartitionMap(AbstractPartitionMap):
 
 	def __init__(self, data, bootloader):
+		from common.bytes import Bytes
 		self.partitions = []
 
 		def last_partition():
 			return self.partitions[-1] if len(self.partitions) > 0 else None
 
+		gpt_offset = Bytes('17KiB')
+
 		if bootloader == 'grub':
 			from ..partitions.unformatted import UnformattedPartition
-			self.grub_boot = UnformattedPartition(2, last_partition())
+			self.grub_boot = UnformattedPartition(Bytes('2MiB'), last_partition())
 			self.grub_boot.flags.append('bios_grub')
 			self.partitions.append(self.grub_boot)
 
 		if 'boot' in data:
-			self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', last_partition())
+			self.boot = GPTPartition(Bytes(data['boot']['size']), data['boot']['filesystem'],
+			                         'boot', last_partition())
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = GPTSwapPartition(data['swap']['size'], last_partition())
+			self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition())
 			self.partitions.append(self.swap)
-		self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', last_partition())
+		self.root = GPTPartition(Bytes(data['root']['size']), data['root']['filesystem'], 'root', last_partition())
 		self.partitions.append(self.root)
 
-		if bootloader == 'extlinux':
-			getattr(self, 'boot', self.root).flags.append('legacy_boot')
+		if hasattr(self, 'grub_boot'):
+			self.partitions[1].size -= self.grub_boot.size
+
+		# Offset for GPT partitioning table
+		self.partitions[0].offset = gpt_offset
+		self.partitions[0].size -= self.partitions[0].offset
 
 		super(GPTPartitionMap, self).__init__(bootloader)
 

base/fs/partitionmaps/msdos.py

@@ -7,23 +7,29 @@ from common.tools import log_check_call
 class MSDOSPartitionMap(AbstractPartitionMap):
 
 	def __init__(self, data, bootloader):
+		from common.bytes import Bytes
 		self.partitions = []
 
 		def last_partition():
 			return self.partitions[-1] if len(self.partitions) > 0 else None
+
+		grub_offset = Bytes('2MiB')
+
 		if 'boot' in data:
-			self.boot = MSDOSPartition(data['boot']['size'], data['boot']['filesystem'], None)
+			self.boot = MSDOSPartition(Bytes(data['boot']['size']), data['boot']['filesystem'], None)
 			self.partitions.append(self.boot)
 		if 'swap' in data:
-			self.swap = MSDOSSwapPartition(data['swap']['size'], last_partition())
+			self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition())
 			self.partitions.append(self.swap)
-		self.root = MSDOSPartition(data['root']['size'], data['root']['filesystem'], last_partition())
+		self.root = MSDOSPartition(Bytes(data['root']['size']), data['root']['filesystem'], last_partition())
 		self.partitions.append(self.root)
 
 		getattr(self, 'boot', self.root).flags.append('boot')
 
 		if bootloader == 'grub':
-			self.partitions[0].offset = 2
+			self.partitions[0].offset = grub_offset
+			self.partitions[0].size -= self.partitions[0].offset
+
 		super(MSDOSPartitionMap, self).__init__(bootloader)
 
 	def _before_create(self, event):

base/fs/partitionmaps/none.py

@@ -4,12 +4,12 @@ from ..partitions.single import SinglePartition
 class NoPartitions(object):
 
 	def __init__(self, data, bootloader):
-		root = data['root']
-		self.root = SinglePartition(root['size'], root['filesystem'])
+		from common.bytes import Bytes
+		self.root = SinglePartition(Bytes(data['root']['size']), data['root']['filesystem'])
 		self.partitions = [self.root]
 
 	def is_blocking(self):
 		return self.root.fsm.current == 'mounted'
 
 	def get_total_size(self):
-		return self.root.size
+		return self.root.get_end()

base/fs/partitions/abstract.py

@@ -37,10 +37,10 @@ class AbstractPartition(FSMProxy):
 			del self.mount_dir
 
 	def __init__(self, size, filesystem):
-		self.size          = size
-		self.filesystem    = filesystem
-		self.device_path   = None
-		self.mounts        = {}
+		self.size        = size
+		self.filesystem  = filesystem
+		self.device_path = None
+		self.mounts      = {}
 
 		cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}}
 		super(AbstractPartition, self).__init__(cfg)

base/fs/partitions/base.py

@@ -16,7 +16,8 @@ class BasePartition(AbstractPartition):
 
 	def __init__(self, size, filesystem, previous):
 		self.previous = previous
-		self.offset = 0
+		from common.bytes import Bytes
+		self.offset = Bytes(0)
 		self.flags = []
 		super(BasePartition, self).__init__(size, filesystem)
 
@@ -40,7 +41,7 @@ class BasePartition(AbstractPartition):
 
 	def _before_create(self, e):
 		from common.tools import log_check_call
-		create_command = ('mkpart primary {start}MiB {end}MiB'
+		create_command = ('mkpart primary {start} {end}'
 		                  .format(start=str(self.get_start()),
 		                          end=str(self.get_end())))
 		log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path,

base/fs/partitions/single.py

@@ -4,4 +4,5 @@ from abstract import AbstractPartition
 class SinglePartition(AbstractPartition):
 
 	def get_start(self):
-		return 0
+		from common.bytes import Bytes
+		return Bytes(0)

base/fs/volume.py

@@ -55,7 +55,7 @@ class Volume(FSMProxy):
 		# The offset at which the volume should begin to be mapped in the new volume
 		start_sector = getattr(e, 'start_sector', 0)
 
-		sectors = getattr(e, 'sectors', self.size * 1024 * 1024 / 512 - start_sector)
+		sectors = getattr(e, 'sectors', int(self.size / 512) - start_sector)
 
 		table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}'
 		         .format(log_start_sec=logical_start_sector,

base/manifest-schema.json

@@ -105,6 +105,10 @@
 			"type":    "string",
 			"pattern": "^/[^\\0]+$"
 		},
+		"bytes": {
+			"type":    "string",
+			"pattern": "^\\d+([KMGT]i?B|B)$"
+		},
 		"no_partitions": {
 			"type": "object",
 			"properties": {
@@ -122,7 +126,7 @@
 				"root": { "$ref": "#/definitions/partition" },
 				"swap": {
 					"type": "object",
-					"properties": { "size": { "type": "integer", "minimum": 1 } },
+					"properties": { "size": { "$ref": "#/definitions/bytes" } },
 					"required": ["size"]
 				}
 			},
@@ -132,7 +136,7 @@
 		"partition": {
 			"type": "object",
 			"properties": {
-				"size": { "type": "integer", "minimum": 1 },
+				"size": { "$ref": "#/definitions/bytes" },
 				"filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] }
 			},
 			"required": ["size", "filesystem"]

common/bytes.py

@@ -0,0 +1,131 @@
+
+
+class Bytes(object):
+
+	units = {'B': 1,
+	         'KiB': 1024,
+	         'MiB': 1024*1024,
+	         'GiB': 1024*1024*1024,
+	         'TiB': 1024*1024*1024*1024,
+	         }
+
+	def __init__(self, qty):
+		if isinstance(qty, (int, long)):
+			self.qty = qty
+		else:
+			self.qty = Bytes.parse(qty)
+
+	@staticmethod
+	def parse(qty_str):
+		import re
+		regex = re.compile('^(?P<qty>\d+)(?P<unit>[KMGT]i?B|B)$')
+		parsed = regex.match(qty_str)
+		if parsed is None:
+			raise UnitError('Unable to parse {str}'.format(str=qty_str))
+
+		qty = int(parsed.group('qty'))
+		unit = parsed.group('unit')
+		if unit[0] in 'KMGT':
+			unit = unit[0] + 'iB'
+		byte_qty = qty * Bytes.units[unit]
+		return byte_qty
+
+	def get_qty_in(self, unit):
+		if unit[0] in 'KMGT':
+			unit = unit[0] + 'iB'
+		if unit not in Bytes.units:
+			raise UnitError('Unrecognized unit `{unit}\''.format(unit=Bytes.magnitude))
+		if self.qty % Bytes.units[unit] != 0:
+			msg = 'Unable to convert {qty} bytes to a whole number in {unit}'.format(qty=self.qty, unit=unit)
+			raise UnitError(msg)
+		return self.qty / Bytes.units[unit]
+
+	def __repr__(self):
+		converted = str(self.get_qty_in('B')) + 'B'
+		if self.qty == 0:
+			return converted
+		for unit in ['TiB', 'GiB', 'MiB', 'KiB']:
+			try:
+				converted = str(self.get_qty_in(unit)) + unit
+				break
+			except UnitError:
+				pass
+		return converted
+
+	def __str__(self):
+		return self.__repr__()
+
+	def __int__(self):
+		return self.qty
+
+	def __long__(self):
+		return self.qty
+
+	def __add__(self, other):
+		if not isinstance(other, Bytes):
+			raise UnitError('Can only add Bytes to Bytes')
+		return Bytes(self.qty + other.qty)
+
+	def __iadd__(self, other):
+		if not isinstance(other, Bytes):
+			raise UnitError('Can only add Bytes to Bytes')
+		self.qty += other.qty
+		return self
+
+	def __sub__(self, other):
+		if not isinstance(other, Bytes):
+			raise UnitError('Can only subtract Bytes from Bytes')
+		return Bytes(self.qty - other.qty)
+
+	def __isub__(self, other):
+		if not isinstance(other, Bytes):
+			raise UnitError('Can only subtract Bytes from Bytes')
+		self.qty -= other.qty
+		return self
+
+	def __mul__(self, other):
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only multiply Bytes with integers')
+		return Bytes(self.qty * other)
+
+	def __imul__(self, other):
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only multiply Bytes with integers')
+		self.qty *= other
+		return self
+
+	def __div__(self, other):
+		if isinstance(other, Bytes):
+			return self.qty / other.qty
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only divide Bytes with integers or Bytes')
+		return Bytes(self.qty / other)
+
+	def __idiv__(self, other):
+		if isinstance(other, Bytes):
+			self.qty /= other.qty
+		else:
+			if not isinstance(other, (int, long)):
+				raise UnitError('Can only divide Bytes with integers or Bytes')
+			self.qty /= other
+		return self
+
+	def __mod__(self, other):
+		if isinstance(other, Bytes):
+			return self.qty % other.qty
+		if not isinstance(other, (int, long)):
+			raise UnitError('Can only take modulus of Bytes with integers or Bytes')
+		return Bytes(self.qty % other)
+
+	def __imod__(self, other):
+		if isinstance(other, Bytes):
+			self.qty %= other.qty
+		else:
+			if not isinstance(other, (int, long)):
+				raise UnitError('Can only divide Bytes with integers or Bytes')
+			self.qty %= other
+		return self
+
+
+class UnitError(Exception):
+	pass

common/fs/loopbackvolume.py

@@ -11,7 +11,8 @@ class LoopbackVolume(Volume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M'])
+		vol_size = str(self.size.get_qty_in('MiB')) + 'M'
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, vol_size])
 
 	def _before_attach(self, e):
 		[self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path])

common/fs/qemuvolume.py

@@ -8,7 +8,8 @@ class QEMUVolume(LoopbackVolume):
 
 	def _before_create(self, e):
 		self.image_path = e.image_path
-		log_check_call(['/usr/bin/qemu-img', 'create', '-f', self.qemu_format, self.image_path, str(self.size) + 'M'])
+		vol_size = str(self.size.get_qty_in('MiB')) + 'M'
+		log_check_call(['/usr/bin/qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size])
 
 	def _check_nbd_module(self):
 		from base.fs.partitionmaps.none import NoPartitions

manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       8192,
+				"size":       "8GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       8192,
+				"size":       "8GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-ebs-debian-official-i386-pvm.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       8192,
+				"size":       "8GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       8192,
+				"size":       "8GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-ebs-partitioned.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "msdos",
 			"root": {
-				"size":       1022,
+				"size":       "1GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-ebs-single.manifest.json

@@ -29,7 +29,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       1024,
+				"size":       "1GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/ec2-s3.manifest.json

@@ -33,7 +33,7 @@
 		"partitions": {
 			"type": "none",
 			"root": {
-				"size":       1024,
+				"size":       "1GiB",
 				"filesystem": "ext4"
 			}
 		}

manifests/virtualbox-vagrant.manifest.json

@@ -22,14 +22,14 @@
 		"partitions": {
 			"type": "msdos",
 			"boot": {
-				"size":       64,
+				"size":       "64MiB",
 				"filesystem": "ext2"
 			},
 			"root": {
-				"size":       1854,
+				"size":       "1856MiB",
 				"filesystem": "ext4"
 			},
-			"swap": {"size": 128}
+			"swap": {"size": "128MiB"}
 		}
 	},
 	"plugins": {

manifests/virtualbox.manifest.json

@@ -22,14 +22,14 @@
 		"partitions": {
 			"type": "msdos",
 			"boot": {
-				"size":       32,
+				"size":       "32MiB",
 				"filesystem": "ext2"
 			},
 			"root": {
-				"size":       990,
+				"size":       "864MiB",
 				"filesystem": "ext4"
 			},
-			"swap": {"size": 128}
+			"swap": {"size": "128MiB"}
 		}
 	}
 }

plugins/prebootstrapped/tasks.py

@@ -33,9 +33,8 @@ class CreateFromSnapshot(Task):
 
 	@classmethod
 	def run(cls, info):
-		volume_size = int(info.volume.size / 1024)
 		snapshot = info.manifest.plugins['prebootstrapped']['snapshot']
-		ebs_volume = info.connection.create_volume(volume_size,
+		ebs_volume = info.connection.create_volume(info.volume.size.get_qty_in('GiB'),
 		                                           info.host['availabilityZone'],
 		                                           snapshot=snapshot)
 		while ebs_volume.volume_state() != 'available':

plugins/vagrant/tasks.py

@@ -163,7 +163,7 @@ class PackageBox(Task):
 		# VHDURI = "http://go.microsoft.com/fwlink/?LinkId=137171"
 		volume_uuid = info.volume.get_uuid()
 		[disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces)
-		attr(disk, 'ovf:capacity', info.volume.size * 1024 * 1024)
+		attr(disk, 'ovf:capacity', info.volume.size.get_qty_in('B'))
 		attr(disk, 'ovf:format', info.volume.ovf_uri)
 		attr(disk, 'ovf:uuid', volume_uuid)
 

providers/ec2/__init__.py

@@ -30,14 +30,14 @@ def validate_manifest(data, validator, error):
 	import os.path
 	validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
 
+	from common.bytes import Bytes
 	if data['volume']['backing'] == 'ebs':
-		volume_size = 2 if data['volume']['partitions']['type'] == 'msdos' else 0
+		volume_size = Bytes('2MiB') if data['volume']['partitions']['type'] == 'msdos' else Bytes(0)
 		for key, partition in data['volume']['partitions'].iteritems():
 			if key != 'type':
-				volume_size += partition['size']
-		if volume_size % 1024 != 0:
-			msg = ('The volume size must be a multiple of 1024 when using EBS backing '
-			       '(MBR partitioned volumes are 2MB larger than specified, for the post-mbr gap)')
+				volume_size += Bytes(partition['size'])
+		if volume_size % Bytes('1GiB') != 0:
+			msg = ('The volume size must be a multiple of 1GiB when using EBS backing')
 			error(msg, ['volume', 'partitions'])
 	else:
 		validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json'))

providers/ec2/ebsvolume.py

@@ -11,8 +11,7 @@ class EBSVolume(Volume):
 	def _before_create(self, e):
 		conn = e.connection
 		zone = e.zone
-		import math
-		size = int(math.ceil(self.size / 1024))
+		size = self.size.get_qty_in('GiB')
 		self.volume = conn.create_volume(size, zone)
 		while self.volume.volume_state() != 'available':
 			time.sleep(5)

providers/ec2/tasks/ami.py

@@ -166,7 +166,7 @@ class RegisterAMI(Task):
 			from boto.ec2.blockdevicemapping import BlockDeviceType
 			from boto.ec2.blockdevicemapping import BlockDeviceMapping
 			block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
-			                               size=info.volume.size / 1024)
+			                               size=info.volume.size.get_qty_in('GiB'))
 			registration_params['block_device_map'] = BlockDeviceMapping()
 			registration_params['block_device_map'][root_dev_name] = block_device