From a840dc28f3e8a9a7f6f6c8e0aaf984b1a47f70b4 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 19 Jan 2014 12:39:07 +0100 Subject: [PATCH] MBR-gap, grub and GPT offset are now subtracted automatically from the first available partition. There is no need to fiddle with sizes while keeping the offsets in mind any longer. Introduced Bytes() class which makes it a lot easier to handle size units. --- base/fs/partitionmaps/gpt.py | 20 ++- base/fs/partitionmaps/msdos.py | 14 +- base/fs/partitionmaps/none.py | 6 +- base/fs/partitions/abstract.py | 8 +- base/fs/partitions/base.py | 5 +- base/fs/partitions/single.py | 3 +- base/fs/volume.py | 2 +- base/manifest-schema.json | 8 +- common/bytes.py | 131 ++++++++++++++++++ common/fs/loopbackvolume.py | 3 +- common/fs/qemuvolume.py | 3 +- ...bs-debian-official-amd64-hvm.manifest.json | 2 +- ...bs-debian-official-amd64-pvm.manifest.json | 2 +- ...ebs-debian-official-i386-pvm.manifest.json | 2 +- ...bs-debian-unstable-amd64-pvm.manifest.json | 2 +- manifests/ec2-ebs-partitioned.manifest.json | 2 +- manifests/ec2-ebs-single.manifest.json | 2 +- manifests/ec2-s3.manifest.json | 2 +- manifests/virtualbox-vagrant.manifest.json | 6 +- manifests/virtualbox.manifest.json | 6 +- plugins/prebootstrapped/tasks.py | 3 +- plugins/vagrant/tasks.py | 2 +- providers/ec2/__init__.py | 10 +- providers/ec2/ebsvolume.py | 3 +- providers/ec2/tasks/ami.py | 2 +- 25 files changed, 200 insertions(+), 49 deletions(-) create mode 100644 common/bytes.py diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py index 55c404b..6187a6f 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/base/fs/partitionmaps/gpt.py @@ -7,28 +7,36 @@ from common.tools import log_check_call class GPTPartitionMap(AbstractPartitionMap): def __init__(self, data, bootloader): + from common.bytes import Bytes self.partitions = [] def last_partition(): return self.partitions[-1] if len(self.partitions) > 0 else None + gpt_offset = Bytes('17KiB') + if bootloader == 'grub': from ..partitions.unformatted import UnformattedPartition - self.grub_boot = UnformattedPartition(2, last_partition()) + self.grub_boot = UnformattedPartition(Bytes('2MiB'), last_partition()) self.grub_boot.flags.append('bios_grub') self.partitions.append(self.grub_boot) if 'boot' in data: - self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', last_partition()) + self.boot = GPTPartition(Bytes(data['boot']['size']), data['boot']['filesystem'], + 'boot', last_partition()) self.partitions.append(self.boot) if 'swap' in data: - self.swap = GPTSwapPartition(data['swap']['size'], last_partition()) + self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition()) self.partitions.append(self.swap) - self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', last_partition()) + self.root = GPTPartition(Bytes(data['root']['size']), data['root']['filesystem'], 'root', last_partition()) self.partitions.append(self.root) - if bootloader == 'extlinux': - getattr(self, 'boot', self.root).flags.append('legacy_boot') + if hasattr(self, 'grub_boot'): + self.partitions[1].size -= self.grub_boot.size + + # Offset for GPT partitioning table + self.partitions[0].offset = gpt_offset + self.partitions[0].size -= self.partitions[0].offset super(GPTPartitionMap, self).__init__(bootloader) diff --git a/base/fs/partitionmaps/msdos.py b/base/fs/partitionmaps/msdos.py index 296eac9..138d630 100644 --- a/base/fs/partitionmaps/msdos.py +++ b/base/fs/partitionmaps/msdos.py @@ -7,23 +7,29 @@ from common.tools import log_check_call class MSDOSPartitionMap(AbstractPartitionMap): def __init__(self, data, bootloader): + from common.bytes import Bytes self.partitions = [] def last_partition(): return self.partitions[-1] if len(self.partitions) > 0 else None + + grub_offset = Bytes('2MiB') + if 'boot' in data: - self.boot = MSDOSPartition(data['boot']['size'], data['boot']['filesystem'], None) + self.boot = MSDOSPartition(Bytes(data['boot']['size']), data['boot']['filesystem'], None) self.partitions.append(self.boot) if 'swap' in data: - self.swap = MSDOSSwapPartition(data['swap']['size'], last_partition()) + self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition()) self.partitions.append(self.swap) - self.root = MSDOSPartition(data['root']['size'], data['root']['filesystem'], last_partition()) + self.root = MSDOSPartition(Bytes(data['root']['size']), data['root']['filesystem'], last_partition()) self.partitions.append(self.root) getattr(self, 'boot', self.root).flags.append('boot') if bootloader == 'grub': - self.partitions[0].offset = 2 + self.partitions[0].offset = grub_offset + self.partitions[0].size -= self.partitions[0].offset + super(MSDOSPartitionMap, self).__init__(bootloader) def _before_create(self, event): diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index 3070067..8fc7650 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -4,12 +4,12 @@ from ..partitions.single import SinglePartition class NoPartitions(object): def __init__(self, data, bootloader): - root = data['root'] - self.root = SinglePartition(root['size'], root['filesystem']) + from common.bytes import Bytes + self.root = SinglePartition(Bytes(data['root']['size']), data['root']['filesystem']) self.partitions = [self.root] def is_blocking(self): return self.root.fsm.current == 'mounted' def get_total_size(self): - return self.root.size + return self.root.get_end() diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py index 4478d21..5684561 100644 --- a/base/fs/partitions/abstract.py +++ b/base/fs/partitions/abstract.py @@ -37,10 +37,10 @@ class AbstractPartition(FSMProxy): del self.mount_dir def __init__(self, size, filesystem): - self.size = size - self.filesystem = filesystem - self.device_path = None - self.mounts = {} + self.size = size + self.filesystem = filesystem + self.device_path = None + self.mounts = {} cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} super(AbstractPartition, self).__init__(cfg) diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index 052378b..216479e 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -16,7 +16,8 @@ class BasePartition(AbstractPartition): def __init__(self, size, filesystem, previous): self.previous = previous - self.offset = 0 + from common.bytes import Bytes + self.offset = Bytes(0) self.flags = [] super(BasePartition, self).__init__(size, filesystem) @@ -40,7 +41,7 @@ class BasePartition(AbstractPartition): def _before_create(self, e): from common.tools import log_check_call - create_command = ('mkpart primary {start}MiB {end}MiB' + create_command = ('mkpart primary {start} {end}' .format(start=str(self.get_start()), end=str(self.get_end()))) log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index c091eb5..828ba91 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -4,4 +4,5 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): def get_start(self): - return 0 + from common.bytes import Bytes + return Bytes(0) diff --git a/base/fs/volume.py b/base/fs/volume.py index cf3353b..631c886 100644 --- a/base/fs/volume.py +++ b/base/fs/volume.py @@ -55,7 +55,7 @@ class Volume(FSMProxy): # The offset at which the volume should begin to be mapped in the new volume start_sector = getattr(e, 'start_sector', 0) - sectors = getattr(e, 'sectors', self.size * 1024 * 1024 / 512 - start_sector) + sectors = getattr(e, 'sectors', int(self.size / 512) - start_sector) table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' .format(log_start_sec=logical_start_sector, diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 7e35893..30e16c2 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -105,6 +105,10 @@ "type": "string", "pattern": "^/[^\\0]+$" }, + "bytes": { + "type": "string", + "pattern": "^\\d+([KMGT]i?B|B)$" + }, "no_partitions": { "type": "object", "properties": { @@ -122,7 +126,7 @@ "root": { "$ref": "#/definitions/partition" }, "swap": { "type": "object", - "properties": { "size": { "type": "integer", "minimum": 1 } }, + "properties": { "size": { "$ref": "#/definitions/bytes" } }, "required": ["size"] } }, @@ -132,7 +136,7 @@ "partition": { "type": "object", "properties": { - "size": { "type": "integer", "minimum": 1 }, + "size": { "$ref": "#/definitions/bytes" }, "filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] } }, "required": ["size", "filesystem"] diff --git a/common/bytes.py b/common/bytes.py new file mode 100644 index 0000000..e6f2b88 --- /dev/null +++ b/common/bytes.py @@ -0,0 +1,131 @@ + + +class Bytes(object): + + units = {'B': 1, + 'KiB': 1024, + 'MiB': 1024*1024, + 'GiB': 1024*1024*1024, + 'TiB': 1024*1024*1024*1024, + } + + def __init__(self, qty): + if isinstance(qty, (int, long)): + self.qty = qty + else: + self.qty = Bytes.parse(qty) + + @staticmethod + def parse(qty_str): + import re + regex = re.compile('^(?P\d+)(?P[KMGT]i?B|B)$') + parsed = regex.match(qty_str) + if parsed is None: + raise UnitError('Unable to parse {str}'.format(str=qty_str)) + + qty = int(parsed.group('qty')) + unit = parsed.group('unit') + if unit[0] in 'KMGT': + unit = unit[0] + 'iB' + byte_qty = qty * Bytes.units[unit] + return byte_qty + + def get_qty_in(self, unit): + if unit[0] in 'KMGT': + unit = unit[0] + 'iB' + if unit not in Bytes.units: + raise UnitError('Unrecognized unit `{unit}\''.format(unit=Bytes.magnitude)) + if self.qty % Bytes.units[unit] != 0: + msg = 'Unable to convert {qty} bytes to a whole number in {unit}'.format(qty=self.qty, unit=unit) + raise UnitError(msg) + return self.qty / Bytes.units[unit] + + def __repr__(self): + converted = str(self.get_qty_in('B')) + 'B' + if self.qty == 0: + return converted + for unit in ['TiB', 'GiB', 'MiB', 'KiB']: + try: + converted = str(self.get_qty_in(unit)) + unit + break + except UnitError: + pass + return converted + + def __str__(self): + return self.__repr__() + + def __int__(self): + return self.qty + + def __long__(self): + return self.qty + + def __add__(self, other): + if not isinstance(other, Bytes): + raise UnitError('Can only add Bytes to Bytes') + return Bytes(self.qty + other.qty) + + def __iadd__(self, other): + if not isinstance(other, Bytes): + raise UnitError('Can only add Bytes to Bytes') + self.qty += other.qty + return self + + def __sub__(self, other): + if not isinstance(other, Bytes): + raise UnitError('Can only subtract Bytes from Bytes') + return Bytes(self.qty - other.qty) + + def __isub__(self, other): + if not isinstance(other, Bytes): + raise UnitError('Can only subtract Bytes from Bytes') + self.qty -= other.qty + return self + + def __mul__(self, other): + if not isinstance(other, (int, long)): + raise UnitError('Can only multiply Bytes with integers') + return Bytes(self.qty * other) + + def __imul__(self, other): + if not isinstance(other, (int, long)): + raise UnitError('Can only multiply Bytes with integers') + self.qty *= other + return self + + def __div__(self, other): + if isinstance(other, Bytes): + return self.qty / other.qty + if not isinstance(other, (int, long)): + raise UnitError('Can only divide Bytes with integers or Bytes') + return Bytes(self.qty / other) + + def __idiv__(self, other): + if isinstance(other, Bytes): + self.qty /= other.qty + else: + if not isinstance(other, (int, long)): + raise UnitError('Can only divide Bytes with integers or Bytes') + self.qty /= other + return self + + def __mod__(self, other): + if isinstance(other, Bytes): + return self.qty % other.qty + if not isinstance(other, (int, long)): + raise UnitError('Can only take modulus of Bytes with integers or Bytes') + return Bytes(self.qty % other) + + def __imod__(self, other): + if isinstance(other, Bytes): + self.qty %= other.qty + else: + if not isinstance(other, (int, long)): + raise UnitError('Can only divide Bytes with integers or Bytes') + self.qty %= other + return self + + +class UnitError(Exception): + pass diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index 4355c41..9f42728 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -11,7 +11,8 @@ class LoopbackVolume(Volume): def _before_create(self, e): self.image_path = e.image_path - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) + vol_size = str(self.size.get_qty_in('MiB')) + 'M' + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, vol_size]) def _before_attach(self, e): [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) diff --git a/common/fs/qemuvolume.py b/common/fs/qemuvolume.py index 0d40599..3383a0c 100644 --- a/common/fs/qemuvolume.py +++ b/common/fs/qemuvolume.py @@ -8,7 +8,8 @@ class QEMUVolume(LoopbackVolume): def _before_create(self, e): self.image_path = e.image_path - log_check_call(['/usr/bin/qemu-img', 'create', '-f', self.qemu_format, self.image_path, str(self.size) + 'M']) + vol_size = str(self.size.get_qty_in('MiB')) + 'M' + log_check_call(['/usr/bin/qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size]) def _check_nbd_module(self): from base.fs.partitionmaps.none import NoPartitions diff --git a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json index ab2159f..93039ff 100644 --- a/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "none", "root": { - "size": 8192, + "size": "8GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json index 4dcd130..9fb8095 100644 --- a/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "none", "root": { - "size": 8192, + "size": "8GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json index daf6a95..d68a6d2 100644 --- a/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official-i386-pvm.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "none", "root": { - "size": 8192, + "size": "8GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json index a99e538..b9a4174 100644 --- a/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-unstable-amd64-pvm.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "none", "root": { - "size": 8192, + "size": "8GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json index 2871e63..441c076 100644 --- a/manifests/ec2-ebs-partitioned.manifest.json +++ b/manifests/ec2-ebs-partitioned.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "msdos", "root": { - "size": 1022, + "size": "1GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-ebs-single.manifest.json b/manifests/ec2-ebs-single.manifest.json index c2deb2c..1e7f883 100644 --- a/manifests/ec2-ebs-single.manifest.json +++ b/manifests/ec2-ebs-single.manifest.json @@ -29,7 +29,7 @@ "partitions": { "type": "none", "root": { - "size": 1024, + "size": "1GiB", "filesystem": "ext4" } } diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json index 09db13c..3351c90 100644 --- a/manifests/ec2-s3.manifest.json +++ b/manifests/ec2-s3.manifest.json @@ -33,7 +33,7 @@ "partitions": { "type": "none", "root": { - "size": 1024, + "size": "1GiB", "filesystem": "ext4" } } diff --git a/manifests/virtualbox-vagrant.manifest.json b/manifests/virtualbox-vagrant.manifest.json index 19cc549..9842757 100644 --- a/manifests/virtualbox-vagrant.manifest.json +++ b/manifests/virtualbox-vagrant.manifest.json @@ -22,14 +22,14 @@ "partitions": { "type": "msdos", "boot": { - "size": 64, + "size": "64MiB", "filesystem": "ext2" }, "root": { - "size": 1854, + "size": "1856MiB", "filesystem": "ext4" }, - "swap": {"size": 128} + "swap": {"size": "128MiB"} } }, "plugins": { diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index 75091e1..062cc9c 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -22,14 +22,14 @@ "partitions": { "type": "msdos", "boot": { - "size": 32, + "size": "32MiB", "filesystem": "ext2" }, "root": { - "size": 990, + "size": "864MiB", "filesystem": "ext4" }, - "swap": {"size": 128} + "swap": {"size": "128MiB"} } } } diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 82c0378..047cf31 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -33,9 +33,8 @@ class CreateFromSnapshot(Task): @classmethod def run(cls, info): - volume_size = int(info.volume.size / 1024) snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - ebs_volume = info.connection.create_volume(volume_size, + ebs_volume = info.connection.create_volume(info.volume.size.get_qty_in('GiB'), info.host['availabilityZone'], snapshot=snapshot) while ebs_volume.volume_state() != 'available': diff --git a/plugins/vagrant/tasks.py b/plugins/vagrant/tasks.py index a8839f7..d676621 100644 --- a/plugins/vagrant/tasks.py +++ b/plugins/vagrant/tasks.py @@ -163,7 +163,7 @@ class PackageBox(Task): # VHDURI = "http://go.microsoft.com/fwlink/?LinkId=137171" volume_uuid = info.volume.get_uuid() [disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces) - attr(disk, 'ovf:capacity', info.volume.size * 1024 * 1024) + attr(disk, 'ovf:capacity', info.volume.size.get_qty_in('B')) attr(disk, 'ovf:format', info.volume.ovf_uri) attr(disk, 'ovf:uuid', volume_uuid) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 4806999..6292e84 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -30,14 +30,14 @@ def validate_manifest(data, validator, error): import os.path validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + from common.bytes import Bytes if data['volume']['backing'] == 'ebs': - volume_size = 2 if data['volume']['partitions']['type'] == 'msdos' else 0 + volume_size = Bytes('2MiB') if data['volume']['partitions']['type'] == 'msdos' else Bytes(0) for key, partition in data['volume']['partitions'].iteritems(): if key != 'type': - volume_size += partition['size'] - if volume_size % 1024 != 0: - msg = ('The volume size must be a multiple of 1024 when using EBS backing ' - '(MBR partitioned volumes are 2MB larger than specified, for the post-mbr gap)') + volume_size += Bytes(partition['size']) + if volume_size % Bytes('1GiB') != 0: + msg = ('The volume size must be a multiple of 1GiB when using EBS backing') error(msg, ['volume', 'partitions']) else: validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json')) diff --git a/providers/ec2/ebsvolume.py b/providers/ec2/ebsvolume.py index 6ac54e2..8cbcdbf 100644 --- a/providers/ec2/ebsvolume.py +++ b/providers/ec2/ebsvolume.py @@ -11,8 +11,7 @@ class EBSVolume(Volume): def _before_create(self, e): conn = e.connection zone = e.zone - import math - size = int(math.ceil(self.size / 1024)) + size = self.size.get_qty_in('GiB') self.volume = conn.create_volume(size, zone) while self.volume.volume_state() != 'available': time.sleep(5) diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index da863b7..000c0db 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -166,7 +166,7 @@ class RegisterAMI(Task): from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.volume.size / 1024) + size=info.volume.size.get_qty_in('GiB')) registration_params['block_device_map'] = BlockDeviceMapping() registration_params['block_device_map'][root_dev_name] = block_device