diff --git a/bootstrapvz/base/fs/__init__.py b/bootstrapvz/base/fs/__init__.py index 6ffe21f..9f7742b 100644 --- a/bootstrapvz/base/fs/__init__.py +++ b/bootstrapvz/base/fs/__init__.py @@ -9,27 +9,33 @@ def load_volume(data, bootloader): :return: The volume that represents all information pertaining to the volume we bootstrap on. :rtype: Volume """ - # Create a mapping between valid partition maps in the manifest and their corresponding classes + # Map valid partition maps in the manifest and their corresponding classes from partitionmaps.gpt import GPTPartitionMap from partitionmaps.msdos import MSDOSPartitionMap from partitionmaps.none import NoPartitions - partition_maps = {'none': NoPartitions, - 'gpt': GPTPartitionMap, - 'msdos': MSDOSPartitionMap, - } - # Instantiate the partition map - partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader) + partition_map = {'none': NoPartitions, + 'gpt': GPTPartitionMap, + 'msdos': MSDOSPartitionMap, + }.get(data['partitions']['type']) - # Create a mapping between valid volume backings in the manifest and their corresponding classes + # Map valid volume backings in the manifest and their corresponding classes from bootstrapvz.common.fs.loopbackvolume import LoopbackVolume from bootstrapvz.providers.ec2.ebsvolume import EBSVolume from bootstrapvz.common.fs.virtualdiskimage import VirtualDiskImage from bootstrapvz.common.fs.virtualmachinedisk import VirtualMachineDisk - volume_backings = {'raw': LoopbackVolume, - 's3': LoopbackVolume, - 'vdi': VirtualDiskImage, - 'vmdk': VirtualMachineDisk, - 'ebs': EBSVolume - } + volume_backing = {'raw': LoopbackVolume, + 's3': LoopbackVolume, + 'vdi': VirtualDiskImage, + 'vmdk': VirtualMachineDisk, + 'ebs': EBSVolume + }.get(data['backing']) + + # Instantiate the partition map + from bootstrapvz.common.bytes import Bytes + # Only operate with a physical sector size of 512 bytes for now, + # not sure if we can change that for some of the virtual disks + sector_size = Bytes('512B') + partition_map = partition_map(data['partitions'], sector_size, bootloader) + # Create the volume with the partition map as an argument - return volume_backings.get(data['backing'])(partition_map) + return volume_backing(partition_map) diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py index 43f9e79..d1e08ec 100644 --- a/bootstrapvz/base/fs/partitionmaps/abstract.py +++ b/bootstrapvz/base/fs/partitionmaps/abstract.py @@ -1,5 +1,6 @@ from abc import ABCMeta from abc import abstractmethod +from ..partitions.gap import PartitionGap from bootstrapvz.common.tools import log_check_call from bootstrapvz.common.fsm_proxy import FSMProxy from ..exceptions import PartitionError @@ -86,6 +87,8 @@ class AbstractPartitionMap(FSMProxy): # Check if any partition was not mapped for idx, partition in enumerate(self.partitions): + if isinstance(partition, PartitionGap): + continue if partition.fsm.current not in ['mapped', 'formatted']: raise PartitionError('kpartx did not map partition #' + str(idx + 1)) @@ -111,6 +114,8 @@ class AbstractPartitionMap(FSMProxy): volume = event.volume # Run through all partitions before unmapping and make sure they can all be unmapped for partition in self.partitions: + if isinstance(partition, PartitionGap): + continue if partition.fsm.cannot('unmap'): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) @@ -118,4 +123,6 @@ class AbstractPartitionMap(FSMProxy): log_check_call(['kpartx', '-ds', volume.device_path]) # Call unmap on all partitions for partition in self.partitions: + if isinstance(partition, PartitionGap): + continue partition.unmap() diff --git a/bootstrapvz/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py index 8472338..be54c32 100644 --- a/bootstrapvz/base/fs/partitionmaps/gpt.py +++ b/bootstrapvz/base/fs/partitionmaps/gpt.py @@ -1,6 +1,7 @@ from abstract import AbstractPartitionMap from ..partitions.gpt import GPTPartition from ..partitions.gpt_swap import GPTSwapPartition +from ..partitions.gap import PartitionGap from bootstrapvz.common.tools import log_check_call @@ -8,12 +9,14 @@ class GPTPartitionMap(AbstractPartitionMap): """Represents a GPT partition map """ - def __init__(self, data, bootloader): + def __init__(self, data, sector_size, bootloader): """ :param dict data: volume.partitions part of the manifest + :param int sector_size: Sectorsize of the volume :param str bootloader: Name of the bootloader we will use for bootstrapping """ - from bootstrapvz.common.bytes import Bytes + from bootstrapvz.common.sectors import Sectors + # List of partitions self.partitions = [] @@ -26,27 +29,27 @@ class GPTPartitionMap(AbstractPartitionMap): # next partition. if bootloader == 'grub': from ..partitions.unformatted import UnformattedPartition - self.grub_boot = UnformattedPartition(Bytes('1007KiB'), last_partition()) + self.grub_boot = UnformattedPartition(Sectors('1007KiB', sector_size), last_partition()) # Mark the partition as a bios_grub partition self.grub_boot.flags.append('bios_grub') self.partitions.append(self.grub_boot) # The boot and swap partitions are optional if 'boot' in data: - self.boot = GPTPartition(Bytes(data['boot']['size']), + self.boot = GPTPartition(Sectors(data['boot']['size'], sector_size), data['boot']['filesystem'], data['boot'].get('format_command', None), 'boot', last_partition()) self.partitions.append(self.boot) if 'swap' in data: - self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition()) + self.swap = GPTSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition()) self.partitions.append(self.swap) - self.root = GPTPartition(Bytes(data['root']['size']), + self.root = GPTPartition(Sectors(data['root']['size'], sector_size), data['root']['filesystem'], data['root'].get('format_command', None), 'root', last_partition()) self.partitions.append(self.root) # We need to move the first partition to make space for the gpt offset - gpt_offset = Bytes('17KiB') + gpt_offset = Sectors('17KiB', sector_size) self.partitions[0].offset += gpt_offset if hasattr(self, 'grub_boot'): @@ -58,6 +61,10 @@ class GPTPartitionMap(AbstractPartitionMap): # Avoid increasing the volume size because of gpt_offset self.partitions[0].size -= gpt_offset + # Leave the last sector unformatted + self.partitions[-1].size -= 1 + self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition())) + super(GPTPartitionMap, self).__init__(bootloader) def _before_create(self, event): @@ -70,4 +77,6 @@ class GPTPartitionMap(AbstractPartitionMap): '--', 'mklabel', 'gpt']) # Create the partitions for partition in self.partitions: + if isinstance(partition, PartitionGap): + continue partition.create(volume) diff --git a/bootstrapvz/base/fs/partitionmaps/msdos.py b/bootstrapvz/base/fs/partitionmaps/msdos.py index 1b726a5..d959e37 100644 --- a/bootstrapvz/base/fs/partitionmaps/msdos.py +++ b/bootstrapvz/base/fs/partitionmaps/msdos.py @@ -1,6 +1,7 @@ from abstract import AbstractPartitionMap from ..partitions.msdos import MSDOSPartition from ..partitions.msdos_swap import MSDOSSwapPartition +from ..partitions.gap import PartitionGap from bootstrapvz.common.tools import log_check_call @@ -9,12 +10,14 @@ class MSDOSPartitionMap(AbstractPartitionMap): Sometimes also called MBR (but that confuses the hell out of me, so ms-dos it is) """ - def __init__(self, data, bootloader): + def __init__(self, data, sector_size, bootloader): """ :param dict data: volume.partitions part of the manifest + :param int sector_size: Sectorsize of the volume :param str bootloader: Name of the bootloader we will use for bootstrapping """ - from bootstrapvz.common.bytes import Bytes + from bootstrapvz.common.sectors import Sectors + # List of partitions self.partitions = [] @@ -24,14 +27,14 @@ class MSDOSPartitionMap(AbstractPartitionMap): # The boot and swap partitions are optional if 'boot' in data: - self.boot = MSDOSPartition(Bytes(data['boot']['size']), + self.boot = MSDOSPartition(Sectors(data['boot']['size'], sector_size), data['boot']['filesystem'], data['boot'].get('format_command', None), last_partition()) self.partitions.append(self.boot) if 'swap' in data: - self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition()) + self.swap = MSDOSSwapPartition(Sectors(data['swap']['size'], sector_size), last_partition()) self.partitions.append(self.swap) - self.root = MSDOSPartition(Bytes(data['root']['size']), + self.root = MSDOSPartition(Sectors(data['root']['size'], sector_size), data['root']['filesystem'], data['root'].get('format_command', None), last_partition()) self.partitions.append(self.root) @@ -44,13 +47,17 @@ class MSDOSPartitionMap(AbstractPartitionMap): # The MBR offset is included in the grub offset, so if we don't use grub # we should reduce the size of the first partition and move it by only 512 bytes. if bootloader == 'grub': - offset = Bytes('2MiB') + offset = Sectors('2MiB', sector_size) else: - offset = Bytes('512B') + offset = Sectors('512B', sector_size) self.partitions[0].offset += offset self.partitions[0].size -= offset + # Leave the last sector unformatted + self.partitions[-1].size -= 1 + self.partitions.append(PartitionGap(Sectors(1, sector_size), last_partition())) + super(MSDOSPartitionMap, self).__init__(bootloader) def _before_create(self, event): @@ -61,4 +68,6 @@ class MSDOSPartitionMap(AbstractPartitionMap): '--', 'mklabel', 'msdos']) # Create the partitions for partition in self.partitions: + if isinstance(partition, PartitionGap): + continue partition.create(volume) diff --git a/bootstrapvz/base/fs/partitionmaps/none.py b/bootstrapvz/base/fs/partitionmaps/none.py index 439fe48..944f8a5 100644 --- a/bootstrapvz/base/fs/partitionmaps/none.py +++ b/bootstrapvz/base/fs/partitionmaps/none.py @@ -7,14 +7,16 @@ class NoPartitions(object): simply always deal with partition maps and then let the base abstract that away. """ - def __init__(self, data, bootloader): + def __init__(self, data, sector_size, bootloader): """ :param dict data: volume.partitions part of the manifest + :param int sector_size: Sectorsize of the volume :param str bootloader: Name of the bootloader we will use for bootstrapping """ - from bootstrapvz.common.bytes import Bytes + from bootstrapvz.common.sectors import Sectors + # In the NoPartitions partitions map we only have a single 'partition' - self.root = SinglePartition(Bytes(data['root']['size']), + self.root = SinglePartition(Sectors(data['root']['size'], sector_size), data['root']['filesystem'], data['root'].get('format_command', None)) self.partitions = [self.root] @@ -29,7 +31,7 @@ class NoPartitions(object): """Returns the total size the partitions occupy :return: The size of all the partitions - :rtype: Bytes + :rtype: Sectors """ return self.root.get_end() diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py index e67e733..019218c 100644 --- a/bootstrapvz/base/fs/partitions/base.py +++ b/bootstrapvz/base/fs/partitions/base.py @@ -28,9 +28,9 @@ class BasePartition(AbstractPartition): # By saving the previous partition we have # a linked list that partitions can go backwards in to find the first partition. self.previous = previous - from bootstrapvz.common.bytes import Bytes - # Initialize the offset to 0 bytes, may be changed later - self.offset = Bytes(0) + from bootstrapvz.common.sectors import Sectors + # Initialize the offset to 0 sectors, may be changed later + self.offset = Sectors(0, size.sector_size) # List of flags that parted should put on the partition self.flags = [] super(BasePartition, self).__init__(size, filesystem, format_command) diff --git a/bootstrapvz/base/fs/partitions/gap.py b/bootstrapvz/base/fs/partitions/gap.py new file mode 100644 index 0000000..9aadb97 --- /dev/null +++ b/bootstrapvz/base/fs/partitions/gap.py @@ -0,0 +1,17 @@ +from base import BasePartition + + +class PartitionGap(BasePartition): + """Represents a non-existent partition + A gap in the partitionmap + """ + + # The states for our state machine. It can neither be create nor mapped. + events = [] + + def __init__(self, size, previous): + """ + :param Bytes size: Size of the partition + :param BasePartition previous: The partition that preceeds this one + """ + super(PartitionGap, self).__init__(size, None, None, previous) diff --git a/bootstrapvz/common/exceptions.py b/bootstrapvz/common/exceptions.py index 2e1930e..909d7d3 100644 --- a/bootstrapvz/common/exceptions.py +++ b/bootstrapvz/common/exceptions.py @@ -34,3 +34,7 @@ class NoMatchesError(Exception): class TooManyMatchesError(Exception): pass + + +class UnitError(Exception): + pass diff --git a/bootstrapvz/common/fs/loopbackvolume.py b/bootstrapvz/common/fs/loopbackvolume.py index 49f905f..2db9edb 100644 --- a/bootstrapvz/common/fs/loopbackvolume.py +++ b/bootstrapvz/common/fs/loopbackvolume.py @@ -11,7 +11,7 @@ class LoopbackVolume(Volume): def _before_create(self, e): self.image_path = e.image_path - size_opt = '--size={mib}M'.format(mib=self.size.get_qty_in('MiB')) + size_opt = '--size={mib}M'.format(mib=self.size.bytes.get_qty_in('MiB')) log_check_call(['truncate', 'create', size_opt, self.image_path]) def _before_attach(self, e): diff --git a/bootstrapvz/common/fs/qemuvolume.py b/bootstrapvz/common/fs/qemuvolume.py index 26d3300..0e8c2f0 100644 --- a/bootstrapvz/common/fs/qemuvolume.py +++ b/bootstrapvz/common/fs/qemuvolume.py @@ -8,7 +8,7 @@ class QEMUVolume(LoopbackVolume): def _before_create(self, e): self.image_path = e.image_path - vol_size = str(self.size.get_qty_in('MiB')) + 'M' + vol_size = str(self.size.bytes.get_qty_in('MiB')) + 'M' log_check_call(['qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size]) def _check_nbd_module(self): diff --git a/bootstrapvz/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py index 8c4ab8a..7fb7511 100644 --- a/bootstrapvz/common/tasks/filesystem.py +++ b/bootstrapvz/common/tasks/filesystem.py @@ -25,9 +25,11 @@ class Format(Task): @classmethod def run(cls, info): from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition + from bootstrapvz.base.fs.partitions.gap import PartitionGap for partition in info.volume.partition_map.partitions: - if not isinstance(partition, UnformattedPartition): - partition.format() + if isinstance(partition, (UnformattedPartition, PartitionGap)): + continue + partition.format() class TuneVolumeFS(Task): @@ -38,12 +40,14 @@ class TuneVolumeFS(Task): @classmethod def run(cls, info): from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition + from bootstrapvz.base.fs.partitions.gap import PartitionGap import re # Disable the time based filesystem check for partition in info.volume.partition_map.partitions: - if not isinstance(partition, UnformattedPartition): - if re.match('^ext[2-4]$', partition.filesystem) is not None: - log_check_call(['tune2fs', '-i', '0', partition.device_path]) + if isinstance(partition, (UnformattedPartition, PartitionGap)): + continue + if re.match('^ext[2-4]$', partition.filesystem) is not None: + log_check_call(['tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): diff --git a/bootstrapvz/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py index d56c666..7937b30 100644 --- a/bootstrapvz/plugins/prebootstrapped/tasks.py +++ b/bootstrapvz/plugins/prebootstrapped/tasks.py @@ -34,7 +34,7 @@ class CreateFromSnapshot(Task): @classmethod def run(cls, info): snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - ebs_volume = info._ec2['connection'].create_volume(info.volume.size.get_qty_in('GiB'), + ebs_volume = info._ec2['connection'].create_volume(info.volume.size.bytes.get_qty_in('GiB'), info._ec2['host']['availabilityZone'], snapshot=snapshot) while ebs_volume.volume_state() != 'available': diff --git a/bootstrapvz/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py index 1810146..9f673cf 100644 --- a/bootstrapvz/plugins/vagrant/tasks.py +++ b/bootstrapvz/plugins/vagrant/tasks.py @@ -184,7 +184,7 @@ class PackageBox(Task): # VHDURI = "http://go.microsoft.com/fwlink/?LinkId=137171" volume_uuid = info.volume.get_uuid() [disk] = root.findall('./ovf:DiskSection/ovf:Disk', namespaces) - attr(disk, 'ovf:capacity', info.volume.size.get_qty_in('B')) + attr(disk, 'ovf:capacity', info.volume.size.bytes.get_qty_in('B')) attr(disk, 'ovf:format', info.volume.ovf_uri) attr(disk, 'ovf:uuid', volume_uuid) diff --git a/bootstrapvz/providers/ec2/ebsvolume.py b/bootstrapvz/providers/ec2/ebsvolume.py index 2cd5549..270bc75 100644 --- a/bootstrapvz/providers/ec2/ebsvolume.py +++ b/bootstrapvz/providers/ec2/ebsvolume.py @@ -11,7 +11,7 @@ class EBSVolume(Volume): def _before_create(self, e): conn = e.connection zone = e.zone - size = self.size.get_qty_in('GiB') + size = self.size.bytes.get_qty_in('GiB') self.volume = conn.create_volume(size, zone) while self.volume.volume_state() != 'available': time.sleep(5) diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py index 7a39f2e..4e2da26 100644 --- a/bootstrapvz/providers/ec2/tasks/ami.py +++ b/bootstrapvz/providers/ec2/tasks/ami.py @@ -109,7 +109,7 @@ class RegisterAMI(Task): from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping block_device = BlockDeviceType(snapshot_id=info._ec2['snapshot'].id, delete_on_termination=True, - size=info.volume.size.get_qty_in('GiB')) + size=info.volume.size.bytes.get_qty_in('GiB')) registration_params['block_device_map'] = BlockDeviceMapping() registration_params['block_device_map'][root_dev_name] = block_device