From ff7c04c12039b20910489f42097b588d52b50c63 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 13:19:45 +0200 Subject: [PATCH] Support for partitions MAJOR refactor. The volume is now abstracted into a model along with a partitionmap and partitions. Volumes and partitions are now controlled via an FSM to ensure that commands are called in the proper sequence. GRUB can now be installed properly onto loop devices by using dmsetup to fake a proper harddisk. --- base/bootstrapinfo.py | 2 + base/fs/__init__.py | 17 +++ base/fs/exceptions.py | 8 ++ base/fs/nopartitions.py | 30 +++++ base/fs/partitionmap.py | 88 ++++++++++++++ base/fs/partitions/__init__.py | 0 base/fs/partitions/abstractpartition.py | 56 +++++++++ base/fs/partitions/partition.py | 60 ++++++++++ base/fs/partitions/singlepartition.py | 20 ++++ base/fs/partitions/swap.py | 11 ++ base/fs/volume.py | 138 ++++++++++++++++++++++ base/manifest-schema.json | 15 ++- common/fs/__init__.py | 11 ++ common/fs/loopbackvolume.py | 90 ++++++++++++++ common/fsm.py | 7 ++ common/tasks/filesystem.py | 130 +++++++++++--------- common/tasks/loopback.py | 53 +-------- common/tasks/parted.py | 53 --------- common/tasks/partitioning.py | 32 +++++ common/tasks/volume.py | 26 ++++ manifests/ec2-ebs-pvm.manifest.json | 10 +- manifests/virtualbox.json | 41 ------- manifests/virtualbox.manifest.json | 50 ++++++++ plugins/prebootstrapped/__init__.py | 32 +++-- plugins/prebootstrapped/tasks.py | 36 ++++-- providers/ec2/__init__.py | 23 ++-- providers/ec2/tasks/ebs.py | 71 +---------- providers/ec2/volume.py | 56 +++++++++ providers/kvm/__init__.py | 2 +- providers/virtualbox/__init__.py | 57 +++++---- providers/virtualbox/manifest-schema.json | 10 +- providers/virtualbox/tasks/boot.py | 57 +++++---- providers/virtualbox/tasks/packages.py | 2 +- providers/virtualbox/volume.py | 11 ++ 34 files changed, 941 insertions(+), 364 deletions(-) create mode 100644 base/fs/__init__.py create mode 100644 base/fs/exceptions.py create mode 100644 base/fs/nopartitions.py create mode 100644 base/fs/partitionmap.py create mode 100644 base/fs/partitions/__init__.py create mode 100644 base/fs/partitions/abstractpartition.py create mode 100644 base/fs/partitions/partition.py create mode 100644 base/fs/partitions/singlepartition.py create mode 100644 base/fs/partitions/swap.py create mode 100644 base/fs/volume.py create mode 100644 common/fs/__init__.py create mode 100644 common/fs/loopbackvolume.py create mode 100644 common/fsm.py delete mode 100644 common/tasks/parted.py create mode 100644 common/tasks/partitioning.py create mode 100644 common/tasks/volume.py delete mode 100644 manifests/virtualbox.json create mode 100644 manifests/virtualbox.manifest.json create mode 100644 providers/ec2/volume.py create mode 100644 providers/virtualbox/volume.py diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index d1c03ec..07c954b 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -3,6 +3,8 @@ class BootstrapInformation(object): def __init__(self, manifest=None, debug=False): self.manifest = manifest + from fs import load_volume + self.volume = load_volume(self.manifest.volume) self.debug = debug import random self.run_id = random.randrange(16 ** 8) diff --git a/base/fs/__init__.py b/base/fs/__init__.py new file mode 100644 index 0000000..6180894 --- /dev/null +++ b/base/fs/__init__.py @@ -0,0 +1,17 @@ + + +def load_volume(data): + from common.fs.loopbackvolume import LoopbackVolume + from providers.ec2.volume import EBSVolume + from providers.virtualbox.volume import VirtualBoxVolume + from partitionmap import PartitionMap + from nopartitions import NoPartitions + partition_maps = {'none': NoPartitions, + 'gpt': PartitionMap, + } + partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) + volume_backings = {'raw': LoopbackVolume, + 'vdi': VirtualBoxVolume, + 'ebs': EBSVolume + } + return volume_backings.get(data['backing'])(partition_map) diff --git a/base/fs/exceptions.py b/base/fs/exceptions.py new file mode 100644 index 0000000..bc38490 --- /dev/null +++ b/base/fs/exceptions.py @@ -0,0 +1,8 @@ + + +class VolumeError(Exception): + pass + + +class PartitionError(Exception): + pass diff --git a/base/fs/nopartitions.py b/base/fs/nopartitions.py new file mode 100644 index 0000000..b8935ca --- /dev/null +++ b/base/fs/nopartitions.py @@ -0,0 +1,30 @@ +from partitions.singlepartition import SinglePartition + + +class NoPartitions(object): + + def __init__(self, data): + root = data['root'] + self.root = SinglePartition(root['size'], root['filesystem']) + self.mount_points = [('/', self.root)] + + def get_total_size(self): + return self.root.size + + def create(self, volume): + pass + + def map(self, volume): + pass + + def unmap(self, volume): + pass + + def format(self): + self.root.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() diff --git a/base/fs/partitionmap.py b/base/fs/partitionmap.py new file mode 100644 index 0000000..49a75ab --- /dev/null +++ b/base/fs/partitionmap.py @@ -0,0 +1,88 @@ +from common.tools import log_check_call +from partitions.partition import Partition +from partitions.swap import Swap +from exceptions import PartitionError + + +class PartitionMap(object): + + def __init__(self, data): + self.boot = None + self.swap = None + self.mount_points = [] + if 'boot' in data: + self.boot = Partition(data['boot']['size'], data['boot']['filesystem'], None) + self.mount_points.append(('/boot', self.boot)) + self.root = Partition(data['root']['size'], data['root']['filesystem'], self.boot) + self.mount_points.append(('/', self.root)) + if 'swap' in data: + self.swap = Swap(data['swap']['size'], self.root) + self.mount_points.append(('none', self.root)) + self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + + def get_total_size(self): + return sum(p.size for p in self.partitions) + + def create(self, volume): + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', volume.device_path, + '--', 'mklabel', 'gpt']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if self.boot is not None: + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set', str(boot_idx), 'bios_grub', 'on']) + + def map(self, volume): + try: + mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) + import re + regexp = re.compile('^(?P.+[^\d](?P\d+)) : ' + '(?P\d) (?P\d+) ' + '{device_path} (?P\d+)$' + .format(device_path=volume.device_path)) + log_check_call(['/sbin/kpartx', '-a', volume.device_path]) + import os.path + for mapping in mappings: + match = regexp.match(mapping) + if match is None: + raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping)) + partition_path = os.path.join('/dev/mapper', match.group('name')) + p_idx = int(match.group('p_idx'))-1 + self.partitions[p_idx].map(partition_path) + + for idx, partition in enumerate(self.partitions): + if not partition.state() in ['mapped', 'formatted']: + raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) + + except PartitionError as e: + for partition in self.partitions: + if partition.state() in ['mapped', 'formatted']: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + raise e + + def unmap(self, volume): + for partition in self.partitions: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + + def format(self): + for partition in self.partitions: + partition.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() + + def mount_boot(self): + import os.path + destination = os.path.join(self.root.mount_dir, 'boot') + self.boot.mount(destination) + + def unmount_boot(self): + self.boot.unmount() diff --git a/base/fs/partitions/__init__.py b/base/fs/partitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/fs/partitions/abstractpartition.py b/base/fs/partitions/abstractpartition.py new file mode 100644 index 0000000..161e134 --- /dev/null +++ b/base/fs/partitions/abstractpartition.py @@ -0,0 +1,56 @@ +from common.tools import log_check_call +from abc import ABCMeta +from fysom import Fysom + + +class AbstractPartition(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + self.size = size + self.filesystem = filesystem + self.device_path = None + self.initial_state = 'nonexistent' + + callbacks.update({'onbeforeformat': self._format, + 'onbeforemount': self._mount, + 'onbeforeunmount': self._unmount, + }) + + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def get_uuid(self): + [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) + return uuid + + def _format(self, e): + mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) + log_check_call([mkfs, self.device_path]) + + def mount(self, destination): + self.fsm.mount(destination=destination) + + def _mount(self, e): + log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) + self.mount_dir = e.destination + + def _unmount(self, e): + log_check_call(['/bin/umount', self.mount_dir]) + del self.mount_dir diff --git a/base/fs/partitions/partition.py b/base/fs/partitions/partition.py new file mode 100644 index 0000000..a62ac0e --- /dev/null +++ b/base/fs/partitions/partition.py @@ -0,0 +1,60 @@ +from common.tools import log_check_call +from abstractpartition import AbstractPartition + + +class Partition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, filesystem, previous, callbacks={}): + self.previous = previous + callbacks.update({'onbeforecreate': self._create, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + }) + super(Partition, self).__init__(size, filesystem, callbacks=callbacks) + + def get_index(self): + if self.previous is None: + return 1 + else: + return self.previous.get_index()+1 + + def get_start(self): + if self.previous is None: + return 0 + else: + return self.previous.get_start() + self.previous.size + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + start = self.get_start() + # maybe use `parted -- name` to set partition name + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', e.volume.device_path, + '--', 'mkpart', 'primary', str(start), str(start + self.size)]) + + def map(self, device_path): + self.fsm.map(device_path=device_path) + + def _map(self, e): + self.device_path = e.device_path + + def _unmap(self, e): + self.device_path = None + +# Partition flags: +# boot, root, swap, hidden, raid, lvm, lba, legacy_boot, palo + + +# Partition tables +# bsd, dvh, gpt, loop, mac, msdos, pc98, sun diff --git a/base/fs/partitions/singlepartition.py b/base/fs/partitions/singlepartition.py new file mode 100644 index 0000000..10bd702 --- /dev/null +++ b/base/fs/partitions/singlepartition.py @@ -0,0 +1,20 @@ +from abstractpartition import AbstractPartition + + +class SinglePartition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + callbacks['oncreate'] = self._create + super(SinglePartition, self).__init__(size, filesystem, callbacks=callbacks) + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + self.device_path = e.volume.device_path diff --git a/base/fs/partitions/swap.py b/base/fs/partitions/swap.py new file mode 100644 index 0000000..80b7531 --- /dev/null +++ b/base/fs/partitions/swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from partition import Partition + + +class Swap(Partition): + + def __init__(self, size, previous): + super(Swap, self).__init__(size, 'swap', previous) + + def _format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/volume.py b/base/fs/volume.py new file mode 100644 index 0000000..8d063f1 --- /dev/null +++ b/base/fs/volume.py @@ -0,0 +1,138 @@ +from abc import ABCMeta +from fysom import Fysom + + +class Volume(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'partition', 'src': 'attached', 'dst': 'partitioned'}, + {'name': 'map', 'src': 'partitioned', 'dst': 'mapped'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': ['formatted', 'mounted'], 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'partitioned_fmt'}, + {'name': 'detach', 'src': 'partitioned_fmt', 'dst': 'detached_fmt'}, + {'name': 'delete', 'src': ['detached', 'detached_prt', 'detached_fmt'], 'dst': 'deleted'}, + + {'name': 'attach', 'src': 'detached_fmt', 'dst': 'partitioned_fmt'}, + {'name': 'map', 'src': 'partitioned_fmt', 'dst': 'formatted'}, + + {'name': 'attach', 'src': 'detached_prt', 'dst': 'partitioned'}, + {'name': 'detach', 'src': 'partitioned', 'dst': 'detached_prt'}, + + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'partitioned'}, + ] + + mount_events = [{'name': 'mount_root', 'src': 'unmounted', 'dst': 'root_mounted'}, + {'name': 'mount_boot', 'src': 'root_mounted', 'dst': 'boot_mounted'}, + {'name': 'mount_specials', 'src': 'boot_mounted', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'boot_mounted'}, + {'name': 'unmount_boot', 'src': 'boot_mounted', 'dst': 'root_mounted'}, + {'name': 'unmount_root', 'src': 'root_mounted', 'dst': 'unmounted'}, + + {'name': 'mount_specials', 'src': 'root_mounted', 'dst': 'specials_mounted_no_boot'}, + {'name': 'mount_boot', 'src': 'specials_mounted_no_boot', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted_no_boot', 'dst': 'root_mounted'}, + {'name': 'unmount_boot', 'src': 'specials_mounted', 'dst': 'specials_mounted_no_boot'}, + ] + + def __init__(self, partition_map, callbacks={}): + self.device_path = None + self.partition_map = partition_map + self.size = self.partition_map.get_total_size() + + callbacks.update({'onbeforepartition': self._partition, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + 'onbeforeformat': self._format, + 'onbeforeunmount': self._unmount, + }) + + mount_callbacks = {'onbeforemount_root': self._mount_root, + 'onbeforemount_boot': self._mount_boot, + 'onbeforemount_specials': self._mount_specials, + 'onbeforeunmount_root': self._unmount_root, + 'onbeforeunmount_boot': self._unmount_boot, + 'onbeforeunmount_specials': self._unmount_specials, + } + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + + self.mount_fsm = Fysom({'initial': 'unmounted', + 'events': self.mount_events, + 'callbacks': mount_callbacks}) + + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + attach_proxy_methods(self, self.mount_events, self.mount_fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def _partition(self, e): + self.partition_map.create(self) + + def _map(self, e): + self.partition_map.map(self) + + def _unmap(self, e): + self.partition_map.unmap(self) + + def _format(self, e): + self.partition_map.format() + + def _unmount(self, e): + if self.mount_fsm.current is 'specials_mounted': + self.unmount_specials() + if self.mount_fsm.current is 'specials_mounted_no_boot': + self.unmount_specials() + if self.mount_fsm.current is 'boot_mounted': + self.unmount_boot() + if self.mount_fsm.current is 'root_mounted': + self.unmount_root() + + def mount_root(self, destination): + self.mount_fsm.mount_root(destination=destination) + + def unmount_root(self): + self.mount_fsm.unmount_root() + self.fsm.unmount() + + def _mount_root(self, e): + self.mount() + self.partition_map.mount_root(e.destination) + + def _unmount_root(self, e): + self.partition_map.unmount_root() + + def _mount_boot(self, e): + self.mount() + self.partition_map.mount_boot() + + def _unmount_boot(self, e): + self.partition_map.unmount_boot() + + def _mount_specials(self, e): + self.mount() + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + + def _unmount_specials(self, e): + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 6c5627d..3a982c4 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -34,14 +34,13 @@ "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { - "type": "object", - "properties": { - "size": { - "type": "integer", - "minimum": 1 - } - }, - "required": ["size"] + "type": "object" + // "properties": { + // "backing": { + // "type": "string" + // } + // }, + // "required": ["backing"] }, "plugins": { "type": "object", diff --git a/common/fs/__init__.py b/common/fs/__init__.py new file mode 100644 index 0000000..20b2f69 --- /dev/null +++ b/common/fs/__init__.py @@ -0,0 +1,11 @@ + + +def get_major_minor_dev_num(device_name): + import re + regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) {device_name}$' + .format(device_name=device_name)) + with open('/proc/partitions') as partitions: + for line in partitions: + match = regexp.match(line) + if match is not None: + return match.group('major'), match.group('minor') diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py new file mode 100644 index 0000000..c8ffb1e --- /dev/null +++ b/common/fs/loopbackvolume.py @@ -0,0 +1,90 @@ +from base.fs.volume import Volume +from common.tools import log_check_call +from base.fs.exceptions import VolumeError + + +# QEMU formats: +# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop + + +class LoopbackVolume(Volume): + + link_dm_events = [{'name': 'link_dm_node', 'src': 'partitioned', 'dst': 'linked'}, + {'name': 'map', 'src': 'linked', 'dst': 'mapped_lnk'}, + {'name': 'format', 'src': 'mapped_lnk', 'dst': 'formatted_lnk'}, + {'name': 'mount', 'src': ['formatted_lnk', 'mounted_lnk'], 'dst': 'mounted_lnk'}, + {'name': 'unmount', 'src': 'mounted_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'formatted_lnk', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'unlink_dm_node', 'src': 'partitioned_fmt_lnk', 'dst': 'partitioned_fmt'}, + + {'name': 'link_dm_node', 'src': 'partitioned_fmt', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'map', 'src': 'partitioned_fmt_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'mapped_lnk', 'dst': 'linked'}, + {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, + ] + + def __init__(self, partition_map, callbacks={}): + callbacks.update({'onbeforecreate': self._create, + 'onbeforeattach': self._attach, + 'onbeforedetach': self._detach, + 'onbeforedelete': self._delete, + 'onbeforelink_dm_node': self._link_dm_node, + 'onbeforeunlink_dm_node': self._unlink_dm_node, + }) + self.events.extend(self.link_dm_events) + super(LoopbackVolume, self).__init__(partition_map, callbacks=callbacks) + + def create(self, image_path): + self.fsm.create(image_path=image_path) + + def _create(self, e): + self.image_path = e.image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) + + def _attach(self, e): + [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) + self.device_path = self.loop_device_path + + def _link_dm_node(self, e): + import re + loop_device_name = re.match('^/dev/(?P.*)$', self.loop_device_path).group('name') + from . import get_major_minor_dev_num + major, minor = get_major_minor_dev_num(loop_device_name) + sectors = self.size*1024*1024/512 + table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' + .format(log_start_sec=0, + sectors=sectors, + major=major, + minor=minor, + start_sec=0)) + import string + import os.path + for letter in string.ascii_lowercase: + dev_name = 'vd' + letter + dev_path = os.path.join('/dev/mapper', dev_name) + if not os.path.exists(dev_path): + self.dm_node_name = dev_name + self.dm_node_path = dev_path + break + + if not hasattr(self, 'dm_node_name'): + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) + self.device_path = self.dm_node_path + + def _unlink_dm_node(self, e): + log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) + del self.dm_node_name + del self.dm_node_path + self.device_path = self.loop_device_path + + def _detach(self, e): + log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _delete(self, e): + from os import remove + remove(self.image_path) + del self.image_path diff --git a/common/fsm.py b/common/fsm.py new file mode 100644 index 0000000..44286e0 --- /dev/null +++ b/common/fsm.py @@ -0,0 +1,7 @@ + + +def attach_proxy_methods(obj, events, fsm): + methods = set([e['name'] for e in events]) + for event in methods: + if not hasattr(obj, event): + setattr(obj, event, lambda e=event: getattr(fsm, e)()) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index c0b5a75..f1d658e 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -1,29 +1,29 @@ from base import Task from common import phases -from common.exceptions import TaskError from common.tools import log_check_call from bootstrap import Bootstrap +import volume -class FormatVolume(Task): +class Format(Task): description = 'Formatting the volume' phase = phases.volume_preparation def run(self, info): - dev_path = info.bootstrap_device['path'] - mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) - log_check_call([mkfs, dev_path]) - info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']} + info.volume.format() class TuneVolumeFS(Task): description = 'Tuning the bootstrap volume filesystem' phase = phases.volume_preparation - after = [FormatVolume] + after = [Format] def run(self, info): + import re # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']]) + for partition in info.volume.partition_map.partitions: + if re.match('^ext[2-4]$', partition.filesystem) is not None: + log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): @@ -36,33 +36,45 @@ class AddXFSProgs(Task): class CreateMountDir(Task): - description = 'Creating mountpoint for the bootstrap volume' + description = 'Creating mountpoint for the root partition' phase = phases.volume_mounting def run(self, info): import os mount_dir = info.manifest.bootstrapper['mount_dir'] info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) - # Works recursively, fails if last part exists, which is exaclty what we want. + # Works recursively, fails if last part exists, which is exactly what we want. os.makedirs(info.root) -class MountVolume(Task): - description = 'Mounting the bootstrap volume' +class MountRoot(Task): + description = 'Mounting the root partition' phase = phases.volume_mounting after = [CreateMountDir] def run(self, info): - with open('/proc/mounts') as mounts: - for mount in mounts: - if info.root in mount: - msg = 'Something is already mounted at {root}'.format(root=info.root) - raise TaskError(msg) + info.volume.mount_root(info.root) - log_check_call(['/bin/mount', - '--types', info.manifest.volume['filesystem'], - info.bootstrap_device['partitions']['root_path'], - info.root]) + +class MountBoot(Task): + description = 'Mounting the boot partition' + phase = phases.volume_mounting + after = [MountRoot] + + def run(self, info): + info.volume.mount_boot() + + +class CreateBootMountDir(Task): + description = 'Creating mountpoint boot partition' + phase = phases.volume_mounting + after = [MountRoot] + before = [MountBoot] + + def run(self, info): + import os + info.boot = os.path.join(info.root, 'boot') + os.makedirs() class MountSpecials(Task): @@ -71,36 +83,40 @@ class MountSpecials(Task): after = [Bootstrap] def run(self, info): - log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + info.volume.mount_specials() + + +class UnmountRoot(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + before = [volume.Detach] + + def run(self, info): + info.volume.unmount_root() + + +class UnmountBoot(Task): + description = 'Unmounting the boot partition' + phase = phases.volume_unmounting + before = [UnmountRoot] + + def run(self, info): + info.volume.unmount_boot() class UnmountSpecials(Task): description = 'Unmunting special block devices' phase = phases.volume_unmounting + before = [UnmountRoot] def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) - log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) - - -class UnmountVolume(Task): - description = 'Unmounting the bootstrap volume' - phase = phases.volume_unmounting - after = [UnmountSpecials] - - def run(self, info): - log_check_call(['/bin/umount', info.root]) + info.volume.unmount_specials() class DeleteMountDir(Task): description = 'Deleting mountpoint for the bootstrap volume' phase = phases.volume_unmounting - after = [UnmountVolume] + after = [UnmountRoot] def run(self, info): import os @@ -108,24 +124,28 @@ class DeleteMountDir(Task): del info.root -class ModifyFstab(Task): - description = 'Adding root volume to the fstab' +class FStab(Task): + description = 'Adding partitions to the fstab' phase = phases.system_modification def run(self, info): import os.path - mount_opts = ['defaults'] - if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if info.manifest.volume['filesystem'].lower() == 'xfs': - mount_opts.append('nobarrier') - fstab_path = os.path.join(info.root, 'etc/fstab') + # device = '/dev/sda' + # if info.manifest.virtualization == 'pvm': + # device = '/dev/xvda' + fstab_lines = [] + for mount_point, partition in info.volume.partition_map.mount_points: + mount_opts = ['defaults'] + if partition.filesystem in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if partition.filesystem == 'xfs': + mount_opts.append('nobarrier') + fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} 1 1' + .format(uuid=partition.get_uuid(), + mountpoint=mount_point, + filesystem=partition.filesystem, + mount_opts=','.join(mount_opts))) - device = '/dev/sda1' - if info.manifest.virtualization == 'pvm': - device = '/dev/xvda1' - with open(fstab_path, 'a') as fstab: - fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' - .format(device=device, - filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts))) + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index b801fc1..2db681c 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from filesystem import UnmountVolume -from common.tools import log_check_call +import volume class Create(Task): description = 'Creating a loopback volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/bin/dd', - 'if=/dev/zero', 'of=' + info.loopback_file, - 'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0']) - - -class CreateQemuImg(Task): - description = 'Creating a loopback volume with qemu' - phase = phases.volume_creation - - def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', - info.loopback_file, str(info.manifest.volume['size']) + 'M']) - - -class Attach(Task): - description = 'Attaching the loopback volume' - phase = phases.volume_creation - after = [Create, CreateQemuImg] - - def run(self, info): - info.bootstrap_device = {} - command = ['/sbin/losetup', '--show', '--find', info.loopback_file] - [info.bootstrap_device['path']] = log_check_call(command) - - -class Detach(Task): - description = 'Detaching the loopback volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']]) - del info.bootstrap_device - - -class Delete(Task): - description = 'Deleting the loopback volume' - phase = phases.cleaning - - def run(self, info): - from os import remove - remove(info.loopback_file) - del info.loopback_file + image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.create(image_path) diff --git a/common/tasks/parted.py b/common/tasks/parted.py deleted file mode 100644 index e242765..0000000 --- a/common/tasks/parted.py +++ /dev/null @@ -1,53 +0,0 @@ -from base import Task -from common import phases -from common.tools import log_check_call -import filesystem -import loopback - - -class PartitionVolume(Task): - description = 'Partitioning the volume' - phase = phases.volume_preparation - - def run(self, info): - # parted - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mklabel', 'msdos']) - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mkpart', 'primary', 'ext4', '32k', '-1']) - log_check_call(['parted', '--script', info.bootstrap_device['path'], - '--', 'set', '1', 'boot', 'on']) - - -class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - after = [PartitionVolume] - - def run(self, info): - root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1' - log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) - info.bootstrap_device['partitions'] = {'root_path': root_partition_path} - - -class FormatPartitions(Task): - description = 'Formatting the partitions' - phase = phases.volume_preparation - before = [filesystem.TuneVolumeFS] - after = [MapPartitions] - - def run(self, info): - # These params will fail for mkfs.xfs - log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), - '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']]) - - -class UnmapPartitions(Task): - description = 'Removing volume partitions mapping' - phase = phases.volume_unmounting - before = [loopback.Detach] - after = [filesystem.UnmountVolume] - - def run(self, info): - log_check_call(['kpartx', '-d', info.bootstrap_device['path']]) - del info.bootstrap_device['partitions']['root_path'] diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py new file mode 100644 index 0000000..181bb63 --- /dev/null +++ b/common/tasks/partitioning.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import filesystem +import volume + + +class PartitionVolume(Task): + description = 'Partitioning the volume' + phase = phases.volume_preparation + + def run(self, info): + info.volume.partition() + + +class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + before = [filesystem.Format] + after = [PartitionVolume] + + def run(self, info): + info.volume.map() + + +class UnmapPartitions(Task): + description = 'Removing volume partitions mapping' + phase = phases.volume_unmounting + before = [volume.Detach] + after = [filesystem.UnmountRoot] + + def run(self, info): + info.volume.unmap() diff --git a/common/tasks/volume.py b/common/tasks/volume.py new file mode 100644 index 0000000..aa2a557 --- /dev/null +++ b/common/tasks/volume.py @@ -0,0 +1,26 @@ +from base import Task +from common import phases + + +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + + def run(self, info): + info.volume.attach() + + +class Detach(Task): + description = 'Detaching the volume' + phase = phases.volume_unmounting + + def run(self, info): + info.volume.detach() + + +class Delete(Task): + description = 'Deleting the volume' + phase = phases.cleaning + + def run(self, info): + info.volume.delete() diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index bb63082..fb0a704 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -23,8 +23,14 @@ }, "volume": { "backing": "ebs", - "filesystem": "ext4", - "size": 8192 + "partition_table": { + "type": null, + "partitions": [ + {"size": 8192, + "filesystem": "ext4", + "flags": ["root"]} + ] + } }, "plugins": { "admin_user": { diff --git a/manifests/virtualbox.json b/manifests/virtualbox.json deleted file mode 100644 index 238ef98..0000000 --- a/manifests/virtualbox.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "provider" : "virtualbox", - "virtualization": "ide", - - "bootstrapper": { - "mount_dir": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "convert_image": { - "enabled": true, - "format": "vdi" - } - } -} diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json new file mode 100644 index 0000000..1b0ab6a --- /dev/null +++ b/manifests/virtualbox.manifest.json @@ -0,0 +1,50 @@ +{ + "provider": "virtualbox", + "virtualization": "ide", + + "bootstrapper": { + "mount_dir": "/mnt/target", + "mirror": "http://http.debian.net/debian/" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "raw", + "loopback_dir": "/tmp", + "partitions": { + "boot": { + "size": 12, + "filesystem": "ext2" + }, + "root": { + "size": 812, + "filesystem": "ext4" + }, + "swap": {"size": 200} + } + }, + "plugins": { + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + }, + "root_password": { + "enabled": true, + "password": "test" + }, + "convert_image": { + "enabled": true, + "format": "vdi" + } + } +} diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index 8fd94ea..dc258a7 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -4,35 +4,31 @@ from tasks import CreateFromSnapshot from tasks import CreateFromImage from providers.ec2.tasks import ebs from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap from common.tasks import filesystem -from common.tasks import parted +from common.tasks import partitioning def tasks(tasklist, manifest): settings = manifest.plugins['prebootstrapped'] + skip_tasks = [filesystem.Format, + partitioning.PartitionVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + filesystem.CreateBootMountDir, + bootstrap.MakeTarball, + bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: tasklist.replace(ebs.Create, CreateFromSnapshot()) - tasklist.remove(filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.remove(*skip_tasks) else: tasklist.add(Snapshot()) else: if 'image' in settings and settings['image'] is not None: - tasklist.add(CreateFromImage()) - tasklist.remove(loopback.Create, - loopback.CreateQemuImg, - parted.PartitionVolume, - parted.FormatPartitions, - filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.replace(loopback.Create, CreateFromImage()) + tasklist.remove(*skip_tasks) else: tasklist.add(CopyImage()) @@ -45,9 +41,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'] == 'ebs': - counter_task(CreateFromSnapshot, ebs.Delete) + counter_task(CreateFromSnapshot, volume.Delete) else: - counter_task(CreateFromImage, loopback.Delete) + counter_task(CreateFromImage, volume.Delete) def validate_manifest(data, schema_validate): diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index bcaebc8..57d0610 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,7 +1,7 @@ from base import Task from common import phases from providers.ec2.tasks import ebs -from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap import time import logging @@ -22,18 +22,25 @@ class Snapshot(ebs.Snapshot): class CreateFromSnapshot(Task): description = 'Creating EBS volume from a snapshot' phase = phases.volume_creation - before = [ebs.Attach] + before = [volume.Attach] def run(self, info): volume_size = int(info.manifest.volume['size'] / 1024) snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - info.volume = info.connection.create_volume(volume_size, - info.host['availabilityZone'], - snapshot=snapshot) + info.volume.volume = info.connection.create_volume(volume_size, + info.host['availabilityZone'], + snapshot=snapshot) while info.volume.volume_state() != 'available': time.sleep(5) info.volume.update() + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) + class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' @@ -45,7 +52,7 @@ class CopyImage(Task): from shutil import copyfile loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.loopback_file, image_copy_path) + copyfile(info.volume.image_path, image_copy_path) msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) log.info(msg) @@ -53,12 +60,19 @@ class CopyImage(Task): class CreateFromImage(Task): description = 'Creating loopback image from a copy' phase = phases.volume_creation - before = [loopback.Attach] + before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] from shutil import copyfile - copyfile(loopback_backup_path, info.loopback_file) + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] + copyfile(loopback_backup_path, info.volume.image_path) + + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 04be78b..ee63e71 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -6,6 +6,7 @@ from tasks import connection from tasks import host from common.tasks import host as common_host from tasks import ami +from common.tasks import volume from tasks import ebs from common.tasks import loopback from common.tasks import filesystem @@ -49,7 +50,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -77,16 +78,16 @@ def tasks(tasklist, manifest): tasklist.add(bootstrap.MakeTarball()) backing_specific_tasks = {'ebs': [ebs.Create(), - ebs.Attach(), - ebs.Detach(), + volume.Attach(), + volume.Detach(), ebs.Snapshot(), - ebs.Delete()], + volume.Delete()], 's3': [loopback.Create(), - loopback.Attach(), - loopback.Detach(), + volume.Attach(), + volume.Detach(), ami.BundleImage(), ami.UploadImage(), - loopback.Delete(), + volume.Delete(), ami.RemoveBundle()]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) @@ -105,11 +106,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, ebs.Delete) - counter_task(ebs.Attach, ebs.Detach) + counter_task(ebs.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, loopback.Delete) - counter_task(loopback.Attach, loopback.Detach) + counter_task(loopback.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index 85ad4b6..bb84a44 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from common.exceptions import TaskError -from common.tasks.filesystem import UnmountVolume -import time +from common.tasks import volume class Create(Task): - description = 'Creating an EBS volume for bootstrapping' + description = 'Creating the EBS volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): - info.volume = info.connection.create_volume(info.manifest.ebs_volume_size, - info.host['availabilityZone']) - while info.volume.volume_state() != 'available': - time.sleep(5) - info.volume.update() - - -class Attach(Task): - description = 'Attaching the EBS volume' - phase = phases.volume_creation - after = [Create] - - def run(self, info): - def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" - for c in xrange(ord(c1), ord(c2) + 1): - yield chr(c) - - import os.path - info.bootstrap_device = {} - for letter in char_range('f', 'z'): - dev_path = os.path.join('/dev', 'xvd' + letter) - if not os.path.exists(dev_path): - info.bootstrap_device['path'] = dev_path - info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) - break - if 'path' not in info.bootstrap_device: - raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - - info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) - while info.volume.attachment_state() != 'attached': - time.sleep(2) - info.volume.update() - - -class Detach(Task): - description = 'Detaching the EBS volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - info.volume.detach() - while info.volume.attachment_state() is not None: - time.sleep(2) - info.volume.update() + info.volume.create(info.connection, info.host['availabilityZone']) class Snapshot(Task): @@ -62,20 +17,4 @@ class Snapshot(Task): phase = phases.image_registration def run(self, info): - info.snapshot = info.volume.create_snapshot() - while info.snapshot.status != 'completed': - time.sleep(2) - info.snapshot.update() - - -class Delete(Task): - description = 'Deleting the EBS volume' - phase = phases.cleaning - - def run(self, info): - info.volume.delete() - del info.volume - - -class VolumeError(TaskError): - pass + info.snapshot = info.volume.snapshot() diff --git a/providers/ec2/volume.py b/providers/ec2/volume.py new file mode 100644 index 0000000..bd1983f --- /dev/null +++ b/providers/ec2/volume.py @@ -0,0 +1,56 @@ +from base.fs.volume import Volume +from base.fs.exceptions import VolumeError +import time + + +class EBSVolume(Volume): + + volume = None + + def create(self, conn, zone): + super(EBSVolume, self).create(self) + import math + # TODO: Warn if volume size is not a multiple of 1024 + size = int(math.ceil(self.partition_map.get_volume_size() / 1024)) + self.volume = conn.create_volume(size, zone) + while self.volume.volume_state() != 'available': + time.sleep(5) + self.volume.update() + self.created = True + + def attach(self, instance_id): + super(EBSVolume, self).attach(self) + import os.path + import string + for letter in string.ascii_lowercase: + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + self.device_path = dev_path + self.ec2_device_path = os.path.join('/dev', 'sd' + letter) + break + + if self.device_path is None: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + self.volume.attach(instance_id, self.ec2_device_path) + while self.volume.attachment_state() != 'attached': + time.sleep(2) + self.volume.update() + + def detach(self): + super(EBSVolume, self).detach(self) + self.volume.detach() + while self.volume.attachment_state() is not None: + time.sleep(2) + self.volume.update() + + def delete(self): + super(EBSVolume, self).delete(self) + self.volume.delete() + + def snapshot(self): + snapshot = self.volume.create_snapshot() + while snapshot.status != 'completed': + time.sleep(2) + snapshot.update() + return snapshot diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py index 6a2603c..724a250 100644 --- a/providers/kvm/__init__.py +++ b/providers/kvm/__init__.py @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 6a2603c..c6382ae 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -2,8 +2,9 @@ from manifest import Manifest from tasks import packages from common.tasks import packages as common_packages from common.tasks import host +from common.tasks import volume as volume_tasks from common.tasks import loopback -from common.tasks import parted +from common.tasks import partitioning from common.tasks import filesystem from common.tasks import bootstrap from common.tasks import locale @@ -14,7 +15,6 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup -from common.tasks import loopback def initialize(): @@ -28,13 +28,13 @@ def tasks(tasklist, manifest): common_packages.ImagePackages(), host.CheckPackages(), - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), + loopback.Create(), + volume_tasks.Attach(), + partitioning.PartitionVolume(), + partitioning.MapPartitions(), + filesystem.Format(), filesystem.CreateMountDir(), - filesystem.MountVolume(), + filesystem.MountRoot(), bootstrap.Bootstrap(), filesystem.MountSpecials(), @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -63,19 +63,32 @@ def tasks(tasklist, manifest): apt.EnableDaemonAutostart(), filesystem.UnmountSpecials(), - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) + filesystem.UnmountRoot(), + partitioning.UnmapPartitions(), + volume_tasks.Detach(), + filesystem.DeleteMountDir(), + volume_tasks.Delete()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + partitions = manifest.volume['partitions'] + import re + for key in ['boot', 'root']: + if key not in partitions: + continue + if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None: + tasklist.add(filesystem.TuneVolumeFS()) + break + for key in ['boot', 'root']: + if key not in partitions: + continue + if partitions[key]['filesystem'] == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + break + + if 'boot' in manifest.volume['partitions']: + tasklist.add(filesystem.MountBoot(), filesystem.UnmountBoot()) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -85,8 +98,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if task in completed and counter not in completed: tasklist.add(counter()) + counter_task(loopback.Create, volume_tasks.Delete) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 52ac2ee..0d51f6e 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -9,13 +9,13 @@ "backing": { "type": "string", "enum": ["raw", "qcow2"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] } + // "filesystem": { + // "type": "string", + // "enum": ["ext2", "ext3", "ext4", "xfs"] + // } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, "required": ["volume"] diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 428f70a..bf96e11 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -1,34 +1,47 @@ from base import Task from common import phases +from common.tasks import apt +from common.fs.loopbackvolume import LoopbackVolume class ConfigureGrub(Task): description = 'Configuring grub' phase = phases.system_modification + after = [apt.AptUpgrade] def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - + import os from common.tools import log_check_call - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/", - info.bootstrap_device['path']]) + boot_dir = os.path.join(info.root, 'boot') + grub_dir = os.path.join(boot_dir, 'grub') + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.link_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() + [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) + device_map_path = os.path.join(grub_dir, 'device.map') + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + + # Install grub + log_check_call(['/usr/sbin/grub-install', + '--root-directory=' + info.root, + '--boot-directory=' + boot_dir, + device_path]) log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + # log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.unlink_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 9bdd204..207f356 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -12,7 +12,7 @@ class HostPackages(Task): def run(self, info): info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py new file mode 100644 index 0000000..3425729 --- /dev/null +++ b/providers/virtualbox/volume.py @@ -0,0 +1,11 @@ +from common.fs.loopbackvolume import LoopbackVolume +from common.tools import log_check_call + + +class VirtualBoxVolume(LoopbackVolume): + + def create(self, image_path): + super(VirtualBoxVolume, self).create(self) + self.image_path = image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + self.created = True