diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index d1c03ec..07c954b 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -3,6 +3,8 @@ class BootstrapInformation(object): def __init__(self, manifest=None, debug=False): self.manifest = manifest + from fs import load_volume + self.volume = load_volume(self.manifest.volume) self.debug = debug import random self.run_id = random.randrange(16 ** 8) diff --git a/base/fs/__init__.py b/base/fs/__init__.py new file mode 100644 index 0000000..6180894 --- /dev/null +++ b/base/fs/__init__.py @@ -0,0 +1,17 @@ + + +def load_volume(data): + from common.fs.loopbackvolume import LoopbackVolume + from providers.ec2.volume import EBSVolume + from providers.virtualbox.volume import VirtualBoxVolume + from partitionmap import PartitionMap + from nopartitions import NoPartitions + partition_maps = {'none': NoPartitions, + 'gpt': PartitionMap, + } + partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) + volume_backings = {'raw': LoopbackVolume, + 'vdi': VirtualBoxVolume, + 'ebs': EBSVolume + } + return volume_backings.get(data['backing'])(partition_map) diff --git a/base/fs/exceptions.py b/base/fs/exceptions.py new file mode 100644 index 0000000..bc38490 --- /dev/null +++ b/base/fs/exceptions.py @@ -0,0 +1,8 @@ + + +class VolumeError(Exception): + pass + + +class PartitionError(Exception): + pass diff --git a/base/fs/nopartitions.py b/base/fs/nopartitions.py new file mode 100644 index 0000000..b8935ca --- /dev/null +++ b/base/fs/nopartitions.py @@ -0,0 +1,30 @@ +from partitions.singlepartition import SinglePartition + + +class NoPartitions(object): + + def __init__(self, data): + root = data['root'] + self.root = SinglePartition(root['size'], root['filesystem']) + self.mount_points = [('/', self.root)] + + def get_total_size(self): + return self.root.size + + def create(self, volume): + pass + + def map(self, volume): + pass + + def unmap(self, volume): + pass + + def format(self): + self.root.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() diff --git a/base/fs/partitionmap.py b/base/fs/partitionmap.py new file mode 100644 index 0000000..49a75ab --- /dev/null +++ b/base/fs/partitionmap.py @@ -0,0 +1,88 @@ +from common.tools import log_check_call +from partitions.partition import Partition +from partitions.swap import Swap +from exceptions import PartitionError + + +class PartitionMap(object): + + def __init__(self, data): + self.boot = None + self.swap = None + self.mount_points = [] + if 'boot' in data: + self.boot = Partition(data['boot']['size'], data['boot']['filesystem'], None) + self.mount_points.append(('/boot', self.boot)) + self.root = Partition(data['root']['size'], data['root']['filesystem'], self.boot) + self.mount_points.append(('/', self.root)) + if 'swap' in data: + self.swap = Swap(data['swap']['size'], self.root) + self.mount_points.append(('none', self.root)) + self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + + def get_total_size(self): + return sum(p.size for p in self.partitions) + + def create(self, volume): + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', volume.device_path, + '--', 'mklabel', 'gpt']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if self.boot is not None: + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set', str(boot_idx), 'bios_grub', 'on']) + + def map(self, volume): + try: + mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) + import re + regexp = re.compile('^(?P.+[^\d](?P\d+)) : ' + '(?P\d) (?P\d+) ' + '{device_path} (?P\d+)$' + .format(device_path=volume.device_path)) + log_check_call(['/sbin/kpartx', '-a', volume.device_path]) + import os.path + for mapping in mappings: + match = regexp.match(mapping) + if match is None: + raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping)) + partition_path = os.path.join('/dev/mapper', match.group('name')) + p_idx = int(match.group('p_idx'))-1 + self.partitions[p_idx].map(partition_path) + + for idx, partition in enumerate(self.partitions): + if not partition.state() in ['mapped', 'formatted']: + raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) + + except PartitionError as e: + for partition in self.partitions: + if partition.state() in ['mapped', 'formatted']: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + raise e + + def unmap(self, volume): + for partition in self.partitions: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + + def format(self): + for partition in self.partitions: + partition.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() + + def mount_boot(self): + import os.path + destination = os.path.join(self.root.mount_dir, 'boot') + self.boot.mount(destination) + + def unmount_boot(self): + self.boot.unmount() diff --git a/base/fs/partitions/__init__.py b/base/fs/partitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/fs/partitions/abstractpartition.py b/base/fs/partitions/abstractpartition.py new file mode 100644 index 0000000..161e134 --- /dev/null +++ b/base/fs/partitions/abstractpartition.py @@ -0,0 +1,56 @@ +from common.tools import log_check_call +from abc import ABCMeta +from fysom import Fysom + + +class AbstractPartition(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + self.size = size + self.filesystem = filesystem + self.device_path = None + self.initial_state = 'nonexistent' + + callbacks.update({'onbeforeformat': self._format, + 'onbeforemount': self._mount, + 'onbeforeunmount': self._unmount, + }) + + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def get_uuid(self): + [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) + return uuid + + def _format(self, e): + mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) + log_check_call([mkfs, self.device_path]) + + def mount(self, destination): + self.fsm.mount(destination=destination) + + def _mount(self, e): + log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) + self.mount_dir = e.destination + + def _unmount(self, e): + log_check_call(['/bin/umount', self.mount_dir]) + del self.mount_dir diff --git a/base/fs/partitions/partition.py b/base/fs/partitions/partition.py new file mode 100644 index 0000000..a62ac0e --- /dev/null +++ b/base/fs/partitions/partition.py @@ -0,0 +1,60 @@ +from common.tools import log_check_call +from abstractpartition import AbstractPartition + + +class Partition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, filesystem, previous, callbacks={}): + self.previous = previous + callbacks.update({'onbeforecreate': self._create, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + }) + super(Partition, self).__init__(size, filesystem, callbacks=callbacks) + + def get_index(self): + if self.previous is None: + return 1 + else: + return self.previous.get_index()+1 + + def get_start(self): + if self.previous is None: + return 0 + else: + return self.previous.get_start() + self.previous.size + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + start = self.get_start() + # maybe use `parted -- name` to set partition name + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', e.volume.device_path, + '--', 'mkpart', 'primary', str(start), str(start + self.size)]) + + def map(self, device_path): + self.fsm.map(device_path=device_path) + + def _map(self, e): + self.device_path = e.device_path + + def _unmap(self, e): + self.device_path = None + +# Partition flags: +# boot, root, swap, hidden, raid, lvm, lba, legacy_boot, palo + + +# Partition tables +# bsd, dvh, gpt, loop, mac, msdos, pc98, sun diff --git a/base/fs/partitions/singlepartition.py b/base/fs/partitions/singlepartition.py new file mode 100644 index 0000000..10bd702 --- /dev/null +++ b/base/fs/partitions/singlepartition.py @@ -0,0 +1,20 @@ +from abstractpartition import AbstractPartition + + +class SinglePartition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + callbacks['oncreate'] = self._create + super(SinglePartition, self).__init__(size, filesystem, callbacks=callbacks) + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + self.device_path = e.volume.device_path diff --git a/base/fs/partitions/swap.py b/base/fs/partitions/swap.py new file mode 100644 index 0000000..80b7531 --- /dev/null +++ b/base/fs/partitions/swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from partition import Partition + + +class Swap(Partition): + + def __init__(self, size, previous): + super(Swap, self).__init__(size, 'swap', previous) + + def _format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/volume.py b/base/fs/volume.py new file mode 100644 index 0000000..8d063f1 --- /dev/null +++ b/base/fs/volume.py @@ -0,0 +1,138 @@ +from abc import ABCMeta +from fysom import Fysom + + +class Volume(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'partition', 'src': 'attached', 'dst': 'partitioned'}, + {'name': 'map', 'src': 'partitioned', 'dst': 'mapped'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': ['formatted', 'mounted'], 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'partitioned_fmt'}, + {'name': 'detach', 'src': 'partitioned_fmt', 'dst': 'detached_fmt'}, + {'name': 'delete', 'src': ['detached', 'detached_prt', 'detached_fmt'], 'dst': 'deleted'}, + + {'name': 'attach', 'src': 'detached_fmt', 'dst': 'partitioned_fmt'}, + {'name': 'map', 'src': 'partitioned_fmt', 'dst': 'formatted'}, + + {'name': 'attach', 'src': 'detached_prt', 'dst': 'partitioned'}, + {'name': 'detach', 'src': 'partitioned', 'dst': 'detached_prt'}, + + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'partitioned'}, + ] + + mount_events = [{'name': 'mount_root', 'src': 'unmounted', 'dst': 'root_mounted'}, + {'name': 'mount_boot', 'src': 'root_mounted', 'dst': 'boot_mounted'}, + {'name': 'mount_specials', 'src': 'boot_mounted', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'boot_mounted'}, + {'name': 'unmount_boot', 'src': 'boot_mounted', 'dst': 'root_mounted'}, + {'name': 'unmount_root', 'src': 'root_mounted', 'dst': 'unmounted'}, + + {'name': 'mount_specials', 'src': 'root_mounted', 'dst': 'specials_mounted_no_boot'}, + {'name': 'mount_boot', 'src': 'specials_mounted_no_boot', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted_no_boot', 'dst': 'root_mounted'}, + {'name': 'unmount_boot', 'src': 'specials_mounted', 'dst': 'specials_mounted_no_boot'}, + ] + + def __init__(self, partition_map, callbacks={}): + self.device_path = None + self.partition_map = partition_map + self.size = self.partition_map.get_total_size() + + callbacks.update({'onbeforepartition': self._partition, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + 'onbeforeformat': self._format, + 'onbeforeunmount': self._unmount, + }) + + mount_callbacks = {'onbeforemount_root': self._mount_root, + 'onbeforemount_boot': self._mount_boot, + 'onbeforemount_specials': self._mount_specials, + 'onbeforeunmount_root': self._unmount_root, + 'onbeforeunmount_boot': self._unmount_boot, + 'onbeforeunmount_specials': self._unmount_specials, + } + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + + self.mount_fsm = Fysom({'initial': 'unmounted', + 'events': self.mount_events, + 'callbacks': mount_callbacks}) + + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + attach_proxy_methods(self, self.mount_events, self.mount_fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def _partition(self, e): + self.partition_map.create(self) + + def _map(self, e): + self.partition_map.map(self) + + def _unmap(self, e): + self.partition_map.unmap(self) + + def _format(self, e): + self.partition_map.format() + + def _unmount(self, e): + if self.mount_fsm.current is 'specials_mounted': + self.unmount_specials() + if self.mount_fsm.current is 'specials_mounted_no_boot': + self.unmount_specials() + if self.mount_fsm.current is 'boot_mounted': + self.unmount_boot() + if self.mount_fsm.current is 'root_mounted': + self.unmount_root() + + def mount_root(self, destination): + self.mount_fsm.mount_root(destination=destination) + + def unmount_root(self): + self.mount_fsm.unmount_root() + self.fsm.unmount() + + def _mount_root(self, e): + self.mount() + self.partition_map.mount_root(e.destination) + + def _unmount_root(self, e): + self.partition_map.unmount_root() + + def _mount_boot(self, e): + self.mount() + self.partition_map.mount_boot() + + def _unmount_boot(self, e): + self.partition_map.unmount_boot() + + def _mount_specials(self, e): + self.mount() + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + + def _unmount_specials(self, e): + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 6c5627d..3a982c4 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -34,14 +34,13 @@ "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { - "type": "object", - "properties": { - "size": { - "type": "integer", - "minimum": 1 - } - }, - "required": ["size"] + "type": "object" + // "properties": { + // "backing": { + // "type": "string" + // } + // }, + // "required": ["backing"] }, "plugins": { "type": "object", diff --git a/common/fs/__init__.py b/common/fs/__init__.py new file mode 100644 index 0000000..20b2f69 --- /dev/null +++ b/common/fs/__init__.py @@ -0,0 +1,11 @@ + + +def get_major_minor_dev_num(device_name): + import re + regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) {device_name}$' + .format(device_name=device_name)) + with open('/proc/partitions') as partitions: + for line in partitions: + match = regexp.match(line) + if match is not None: + return match.group('major'), match.group('minor') diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py new file mode 100644 index 0000000..c8ffb1e --- /dev/null +++ b/common/fs/loopbackvolume.py @@ -0,0 +1,90 @@ +from base.fs.volume import Volume +from common.tools import log_check_call +from base.fs.exceptions import VolumeError + + +# QEMU formats: +# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop + + +class LoopbackVolume(Volume): + + link_dm_events = [{'name': 'link_dm_node', 'src': 'partitioned', 'dst': 'linked'}, + {'name': 'map', 'src': 'linked', 'dst': 'mapped_lnk'}, + {'name': 'format', 'src': 'mapped_lnk', 'dst': 'formatted_lnk'}, + {'name': 'mount', 'src': ['formatted_lnk', 'mounted_lnk'], 'dst': 'mounted_lnk'}, + {'name': 'unmount', 'src': 'mounted_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'formatted_lnk', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'unlink_dm_node', 'src': 'partitioned_fmt_lnk', 'dst': 'partitioned_fmt'}, + + {'name': 'link_dm_node', 'src': 'partitioned_fmt', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'map', 'src': 'partitioned_fmt_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'mapped_lnk', 'dst': 'linked'}, + {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, + ] + + def __init__(self, partition_map, callbacks={}): + callbacks.update({'onbeforecreate': self._create, + 'onbeforeattach': self._attach, + 'onbeforedetach': self._detach, + 'onbeforedelete': self._delete, + 'onbeforelink_dm_node': self._link_dm_node, + 'onbeforeunlink_dm_node': self._unlink_dm_node, + }) + self.events.extend(self.link_dm_events) + super(LoopbackVolume, self).__init__(partition_map, callbacks=callbacks) + + def create(self, image_path): + self.fsm.create(image_path=image_path) + + def _create(self, e): + self.image_path = e.image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) + + def _attach(self, e): + [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) + self.device_path = self.loop_device_path + + def _link_dm_node(self, e): + import re + loop_device_name = re.match('^/dev/(?P.*)$', self.loop_device_path).group('name') + from . import get_major_minor_dev_num + major, minor = get_major_minor_dev_num(loop_device_name) + sectors = self.size*1024*1024/512 + table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' + .format(log_start_sec=0, + sectors=sectors, + major=major, + minor=minor, + start_sec=0)) + import string + import os.path + for letter in string.ascii_lowercase: + dev_name = 'vd' + letter + dev_path = os.path.join('/dev/mapper', dev_name) + if not os.path.exists(dev_path): + self.dm_node_name = dev_name + self.dm_node_path = dev_path + break + + if not hasattr(self, 'dm_node_name'): + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) + self.device_path = self.dm_node_path + + def _unlink_dm_node(self, e): + log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) + del self.dm_node_name + del self.dm_node_path + self.device_path = self.loop_device_path + + def _detach(self, e): + log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _delete(self, e): + from os import remove + remove(self.image_path) + del self.image_path diff --git a/common/fsm.py b/common/fsm.py new file mode 100644 index 0000000..44286e0 --- /dev/null +++ b/common/fsm.py @@ -0,0 +1,7 @@ + + +def attach_proxy_methods(obj, events, fsm): + methods = set([e['name'] for e in events]) + for event in methods: + if not hasattr(obj, event): + setattr(obj, event, lambda e=event: getattr(fsm, e)()) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index c0b5a75..f1d658e 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -1,29 +1,29 @@ from base import Task from common import phases -from common.exceptions import TaskError from common.tools import log_check_call from bootstrap import Bootstrap +import volume -class FormatVolume(Task): +class Format(Task): description = 'Formatting the volume' phase = phases.volume_preparation def run(self, info): - dev_path = info.bootstrap_device['path'] - mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) - log_check_call([mkfs, dev_path]) - info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']} + info.volume.format() class TuneVolumeFS(Task): description = 'Tuning the bootstrap volume filesystem' phase = phases.volume_preparation - after = [FormatVolume] + after = [Format] def run(self, info): + import re # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']]) + for partition in info.volume.partition_map.partitions: + if re.match('^ext[2-4]$', partition.filesystem) is not None: + log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): @@ -36,33 +36,45 @@ class AddXFSProgs(Task): class CreateMountDir(Task): - description = 'Creating mountpoint for the bootstrap volume' + description = 'Creating mountpoint for the root partition' phase = phases.volume_mounting def run(self, info): import os mount_dir = info.manifest.bootstrapper['mount_dir'] info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) - # Works recursively, fails if last part exists, which is exaclty what we want. + # Works recursively, fails if last part exists, which is exactly what we want. os.makedirs(info.root) -class MountVolume(Task): - description = 'Mounting the bootstrap volume' +class MountRoot(Task): + description = 'Mounting the root partition' phase = phases.volume_mounting after = [CreateMountDir] def run(self, info): - with open('/proc/mounts') as mounts: - for mount in mounts: - if info.root in mount: - msg = 'Something is already mounted at {root}'.format(root=info.root) - raise TaskError(msg) + info.volume.mount_root(info.root) - log_check_call(['/bin/mount', - '--types', info.manifest.volume['filesystem'], - info.bootstrap_device['partitions']['root_path'], - info.root]) + +class MountBoot(Task): + description = 'Mounting the boot partition' + phase = phases.volume_mounting + after = [MountRoot] + + def run(self, info): + info.volume.mount_boot() + + +class CreateBootMountDir(Task): + description = 'Creating mountpoint boot partition' + phase = phases.volume_mounting + after = [MountRoot] + before = [MountBoot] + + def run(self, info): + import os + info.boot = os.path.join(info.root, 'boot') + os.makedirs() class MountSpecials(Task): @@ -71,36 +83,40 @@ class MountSpecials(Task): after = [Bootstrap] def run(self, info): - log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + info.volume.mount_specials() + + +class UnmountRoot(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + before = [volume.Detach] + + def run(self, info): + info.volume.unmount_root() + + +class UnmountBoot(Task): + description = 'Unmounting the boot partition' + phase = phases.volume_unmounting + before = [UnmountRoot] + + def run(self, info): + info.volume.unmount_boot() class UnmountSpecials(Task): description = 'Unmunting special block devices' phase = phases.volume_unmounting + before = [UnmountRoot] def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) - log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) - - -class UnmountVolume(Task): - description = 'Unmounting the bootstrap volume' - phase = phases.volume_unmounting - after = [UnmountSpecials] - - def run(self, info): - log_check_call(['/bin/umount', info.root]) + info.volume.unmount_specials() class DeleteMountDir(Task): description = 'Deleting mountpoint for the bootstrap volume' phase = phases.volume_unmounting - after = [UnmountVolume] + after = [UnmountRoot] def run(self, info): import os @@ -108,24 +124,28 @@ class DeleteMountDir(Task): del info.root -class ModifyFstab(Task): - description = 'Adding root volume to the fstab' +class FStab(Task): + description = 'Adding partitions to the fstab' phase = phases.system_modification def run(self, info): import os.path - mount_opts = ['defaults'] - if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if info.manifest.volume['filesystem'].lower() == 'xfs': - mount_opts.append('nobarrier') - fstab_path = os.path.join(info.root, 'etc/fstab') + # device = '/dev/sda' + # if info.manifest.virtualization == 'pvm': + # device = '/dev/xvda' + fstab_lines = [] + for mount_point, partition in info.volume.partition_map.mount_points: + mount_opts = ['defaults'] + if partition.filesystem in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if partition.filesystem == 'xfs': + mount_opts.append('nobarrier') + fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} 1 1' + .format(uuid=partition.get_uuid(), + mountpoint=mount_point, + filesystem=partition.filesystem, + mount_opts=','.join(mount_opts))) - device = '/dev/sda1' - if info.manifest.virtualization == 'pvm': - device = '/dev/xvda1' - with open(fstab_path, 'a') as fstab: - fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' - .format(device=device, - filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts))) + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index b801fc1..2db681c 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from filesystem import UnmountVolume -from common.tools import log_check_call +import volume class Create(Task): description = 'Creating a loopback volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/bin/dd', - 'if=/dev/zero', 'of=' + info.loopback_file, - 'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0']) - - -class CreateQemuImg(Task): - description = 'Creating a loopback volume with qemu' - phase = phases.volume_creation - - def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', - info.loopback_file, str(info.manifest.volume['size']) + 'M']) - - -class Attach(Task): - description = 'Attaching the loopback volume' - phase = phases.volume_creation - after = [Create, CreateQemuImg] - - def run(self, info): - info.bootstrap_device = {} - command = ['/sbin/losetup', '--show', '--find', info.loopback_file] - [info.bootstrap_device['path']] = log_check_call(command) - - -class Detach(Task): - description = 'Detaching the loopback volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']]) - del info.bootstrap_device - - -class Delete(Task): - description = 'Deleting the loopback volume' - phase = phases.cleaning - - def run(self, info): - from os import remove - remove(info.loopback_file) - del info.loopback_file + image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.create(image_path) diff --git a/common/tasks/parted.py b/common/tasks/parted.py deleted file mode 100644 index e242765..0000000 --- a/common/tasks/parted.py +++ /dev/null @@ -1,53 +0,0 @@ -from base import Task -from common import phases -from common.tools import log_check_call -import filesystem -import loopback - - -class PartitionVolume(Task): - description = 'Partitioning the volume' - phase = phases.volume_preparation - - def run(self, info): - # parted - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mklabel', 'msdos']) - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mkpart', 'primary', 'ext4', '32k', '-1']) - log_check_call(['parted', '--script', info.bootstrap_device['path'], - '--', 'set', '1', 'boot', 'on']) - - -class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - after = [PartitionVolume] - - def run(self, info): - root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1' - log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) - info.bootstrap_device['partitions'] = {'root_path': root_partition_path} - - -class FormatPartitions(Task): - description = 'Formatting the partitions' - phase = phases.volume_preparation - before = [filesystem.TuneVolumeFS] - after = [MapPartitions] - - def run(self, info): - # These params will fail for mkfs.xfs - log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), - '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']]) - - -class UnmapPartitions(Task): - description = 'Removing volume partitions mapping' - phase = phases.volume_unmounting - before = [loopback.Detach] - after = [filesystem.UnmountVolume] - - def run(self, info): - log_check_call(['kpartx', '-d', info.bootstrap_device['path']]) - del info.bootstrap_device['partitions']['root_path'] diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py new file mode 100644 index 0000000..181bb63 --- /dev/null +++ b/common/tasks/partitioning.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import filesystem +import volume + + +class PartitionVolume(Task): + description = 'Partitioning the volume' + phase = phases.volume_preparation + + def run(self, info): + info.volume.partition() + + +class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + before = [filesystem.Format] + after = [PartitionVolume] + + def run(self, info): + info.volume.map() + + +class UnmapPartitions(Task): + description = 'Removing volume partitions mapping' + phase = phases.volume_unmounting + before = [volume.Detach] + after = [filesystem.UnmountRoot] + + def run(self, info): + info.volume.unmap() diff --git a/common/tasks/volume.py b/common/tasks/volume.py new file mode 100644 index 0000000..aa2a557 --- /dev/null +++ b/common/tasks/volume.py @@ -0,0 +1,26 @@ +from base import Task +from common import phases + + +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + + def run(self, info): + info.volume.attach() + + +class Detach(Task): + description = 'Detaching the volume' + phase = phases.volume_unmounting + + def run(self, info): + info.volume.detach() + + +class Delete(Task): + description = 'Deleting the volume' + phase = phases.cleaning + + def run(self, info): + info.volume.delete() diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index bb63082..fb0a704 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -23,8 +23,14 @@ }, "volume": { "backing": "ebs", - "filesystem": "ext4", - "size": 8192 + "partition_table": { + "type": null, + "partitions": [ + {"size": 8192, + "filesystem": "ext4", + "flags": ["root"]} + ] + } }, "plugins": { "admin_user": { diff --git a/manifests/virtualbox.json b/manifests/virtualbox.json deleted file mode 100644 index 238ef98..0000000 --- a/manifests/virtualbox.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "provider" : "virtualbox", - "virtualization": "ide", - - "bootstrapper": { - "mount_dir": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "convert_image": { - "enabled": true, - "format": "vdi" - } - } -} diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json new file mode 100644 index 0000000..1b0ab6a --- /dev/null +++ b/manifests/virtualbox.manifest.json @@ -0,0 +1,50 @@ +{ + "provider": "virtualbox", + "virtualization": "ide", + + "bootstrapper": { + "mount_dir": "/mnt/target", + "mirror": "http://http.debian.net/debian/" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "raw", + "loopback_dir": "/tmp", + "partitions": { + "boot": { + "size": 12, + "filesystem": "ext2" + }, + "root": { + "size": 812, + "filesystem": "ext4" + }, + "swap": {"size": 200} + } + }, + "plugins": { + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + }, + "root_password": { + "enabled": true, + "password": "test" + }, + "convert_image": { + "enabled": true, + "format": "vdi" + } + } +} diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index 8fd94ea..dc258a7 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -4,35 +4,31 @@ from tasks import CreateFromSnapshot from tasks import CreateFromImage from providers.ec2.tasks import ebs from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap from common.tasks import filesystem -from common.tasks import parted +from common.tasks import partitioning def tasks(tasklist, manifest): settings = manifest.plugins['prebootstrapped'] + skip_tasks = [filesystem.Format, + partitioning.PartitionVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + filesystem.CreateBootMountDir, + bootstrap.MakeTarball, + bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: tasklist.replace(ebs.Create, CreateFromSnapshot()) - tasklist.remove(filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.remove(*skip_tasks) else: tasklist.add(Snapshot()) else: if 'image' in settings and settings['image'] is not None: - tasklist.add(CreateFromImage()) - tasklist.remove(loopback.Create, - loopback.CreateQemuImg, - parted.PartitionVolume, - parted.FormatPartitions, - filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.replace(loopback.Create, CreateFromImage()) + tasklist.remove(*skip_tasks) else: tasklist.add(CopyImage()) @@ -45,9 +41,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'] == 'ebs': - counter_task(CreateFromSnapshot, ebs.Delete) + counter_task(CreateFromSnapshot, volume.Delete) else: - counter_task(CreateFromImage, loopback.Delete) + counter_task(CreateFromImage, volume.Delete) def validate_manifest(data, schema_validate): diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index bcaebc8..57d0610 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,7 +1,7 @@ from base import Task from common import phases from providers.ec2.tasks import ebs -from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap import time import logging @@ -22,18 +22,25 @@ class Snapshot(ebs.Snapshot): class CreateFromSnapshot(Task): description = 'Creating EBS volume from a snapshot' phase = phases.volume_creation - before = [ebs.Attach] + before = [volume.Attach] def run(self, info): volume_size = int(info.manifest.volume['size'] / 1024) snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - info.volume = info.connection.create_volume(volume_size, - info.host['availabilityZone'], - snapshot=snapshot) + info.volume.volume = info.connection.create_volume(volume_size, + info.host['availabilityZone'], + snapshot=snapshot) while info.volume.volume_state() != 'available': time.sleep(5) info.volume.update() + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) + class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' @@ -45,7 +52,7 @@ class CopyImage(Task): from shutil import copyfile loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.loopback_file, image_copy_path) + copyfile(info.volume.image_path, image_copy_path) msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) log.info(msg) @@ -53,12 +60,19 @@ class CopyImage(Task): class CreateFromImage(Task): description = 'Creating loopback image from a copy' phase = phases.volume_creation - before = [loopback.Attach] + before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] from shutil import copyfile - copyfile(loopback_backup_path, info.loopback_file) + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] + copyfile(loopback_backup_path, info.volume.image_path) + + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 04be78b..ee63e71 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -6,6 +6,7 @@ from tasks import connection from tasks import host from common.tasks import host as common_host from tasks import ami +from common.tasks import volume from tasks import ebs from common.tasks import loopback from common.tasks import filesystem @@ -49,7 +50,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -77,16 +78,16 @@ def tasks(tasklist, manifest): tasklist.add(bootstrap.MakeTarball()) backing_specific_tasks = {'ebs': [ebs.Create(), - ebs.Attach(), - ebs.Detach(), + volume.Attach(), + volume.Detach(), ebs.Snapshot(), - ebs.Delete()], + volume.Delete()], 's3': [loopback.Create(), - loopback.Attach(), - loopback.Detach(), + volume.Attach(), + volume.Detach(), ami.BundleImage(), ami.UploadImage(), - loopback.Delete(), + volume.Delete(), ami.RemoveBundle()]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) @@ -105,11 +106,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, ebs.Delete) - counter_task(ebs.Attach, ebs.Detach) + counter_task(ebs.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, loopback.Delete) - counter_task(loopback.Attach, loopback.Detach) + counter_task(loopback.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index 85ad4b6..bb84a44 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from common.exceptions import TaskError -from common.tasks.filesystem import UnmountVolume -import time +from common.tasks import volume class Create(Task): - description = 'Creating an EBS volume for bootstrapping' + description = 'Creating the EBS volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): - info.volume = info.connection.create_volume(info.manifest.ebs_volume_size, - info.host['availabilityZone']) - while info.volume.volume_state() != 'available': - time.sleep(5) - info.volume.update() - - -class Attach(Task): - description = 'Attaching the EBS volume' - phase = phases.volume_creation - after = [Create] - - def run(self, info): - def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" - for c in xrange(ord(c1), ord(c2) + 1): - yield chr(c) - - import os.path - info.bootstrap_device = {} - for letter in char_range('f', 'z'): - dev_path = os.path.join('/dev', 'xvd' + letter) - if not os.path.exists(dev_path): - info.bootstrap_device['path'] = dev_path - info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) - break - if 'path' not in info.bootstrap_device: - raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - - info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) - while info.volume.attachment_state() != 'attached': - time.sleep(2) - info.volume.update() - - -class Detach(Task): - description = 'Detaching the EBS volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - info.volume.detach() - while info.volume.attachment_state() is not None: - time.sleep(2) - info.volume.update() + info.volume.create(info.connection, info.host['availabilityZone']) class Snapshot(Task): @@ -62,20 +17,4 @@ class Snapshot(Task): phase = phases.image_registration def run(self, info): - info.snapshot = info.volume.create_snapshot() - while info.snapshot.status != 'completed': - time.sleep(2) - info.snapshot.update() - - -class Delete(Task): - description = 'Deleting the EBS volume' - phase = phases.cleaning - - def run(self, info): - info.volume.delete() - del info.volume - - -class VolumeError(TaskError): - pass + info.snapshot = info.volume.snapshot() diff --git a/providers/ec2/volume.py b/providers/ec2/volume.py new file mode 100644 index 0000000..bd1983f --- /dev/null +++ b/providers/ec2/volume.py @@ -0,0 +1,56 @@ +from base.fs.volume import Volume +from base.fs.exceptions import VolumeError +import time + + +class EBSVolume(Volume): + + volume = None + + def create(self, conn, zone): + super(EBSVolume, self).create(self) + import math + # TODO: Warn if volume size is not a multiple of 1024 + size = int(math.ceil(self.partition_map.get_volume_size() / 1024)) + self.volume = conn.create_volume(size, zone) + while self.volume.volume_state() != 'available': + time.sleep(5) + self.volume.update() + self.created = True + + def attach(self, instance_id): + super(EBSVolume, self).attach(self) + import os.path + import string + for letter in string.ascii_lowercase: + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + self.device_path = dev_path + self.ec2_device_path = os.path.join('/dev', 'sd' + letter) + break + + if self.device_path is None: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + self.volume.attach(instance_id, self.ec2_device_path) + while self.volume.attachment_state() != 'attached': + time.sleep(2) + self.volume.update() + + def detach(self): + super(EBSVolume, self).detach(self) + self.volume.detach() + while self.volume.attachment_state() is not None: + time.sleep(2) + self.volume.update() + + def delete(self): + super(EBSVolume, self).delete(self) + self.volume.delete() + + def snapshot(self): + snapshot = self.volume.create_snapshot() + while snapshot.status != 'completed': + time.sleep(2) + snapshot.update() + return snapshot diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py index 6a2603c..724a250 100644 --- a/providers/kvm/__init__.py +++ b/providers/kvm/__init__.py @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 6a2603c..c6382ae 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -2,8 +2,9 @@ from manifest import Manifest from tasks import packages from common.tasks import packages as common_packages from common.tasks import host +from common.tasks import volume as volume_tasks from common.tasks import loopback -from common.tasks import parted +from common.tasks import partitioning from common.tasks import filesystem from common.tasks import bootstrap from common.tasks import locale @@ -14,7 +15,6 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup -from common.tasks import loopback def initialize(): @@ -28,13 +28,13 @@ def tasks(tasklist, manifest): common_packages.ImagePackages(), host.CheckPackages(), - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), + loopback.Create(), + volume_tasks.Attach(), + partitioning.PartitionVolume(), + partitioning.MapPartitions(), + filesystem.Format(), filesystem.CreateMountDir(), - filesystem.MountVolume(), + filesystem.MountRoot(), bootstrap.Bootstrap(), filesystem.MountSpecials(), @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -63,19 +63,32 @@ def tasks(tasklist, manifest): apt.EnableDaemonAutostart(), filesystem.UnmountSpecials(), - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) + filesystem.UnmountRoot(), + partitioning.UnmapPartitions(), + volume_tasks.Detach(), + filesystem.DeleteMountDir(), + volume_tasks.Delete()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + partitions = manifest.volume['partitions'] + import re + for key in ['boot', 'root']: + if key not in partitions: + continue + if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None: + tasklist.add(filesystem.TuneVolumeFS()) + break + for key in ['boot', 'root']: + if key not in partitions: + continue + if partitions[key]['filesystem'] == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + break + + if 'boot' in manifest.volume['partitions']: + tasklist.add(filesystem.MountBoot(), filesystem.UnmountBoot()) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -85,8 +98,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if task in completed and counter not in completed: tasklist.add(counter()) + counter_task(loopback.Create, volume_tasks.Delete) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 52ac2ee..0d51f6e 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -9,13 +9,13 @@ "backing": { "type": "string", "enum": ["raw", "qcow2"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] } + // "filesystem": { + // "type": "string", + // "enum": ["ext2", "ext3", "ext4", "xfs"] + // } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, "required": ["volume"] diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 428f70a..bf96e11 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -1,34 +1,47 @@ from base import Task from common import phases +from common.tasks import apt +from common.fs.loopbackvolume import LoopbackVolume class ConfigureGrub(Task): description = 'Configuring grub' phase = phases.system_modification + after = [apt.AptUpgrade] def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - + import os from common.tools import log_check_call - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/", - info.bootstrap_device['path']]) + boot_dir = os.path.join(info.root, 'boot') + grub_dir = os.path.join(boot_dir, 'grub') + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.link_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() + [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) + device_map_path = os.path.join(grub_dir, 'device.map') + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + + # Install grub + log_check_call(['/usr/sbin/grub-install', + '--root-directory=' + info.root, + '--boot-directory=' + boot_dir, + device_path]) log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + # log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.unlink_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 9bdd204..207f356 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -12,7 +12,7 @@ class HostPackages(Task): def run(self, info): info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py new file mode 100644 index 0000000..3425729 --- /dev/null +++ b/providers/virtualbox/volume.py @@ -0,0 +1,11 @@ +from common.fs.loopbackvolume import LoopbackVolume +from common.tools import log_check_call + + +class VirtualBoxVolume(LoopbackVolume): + + def create(self, image_path): + super(VirtualBoxVolume, self).create(self) + self.image_path = image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + self.created = True