From c1e931df5e802041518c3abce82401b922c0a2a9 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 13 Sep 2013 01:32:08 +0200 Subject: [PATCH 01/63] Fix bug where log_call would not read all lines --- common/tools.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/common/tools.py b/common/tools.py index 468394c..ca83783 100644 --- a/common/tools.py +++ b/common/tools.py @@ -31,13 +31,11 @@ def log_call(command, stdin=None): ret = select.select(reads, [], []) for fd in ret[0]: if fd == process.stdout.fileno(): - line = process.stdout.readline() - if line != '': + for line in iter(process.stdout.readline, ''): log.debug(line.strip()) stdout.append(line.strip()) if fd == process.stderr.fileno(): - line = process.stderr.readline() - if line != '': + for line in iter(process.stderr.readline, ''): log.error(line.strip()) stderr.append(line.strip()) if process.poll() is not None: From 77c8b3615117ab010ce2d7e319002f914f5b19c7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 14 Sep 2013 23:29:12 +0200 Subject: [PATCH 02/63] Added pause-on-error switch for debugging --- base/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/base/main.py b/base/main.py index e5dcd64..398adc2 100644 --- a/base/main.py +++ b/base/main.py @@ -15,6 +15,8 @@ def get_args(): parser = ArgumentParser(description='Bootstrap Debian for the cloud.') parser.add_argument('--debug', action='store_true', help='Print debugging information') + parser.add_argument('--pause-on-error', action='store_true', + help='Pause on error, before rollback') parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') return parser.parse_args() @@ -37,6 +39,8 @@ def run(args): log.info('Successfully completed bootstrapping') except (Exception, KeyboardInterrupt) as e: log.exception(e) + if args.pause_on_error: + raw_input("Press Enter to commence rollback") log.error('Rolling back') rollback_tasklist = TaskList() provider.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) From ff7c04c12039b20910489f42097b588d52b50c63 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 13:19:45 +0200 Subject: [PATCH 03/63] Support for partitions MAJOR refactor. The volume is now abstracted into a model along with a partitionmap and partitions. Volumes and partitions are now controlled via an FSM to ensure that commands are called in the proper sequence. GRUB can now be installed properly onto loop devices by using dmsetup to fake a proper harddisk. --- base/bootstrapinfo.py | 2 + base/fs/__init__.py | 17 +++ base/fs/exceptions.py | 8 ++ base/fs/nopartitions.py | 30 +++++ base/fs/partitionmap.py | 88 ++++++++++++++ base/fs/partitions/__init__.py | 0 base/fs/partitions/abstractpartition.py | 56 +++++++++ base/fs/partitions/partition.py | 60 ++++++++++ base/fs/partitions/singlepartition.py | 20 ++++ base/fs/partitions/swap.py | 11 ++ base/fs/volume.py | 138 ++++++++++++++++++++++ base/manifest-schema.json | 15 ++- common/fs/__init__.py | 11 ++ common/fs/loopbackvolume.py | 90 ++++++++++++++ common/fsm.py | 7 ++ common/tasks/filesystem.py | 130 +++++++++++--------- common/tasks/loopback.py | 53 +-------- common/tasks/parted.py | 53 --------- common/tasks/partitioning.py | 32 +++++ common/tasks/volume.py | 26 ++++ manifests/ec2-ebs-pvm.manifest.json | 10 +- manifests/virtualbox.json | 41 ------- manifests/virtualbox.manifest.json | 50 ++++++++ plugins/prebootstrapped/__init__.py | 32 +++-- plugins/prebootstrapped/tasks.py | 36 ++++-- providers/ec2/__init__.py | 23 ++-- providers/ec2/tasks/ebs.py | 71 +---------- providers/ec2/volume.py | 56 +++++++++ providers/kvm/__init__.py | 2 +- providers/virtualbox/__init__.py | 57 +++++---- providers/virtualbox/manifest-schema.json | 10 +- providers/virtualbox/tasks/boot.py | 57 +++++---- providers/virtualbox/tasks/packages.py | 2 +- providers/virtualbox/volume.py | 11 ++ 34 files changed, 941 insertions(+), 364 deletions(-) create mode 100644 base/fs/__init__.py create mode 100644 base/fs/exceptions.py create mode 100644 base/fs/nopartitions.py create mode 100644 base/fs/partitionmap.py create mode 100644 base/fs/partitions/__init__.py create mode 100644 base/fs/partitions/abstractpartition.py create mode 100644 base/fs/partitions/partition.py create mode 100644 base/fs/partitions/singlepartition.py create mode 100644 base/fs/partitions/swap.py create mode 100644 base/fs/volume.py create mode 100644 common/fs/__init__.py create mode 100644 common/fs/loopbackvolume.py create mode 100644 common/fsm.py delete mode 100644 common/tasks/parted.py create mode 100644 common/tasks/partitioning.py create mode 100644 common/tasks/volume.py delete mode 100644 manifests/virtualbox.json create mode 100644 manifests/virtualbox.manifest.json create mode 100644 providers/ec2/volume.py create mode 100644 providers/virtualbox/volume.py diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index d1c03ec..07c954b 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -3,6 +3,8 @@ class BootstrapInformation(object): def __init__(self, manifest=None, debug=False): self.manifest = manifest + from fs import load_volume + self.volume = load_volume(self.manifest.volume) self.debug = debug import random self.run_id = random.randrange(16 ** 8) diff --git a/base/fs/__init__.py b/base/fs/__init__.py new file mode 100644 index 0000000..6180894 --- /dev/null +++ b/base/fs/__init__.py @@ -0,0 +1,17 @@ + + +def load_volume(data): + from common.fs.loopbackvolume import LoopbackVolume + from providers.ec2.volume import EBSVolume + from providers.virtualbox.volume import VirtualBoxVolume + from partitionmap import PartitionMap + from nopartitions import NoPartitions + partition_maps = {'none': NoPartitions, + 'gpt': PartitionMap, + } + partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) + volume_backings = {'raw': LoopbackVolume, + 'vdi': VirtualBoxVolume, + 'ebs': EBSVolume + } + return volume_backings.get(data['backing'])(partition_map) diff --git a/base/fs/exceptions.py b/base/fs/exceptions.py new file mode 100644 index 0000000..bc38490 --- /dev/null +++ b/base/fs/exceptions.py @@ -0,0 +1,8 @@ + + +class VolumeError(Exception): + pass + + +class PartitionError(Exception): + pass diff --git a/base/fs/nopartitions.py b/base/fs/nopartitions.py new file mode 100644 index 0000000..b8935ca --- /dev/null +++ b/base/fs/nopartitions.py @@ -0,0 +1,30 @@ +from partitions.singlepartition import SinglePartition + + +class NoPartitions(object): + + def __init__(self, data): + root = data['root'] + self.root = SinglePartition(root['size'], root['filesystem']) + self.mount_points = [('/', self.root)] + + def get_total_size(self): + return self.root.size + + def create(self, volume): + pass + + def map(self, volume): + pass + + def unmap(self, volume): + pass + + def format(self): + self.root.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() diff --git a/base/fs/partitionmap.py b/base/fs/partitionmap.py new file mode 100644 index 0000000..49a75ab --- /dev/null +++ b/base/fs/partitionmap.py @@ -0,0 +1,88 @@ +from common.tools import log_check_call +from partitions.partition import Partition +from partitions.swap import Swap +from exceptions import PartitionError + + +class PartitionMap(object): + + def __init__(self, data): + self.boot = None + self.swap = None + self.mount_points = [] + if 'boot' in data: + self.boot = Partition(data['boot']['size'], data['boot']['filesystem'], None) + self.mount_points.append(('/boot', self.boot)) + self.root = Partition(data['root']['size'], data['root']['filesystem'], self.boot) + self.mount_points.append(('/', self.root)) + if 'swap' in data: + self.swap = Swap(data['swap']['size'], self.root) + self.mount_points.append(('none', self.root)) + self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + + def get_total_size(self): + return sum(p.size for p in self.partitions) + + def create(self, volume): + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', volume.device_path, + '--', 'mklabel', 'gpt']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if self.boot is not None: + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set', str(boot_idx), 'bios_grub', 'on']) + + def map(self, volume): + try: + mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) + import re + regexp = re.compile('^(?P.+[^\d](?P\d+)) : ' + '(?P\d) (?P\d+) ' + '{device_path} (?P\d+)$' + .format(device_path=volume.device_path)) + log_check_call(['/sbin/kpartx', '-a', volume.device_path]) + import os.path + for mapping in mappings: + match = regexp.match(mapping) + if match is None: + raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping)) + partition_path = os.path.join('/dev/mapper', match.group('name')) + p_idx = int(match.group('p_idx'))-1 + self.partitions[p_idx].map(partition_path) + + for idx, partition in enumerate(self.partitions): + if not partition.state() in ['mapped', 'formatted']: + raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) + + except PartitionError as e: + for partition in self.partitions: + if partition.state() in ['mapped', 'formatted']: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + raise e + + def unmap(self, volume): + for partition in self.partitions: + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + + def format(self): + for partition in self.partitions: + partition.format() + + def mount_root(self, destination): + self.root.mount(destination) + + def unmount_root(self): + self.root.unmount() + + def mount_boot(self): + import os.path + destination = os.path.join(self.root.mount_dir, 'boot') + self.boot.mount(destination) + + def unmount_boot(self): + self.boot.unmount() diff --git a/base/fs/partitions/__init__.py b/base/fs/partitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/fs/partitions/abstractpartition.py b/base/fs/partitions/abstractpartition.py new file mode 100644 index 0000000..161e134 --- /dev/null +++ b/base/fs/partitions/abstractpartition.py @@ -0,0 +1,56 @@ +from common.tools import log_check_call +from abc import ABCMeta +from fysom import Fysom + + +class AbstractPartition(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + self.size = size + self.filesystem = filesystem + self.device_path = None + self.initial_state = 'nonexistent' + + callbacks.update({'onbeforeformat': self._format, + 'onbeforemount': self._mount, + 'onbeforeunmount': self._unmount, + }) + + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def get_uuid(self): + [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) + return uuid + + def _format(self, e): + mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) + log_check_call([mkfs, self.device_path]) + + def mount(self, destination): + self.fsm.mount(destination=destination) + + def _mount(self, e): + log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) + self.mount_dir = e.destination + + def _unmount(self, e): + log_check_call(['/bin/umount', self.mount_dir]) + del self.mount_dir diff --git a/base/fs/partitions/partition.py b/base/fs/partitions/partition.py new file mode 100644 index 0000000..a62ac0e --- /dev/null +++ b/base/fs/partitions/partition.py @@ -0,0 +1,60 @@ +from common.tools import log_check_call +from abstractpartition import AbstractPartition + + +class Partition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, filesystem, previous, callbacks={}): + self.previous = previous + callbacks.update({'onbeforecreate': self._create, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + }) + super(Partition, self).__init__(size, filesystem, callbacks=callbacks) + + def get_index(self): + if self.previous is None: + return 1 + else: + return self.previous.get_index()+1 + + def get_start(self): + if self.previous is None: + return 0 + else: + return self.previous.get_start() + self.previous.size + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + start = self.get_start() + # maybe use `parted -- name` to set partition name + log_check_call(['/sbin/parted', '--script', '--align', 'optimal', e.volume.device_path, + '--', 'mkpart', 'primary', str(start), str(start + self.size)]) + + def map(self, device_path): + self.fsm.map(device_path=device_path) + + def _map(self, e): + self.device_path = e.device_path + + def _unmap(self, e): + self.device_path = None + +# Partition flags: +# boot, root, swap, hidden, raid, lvm, lba, legacy_boot, palo + + +# Partition tables +# bsd, dvh, gpt, loop, mac, msdos, pc98, sun diff --git a/base/fs/partitions/singlepartition.py b/base/fs/partitions/singlepartition.py new file mode 100644 index 0000000..10bd702 --- /dev/null +++ b/base/fs/partitions/singlepartition.py @@ -0,0 +1,20 @@ +from abstractpartition import AbstractPartition + + +class SinglePartition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem, callbacks={}): + callbacks['oncreate'] = self._create + super(SinglePartition, self).__init__(size, filesystem, callbacks=callbacks) + + def create(self, volume): + self.fsm.create(volume=volume) + + def _create(self, e): + self.device_path = e.volume.device_path diff --git a/base/fs/partitions/swap.py b/base/fs/partitions/swap.py new file mode 100644 index 0000000..80b7531 --- /dev/null +++ b/base/fs/partitions/swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from partition import Partition + + +class Swap(Partition): + + def __init__(self, size, previous): + super(Swap, self).__init__(size, 'swap', previous) + + def _format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/volume.py b/base/fs/volume.py new file mode 100644 index 0000000..8d063f1 --- /dev/null +++ b/base/fs/volume.py @@ -0,0 +1,138 @@ +from abc import ABCMeta +from fysom import Fysom + + +class Volume(object): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'partition', 'src': 'attached', 'dst': 'partitioned'}, + {'name': 'map', 'src': 'partitioned', 'dst': 'mapped'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': ['formatted', 'mounted'], 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'partitioned_fmt'}, + {'name': 'detach', 'src': 'partitioned_fmt', 'dst': 'detached_fmt'}, + {'name': 'delete', 'src': ['detached', 'detached_prt', 'detached_fmt'], 'dst': 'deleted'}, + + {'name': 'attach', 'src': 'detached_fmt', 'dst': 'partitioned_fmt'}, + {'name': 'map', 'src': 'partitioned_fmt', 'dst': 'formatted'}, + + {'name': 'attach', 'src': 'detached_prt', 'dst': 'partitioned'}, + {'name': 'detach', 'src': 'partitioned', 'dst': 'detached_prt'}, + + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'partitioned'}, + ] + + mount_events = [{'name': 'mount_root', 'src': 'unmounted', 'dst': 'root_mounted'}, + {'name': 'mount_boot', 'src': 'root_mounted', 'dst': 'boot_mounted'}, + {'name': 'mount_specials', 'src': 'boot_mounted', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'boot_mounted'}, + {'name': 'unmount_boot', 'src': 'boot_mounted', 'dst': 'root_mounted'}, + {'name': 'unmount_root', 'src': 'root_mounted', 'dst': 'unmounted'}, + + {'name': 'mount_specials', 'src': 'root_mounted', 'dst': 'specials_mounted_no_boot'}, + {'name': 'mount_boot', 'src': 'specials_mounted_no_boot', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted_no_boot', 'dst': 'root_mounted'}, + {'name': 'unmount_boot', 'src': 'specials_mounted', 'dst': 'specials_mounted_no_boot'}, + ] + + def __init__(self, partition_map, callbacks={}): + self.device_path = None + self.partition_map = partition_map + self.size = self.partition_map.get_total_size() + + callbacks.update({'onbeforepartition': self._partition, + 'onbeforemap': self._map, + 'onbeforeunmap': self._unmap, + 'onbeforeformat': self._format, + 'onbeforeunmount': self._unmount, + }) + + mount_callbacks = {'onbeforemount_root': self._mount_root, + 'onbeforemount_boot': self._mount_boot, + 'onbeforemount_specials': self._mount_specials, + 'onbeforeunmount_root': self._unmount_root, + 'onbeforeunmount_boot': self._unmount_boot, + 'onbeforeunmount_specials': self._unmount_specials, + } + self.fsm = Fysom({'initial': 'nonexistent', + 'events': self.events, + 'callbacks': callbacks}) + + self.mount_fsm = Fysom({'initial': 'unmounted', + 'events': self.mount_events, + 'callbacks': mount_callbacks}) + + from common.fsm import attach_proxy_methods + attach_proxy_methods(self, self.events, self.fsm) + attach_proxy_methods(self, self.mount_events, self.mount_fsm) + + def state(self): + return self.fsm.current + + def force_state(self, state): + self.fsm.current = state + + def _partition(self, e): + self.partition_map.create(self) + + def _map(self, e): + self.partition_map.map(self) + + def _unmap(self, e): + self.partition_map.unmap(self) + + def _format(self, e): + self.partition_map.format() + + def _unmount(self, e): + if self.mount_fsm.current is 'specials_mounted': + self.unmount_specials() + if self.mount_fsm.current is 'specials_mounted_no_boot': + self.unmount_specials() + if self.mount_fsm.current is 'boot_mounted': + self.unmount_boot() + if self.mount_fsm.current is 'root_mounted': + self.unmount_root() + + def mount_root(self, destination): + self.mount_fsm.mount_root(destination=destination) + + def unmount_root(self): + self.mount_fsm.unmount_root() + self.fsm.unmount() + + def _mount_root(self, e): + self.mount() + self.partition_map.mount_root(e.destination) + + def _unmount_root(self, e): + self.partition_map.unmount_root() + + def _mount_boot(self, e): + self.mount() + self.partition_map.mount_boot() + + def _unmount_boot(self, e): + self.partition_map.unmount_boot() + + def _mount_specials(self, e): + self.mount() + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + + def _unmount_specials(self, e): + from common.tools import log_check_call + root = self.partition_map.root.mount_dir + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 6c5627d..3a982c4 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -34,14 +34,13 @@ "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { - "type": "object", - "properties": { - "size": { - "type": "integer", - "minimum": 1 - } - }, - "required": ["size"] + "type": "object" + // "properties": { + // "backing": { + // "type": "string" + // } + // }, + // "required": ["backing"] }, "plugins": { "type": "object", diff --git a/common/fs/__init__.py b/common/fs/__init__.py new file mode 100644 index 0000000..20b2f69 --- /dev/null +++ b/common/fs/__init__.py @@ -0,0 +1,11 @@ + + +def get_major_minor_dev_num(device_name): + import re + regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) {device_name}$' + .format(device_name=device_name)) + with open('/proc/partitions') as partitions: + for line in partitions: + match = regexp.match(line) + if match is not None: + return match.group('major'), match.group('minor') diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py new file mode 100644 index 0000000..c8ffb1e --- /dev/null +++ b/common/fs/loopbackvolume.py @@ -0,0 +1,90 @@ +from base.fs.volume import Volume +from common.tools import log_check_call +from base.fs.exceptions import VolumeError + + +# QEMU formats: +# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop + + +class LoopbackVolume(Volume): + + link_dm_events = [{'name': 'link_dm_node', 'src': 'partitioned', 'dst': 'linked'}, + {'name': 'map', 'src': 'linked', 'dst': 'mapped_lnk'}, + {'name': 'format', 'src': 'mapped_lnk', 'dst': 'formatted_lnk'}, + {'name': 'mount', 'src': ['formatted_lnk', 'mounted_lnk'], 'dst': 'mounted_lnk'}, + {'name': 'unmount', 'src': 'mounted_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'formatted_lnk', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'unlink_dm_node', 'src': 'partitioned_fmt_lnk', 'dst': 'partitioned_fmt'}, + + {'name': 'link_dm_node', 'src': 'partitioned_fmt', 'dst': 'partitioned_fmt_lnk'}, + {'name': 'map', 'src': 'partitioned_fmt_lnk', 'dst': 'formatted_lnk'}, + {'name': 'unmap', 'src': 'mapped_lnk', 'dst': 'linked'}, + {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, + ] + + def __init__(self, partition_map, callbacks={}): + callbacks.update({'onbeforecreate': self._create, + 'onbeforeattach': self._attach, + 'onbeforedetach': self._detach, + 'onbeforedelete': self._delete, + 'onbeforelink_dm_node': self._link_dm_node, + 'onbeforeunlink_dm_node': self._unlink_dm_node, + }) + self.events.extend(self.link_dm_events) + super(LoopbackVolume, self).__init__(partition_map, callbacks=callbacks) + + def create(self, image_path): + self.fsm.create(image_path=image_path) + + def _create(self, e): + self.image_path = e.image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) + + def _attach(self, e): + [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) + self.device_path = self.loop_device_path + + def _link_dm_node(self, e): + import re + loop_device_name = re.match('^/dev/(?P.*)$', self.loop_device_path).group('name') + from . import get_major_minor_dev_num + major, minor = get_major_minor_dev_num(loop_device_name) + sectors = self.size*1024*1024/512 + table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' + .format(log_start_sec=0, + sectors=sectors, + major=major, + minor=minor, + start_sec=0)) + import string + import os.path + for letter in string.ascii_lowercase: + dev_name = 'vd' + letter + dev_path = os.path.join('/dev/mapper', dev_name) + if not os.path.exists(dev_path): + self.dm_node_name = dev_name + self.dm_node_path = dev_path + break + + if not hasattr(self, 'dm_node_name'): + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) + self.device_path = self.dm_node_path + + def _unlink_dm_node(self, e): + log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) + del self.dm_node_name + del self.dm_node_path + self.device_path = self.loop_device_path + + def _detach(self, e): + log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _delete(self, e): + from os import remove + remove(self.image_path) + del self.image_path diff --git a/common/fsm.py b/common/fsm.py new file mode 100644 index 0000000..44286e0 --- /dev/null +++ b/common/fsm.py @@ -0,0 +1,7 @@ + + +def attach_proxy_methods(obj, events, fsm): + methods = set([e['name'] for e in events]) + for event in methods: + if not hasattr(obj, event): + setattr(obj, event, lambda e=event: getattr(fsm, e)()) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index c0b5a75..f1d658e 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -1,29 +1,29 @@ from base import Task from common import phases -from common.exceptions import TaskError from common.tools import log_check_call from bootstrap import Bootstrap +import volume -class FormatVolume(Task): +class Format(Task): description = 'Formatting the volume' phase = phases.volume_preparation def run(self, info): - dev_path = info.bootstrap_device['path'] - mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) - log_check_call([mkfs, dev_path]) - info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']} + info.volume.format() class TuneVolumeFS(Task): description = 'Tuning the bootstrap volume filesystem' phase = phases.volume_preparation - after = [FormatVolume] + after = [Format] def run(self, info): + import re # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']]) + for partition in info.volume.partition_map.partitions: + if re.match('^ext[2-4]$', partition.filesystem) is not None: + log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): @@ -36,33 +36,45 @@ class AddXFSProgs(Task): class CreateMountDir(Task): - description = 'Creating mountpoint for the bootstrap volume' + description = 'Creating mountpoint for the root partition' phase = phases.volume_mounting def run(self, info): import os mount_dir = info.manifest.bootstrapper['mount_dir'] info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) - # Works recursively, fails if last part exists, which is exaclty what we want. + # Works recursively, fails if last part exists, which is exactly what we want. os.makedirs(info.root) -class MountVolume(Task): - description = 'Mounting the bootstrap volume' +class MountRoot(Task): + description = 'Mounting the root partition' phase = phases.volume_mounting after = [CreateMountDir] def run(self, info): - with open('/proc/mounts') as mounts: - for mount in mounts: - if info.root in mount: - msg = 'Something is already mounted at {root}'.format(root=info.root) - raise TaskError(msg) + info.volume.mount_root(info.root) - log_check_call(['/bin/mount', - '--types', info.manifest.volume['filesystem'], - info.bootstrap_device['partitions']['root_path'], - info.root]) + +class MountBoot(Task): + description = 'Mounting the boot partition' + phase = phases.volume_mounting + after = [MountRoot] + + def run(self, info): + info.volume.mount_boot() + + +class CreateBootMountDir(Task): + description = 'Creating mountpoint boot partition' + phase = phases.volume_mounting + after = [MountRoot] + before = [MountBoot] + + def run(self, info): + import os + info.boot = os.path.join(info.root, 'boot') + os.makedirs() class MountSpecials(Task): @@ -71,36 +83,40 @@ class MountSpecials(Task): after = [Bootstrap] def run(self, info): - log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + info.volume.mount_specials() + + +class UnmountRoot(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + before = [volume.Detach] + + def run(self, info): + info.volume.unmount_root() + + +class UnmountBoot(Task): + description = 'Unmounting the boot partition' + phase = phases.volume_unmounting + before = [UnmountRoot] + + def run(self, info): + info.volume.unmount_boot() class UnmountSpecials(Task): description = 'Unmunting special block devices' phase = phases.volume_unmounting + before = [UnmountRoot] def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) - log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) - - -class UnmountVolume(Task): - description = 'Unmounting the bootstrap volume' - phase = phases.volume_unmounting - after = [UnmountSpecials] - - def run(self, info): - log_check_call(['/bin/umount', info.root]) + info.volume.unmount_specials() class DeleteMountDir(Task): description = 'Deleting mountpoint for the bootstrap volume' phase = phases.volume_unmounting - after = [UnmountVolume] + after = [UnmountRoot] def run(self, info): import os @@ -108,24 +124,28 @@ class DeleteMountDir(Task): del info.root -class ModifyFstab(Task): - description = 'Adding root volume to the fstab' +class FStab(Task): + description = 'Adding partitions to the fstab' phase = phases.system_modification def run(self, info): import os.path - mount_opts = ['defaults'] - if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if info.manifest.volume['filesystem'].lower() == 'xfs': - mount_opts.append('nobarrier') - fstab_path = os.path.join(info.root, 'etc/fstab') + # device = '/dev/sda' + # if info.manifest.virtualization == 'pvm': + # device = '/dev/xvda' + fstab_lines = [] + for mount_point, partition in info.volume.partition_map.mount_points: + mount_opts = ['defaults'] + if partition.filesystem in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if partition.filesystem == 'xfs': + mount_opts.append('nobarrier') + fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} 1 1' + .format(uuid=partition.get_uuid(), + mountpoint=mount_point, + filesystem=partition.filesystem, + mount_opts=','.join(mount_opts))) - device = '/dev/sda1' - if info.manifest.virtualization == 'pvm': - device = '/dev/xvda1' - with open(fstab_path, 'a') as fstab: - fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' - .format(device=device, - filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts))) + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index b801fc1..2db681c 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from filesystem import UnmountVolume -from common.tools import log_check_call +import volume class Create(Task): description = 'Creating a loopback volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/bin/dd', - 'if=/dev/zero', 'of=' + info.loopback_file, - 'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0']) - - -class CreateQemuImg(Task): - description = 'Creating a loopback volume with qemu' - phase = phases.volume_creation - - def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', - info.loopback_file, str(info.manifest.volume['size']) + 'M']) - - -class Attach(Task): - description = 'Attaching the loopback volume' - phase = phases.volume_creation - after = [Create, CreateQemuImg] - - def run(self, info): - info.bootstrap_device = {} - command = ['/sbin/losetup', '--show', '--find', info.loopback_file] - [info.bootstrap_device['path']] = log_check_call(command) - - -class Detach(Task): - description = 'Detaching the loopback volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']]) - del info.bootstrap_device - - -class Delete(Task): - description = 'Deleting the loopback volume' - phase = phases.cleaning - - def run(self, info): - from os import remove - remove(info.loopback_file) - del info.loopback_file + image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.create(image_path) diff --git a/common/tasks/parted.py b/common/tasks/parted.py deleted file mode 100644 index e242765..0000000 --- a/common/tasks/parted.py +++ /dev/null @@ -1,53 +0,0 @@ -from base import Task -from common import phases -from common.tools import log_check_call -import filesystem -import loopback - - -class PartitionVolume(Task): - description = 'Partitioning the volume' - phase = phases.volume_preparation - - def run(self, info): - # parted - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mklabel', 'msdos']) - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mkpart', 'primary', 'ext4', '32k', '-1']) - log_check_call(['parted', '--script', info.bootstrap_device['path'], - '--', 'set', '1', 'boot', 'on']) - - -class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - after = [PartitionVolume] - - def run(self, info): - root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1' - log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) - info.bootstrap_device['partitions'] = {'root_path': root_partition_path} - - -class FormatPartitions(Task): - description = 'Formatting the partitions' - phase = phases.volume_preparation - before = [filesystem.TuneVolumeFS] - after = [MapPartitions] - - def run(self, info): - # These params will fail for mkfs.xfs - log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), - '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']]) - - -class UnmapPartitions(Task): - description = 'Removing volume partitions mapping' - phase = phases.volume_unmounting - before = [loopback.Detach] - after = [filesystem.UnmountVolume] - - def run(self, info): - log_check_call(['kpartx', '-d', info.bootstrap_device['path']]) - del info.bootstrap_device['partitions']['root_path'] diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py new file mode 100644 index 0000000..181bb63 --- /dev/null +++ b/common/tasks/partitioning.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import filesystem +import volume + + +class PartitionVolume(Task): + description = 'Partitioning the volume' + phase = phases.volume_preparation + + def run(self, info): + info.volume.partition() + + +class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + before = [filesystem.Format] + after = [PartitionVolume] + + def run(self, info): + info.volume.map() + + +class UnmapPartitions(Task): + description = 'Removing volume partitions mapping' + phase = phases.volume_unmounting + before = [volume.Detach] + after = [filesystem.UnmountRoot] + + def run(self, info): + info.volume.unmap() diff --git a/common/tasks/volume.py b/common/tasks/volume.py new file mode 100644 index 0000000..aa2a557 --- /dev/null +++ b/common/tasks/volume.py @@ -0,0 +1,26 @@ +from base import Task +from common import phases + + +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + + def run(self, info): + info.volume.attach() + + +class Detach(Task): + description = 'Detaching the volume' + phase = phases.volume_unmounting + + def run(self, info): + info.volume.detach() + + +class Delete(Task): + description = 'Deleting the volume' + phase = phases.cleaning + + def run(self, info): + info.volume.delete() diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index bb63082..fb0a704 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -23,8 +23,14 @@ }, "volume": { "backing": "ebs", - "filesystem": "ext4", - "size": 8192 + "partition_table": { + "type": null, + "partitions": [ + {"size": 8192, + "filesystem": "ext4", + "flags": ["root"]} + ] + } }, "plugins": { "admin_user": { diff --git a/manifests/virtualbox.json b/manifests/virtualbox.json deleted file mode 100644 index 238ef98..0000000 --- a/manifests/virtualbox.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "provider" : "virtualbox", - "virtualization": "ide", - - "bootstrapper": { - "mount_dir": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "convert_image": { - "enabled": true, - "format": "vdi" - } - } -} diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json new file mode 100644 index 0000000..1b0ab6a --- /dev/null +++ b/manifests/virtualbox.manifest.json @@ -0,0 +1,50 @@ +{ + "provider": "virtualbox", + "virtualization": "ide", + + "bootstrapper": { + "mount_dir": "/mnt/target", + "mirror": "http://http.debian.net/debian/" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "raw", + "loopback_dir": "/tmp", + "partitions": { + "boot": { + "size": 12, + "filesystem": "ext2" + }, + "root": { + "size": 812, + "filesystem": "ext4" + }, + "swap": {"size": 200} + } + }, + "plugins": { + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + }, + "root_password": { + "enabled": true, + "password": "test" + }, + "convert_image": { + "enabled": true, + "format": "vdi" + } + } +} diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index 8fd94ea..dc258a7 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -4,35 +4,31 @@ from tasks import CreateFromSnapshot from tasks import CreateFromImage from providers.ec2.tasks import ebs from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap from common.tasks import filesystem -from common.tasks import parted +from common.tasks import partitioning def tasks(tasklist, manifest): settings = manifest.plugins['prebootstrapped'] + skip_tasks = [filesystem.Format, + partitioning.PartitionVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + filesystem.CreateBootMountDir, + bootstrap.MakeTarball, + bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: tasklist.replace(ebs.Create, CreateFromSnapshot()) - tasklist.remove(filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.remove(*skip_tasks) else: tasklist.add(Snapshot()) else: if 'image' in settings and settings['image'] is not None: - tasklist.add(CreateFromImage()) - tasklist.remove(loopback.Create, - loopback.CreateQemuImg, - parted.PartitionVolume, - parted.FormatPartitions, - filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.replace(loopback.Create, CreateFromImage()) + tasklist.remove(*skip_tasks) else: tasklist.add(CopyImage()) @@ -45,9 +41,9 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'] == 'ebs': - counter_task(CreateFromSnapshot, ebs.Delete) + counter_task(CreateFromSnapshot, volume.Delete) else: - counter_task(CreateFromImage, loopback.Delete) + counter_task(CreateFromImage, volume.Delete) def validate_manifest(data, schema_validate): diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index bcaebc8..57d0610 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,7 +1,7 @@ from base import Task from common import phases from providers.ec2.tasks import ebs -from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap import time import logging @@ -22,18 +22,25 @@ class Snapshot(ebs.Snapshot): class CreateFromSnapshot(Task): description = 'Creating EBS volume from a snapshot' phase = phases.volume_creation - before = [ebs.Attach] + before = [volume.Attach] def run(self, info): volume_size = int(info.manifest.volume['size'] / 1024) snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - info.volume = info.connection.create_volume(volume_size, - info.host['availabilityZone'], - snapshot=snapshot) + info.volume.volume = info.connection.create_volume(volume_size, + info.host['availabilityZone'], + snapshot=snapshot) while info.volume.volume_state() != 'available': time.sleep(5) info.volume.update() + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) + class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' @@ -45,7 +52,7 @@ class CopyImage(Task): from shutil import copyfile loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.loopback_file, image_copy_path) + copyfile(info.volume.image_path, image_copy_path) msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) log.info(msg) @@ -53,12 +60,19 @@ class CopyImage(Task): class CreateFromImage(Task): description = 'Creating loopback image from a copy' phase = phases.volume_creation - before = [loopback.Attach] + before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] from shutil import copyfile - copyfile(loopback_backup_path, info.loopback_file) + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] + copyfile(loopback_backup_path, info.volume.image_path) + + info.volume.force_state('detached_fmt') + partitions_state = 'formatted' + if 'partitions' in info.manifest.volume: + partitions_state = 'unmapped_fmt' + for partition in info.volume.partition_map.partitions: + partition.force_state(partitions_state) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 04be78b..ee63e71 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -6,6 +6,7 @@ from tasks import connection from tasks import host from common.tasks import host as common_host from tasks import ami +from common.tasks import volume from tasks import ebs from common.tasks import loopback from common.tasks import filesystem @@ -49,7 +50,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -77,16 +78,16 @@ def tasks(tasklist, manifest): tasklist.add(bootstrap.MakeTarball()) backing_specific_tasks = {'ebs': [ebs.Create(), - ebs.Attach(), - ebs.Detach(), + volume.Attach(), + volume.Detach(), ebs.Snapshot(), - ebs.Delete()], + volume.Delete()], 's3': [loopback.Create(), - loopback.Attach(), - loopback.Detach(), + volume.Attach(), + volume.Detach(), ami.BundleImage(), ami.UploadImage(), - loopback.Delete(), + volume.Delete(), ami.RemoveBundle()]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) @@ -105,11 +106,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest): tasklist.add(counter()) if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, ebs.Delete) - counter_task(ebs.Attach, ebs.Detach) + counter_task(ebs.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, loopback.Delete) - counter_task(loopback.Attach, loopback.Detach) + counter_task(loopback.Create, volume.Delete) + counter_task(volume.Attach, volume.Detach) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index 85ad4b6..bb84a44 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,60 +1,15 @@ from base import Task from common import phases -from common.exceptions import TaskError -from common.tasks.filesystem import UnmountVolume -import time +from common.tasks import volume class Create(Task): - description = 'Creating an EBS volume for bootstrapping' + description = 'Creating the EBS volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): - info.volume = info.connection.create_volume(info.manifest.ebs_volume_size, - info.host['availabilityZone']) - while info.volume.volume_state() != 'available': - time.sleep(5) - info.volume.update() - - -class Attach(Task): - description = 'Attaching the EBS volume' - phase = phases.volume_creation - after = [Create] - - def run(self, info): - def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" - for c in xrange(ord(c1), ord(c2) + 1): - yield chr(c) - - import os.path - info.bootstrap_device = {} - for letter in char_range('f', 'z'): - dev_path = os.path.join('/dev', 'xvd' + letter) - if not os.path.exists(dev_path): - info.bootstrap_device['path'] = dev_path - info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) - break - if 'path' not in info.bootstrap_device: - raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - - info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) - while info.volume.attachment_state() != 'attached': - time.sleep(2) - info.volume.update() - - -class Detach(Task): - description = 'Detaching the EBS volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - info.volume.detach() - while info.volume.attachment_state() is not None: - time.sleep(2) - info.volume.update() + info.volume.create(info.connection, info.host['availabilityZone']) class Snapshot(Task): @@ -62,20 +17,4 @@ class Snapshot(Task): phase = phases.image_registration def run(self, info): - info.snapshot = info.volume.create_snapshot() - while info.snapshot.status != 'completed': - time.sleep(2) - info.snapshot.update() - - -class Delete(Task): - description = 'Deleting the EBS volume' - phase = phases.cleaning - - def run(self, info): - info.volume.delete() - del info.volume - - -class VolumeError(TaskError): - pass + info.snapshot = info.volume.snapshot() diff --git a/providers/ec2/volume.py b/providers/ec2/volume.py new file mode 100644 index 0000000..bd1983f --- /dev/null +++ b/providers/ec2/volume.py @@ -0,0 +1,56 @@ +from base.fs.volume import Volume +from base.fs.exceptions import VolumeError +import time + + +class EBSVolume(Volume): + + volume = None + + def create(self, conn, zone): + super(EBSVolume, self).create(self) + import math + # TODO: Warn if volume size is not a multiple of 1024 + size = int(math.ceil(self.partition_map.get_volume_size() / 1024)) + self.volume = conn.create_volume(size, zone) + while self.volume.volume_state() != 'available': + time.sleep(5) + self.volume.update() + self.created = True + + def attach(self, instance_id): + super(EBSVolume, self).attach(self) + import os.path + import string + for letter in string.ascii_lowercase: + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + self.device_path = dev_path + self.ec2_device_path = os.path.join('/dev', 'sd' + letter) + break + + if self.device_path is None: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + self.volume.attach(instance_id, self.ec2_device_path) + while self.volume.attachment_state() != 'attached': + time.sleep(2) + self.volume.update() + + def detach(self): + super(EBSVolume, self).detach(self) + self.volume.detach() + while self.volume.attachment_state() is not None: + time.sleep(2) + self.volume.update() + + def delete(self): + super(EBSVolume, self).delete(self) + self.volume.delete() + + def snapshot(self): + snapshot = self.volume.create_snapshot() + while snapshot.status != 'completed': + time.sleep(2) + snapshot.update() + return snapshot diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py index 6a2603c..724a250 100644 --- a/providers/kvm/__init__.py +++ b/providers/kvm/__init__.py @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 6a2603c..c6382ae 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -2,8 +2,9 @@ from manifest import Manifest from tasks import packages from common.tasks import packages as common_packages from common.tasks import host +from common.tasks import volume as volume_tasks from common.tasks import loopback -from common.tasks import parted +from common.tasks import partitioning from common.tasks import filesystem from common.tasks import bootstrap from common.tasks import locale @@ -14,7 +15,6 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup -from common.tasks import loopback def initialize(): @@ -28,13 +28,13 @@ def tasks(tasklist, manifest): common_packages.ImagePackages(), host.CheckPackages(), - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), + loopback.Create(), + volume_tasks.Attach(), + partitioning.PartitionVolume(), + partitioning.MapPartitions(), + filesystem.Format(), filesystem.CreateMountDir(), - filesystem.MountVolume(), + filesystem.MountRoot(), bootstrap.Bootstrap(), filesystem.MountSpecials(), @@ -44,7 +44,7 @@ def tasks(tasklist, manifest): apt.AptSources(), apt.AptUpgrade(), boot.ConfigureGrub(), - filesystem.ModifyFstab(), + filesystem.FStab(), common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), @@ -63,19 +63,32 @@ def tasks(tasklist, manifest): apt.EnableDaemonAutostart(), filesystem.UnmountSpecials(), - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) + filesystem.UnmountRoot(), + partitioning.UnmapPartitions(), + volume_tasks.Detach(), + filesystem.DeleteMountDir(), + volume_tasks.Delete()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + partitions = manifest.volume['partitions'] + import re + for key in ['boot', 'root']: + if key not in partitions: + continue + if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None: + tasklist.add(filesystem.TuneVolumeFS()) + break + for key in ['boot', 'root']: + if key not in partitions: + continue + if partitions[key]['filesystem'] == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + break + + if 'boot' in manifest.volume['partitions']: + tasklist.add(filesystem.MountBoot(), filesystem.UnmountBoot()) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -85,8 +98,10 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if task in completed and counter not in completed: tasklist.add(counter()) + counter_task(loopback.Create, volume_tasks.Delete) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 52ac2ee..0d51f6e 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -9,13 +9,13 @@ "backing": { "type": "string", "enum": ["raw", "qcow2"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] } + // "filesystem": { + // "type": "string", + // "enum": ["ext2", "ext3", "ext4", "xfs"] + // } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, "required": ["volume"] diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 428f70a..bf96e11 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -1,34 +1,47 @@ from base import Task from common import phases +from common.tasks import apt +from common.fs.loopbackvolume import LoopbackVolume class ConfigureGrub(Task): description = 'Configuring grub' phase = phases.system_modification + after = [apt.AptUpgrade] def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - + import os from common.tools import log_check_call - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/", - info.bootstrap_device['path']]) + boot_dir = os.path.join(info.root, 'boot') + grub_dir = os.path.join(boot_dir, 'grub') + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.link_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() + [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) + device_map_path = os.path.join(grub_dir, 'device.map') + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + + # Install grub + log_check_call(['/usr/sbin/grub-install', + '--root-directory=' + info.root, + '--boot-directory=' + boot_dir, + device_path]) log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + # log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) + + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.unlink_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 9bdd204..207f356 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -12,7 +12,7 @@ class HostPackages(Task): def run(self, info): info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py new file mode 100644 index 0000000..3425729 --- /dev/null +++ b/providers/virtualbox/volume.py @@ -0,0 +1,11 @@ +from common.fs.loopbackvolume import LoopbackVolume +from common.tools import log_check_call + + +class VirtualBoxVolume(LoopbackVolume): + + def create(self, image_path): + super(VirtualBoxVolume, self).create(self) + self.image_path = image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + self.created = True From 9cb4b3e375469667faed59ba52f4876897e52c9c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 16:11:45 +0200 Subject: [PATCH 04/63] Rename mount_dir to workspace We'll be using it for a bucnh of different things in the future --- base/manifest-schema.json | 4 ++-- common/tasks/filesystem.py | 4 ++-- common/tasks/loopback.py | 15 +++++++++++++++ manifests/ec2-ebs-pvm.manifest.json | 2 +- manifests/ec2-s3-pvm.manifest.json | 2 +- manifests/kvm-virtio.manifest.json | 2 +- manifests/one-ide.manifest.json | 2 +- manifests/one-virtio.manifest.json | 2 +- manifests/virtualbox.manifest.json | 2 +- plugins/convert_image/tasks.py | 2 ++ providers/virtualbox/__init__.py | 2 +- 11 files changed, 28 insertions(+), 11 deletions(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 3a982c4..e66ef96 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -9,12 +9,12 @@ "bootstrapper": { "type": "object", "properties": { - "mount_dir": { "type": "string" }, + "workspace": { "type": "string" }, "mirror": { "type": "string" }, "tarball": { "type": "boolean" }, "tarball_dir": { "type": "string" } }, - "required": ["mount_dir"] + "required": ["workspace"] }, "system": { "type": "object", diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index f1d658e..82333b6 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -41,8 +41,8 @@ class CreateMountDir(Task): def run(self, info): import os - mount_dir = info.manifest.bootstrapper['mount_dir'] - info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) + workspace = info.manifest.bootstrapper['workspace'] + info.root = '{workspace}/{id:x}'.format(workspace=workspace, id=info.run_id) # Works recursively, fails if last part exists, which is exactly what we want. os.makedirs(info.root) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 2db681c..81d6f10 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -13,3 +13,18 @@ class Create(Task): import os.path image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) info.volume.create(image_path) + + +class MoveImage(Task): + description = 'Moving volume image' + phase = phases.image_registration + + def run(self, info): + import os.path + image_basename = os.path.basename(info.volume.image_path) + destination = os.path.join(info.manifest.bootstrapper['workspace'], image_basename) + import shutil + shutil.move(info.volume.image_path, destination) + import logging + log = logging.getLogger(__name__) + log.info('The volume image has been moved to {image_path}'.format(image_path=destination)) diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index fb0a704..d932077 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -7,7 +7,7 @@ }, "bootstrapper": { - "mount_dir": "/target", + "workspace": "/target", "mirror": "http://http.debian.net/debian" }, "image": { diff --git a/manifests/ec2-s3-pvm.manifest.json b/manifests/ec2-s3-pvm.manifest.json index 006de3b..9c8a5a5 100644 --- a/manifests/ec2-s3-pvm.manifest.json +++ b/manifests/ec2-s3-pvm.manifest.json @@ -10,7 +10,7 @@ }, "bootstrapper": { - "mount_dir": "/target", + "workspace": "/target", "mirror": "http://http.debian.net/debian" }, "image": { diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json index c4b828a..d14a11d 100644 --- a/manifests/kvm-virtio.manifest.json +++ b/manifests/kvm-virtio.manifest.json @@ -3,7 +3,7 @@ "virtualization": "virtio", "bootstrapper": { - "mount_dir": "/mnt/target", + "workspace": "/mnt/target", "mirror" : "http://ftp.fr.debian.org/debian/" }, "image": { diff --git a/manifests/one-ide.manifest.json b/manifests/one-ide.manifest.json index 1593005..05264f4 100644 --- a/manifests/one-ide.manifest.json +++ b/manifests/one-ide.manifest.json @@ -3,7 +3,7 @@ "virtualization": "ide", "bootstrapper": { - "mount_dir": "/target" + "workspace": "/target" }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", diff --git a/manifests/one-virtio.manifest.json b/manifests/one-virtio.manifest.json index 62a81f8..930a16d 100644 --- a/manifests/one-virtio.manifest.json +++ b/manifests/one-virtio.manifest.json @@ -3,7 +3,7 @@ "virtualization": "virtio", "bootstrapper": { - "mount_dir": "/target" + "workspace": "/target" }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index 1b0ab6a..b376127 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -3,7 +3,7 @@ "virtualization": "ide", "bootstrapper": { - "mount_dir": "/mnt/target", + "workspace": "/mnt/target", "mirror": "http://http.debian.net/debian/" }, "image": { diff --git a/plugins/convert_image/tasks.py b/plugins/convert_image/tasks.py index c547417..c573d08 100644 --- a/plugins/convert_image/tasks.py +++ b/plugins/convert_image/tasks.py @@ -1,10 +1,12 @@ from base import Task from common import phases +from common.tasks import loopback class ConvertImage(Task): description = 'Converting raw image' phase = phases.image_registration + before = [loopback.MoveImage] def run(self, info): from common.tools import log_check_call diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index c6382ae..b5a0fce 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -67,7 +67,7 @@ def tasks(tasklist, manifest): partitioning.UnmapPartitions(), volume_tasks.Detach(), filesystem.DeleteMountDir(), - volume_tasks.Delete()) + loopback.MoveImage()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) From 0eb5eecfc2082cb8daec6eb7b10b0a88414fcb8b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 16:59:56 +0200 Subject: [PATCH 05/63] Place everything in info.workspace instead of spreading files all around the disk --- base/bootstrapinfo.py | 3 +++ base/manifest-schema.json | 3 +-- base/manifest.py | 4 ---- common/fs/loopbackvolume.py | 2 ++ common/tasks/bootstrap.py | 2 +- common/tasks/filesystem.py | 4 +--- common/tasks/loopback.py | 7 +++---- common/tasks/volume.py | 2 ++ common/tasks/workspace.py | 20 ++++++++++++++++++++ manifests/kvm-virtio.manifest.json | 3 +-- manifests/virtualbox.manifest.json | 1 - plugins/prebootstrapped/tasks.py | 9 ++++----- providers/ec2/__init__.py | 2 +- providers/ec2/manifest.py | 4 ---- providers/ec2/tasks/ami.py | 4 +++- providers/kvm/__init__.py | 2 +- providers/kvm/manifest.py | 2 -- providers/virtualbox/__init__.py | 10 +++++++--- providers/virtualbox/manifest-schema.json | 2 +- providers/virtualbox/manifest.py | 2 -- providers/virtualbox/volume.py | 8 ++++---- 21 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 common/tasks/workspace.py diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index 07c954b..ff425b9 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -8,3 +8,6 @@ class BootstrapInformation(object): self.debug = debug import random self.run_id = random.randrange(16 ** 8) + import os.path + workspace_dirname = '{id:x}'.format(id=self.run_id) + self.workspace = os.path.join(manifest.bootstrapper['workspace'], workspace_dirname) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index e66ef96..1d751b3 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -11,8 +11,7 @@ "properties": { "workspace": { "type": "string" }, "mirror": { "type": "string" }, - "tarball": { "type": "boolean" }, - "tarball_dir": { "type": "string" } + "tarball": { "type": "boolean" } }, "required": ["workspace"] }, diff --git a/base/manifest.py b/base/manifest.py index 7aebdae..df17372 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -49,10 +49,6 @@ class Manifest(object): self.bootstrapper = data['bootstrapper'] if 'mirror' not in self.bootstrapper: self.bootstrapper['mirror'] = 'http://http.debian.net/debian' - if 'tarball' not in self.bootstrapper: - self.bootstrapper['tarball'] = False - if 'tarball_dir' not in self.bootstrapper and self.bootstrapper['tarball']: - self.bootstrapper['tarball_dir'] = '/tmp' self.volume = data['volume'] self.system = data['system'] self.plugins = data['plugins'] if 'plugins' in data else {} diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index c8ffb1e..f47807b 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -23,6 +23,8 @@ class LoopbackVolume(Volume): {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, ] + extension = 'raw' + def __init__(self, partition_map, callbacks={}): callbacks.update({'onbeforecreate': self._create, 'onbeforeattach': self._attach, diff --git a/common/tasks/bootstrap.py b/common/tasks/bootstrap.py index 68ffc5f..42e1da9 100644 --- a/common/tasks/bootstrap.py +++ b/common/tasks/bootstrap.py @@ -29,7 +29,7 @@ class MakeTarball(Task): hash_args = [arg for arg in arguments if arg != info.root] tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) - info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + info.tarball = os.path.join(info.manifest.bootstrapper['workspace'], tarball_filename) if os.path.isfile(info.tarball): log.debug('Found matching tarball, skipping download') else: diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 82333b6..a0e4075 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -41,9 +41,7 @@ class CreateMountDir(Task): def run(self, info): import os - workspace = info.manifest.bootstrapper['workspace'] - info.root = '{workspace}/{id:x}'.format(workspace=workspace, id=info.run_id) - # Works recursively, fails if last part exists, which is exactly what we want. + info.root = os.path.join(info.workspace, 'root') os.makedirs(info.root) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 81d6f10..c453d6a 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -9,9 +9,8 @@ class Create(Task): before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) info.volume.create(image_path) @@ -21,8 +20,8 @@ class MoveImage(Task): def run(self, info): import os.path - image_basename = os.path.basename(info.volume.image_path) - destination = os.path.join(info.manifest.bootstrapper['workspace'], image_basename) + filename = 'loopback-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) + destination = os.path.join(info.bootstrapper['workspace'], filename) import shutil shutil.move(info.volume.image_path, destination) import logging diff --git a/common/tasks/volume.py b/common/tasks/volume.py index aa2a557..64be8eb 100644 --- a/common/tasks/volume.py +++ b/common/tasks/volume.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.tasks import workspace class Attach(Task): @@ -21,6 +22,7 @@ class Detach(Task): class Delete(Task): description = 'Deleting the volume' phase = phases.cleaning + before = [workspace.DeleteWorkspace] def run(self, info): info.volume.delete() diff --git a/common/tasks/workspace.py b/common/tasks/workspace.py new file mode 100644 index 0000000..836443b --- /dev/null +++ b/common/tasks/workspace.py @@ -0,0 +1,20 @@ +from base import Task +from common import phases + + +class CreateWorkspace(Task): + description = 'Creating workspace' + phase = phases.preparation + + def run(self, info): + import os + os.makedirs(info.workspace) + + +class DeleteWorkspace(Task): + description = 'Deleting workspace' + phase = phases.cleaning + + def run(self, info): + import os + os.rmdir(info.workspace) diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json index d14a11d..7076d92 100644 --- a/manifests/kvm-virtio.manifest.json +++ b/manifests/kvm-virtio.manifest.json @@ -20,8 +20,7 @@ "volume": { "backing" : "raw", "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" + "size" : 1024 }, "plugins": { "user_packages": { diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index b376127..d61e30f 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -19,7 +19,6 @@ }, "volume": { "backing": "raw", - "loopback_dir": "/tmp", "partitions": { "boot": { "size": 12, diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 57d0610..f4eb1fa 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -51,9 +51,9 @@ class CopyImage(Task): import os.path from shutil import copyfile loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) - image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.volume.image_path, image_copy_path) - msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) + destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) + copyfile(info.volume.image_path, destination) + msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination) log.info(msg) @@ -65,8 +65,7 @@ class CreateFromImage(Task): def run(self, info): import os.path from shutil import copyfile - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] copyfile(loopback_backup_path, info.volume.image_path) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index ee63e71..533ccf7 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -74,7 +74,7 @@ def tasks(tasklist, manifest): filesystem.DeleteMountDir(), ami.RegisterAMI()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) backing_specific_tasks = {'ebs': [ebs.Create(), diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index 1d35897..abab855 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -23,7 +23,3 @@ class Manifest(base.Manifest): self.image = data['image'] if data['volume']['backing'] == 'ebs': self.ebs_volume_size = data['volume']['size'] / 1024 - if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3': - self.volume['loopback_dir'] = '/tmp' - if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3': - self.image['bundle_dir'] = '/tmp' diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index be82f43..81c99ed 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -3,6 +3,7 @@ from common import phases from common.exceptions import TaskError from common.tools import log_check_call from ebs import Snapshot +from common.tasks import workspace from connection import Connect import os.path @@ -45,7 +46,7 @@ class BundleImage(Task): def run(self, info): bundle_name = 'bundle-{id:x}'.format(id=info.run_id) - info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name) + info.bundle_path = os.path.join(info.workspace, bundle_name) log_check_call(['/usr/bin/euca-bundle-image', '--image', info.loopback_file, '--user', info.credentials['user-id'], @@ -80,6 +81,7 @@ class UploadImage(Task): class RemoveBundle(Task): description = 'Removing the bundle files' phase = phases.cleaning + before = [workspace.DeleteWorkspace] def run(self, info): from shutil import rmtree diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py index 724a250..8337e7b 100644 --- a/providers/kvm/__init__.py +++ b/providers/kvm/__init__.py @@ -68,7 +68,7 @@ def tasks(tasklist, manifest): loopback.Detach(), filesystem.DeleteMountDir()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], diff --git a/providers/kvm/manifest.py b/providers/kvm/manifest.py index 0e7b141..8ae196b 100644 --- a/providers/kvm/manifest.py +++ b/providers/kvm/manifest.py @@ -12,5 +12,3 @@ class Manifest(base.Manifest): super(Manifest, self).parse(data) self.image = data['image'] self.virtualization = data['virtualization'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index b5a0fce..eff464c 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -15,6 +15,7 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup +from common.tasks import workspace def initialize(): @@ -22,7 +23,8 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), + tasklist.add(workspace.CreateWorkspace(), + packages.HostPackages(), common_packages.HostPackages(), packages.ImagePackages(), common_packages.ImagePackages(), @@ -67,9 +69,10 @@ def tasks(tasklist, manifest): partitioning.UnmapPartitions(), volume_tasks.Detach(), filesystem.DeleteMountDir(), - loopback.MoveImage()) + loopback.MoveImage(), + workspace.DeleteWorkspace()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) partitions = manifest.volume['partitions'] @@ -105,3 +108,4 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) counter_task(filesystem.MountBoot, filesystem.UnmountBoot) counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 0d51f6e..5e2b58b 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -8,7 +8,7 @@ "properties": { "backing": { "type": "string", - "enum": ["raw", "qcow2"] + "enum": ["raw", "vdi", "qcow2"] } // "filesystem": { // "type": "string", diff --git a/providers/virtualbox/manifest.py b/providers/virtualbox/manifest.py index 8e5bb11..8b5fbc2 100644 --- a/providers/virtualbox/manifest.py +++ b/providers/virtualbox/manifest.py @@ -12,5 +12,3 @@ class Manifest(base.Manifest): super(Manifest, self).parse(data) self.virtualization = None self.image = data['image'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index 3425729..1af3d66 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -4,8 +4,8 @@ from common.tools import log_check_call class VirtualBoxVolume(LoopbackVolume): - def create(self, image_path): - super(VirtualBoxVolume, self).create(self) - self.image_path = image_path + extension = 'vdi' + + def _create(self, e): + self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) - self.created = True From c917c3bd65d28f686f58a10d5fc8e4c96bb1826c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 17:04:58 +0200 Subject: [PATCH 06/63] Log commands executed --- common/tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common/tools.py b/common/tools.py index ca83783..b6b2547 100644 --- a/common/tools.py +++ b/common/tools.py @@ -16,6 +16,7 @@ def log_call(command, stdin=None): from os.path import realpath command_log = realpath(command[0]).replace('/', '.') log = logging.getLogger(__name__ + command_log) + log.debug('Executing: {command}'.format(command=' '.join(command))) if stdin is not None: process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 6fb60a33f0d1bc1198fa62cd5d0a86110167a75c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 17:07:57 +0200 Subject: [PATCH 07/63] Get the loop device name in a less dramatic way --- common/fs/loopbackvolume.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index f47807b..d945244 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -48,8 +48,8 @@ class LoopbackVolume(Volume): self.device_path = self.loop_device_path def _link_dm_node(self, e): - import re - loop_device_name = re.match('^/dev/(?P.*)$', self.loop_device_path).group('name') + import os.path + loop_device_name = os.path.basename(self.loop_device_path) from . import get_major_minor_dev_num major, minor = get_major_minor_dev_num(loop_device_name) sectors = self.size*1024*1024/512 From efecc153827a793fcb6ce7ea49dcaba8cff0671d Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 18:26:22 +0200 Subject: [PATCH 08/63] VDI images working now *Helpful error messages included --- base/fs/nopartitions.py | 1 + common/fs/__init__.py | 17 ++++++---- common/fs/loopbackvolume.py | 10 +++--- common/tasks/filesystem.py | 6 ++-- providers/virtualbox/__init__.py | 6 ++-- providers/virtualbox/volume.py | 56 ++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/base/fs/nopartitions.py b/base/fs/nopartitions.py index b8935ca..0133ffc 100644 --- a/base/fs/nopartitions.py +++ b/base/fs/nopartitions.py @@ -6,6 +6,7 @@ class NoPartitions(object): def __init__(self, data): root = data['root'] self.root = SinglePartition(root['size'], root['filesystem']) + self.partitions = [self.root] self.mount_points = [('/', self.root)] def get_total_size(self): diff --git a/common/fs/__init__.py b/common/fs/__init__.py index 20b2f69..a609034 100644 --- a/common/fs/__init__.py +++ b/common/fs/__init__.py @@ -1,11 +1,16 @@ -def get_major_minor_dev_num(device_name): +def get_partitions(): import re - regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) {device_name}$' - .format(device_name=device_name)) - with open('/proc/partitions') as partitions: + regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) (?P\S+)$') + matches = {} + path = '/proc/partitions' + with open(path) as partitions: + next(partitions) + next(partitions) for line in partitions: match = regexp.match(line) - if match is not None: - return match.group('major'), match.group('minor') + if match is None: + raise RuntimeError('Unable to parse {line} in {path}'.format(line=line, path=path)) + matches[match.group('dev_name')] = match.groupdict() + return matches diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index d945244..0dcc35b 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -49,15 +49,17 @@ class LoopbackVolume(Volume): def _link_dm_node(self, e): import os.path + from . import get_partitions + proc_partitions = get_partitions() loop_device_name = os.path.basename(self.loop_device_path) - from . import get_major_minor_dev_num - major, minor = get_major_minor_dev_num(loop_device_name) + loop_device_partition = proc_partitions[loop_device_name] + sectors = self.size*1024*1024/512 table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' .format(log_start_sec=0, sectors=sectors, - major=major, - minor=minor, + major=loop_device_partition['major'], + minor=loop_device_partition['minor'], start_sec=0)) import string import os.path diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index a0e4075..8af2c49 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -64,15 +64,15 @@ class MountBoot(Task): class CreateBootMountDir(Task): - description = 'Creating mountpoint boot partition' + description = 'Creating mountpoint for the boot partition' phase = phases.volume_mounting after = [MountRoot] before = [MountBoot] def run(self, info): import os - info.boot = os.path.join(info.root, 'boot') - os.makedirs() + boot_dir = os.path.join(info.root, 'boot') + os.makedirs(boot_dir) class MountSpecials(Task): diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index eff464c..18c9fef 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -91,7 +91,9 @@ def tasks(tasklist, manifest): break if 'boot' in manifest.volume['partitions']: - tasklist.add(filesystem.MountBoot(), filesystem.UnmountBoot()) + tasklist.add(filesystem.CreateBootMountDir(), + filesystem.MountBoot(), + filesystem.UnmountBoot()) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -105,7 +107,7 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) counter_task(filesystem.MountRoot, filesystem.UnmountRoot) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) counter_task(volume_tasks.Attach, volume_tasks.Detach) counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index 1af3d66..3f8e6da 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -1,5 +1,7 @@ from common.fs.loopbackvolume import LoopbackVolume +from base.fs.exceptions import VolumeError from common.tools import log_check_call +from common.fs import get_partitions class VirtualBoxVolume(LoopbackVolume): @@ -9,3 +11,57 @@ class VirtualBoxVolume(LoopbackVolume): def _create(self, e): self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + + def _attach(self, e): + num_partitions = len(self.partition_map.partitions) + if not self._module_loaded('nbd'): + msg = ('The kernel module `nbd\' must be loaded ' + '(`modprobe nbd max_part={num_partitions}\') to attach .vdi images' + .format(num_partitions=num_partitions)) + raise VolumeError(msg) + nbd_max_part = int(self._module_param('nbd', 'max_part')) + if nbd_max_part < num_partitions: + msg = ('The kernel module `nbd\' was loaded with the max_part ' + 'parameter set to {max_part}, which is below ' + 'the amount of partitions for this volume ({num_partitions}). ' + 'Reload the nbd kernel module with max_part set to at least {num_partitions} ' + '(`rmmod nbd; modprobe nbd max_part={num_partitions}\').' + .format(max_part=nbd_max_part, num_partitions=num_partitions)) + raise VolumeError(msg) + self.loop_device_path = self._find_free_nbd_device() + log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path]) + self.device_path = self.loop_device_path + + def _detach(self, e): + log_check_call(['/usr/bin/qemu-nbd', '--disconnect', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _module_loaded(self, module): + import re + regexp = re.compile('^{module} +'.format(module=module)) + with open('/proc/modules') as loaded_modules: + for line in loaded_modules: + match = regexp.match(line) + if match is not None: + return True + return False + + def _module_param(self, module, param): + import os.path + param_path = os.path.join('/sys/module', module, 'parameters', param) + with open(param_path) as param: + return param.read().strip() + + # From http://lists.gnu.org/archive/html/qemu-devel/2011-11/msg02201.html + # Apparently it's not in the current qemu-nbd shipped with wheezy + def _is_nbd_used(self, device_name): + return device_name in get_partitions() + + def _find_free_nbd_device(self): + import os.path + for i in xrange(0, 15): + device_name = 'nbd' + str(i) + if not self._is_nbd_used(device_name): + return os.path.join('/dev', device_name) + raise VolumeError('Unable to find free nbd device.') From a9c465cf8c916a68dda2e964b772a5f0d04bf945 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 18:53:03 +0200 Subject: [PATCH 09/63] Minor fixes to paths --- common/tasks/loopback.py | 2 +- plugins/prebootstrapped/tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index c453d6a..315fc5a 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -21,7 +21,7 @@ class MoveImage(Task): def run(self, info): import os.path filename = 'loopback-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) - destination = os.path.join(info.bootstrapper['workspace'], filename) + destination = os.path.join(info.manifest.bootstrapper['workspace'], filename) import shutil shutil.move(info.volume.image_path, destination) import logging diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index f4eb1fa..3899336 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -50,7 +50,7 @@ class CopyImage(Task): def run(self, info): import os.path from shutil import copyfile - loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) + loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension) destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) copyfile(info.volume.image_path, destination) msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination) From 6f05774d2c49f6d0afea2796a0a0150d9532e4e2 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 22:57:15 +0200 Subject: [PATCH 10/63] Try fixing grub, added some comments --- providers/virtualbox/tasks/boot.py | 8 ++++++++ providers/virtualbox/volume.py | 1 + 2 files changed, 9 insertions(+) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index bf96e11..bb9a0c5 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -17,6 +17,9 @@ class ConfigureGrub(Task): grub_dir = os.path.join(boot_dir, 'grub') if isinstance(info.volume, LoopbackVolume): + # GRUB cannot deal with installing to loopback devices + # so we fake a real harddisk with dmsetup. + # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ info.volume.unmount() info.volume.unmap() info.volume.link_dm_node() @@ -45,3 +48,8 @@ class ConfigureGrub(Task): info.volume.mount_root(info.root) info.volume.mount_boot() info.volume.mount_specials() + + # Best guess right now... + device_map_path = os.path.join(grub_dir, 'device.map') + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) /dev/mapper/vda\n') diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index 3f8e6da..b6d6b0d 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -21,6 +21,7 @@ class VirtualBoxVolume(LoopbackVolume): raise VolumeError(msg) nbd_max_part = int(self._module_param('nbd', 'max_part')) if nbd_max_part < num_partitions: + # Found here: http://bethesignal.org/blog/2011/01/05/how-to-mount-virtualbox-vdi-image/ msg = ('The kernel module `nbd\' was loaded with the max_part ' 'parameter set to {max_part}, which is below ' 'the amount of partitions for this volume ({num_partitions}). ' From 4f1533b475482884889f8a800d622a3f0b339555 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 22:57:25 +0200 Subject: [PATCH 11/63] Document some more dependencies --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index ea4d99d..67db323 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,17 @@ Pull requests are also welcome! Dependencies ------------ You will need to run debian wheezy with **python 2.7** and **debootstrap** installed. +Other depencies include: +* qemu-utils +* parted +* grub2 +* euca2ools +* xfsprogs (If you want to use XFS as a filesystem) Also the following python libraries are required: * **boto** * **jsomschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip) * **termcolor** +* **fysom** Bootstrapping instance store AMIs requires **euca2ools** to be installed. From 8073edc9021280271431a6de65e64df4c12daa08 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 23:01:52 +0200 Subject: [PATCH 12/63] Fix manifest a bit --- manifests/virtualbox.manifest.json | 36 +++++++++--------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index d61e30f..3b33999 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -1,14 +1,12 @@ { - "provider": "virtualbox", - "virtualization": "ide", - + "provider": "virtualbox", "bootstrapper": { - "workspace": "/mnt/target", - "mirror": "http://http.debian.net/debian/" + "workspace": "/target", + "tarball": true }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" + "name": "debian-{release}-{architecture}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture}" }, "system": { "release": "wheezy", @@ -18,32 +16,18 @@ "charmap": "UTF-8" }, "volume": { - "backing": "raw", + "backing": "vdi", "partitions": { + "type": "gpt", "boot": { - "size": 12, + "size": 64, "filesystem": "ext2" }, "root": { - "size": 812, + "size": 832, "filesystem": "ext4" }, - "swap": {"size": 200} - } - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "convert_image": { - "enabled": true, - "format": "vdi" + "swap": {"size": 128} } } } From d6502089e21bc215477bc9ef91d8d9621bcd1f89 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 18 Sep 2013 00:46:58 +0200 Subject: [PATCH 13/63] Implemented both MBR and GPT partitioning. VirtualBox seems to not like GPT --- base/fs/__init__.py | 8 ++-- base/fs/partitionmaps/__init__.py | 0 .../abstract.py} | 38 +++++------------ base/fs/partitionmaps/gpt.py | 35 ++++++++++++++++ base/fs/partitionmaps/mbr.py | 33 +++++++++++++++ .../none.py} | 2 +- .../{abstractpartition.py => abstract.py} | 14 +++++-- base/fs/partitions/{partition.py => base.py} | 28 +++---------- base/fs/partitions/gpt.py | 19 +++++++++ base/fs/partitions/gpt_swap.py | 11 +++++ base/fs/partitions/mbr.py | 13 ++++++ base/fs/partitions/{swap.py => mbr_swap.py} | 6 +-- .../{singlepartition.py => single.py} | 9 +--- common/tasks/host.py | 1 + manifests/virtualbox.manifest.json | 24 +++++------ providers/virtualbox/tasks/boot.py | 42 ++++++++++++------- 16 files changed, 187 insertions(+), 96 deletions(-) create mode 100644 base/fs/partitionmaps/__init__.py rename base/fs/{partitionmap.py => partitionmaps/abstract.py} (61%) create mode 100644 base/fs/partitionmaps/gpt.py create mode 100644 base/fs/partitionmaps/mbr.py rename base/fs/{nopartitions.py => partitionmaps/none.py} (90%) rename base/fs/partitions/{abstractpartition.py => abstract.py} (86%) rename base/fs/partitions/{partition.py => base.py} (58%) create mode 100644 base/fs/partitions/gpt.py create mode 100644 base/fs/partitions/gpt_swap.py create mode 100644 base/fs/partitions/mbr.py rename base/fs/partitions/{swap.py => mbr_swap.py} (54%) rename base/fs/partitions/{singlepartition.py => single.py} (58%) diff --git a/base/fs/__init__.py b/base/fs/__init__.py index 6180894..6b4763b 100644 --- a/base/fs/__init__.py +++ b/base/fs/__init__.py @@ -4,10 +4,12 @@ def load_volume(data): from common.fs.loopbackvolume import LoopbackVolume from providers.ec2.volume import EBSVolume from providers.virtualbox.volume import VirtualBoxVolume - from partitionmap import PartitionMap - from nopartitions import NoPartitions + from partitionmaps.gpt import GPTPartitionMap + from partitionmaps.mbr import MBRPartitionMap + from partitionmaps.none import NoPartitions partition_maps = {'none': NoPartitions, - 'gpt': PartitionMap, + 'gpt': GPTPartitionMap, + 'mbr': MBRPartitionMap, } partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) volume_backings = {'raw': LoopbackVolume, diff --git a/base/fs/partitionmaps/__init__.py b/base/fs/partitionmaps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/fs/partitionmap.py b/base/fs/partitionmaps/abstract.py similarity index 61% rename from base/fs/partitionmap.py rename to base/fs/partitionmaps/abstract.py index 49a75ab..167ddea 100644 --- a/base/fs/partitionmap.py +++ b/base/fs/partitionmaps/abstract.py @@ -1,39 +1,23 @@ +from abc import ABCMeta +from abc import abstractmethod from common.tools import log_check_call -from partitions.partition import Partition -from partitions.swap import Swap -from exceptions import PartitionError +from ..exceptions import PartitionError -class PartitionMap(object): +class AbstractPartitionMap(object): + __metaclass__ = ABCMeta + + @abstractmethod def __init__(self, data): - self.boot = None - self.swap = None - self.mount_points = [] - if 'boot' in data: - self.boot = Partition(data['boot']['size'], data['boot']['filesystem'], None) - self.mount_points.append(('/boot', self.boot)) - self.root = Partition(data['root']['size'], data['root']['filesystem'], self.boot) - self.mount_points.append(('/', self.root)) - if 'swap' in data: - self.swap = Swap(data['swap']['size'], self.root) - self.mount_points.append(('none', self.root)) - self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + pass def get_total_size(self): - return sum(p.size for p in self.partitions) + return sum(p.size for p in self.partitions) + 1 + @abstractmethod def create(self, volume): - log_check_call(['/sbin/parted', '--script', '--align', 'optimal', volume.device_path, - '--', 'mklabel', 'gpt']) - for partition in self.partitions: - partition.create(volume) - - boot_idx = self.root.get_index() - if self.boot is not None: - boot_idx = self.boot.get_index() - log_check_call(['/sbin/parted', '--script', volume.device_path, - '--', 'set', str(boot_idx), 'bios_grub', 'on']) + pass def map(self, volume): try: diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py new file mode 100644 index 0000000..76bbd0a --- /dev/null +++ b/base/fs/partitionmaps/gpt.py @@ -0,0 +1,35 @@ +from abstract import AbstractPartitionMap +from ..partitions.gpt import GPTPartition +from ..partitions.gpt_swap import GPTSwapPartition +from common.tools import log_check_call + + +class GPTPartitionMap(AbstractPartitionMap): + + def __init__(self, data): + self.boot = None + self.swap = None + self.mount_points = [] + if 'boot' in data: + self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None) + self.mount_points.append(('/boot', self.boot)) + self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', self.boot) + self.mount_points.append(('/', self.root)) + if 'swap' in data: + self.swap = GPTSwapPartition(data['swap']['size'], self.root) + self.mount_points.append(('none', self.root)) + self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + + def create(self, volume): + log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, + '--', 'mklabel', 'gpt']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if self.boot is not None: + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' bios_grub on']) + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' boot on']) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py new file mode 100644 index 0000000..2e635c4 --- /dev/null +++ b/base/fs/partitionmaps/mbr.py @@ -0,0 +1,33 @@ +from abstract import AbstractPartitionMap +from ..partitions.mbr import MBRPartition +from ..partitions.mbr_swap import MBRSwapPartition +from common.tools import log_check_call + + +class MBRPartitionMap(AbstractPartitionMap): + + def __init__(self, data): + self.boot = None + self.swap = None + self.mount_points = [] + if 'boot' in data: + self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None) + self.mount_points.append(('/boot', self.boot)) + self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], self.boot) + self.mount_points.append(('/', self.root)) + if 'swap' in data: + self.swap = MBRSwapPartition(data['swap']['size'], self.root) + self.mount_points.append(('none', self.root)) + self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + + def create(self, volume): + log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, + '--', 'mklabel', 'msdos']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if self.boot is not None: + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' boot on']) diff --git a/base/fs/nopartitions.py b/base/fs/partitionmaps/none.py similarity index 90% rename from base/fs/nopartitions.py rename to base/fs/partitionmaps/none.py index 0133ffc..a04f742 100644 --- a/base/fs/nopartitions.py +++ b/base/fs/partitionmaps/none.py @@ -1,4 +1,4 @@ -from partitions.singlepartition import SinglePartition +from ..partitions.single import SinglePartition class NoPartitions(object): diff --git a/base/fs/partitions/abstractpartition.py b/base/fs/partitions/abstract.py similarity index 86% rename from base/fs/partitions/abstractpartition.py rename to base/fs/partitions/abstract.py index 161e134..15b7f0d 100644 --- a/base/fs/partitions/abstractpartition.py +++ b/base/fs/partitions/abstract.py @@ -1,5 +1,6 @@ -from common.tools import log_check_call from abc import ABCMeta +from abc import abstractmethod +from common.tools import log_check_call from fysom import Fysom @@ -17,9 +18,9 @@ class AbstractPartition(object): self.size = size self.filesystem = filesystem self.device_path = None - self.initial_state = 'nonexistent' - callbacks.update({'onbeforeformat': self._format, + callbacks.update({'onbeforecreate': self._create, + 'onbeforeformat': self._format, 'onbeforemount': self._mount, 'onbeforeunmount': self._unmount, }) @@ -40,6 +41,13 @@ class AbstractPartition(object): [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) return uuid + def create(self, volume): + self.fsm.create(volume=volume) + + @abstractmethod + def _create(self, e): + pass + def _format(self, e): mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) log_check_call([mkfs, self.device_path]) diff --git a/base/fs/partitions/partition.py b/base/fs/partitions/base.py similarity index 58% rename from base/fs/partitions/partition.py rename to base/fs/partitions/base.py index a62ac0e..f79ebb3 100644 --- a/base/fs/partitions/partition.py +++ b/base/fs/partitions/base.py @@ -1,8 +1,7 @@ -from common.tools import log_check_call -from abstractpartition import AbstractPartition +from abstract import AbstractPartition -class Partition(AbstractPartition): +class BasePartition(AbstractPartition): events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, @@ -16,11 +15,10 @@ class Partition(AbstractPartition): def __init__(self, size, filesystem, previous, callbacks={}): self.previous = previous - callbacks.update({'onbeforecreate': self._create, - 'onbeforemap': self._map, + callbacks.update({'onbeforemap': self._map, 'onbeforeunmap': self._unmap, }) - super(Partition, self).__init__(size, filesystem, callbacks=callbacks) + super(BasePartition, self).__init__(size, filesystem, callbacks=callbacks) def get_index(self): if self.previous is None: @@ -30,19 +28,10 @@ class Partition(AbstractPartition): def get_start(self): if self.previous is None: - return 0 + return 1 else: return self.previous.get_start() + self.previous.size - def create(self, volume): - self.fsm.create(volume=volume) - - def _create(self, e): - start = self.get_start() - # maybe use `parted -- name` to set partition name - log_check_call(['/sbin/parted', '--script', '--align', 'optimal', e.volume.device_path, - '--', 'mkpart', 'primary', str(start), str(start + self.size)]) - def map(self, device_path): self.fsm.map(device_path=device_path) @@ -51,10 +40,3 @@ class Partition(AbstractPartition): def _unmap(self, e): self.device_path = None - -# Partition flags: -# boot, root, swap, hidden, raid, lvm, lba, legacy_boot, palo - - -# Partition tables -# bsd, dvh, gpt, loop, mac, msdos, pc98, sun diff --git a/base/fs/partitions/gpt.py b/base/fs/partitions/gpt.py new file mode 100644 index 0000000..01987ad --- /dev/null +++ b/base/fs/partitions/gpt.py @@ -0,0 +1,19 @@ +from common.tools import log_check_call +from base import BasePartition + + +class GPTPartition(BasePartition): + + def __init__(self, size, filesystem, name, previous, callbacks={}): + self.name = name + super(GPTPartition, self).__init__(size, filesystem, previous, callbacks=callbacks) + + def _create(self, e): + start = self.get_start() + # {name} only works for gpt, for msdos that becomes the part-type (primary, extended, logical) + parted_command = ('mkpart primary {start}MiB {end}MiB' + .format(name=self.name, + start=str(start), + end=str(start + self.size))) + log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, + '--', parted_command]) diff --git a/base/fs/partitions/gpt_swap.py b/base/fs/partitions/gpt_swap.py new file mode 100644 index 0000000..030e58b --- /dev/null +++ b/base/fs/partitions/gpt_swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from gpt import GPTPartition + + +class GPTSwapPartition(GPTPartition): + + def __init__(self, size, previous): + super(GPTSwapPartition, self).__init__(size, 'swap', 'swap', previous) + + def _format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/mbr.py b/base/fs/partitions/mbr.py new file mode 100644 index 0000000..9d4fbbd --- /dev/null +++ b/base/fs/partitions/mbr.py @@ -0,0 +1,13 @@ +from common.tools import log_check_call +from base import BasePartition + + +class MBRPartition(BasePartition): + + def _create(self, e): + start = self.get_start() + parted_command = ('mkpart primary {start}MiB {end}MiB' + .format(start=str(start), + end=str(start + self.size))) + log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, + '--', parted_command]) diff --git a/base/fs/partitions/swap.py b/base/fs/partitions/mbr_swap.py similarity index 54% rename from base/fs/partitions/swap.py rename to base/fs/partitions/mbr_swap.py index 80b7531..0c5c1a1 100644 --- a/base/fs/partitions/swap.py +++ b/base/fs/partitions/mbr_swap.py @@ -1,11 +1,11 @@ from common.tools import log_check_call -from partition import Partition +from mbr import MBRPartition -class Swap(Partition): +class MBRSwapPartition(MBRPartition): def __init__(self, size, previous): - super(Swap, self).__init__(size, 'swap', previous) + super(MBRSwapPartition, self).__init__(size, 'swap', previous) def _format(self, e): log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/singlepartition.py b/base/fs/partitions/single.py similarity index 58% rename from base/fs/partitions/singlepartition.py rename to base/fs/partitions/single.py index 10bd702..c43236a 100644 --- a/base/fs/partitions/singlepartition.py +++ b/base/fs/partitions/single.py @@ -1,4 +1,4 @@ -from abstractpartition import AbstractPartition +from abstract import AbstractPartition class SinglePartition(AbstractPartition): @@ -9,12 +9,5 @@ class SinglePartition(AbstractPartition): {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, ] - def __init__(self, size, filesystem, callbacks={}): - callbacks['oncreate'] = self._create - super(SinglePartition, self).__init__(size, filesystem, callbacks=callbacks) - - def create(self, volume): - self.fsm.create(volume=volume) - def _create(self, e): self.device_path = e.volume.device_path diff --git a/common/tasks/host.py b/common/tasks/host.py index c377ecc..aa76df0 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -14,6 +14,7 @@ class CheckPackages(Task): from subprocess import CalledProcessError for package in info.host_packages: try: + # Use "dpkg-query -W -f='${Status} ${Version}\n' package" instead log_check_call(['/usr/bin/dpkg', '--status', package]) except CalledProcessError: msg = "The package ``{0}\'\' is not installed".format(package) diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index 3b33999..d3ce81f 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -1,33 +1,33 @@ { - "provider": "virtualbox", + "provider" : "virtualbox", "bootstrapper": { "workspace": "/target", "tarball": true }, "image": { - "name": "debian-{release}-{architecture}-{%y}{%m}{%d}", + "name" : "debian-{release}-{architecture}-{%y}{%m}{%d}", "description": "Debian {release} {architecture}" }, "system": { - "release": "wheezy", + "release" : "wheezy", "architecture": "amd64", - "timezone": "UTC", - "locale": "en_US", - "charmap": "UTF-8" + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8" }, "volume": { - "backing": "vdi", + "backing": "vdi", "partitions": { - "type": "gpt", + "type": "mbr", "boot": { - "size": 64, + "size": 32, "filesystem": "ext2" }, "root": { - "size": 832, + "size": 991, "filesystem": "ext4" - }, - "swap": {"size": 128} + } + // ,"swap": {"size": 128} } } } diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index bb9a0c5..516b6a6 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -16,6 +16,7 @@ class ConfigureGrub(Task): boot_dir = os.path.join(info.root, 'boot') grub_dir = os.path.join(boot_dir, 'grub') + # if type(info.volume) is LoopbackVolume: if isinstance(info.volume, LoopbackVolume): # GRUB cannot deal with installing to loopback devices # so we fake a real harddisk with dmsetup. @@ -27,18 +28,32 @@ class ConfigureGrub(Task): info.volume.mount_root(info.root) info.volume.mount_boot() info.volume.mount_specials() - [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) - device_map_path = os.path.join(grub_dir, 'device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + try: + [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) + device_map_path = os.path.join(grub_dir, 'device.map') + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + for idx, partition in enumerate(info.volume.partition_map.partitions): + [partition_path] = log_check_call(['readlink', '-f', partition.device_path]) + device_map.write('(hd0,gpt{idx}) {device_path}\n'.format(device_path=partition_path, idx=idx+1)) - # Install grub - log_check_call(['/usr/sbin/grub-install', - '--root-directory=' + info.root, - '--boot-directory=' + boot_dir, - device_path]) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - # log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) + # Install grub + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/sbin/grub-install', + # '--root-directory=' + info.root, + # '--boot-directory=' + boot_dir, + device_path]) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + except Exception as e: + if isinstance(info.volume, LoopbackVolume): + info.volume.unmount() + info.volume.unmap() + info.volume.unlink_dm_node() + info.volume.map() + info.volume.mount_root(info.root) + info.volume.mount_boot() + info.volume.mount_specials() + raise e if isinstance(info.volume, LoopbackVolume): info.volume.unmount() @@ -48,8 +63,3 @@ class ConfigureGrub(Task): info.volume.mount_root(info.root) info.volume.mount_boot() info.volume.mount_specials() - - # Best guess right now... - device_map_path = os.path.join(grub_dir, 'device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/mapper/vda\n') From c744196d2fcd5563158899c4e116e092a61106c6 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 20 Sep 2013 16:08:53 +0000 Subject: [PATCH 14/63] Fix required packages --- providers/virtualbox/tasks/packages.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 207f356..e1c0d65 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -11,7 +11,7 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) + info.host_packages.update(['qemu-utils', 'parted', 'kpartx', 'sysv-rc']) if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') @@ -25,20 +25,7 @@ class ImagePackages(Task): manifest = info.manifest include, exclude = info.img_packages # Add some basic packages we are going to need - include.update(['parted', - 'kpartx', - # Needed for the init scripts - 'file', - # isc-dhcp-client doesn't work properly with ec2 - 'dhcpcd', - 'chkconfig', - 'openssh-client', - 'grub2' - ]) - - exclude.update(['isc-dhcp-client', - 'isc-dhcp-common', - ]) + include.update(['grub2']) # In squeeze, we need a special kernel flavor for xen kernels = {'squeeze': {'amd64': 'linux-image-amd64', From 53991bdad71632847daf7d000a6a8bf4a9e02ca9 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 15:39:22 +0200 Subject: [PATCH 15/63] Rename loopback image to volume --- common/tasks/loopback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 315fc5a..8a0b2c9 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -20,7 +20,7 @@ class MoveImage(Task): def run(self, info): import os.path - filename = 'loopback-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) + filename = 'volume-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) destination = os.path.join(info.manifest.bootstrapper['workspace'], filename) import shutil shutil.move(info.volume.image_path, destination) From 5dcec10d6555201058b5492bd29693c333b0789e Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 15:40:51 +0200 Subject: [PATCH 16/63] Minimize required image packages --- common/tasks/packages.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/common/tasks/packages.py b/common/tasks/packages.py index 2d868d9..abccdf5 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -17,11 +17,8 @@ class ImagePackages(Task): def run(self, info): # Add some basic packages we are going to need - include = set(['udev', - 'openssh-server', - # We could bootstrap without locales, but things just suck without them, error messages etc. - 'locales', - ]) + # We could bootstrap without locales, but things just suck without them, error messages etc. + include = set(['locales']) exclude = set() info.img_packages = include, exclude From 885ace7b48d666ac7ff6012ce8382bfe089302ac Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 15:41:43 +0200 Subject: [PATCH 17/63] Updates to networking in vbox --- common/tasks/network.py | 19 +++++++++---------- providers/ec2/__init__.py | 9 +++++---- providers/ec2/tasks/network.py | 15 +++++++++++++++ providers/virtualbox/__init__.py | 5 +---- 4 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 providers/ec2/tasks/network.py diff --git a/common/tasks/network.py b/common/tasks/network.py index 0910608..e3129d6 100644 --- a/common/tasks/network.py +++ b/common/tasks/network.py @@ -12,6 +12,15 @@ class RemoveDNSInfo(Task): remove(os.path.join(info.root, 'etc/resolv.conf')) +class RemoveHostname(Task): + description = 'Removing the hostname file' + phase = phases.system_modification + + def run(self, info): + from os import remove + remove(os.path.join(info.root, 'etc/hostname')) + + class ConfigureNetworkIF(Task): description = 'Configuring network interfaces' phase = phases.system_modification @@ -26,13 +35,3 @@ class ConfigureNetworkIF(Task): 'iface eth0 inet dhcp\n'} with open(interfaces_path, 'a') as interfaces: interfaces.write(if_config.get(info.manifest.system['release'])) - - -class ConfigureDHCP(Task): - description = 'Configuring the DHCP client' - phase = phases.system_modification - - def run(self, info): - from common.tools import sed_i - dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') - sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 533ccf7..68c47c5 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -16,7 +16,8 @@ from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security -from common.tasks import network +from tasks import network +from common.tasks import network as common_network from tasks import initd from common.tasks import initd as common_initd from common.tasks import cleanup @@ -56,9 +57,9 @@ def tasks(tasklist, manifest): security.EnableShadowConfig(), security.DisableSSHPasswordAuthentication(), security.DisableSSHDNSLookup(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.ConfigureDHCP(), + common_network.RemoveDNSInfo(), + common_network.ConfigureNetworkIF(), + network.EnableDHCPCDDNS(), common_initd.ResolveInitScripts(), initd.AddEC2InitScripts(), common_initd.InstallInitScripts(), diff --git a/providers/ec2/tasks/network.py b/providers/ec2/tasks/network.py new file mode 100644 index 0000000..3f00fb2 --- /dev/null +++ b/providers/ec2/tasks/network.py @@ -0,0 +1,15 @@ +from base import Task +from common import phases +import os.path + + +class EnableDHCPCDDNS(Task): + description = 'Configuring the DHCP client to set the nameservers' + phase = phases.system_modification + + def run(self, info): + # The dhcp client that ships with debian sets the DNS servers per default. + # For dhcpcd we need to configure it to do that. + from common.tools import sed_i + dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') + sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 18c9fef..0eb7475 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -50,15 +50,12 @@ def tasks(tasklist, manifest): common_boot.BlackListModules(), common_boot.DisableGetTTYs(), security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), network.RemoveDNSInfo(), network.ConfigureNetworkIF(), - network.ConfigureDHCP(), + network.RemoveHostname(), initd.ResolveInitScripts(), initd.InstallInitScripts(), cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), cleanup.CleanTMP(), apt.PurgeUnusedPackages(), apt.AptClean(), From 4b2711893f84830089f3444cf2704074e089858c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 15:43:50 +0200 Subject: [PATCH 18/63] Dont require package sysv-rc to be installed --- providers/virtualbox/tasks/packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index e1c0d65..871152a 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -11,7 +11,7 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'kpartx', 'sysv-rc']) + info.host_packages.update(['qemu-utils', 'parted', 'kpartx']) if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') From 8aefa936463ecc3a6394d23d29ccf2ee6a705faf Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 17:02:34 +0200 Subject: [PATCH 19/63] Remove map and unmap methods on singlepartitionmap --- base/fs/partitionmaps/none.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index a04f742..ab58fda 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -13,13 +13,7 @@ class NoPartitions(object): return self.root.size def create(self, volume): - pass - - def map(self, volume): - pass - - def unmap(self, volume): - pass + self.root.create(volume) def format(self): self.root.format() From 738ba47c6557aad7de04f754d0f53cb8f036c068 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 17:31:43 +0200 Subject: [PATCH 20/63] Add references instead of instances to the tasklist --- base/task.py | 20 ----- base/tasklist.py | 37 ++++++---- plugins/admin_user/__init__.py | 10 +-- plugins/backports/__init__.py | 4 +- plugins/build_metadata/__init__.py | 2 +- plugins/convert_image/__init__.py | 2 +- plugins/opennebula/__init__.py | 2 +- plugins/prebootstrapped/__init__.py | 10 +-- plugins/root_password/__init__.py | 2 +- plugins/unattended_upgrades/__init__.py | 4 +- plugins/user_packages/__init__.py | 4 +- providers/virtualbox/__init__.py | 98 ++++++++++++------------- 12 files changed, 93 insertions(+), 102 deletions(-) diff --git a/base/task.py b/base/task.py index c3841a6..8a10462 100644 --- a/base/task.py +++ b/base/task.py @@ -1,4 +1,3 @@ -from common.exceptions import TaskListError class Task(object): @@ -6,27 +5,8 @@ class Task(object): before = [] after = [] - def __init__(self): - self._check_ordering() - def __str__(self): return '{module}.{task}'.format(module=self.__module__, task=self.__class__.__name__) def __repr__(self): return self.__str__() - - def _check_ordering(self): - def name(ref): - return '{module}.{task}'.format(module=ref.__module__, task=ref.__class__.__name__) - for task in self.before: - if self.phase > task.phase: - msg = ("The task {self} is specified as running before {other}, " - "but its phase '{phase}' lies after the phase '{other_phase}'" - .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase)) - raise TaskListError(msg) - for task in self.after: - if self.phase < task.phase: - msg = ("The task {self} is specified as running after {other}, " - "but its phase '{phase}' lies before the phase '{other_phase}'" - .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase)) - raise TaskListError(msg) diff --git a/base/tasklist.py b/base/tasklist.py index d786abd..68b1427 100644 --- a/base/tasklist.py +++ b/base/tasklist.py @@ -13,23 +13,19 @@ class TaskList(object): self.tasks.update(args) def remove(self, *args): - for task_type in args: - task = self.get(task_type) - if task is not None: - self.tasks.discard(task) + for task in args: + self.tasks.discard(task) def replace(self, task, replacement): self.remove(task) self.add(replacement) - def get(self, ref): - return next((task for task in self.tasks if type(task) is ref), None) - def run(self, bootstrap_info): task_list = self.create_list(self.tasks) log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list))) - for task in task_list: + for task_type in task_list: + task = task_type() if hasattr(task, 'description'): log.info(task.description) else: @@ -41,12 +37,13 @@ class TaskList(object): from common.phases import order graph = {} for task in tasks: - successors = [] - successors.extend([self.get(succ) for succ in task.before]) - successors.extend(filter(lambda succ: type(task) in succ.after, tasks)) + self.check_ordering(task) + successors = set() + successors.update(task.before) + successors.update(filter(lambda succ: task in succ.after, tasks)) succeeding_phases = order[order.index(task.phase) + 1:] - successors.extend(filter(lambda succ: succ.phase in succeeding_phases, tasks)) - graph[task] = filter(lambda succ: succ in self.tasks, successors) + successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks)) + graph[task] = filter(lambda succ: succ in tasks, successors) components = self.strongly_connected_components(graph) cycles_found = 0 @@ -63,6 +60,20 @@ class TaskList(object): return sorted_tasks + def check_ordering(self, task): + for successor in task.before: + if successor.phase > successor.phase: + msg = ("The task {task} is specified as running before {other}, " + "but its phase '{phase}' lies after the phase '{other_phase}'" + .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase)) + raise TaskListError(msg) + for predecessor in task.after: + if task.phase < predecessor.phase: + msg = ("The task {task} is specified as running after {other}, " + "but its phase '{phase}' lies before the phase '{other_phase}'" + .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase)) + raise TaskListError(msg) + def strongly_connected_components(self, graph): # Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py # Find the strongly connected components in a graph using Tarjan's algorithm. diff --git a/plugins/admin_user/__init__.py b/plugins/admin_user/__init__.py index 8c396c9..2916bd0 100644 --- a/plugins/admin_user/__init__.py +++ b/plugins/admin_user/__init__.py @@ -2,11 +2,11 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AddSudoPackage()) - tasklist.add(tasks.CreateAdminUser()) - tasklist.add(tasks.PasswordlessSudo()) - tasklist.add(tasks.AdminUserCredentials()) - tasklist.add(tasks.DisableRootLogin()) + tasklist.add(tasks.AddSudoPackage. + tasks.CreateAdminUser, + tasks.PasswordlessSudo, + tasks.AdminUserCredentials, + tasks.DisableRootLogin) def validate_manifest(data, schema_validate): diff --git a/plugins/backports/__init__.py b/plugins/backports/__init__.py index bdeae13..834c2d8 100644 --- a/plugins/backports/__init__.py +++ b/plugins/backports/__init__.py @@ -2,8 +2,8 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AptSourcesBackports()) - tasklist.add(tasks.AddBackportsPackages()) + tasklist.add(tasks.AptSourcesBackports, + tasks.AddBackportsPackages) def validate_manifest(data, schema_validate): diff --git a/plugins/build_metadata/__init__.py b/plugins/build_metadata/__init__.py index a8db8ba..71d1abf 100644 --- a/plugins/build_metadata/__init__.py +++ b/plugins/build_metadata/__init__.py @@ -2,4 +2,4 @@ def tasks(tasklist, manifest): from tasks import WriteMetadata - tasklist.add(WriteMetadata()) + tasklist.add(WriteMetadata diff --git a/plugins/convert_image/__init__.py b/plugins/convert_image/__init__.py index a36d8f4..f37792b 100644 --- a/plugins/convert_image/__init__.py +++ b/plugins/convert_image/__init__.py @@ -2,7 +2,7 @@ def tasks(tasklist, manifest): from tasks import ConvertImage - tasklist.add(ConvertImage()) + tasklist.add(ConvertImage) def validate_manifest(data, schema_validate): diff --git a/plugins/opennebula/__init__.py b/plugins/opennebula/__init__.py index a9ae730..6ffcec9 100644 --- a/plugins/opennebula/__init__.py +++ b/plugins/opennebula/__init__.py @@ -2,4 +2,4 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.OpenNebulaContext()) + tasklist.add(tasks.OpenNebulaContext) diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index dc258a7..ff6aec9 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -21,16 +21,16 @@ def tasks(tasklist, manifest): bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: - tasklist.replace(ebs.Create, CreateFromSnapshot()) + tasklist.replace(ebs.Create, CreateFromSnapshot) tasklist.remove(*skip_tasks) else: - tasklist.add(Snapshot()) + tasklist.add(Snapshot) else: if 'image' in settings and settings['image'] is not None: - tasklist.replace(loopback.Create, CreateFromImage()) + tasklist.replace(loopback.Create, CreateFromImage) tasklist.remove(*skip_tasks) else: - tasklist.add(CopyImage()) + tasklist.add(CopyImage) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -38,7 +38,7 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) if manifest.volume['backing'] == 'ebs': counter_task(CreateFromSnapshot, volume.Delete) diff --git a/plugins/root_password/__init__.py b/plugins/root_password/__init__.py index f3d4e87..34b1248 100644 --- a/plugins/root_password/__init__.py +++ b/plugins/root_password/__init__.py @@ -3,7 +3,7 @@ def tasks(tasklist, manifest): from common.tasks.security import DisableSSHPasswordAuthentication from tasks import SetRootPassword - tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword()) + tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword) def validate_manifest(data, schema_validate): diff --git a/plugins/unattended_upgrades/__init__.py b/plugins/unattended_upgrades/__init__.py index aa1d239..8025744 100644 --- a/plugins/unattended_upgrades/__init__.py +++ b/plugins/unattended_upgrades/__init__.py @@ -2,8 +2,8 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AddUnattendedUpgradesPackage()) - tasklist.add(tasks.EnablePeriodicUpgrades()) + tasklist.add(tasks.AddUnattendedUpgradesPackage, + tasks.EnablePeriodicUpgrades) def validate_manifest(data, schema_validate): diff --git a/plugins/user_packages/__init__.py b/plugins/user_packages/__init__.py index e223e0a..218975c 100644 --- a/plugins/user_packages/__init__.py +++ b/plugins/user_packages/__init__.py @@ -2,5 +2,5 @@ def tasks(tasklist, manifest): from user_packages import AddUserPackages, AddLocalUserPackages - tasklist.add(AddUserPackages()) - tasklist.add(AddLocalUserPackages()) + tasklist.add(AddUserPackages, + AddLocalUserPackages) diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 0eb7475..4d095b2 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -23,54 +23,54 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(workspace.CreateWorkspace(), - packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - host.CheckPackages(), + tasklist.add(workspace.CreateWorkspace, + packages.HostPackages, + common_packages.HostPackages, + packages.ImagePackages, + common_packages.ImagePackages, + host.CheckPackages, - loopback.Create(), - volume_tasks.Attach(), - partitioning.PartitionVolume(), - partitioning.MapPartitions(), - filesystem.Format(), - filesystem.CreateMountDir(), - filesystem.MountRoot(), + loopback.Create, + volume_tasks.Attach, + partitioning.PartitionVolume, + partitioning.MapPartitions, + filesystem.Format, + filesystem.CreateMountDir, + filesystem.MountRoot, - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.FStab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.RemoveHostname(), - initd.ResolveInitScripts(), - initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), + bootstrap.Bootstrap, + filesystem.MountSpecials, + locale.GenerateLocale, + locale.SetTimezone, + apt.DisableDaemonAutostart, + apt.AptSources, + apt.AptUpgrade, + boot.ConfigureGrub, + filesystem.FStab, + common_boot.BlackListModules, + common_boot.DisableGetTTYs, + security.EnableShadowConfig, + network.RemoveDNSInfo, + network.ConfigureNetworkIF, + network.RemoveHostname, + initd.ResolveInitScripts, + initd.InstallInitScripts, + cleanup.ClearMOTD, + cleanup.CleanTMP, + apt.PurgeUnusedPackages, + apt.AptClean, + apt.EnableDaemonAutostart, + filesystem.UnmountSpecials, - filesystem.UnmountRoot(), - partitioning.UnmapPartitions(), - volume_tasks.Detach(), - filesystem.DeleteMountDir(), - loopback.MoveImage(), - workspace.DeleteWorkspace()) + filesystem.UnmountRoot, + partitioning.UnmapPartitions, + volume_tasks.Detach, + filesystem.DeleteMountDir, + loopback.MoveImage, + workspace.DeleteWorkspace) if manifest.bootstrapper.get('tarball', False): - tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.MakeTarball) partitions = manifest.volume['partitions'] import re @@ -78,19 +78,19 @@ def tasks(tasklist, manifest): if key not in partitions: continue if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None: - tasklist.add(filesystem.TuneVolumeFS()) + tasklist.add(filesystem.TuneVolumeFS) break for key in ['boot', 'root']: if key not in partitions: continue if partitions[key]['filesystem'] == 'xfs': - tasklist.add(filesystem.AddXFSProgs()) + tasklist.add(filesystem.AddXFSProgs) break if 'boot' in manifest.volume['partitions']: - tasklist.add(filesystem.CreateBootMountDir(), - filesystem.MountBoot(), - filesystem.UnmountBoot()) + tasklist.add(filesystem.CreateBootMountDir, + filesystem.MountBoot, + filesystem.UnmountBoot) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -98,7 +98,7 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) counter_task(loopback.Create, volume_tasks.Delete) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) From c756eb3f74baaed15e73809f3ea858e8aa877b84 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 20:46:25 +0200 Subject: [PATCH 21/63] Introduce task-sets to combat the unwieldy amount of tasks --- common/task_sets.py | 62 +++++++++++++++++++++++++++++ providers/virtualbox/__init__.py | 68 +++++++++++--------------------- 2 files changed, 85 insertions(+), 45 deletions(-) create mode 100644 common/task_sets.py diff --git a/common/task_sets.py b/common/task_sets.py new file mode 100644 index 0000000..2d30ba8 --- /dev/null +++ b/common/task_sets.py @@ -0,0 +1,62 @@ +from common.tasks import workspace +from common.tasks import packages +from common.tasks import host +from common.tasks import volume +from common.tasks import filesystem +from common.tasks import partitioning +from common.tasks import cleanup +from common.tasks import apt +from common.tasks import security +from common.tasks import locale + +base_set = [workspace.CreateWorkspace, + packages.HostPackages, + packages.ImagePackages, + host.CheckPackages, + workspace.DeleteWorkspace, + ] + +volume_set = [volume.Attach, + volume.Detach, + ] + +partitioning_set = [partitioning.PartitionVolume, + partitioning.MapPartitions, + partitioning.UnmapPartitions, + ] + +boot_partition_set = [filesystem.CreateBootMountDir, + filesystem.MountBoot, + filesystem.UnmountBoot, + ] + +mounting_set = [filesystem.CreateMountDir, + filesystem.MountRoot, + filesystem.MountSpecials, + filesystem.UnmountSpecials, + filesystem.UnmountRoot, + filesystem.DeleteMountDir, + ] + +ssh_set = [security.DisableSSHPasswordAuthentication, + security.DisableSSHDNSLookup, + cleanup.ShredHostkeys, + ] + +apt_set = [apt.DisableDaemonAutostart, + apt.AptSources, + apt.AptUpgrade, + apt.PurgeUnusedPackages, + apt.AptClean, + apt.EnableDaemonAutostart, + ] + +locale_set = [locale.GenerateLocale, + locale.SetTimezone, + ] + +fs_specific_set = {'ext2': [filesystem.TuneVolumeFS], + 'ext3': [filesystem.TuneVolumeFS], + 'ext4': [filesystem.TuneVolumeFS], + 'xfs': [filesystem.AddXFSProgs], + } diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 4d095b2..11b9b2b 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -1,14 +1,10 @@ from manifest import Manifest from tasks import packages -from common.tasks import packages as common_packages -from common.tasks import host from common.tasks import volume as volume_tasks from common.tasks import loopback from common.tasks import partitioning from common.tasks import filesystem from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security @@ -23,28 +19,28 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(workspace.CreateWorkspace, - packages.HostPackages, - common_packages.HostPackages, + from common.task_sets import base_set + from common.task_sets import volume_set + from common.task_sets import mounting_set + from common.task_sets import apt_set + from common.task_sets import locale_set + tasklist.add(*base_set) + tasklist.add(*volume_set) + tasklist.add(*mounting_set) + tasklist.add(*apt_set) + tasklist.add(*locale_set) + + if manifest.volume['partitions']['type'] != 'none': + from common.task_sets import partitioning_set + tasklist.add(*partitioning_set) + + tasklist.add(packages.HostPackages, packages.ImagePackages, - common_packages.ImagePackages, - host.CheckPackages, loopback.Create, - volume_tasks.Attach, - partitioning.PartitionVolume, - partitioning.MapPartitions, filesystem.Format, - filesystem.CreateMountDir, - filesystem.MountRoot, bootstrap.Bootstrap, - filesystem.MountSpecials, - locale.GenerateLocale, - locale.SetTimezone, - apt.DisableDaemonAutostart, - apt.AptSources, - apt.AptUpgrade, boot.ConfigureGrub, filesystem.FStab, common_boot.BlackListModules, @@ -57,40 +53,22 @@ def tasks(tasklist, manifest): initd.InstallInitScripts, cleanup.ClearMOTD, cleanup.CleanTMP, - apt.PurgeUnusedPackages, - apt.AptClean, - apt.EnableDaemonAutostart, - filesystem.UnmountSpecials, - filesystem.UnmountRoot, - partitioning.UnmapPartitions, - volume_tasks.Detach, - filesystem.DeleteMountDir, loopback.MoveImage, workspace.DeleteWorkspace) if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball) - partitions = manifest.volume['partitions'] - import re - for key in ['boot', 'root']: - if key not in partitions: - continue - if re.match('^ext[2-4]$', partitions[key]['filesystem']) is not None: - tasklist.add(filesystem.TuneVolumeFS) - break - for key in ['boot', 'root']: - if key not in partitions: - continue - if partitions[key]['filesystem'] == 'xfs': - tasklist.add(filesystem.AddXFSProgs) - break + from common.task_sets import fs_specific_set + for partition in manifest.volume['partitions']: + if 'filesystem' in partition: + fs_tasks = fs_specific_set.get(partition['filesystem'], []) + tasklist.add(fs_tasks) if 'boot' in manifest.volume['partitions']: - tasklist.add(filesystem.CreateBootMountDir, - filesystem.MountBoot, - filesystem.UnmountBoot) + from common.task_sets import boot_partition_set + tasklist.add(*boot_partition_set) def rollback_tasks(tasklist, tasks_completed, manifest): From ecdc255752799a405b2ae485bcecfaef2b5bb78b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 22 Sep 2013 21:20:09 +0200 Subject: [PATCH 22/63] Simplify FSM framework. Only model relevant states --- base/fs/partitionmaps/abstract.py | 56 +++++++------ base/fs/partitionmaps/gpt.py | 4 +- base/fs/partitionmaps/mbr.py | 5 +- base/fs/partitionmaps/none.py | 23 +++--- base/fs/partitions/abstract.py | 34 +++----- base/fs/partitions/base.py | 17 ++-- base/fs/partitions/gpt.py | 6 +- base/fs/partitions/gpt_swap.py | 2 +- base/fs/partitions/mbr.py | 2 +- base/fs/partitions/mbr_swap.py | 2 +- base/fs/partitions/single.py | 8 +- base/fs/volume.py | 130 +++++------------------------- common/fs/loopbackvolume.py | 44 ++++------ common/fsm.py | 7 -- common/fsm_proxy.py | 51 ++++++++++++ common/tasks/filesystem.py | 15 ++-- common/tasks/partitioning.py | 6 +- providers/ec2/volume.py | 10 ++- providers/virtualbox/volume.py | 6 +- 19 files changed, 183 insertions(+), 245 deletions(-) delete mode 100644 common/fsm.py create mode 100644 common/fsm_proxy.py diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py index 167ddea..5871027 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/base/fs/partitionmaps/abstract.py @@ -1,25 +1,41 @@ from abc import ABCMeta from abc import abstractmethod from common.tools import log_check_call +from common.fsm_proxy import FSMProxy from ..exceptions import PartitionError -class AbstractPartitionMap(object): +class AbstractPartitionMap(FSMProxy): __metaclass__ = ABCMeta - @abstractmethod - def __init__(self, data): - pass + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self): + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(AbstractPartitionMap, self).__init__(cfg) + + def is_blocking(self): + return self.is_state('mapped') def get_total_size(self): return sum(p.size for p in self.partitions) + 1 - @abstractmethod def create(self, volume): + self.fsm.create(volume=volume) + + @abstractmethod + def _before_create(self, event): pass def map(self, volume): + self.fsm.map(volume=volume) + + def _before_map(self, event): + volume = event.volume try: mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) import re @@ -38,35 +54,23 @@ class AbstractPartitionMap(object): self.partitions[p_idx].map(partition_path) for idx, partition in enumerate(self.partitions): - if not partition.state() in ['mapped', 'formatted']: + if not partition.is_state('mapped'): raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) except PartitionError as e: for partition in self.partitions: - if partition.state() in ['mapped', 'formatted']: + if not partition.is_state('mapped'): partition.unmap() log_check_call(['/sbin/kpartx', '-d', volume.device_path]) raise e def unmap(self, volume): + self.fsm.unmap(volume=volume) + + def _before_unmap(self, event): + volume = event.volume for partition in self.partitions: - partition.unmap() + if partition.is_blocking(): + msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) + raise PartitionError(msg) log_check_call(['/sbin/kpartx', '-d', volume.device_path]) - - def format(self): - for partition in self.partitions: - partition.format() - - def mount_root(self, destination): - self.root.mount(destination) - - def unmount_root(self): - self.root.unmount() - - def mount_boot(self): - import os.path - destination = os.path.join(self.root.mount_dir, 'boot') - self.boot.mount(destination) - - def unmount_boot(self): - self.boot.unmount() diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py index 76bbd0a..348fb0c 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/base/fs/partitionmaps/gpt.py @@ -20,7 +20,9 @@ class GPTPartitionMap(AbstractPartitionMap): self.mount_points.append(('none', self.root)) self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) - def create(self, volume): + super(GPTPartitionMap, self).__init__() + + def _before_create(self, volume): log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, '--', 'mklabel', 'gpt']) for partition in self.partitions: diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py index 2e635c4..942f5b4 100644 --- a/base/fs/partitionmaps/mbr.py +++ b/base/fs/partitionmaps/mbr.py @@ -20,7 +20,10 @@ class MBRPartitionMap(AbstractPartitionMap): self.mount_points.append(('none', self.root)) self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) - def create(self, volume): + super(MBRPartitionMap, self).__init__() + + def _before_create(self, event): + volume = event.volume log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, '--', 'mklabel', 'msdos']) for partition in self.partitions: diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index ab58fda..fd80f96 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -1,7 +1,10 @@ from ..partitions.single import SinglePartition +from common.fsm_proxy import FSMProxy -class NoPartitions(object): +class NoPartitions(FSMProxy): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}] def __init__(self, data): root = data['root'] @@ -9,17 +12,17 @@ class NoPartitions(object): self.partitions = [self.root] self.mount_points = [('/', self.root)] + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(NoPartitions, self).__init__(cfg) + + def is_blocking(self): + return self.root.is_blocking() + def get_total_size(self): return self.root.size def create(self, volume): - self.root.create(volume) + self.fsm.create(volume=volume) - def format(self): - self.root.format() - - def mount_root(self, destination): - self.root.mount(destination) - - def unmount_root(self): - self.root.unmount() + def _before_create(self, event): + self.root.create(event.volume) diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py index 15b7f0d..14a0b1a 100644 --- a/base/fs/partitions/abstract.py +++ b/base/fs/partitions/abstract.py @@ -1,10 +1,10 @@ from abc import ABCMeta from abc import abstractmethod from common.tools import log_check_call -from fysom import Fysom +from common.fsm_proxy import FSMProxy -class AbstractPartition(object): +class AbstractPartition(FSMProxy): __metaclass__ = ABCMeta @@ -14,28 +14,16 @@ class AbstractPartition(object): {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, ] - def __init__(self, size, filesystem, callbacks={}): + def __init__(self, size, filesystem): self.size = size self.filesystem = filesystem self.device_path = None - callbacks.update({'onbeforecreate': self._create, - 'onbeforeformat': self._format, - 'onbeforemount': self._mount, - 'onbeforeunmount': self._unmount, - }) + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(AbstractPartition, self).__init__(cfg) - self.fsm = Fysom({'initial': 'nonexistent', - 'events': self.events, - 'callbacks': callbacks}) - from common.fsm import attach_proxy_methods - attach_proxy_methods(self, self.events, self.fsm) - - def state(self): - return self.fsm.current - - def force_state(self, state): - self.fsm.current = state + def is_blocking(self): + return self.is_state('mounted') def get_uuid(self): [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) @@ -45,20 +33,20 @@ class AbstractPartition(object): self.fsm.create(volume=volume) @abstractmethod - def _create(self, e): + def _before_create(self, e): pass - def _format(self, e): + def _before_format(self, e): mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) log_check_call([mkfs, self.device_path]) def mount(self, destination): self.fsm.mount(destination=destination) - def _mount(self, e): + def _before_mount(self, e): log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) self.mount_dir = e.destination - def _unmount(self, e): + def _before_unmount(self, e): log_check_call(['/bin/umount', self.mount_dir]) del self.mount_dir diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index f79ebb3..65d97ee 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -5,20 +5,21 @@ class BasePartition(AbstractPartition): events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, - {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, ] - def __init__(self, size, filesystem, previous, callbacks={}): + def __init__(self, size, filesystem, previous): self.previous = previous - callbacks.update({'onbeforemap': self._map, - 'onbeforeunmap': self._unmap, - }) - super(BasePartition, self).__init__(size, filesystem, callbacks=callbacks) + super(BasePartition, self).__init__(size, filesystem) + + def is_blocking(self): + return self.get_state() in ['mapped', 'mounted', 'formatted'] def get_index(self): if self.previous is None: @@ -35,8 +36,8 @@ class BasePartition(AbstractPartition): def map(self, device_path): self.fsm.map(device_path=device_path) - def _map(self, e): + def _before_map(self, e): self.device_path = e.device_path - def _unmap(self, e): + def _before_unmap(self, e): self.device_path = None diff --git a/base/fs/partitions/gpt.py b/base/fs/partitions/gpt.py index 01987ad..a50f76d 100644 --- a/base/fs/partitions/gpt.py +++ b/base/fs/partitions/gpt.py @@ -4,11 +4,11 @@ from base import BasePartition class GPTPartition(BasePartition): - def __init__(self, size, filesystem, name, previous, callbacks={}): + def __init__(self, size, filesystem, name, previous): self.name = name - super(GPTPartition, self).__init__(size, filesystem, previous, callbacks=callbacks) + super(GPTPartition, self).__init__(size, filesystem, previous) - def _create(self, e): + def _before_create(self, e): start = self.get_start() # {name} only works for gpt, for msdos that becomes the part-type (primary, extended, logical) parted_command = ('mkpart primary {start}MiB {end}MiB' diff --git a/base/fs/partitions/gpt_swap.py b/base/fs/partitions/gpt_swap.py index 030e58b..0217770 100644 --- a/base/fs/partitions/gpt_swap.py +++ b/base/fs/partitions/gpt_swap.py @@ -7,5 +7,5 @@ class GPTSwapPartition(GPTPartition): def __init__(self, size, previous): super(GPTSwapPartition, self).__init__(size, 'swap', 'swap', previous) - def _format(self, e): + def _before_format(self, e): log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/mbr.py b/base/fs/partitions/mbr.py index 9d4fbbd..511cd10 100644 --- a/base/fs/partitions/mbr.py +++ b/base/fs/partitions/mbr.py @@ -4,7 +4,7 @@ from base import BasePartition class MBRPartition(BasePartition): - def _create(self, e): + def _before_create(self, e): start = self.get_start() parted_command = ('mkpart primary {start}MiB {end}MiB' .format(start=str(start), diff --git a/base/fs/partitions/mbr_swap.py b/base/fs/partitions/mbr_swap.py index 0c5c1a1..4e62e4b 100644 --- a/base/fs/partitions/mbr_swap.py +++ b/base/fs/partitions/mbr_swap.py @@ -7,5 +7,5 @@ class MBRSwapPartition(MBRPartition): def __init__(self, size, previous): super(MBRSwapPartition, self).__init__(size, 'swap', previous) - def _format(self, e): + def _before_format(self, e): log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index c43236a..b7905ea 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -3,11 +3,5 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): - events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, - {'name': 'format', 'src': 'created', 'dst': 'formatted'}, - {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, - {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, - ] - - def _create(self, e): + def _before_create(self, e): self.device_path = e.volume.device_path diff --git a/base/fs/volume.py b/base/fs/volume.py index 8d063f1..4a4958e 100644 --- a/base/fs/volume.py +++ b/base/fs/volume.py @@ -1,138 +1,48 @@ from abc import ABCMeta -from fysom import Fysom +from common.fsm_proxy import FSMProxy +from common.tools import log_check_call +from exceptions import VolumeError -class Volume(object): +class Volume(FSMProxy): __metaclass__ = ABCMeta events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, - {'name': 'partition', 'src': 'attached', 'dst': 'partitioned'}, - {'name': 'map', 'src': 'partitioned', 'dst': 'mapped'}, - {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, - {'name': 'mount', 'src': ['formatted', 'mounted'], 'dst': 'mounted'}, - {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, - {'name': 'unmap', 'src': 'formatted', 'dst': 'partitioned_fmt'}, - {'name': 'detach', 'src': 'partitioned_fmt', 'dst': 'detached_fmt'}, - {'name': 'delete', 'src': ['detached', 'detached_prt', 'detached_fmt'], 'dst': 'deleted'}, - - {'name': 'attach', 'src': 'detached_fmt', 'dst': 'partitioned_fmt'}, - {'name': 'map', 'src': 'partitioned_fmt', 'dst': 'formatted'}, - - {'name': 'attach', 'src': 'detached_prt', 'dst': 'partitioned'}, - {'name': 'detach', 'src': 'partitioned', 'dst': 'detached_prt'}, - + {'name': 'mount_specials', 'src': 'attached', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'attached'}, {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, - {'name': 'unmap', 'src': 'mapped', 'dst': 'partitioned'}, + {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, ] - mount_events = [{'name': 'mount_root', 'src': 'unmounted', 'dst': 'root_mounted'}, - {'name': 'mount_boot', 'src': 'root_mounted', 'dst': 'boot_mounted'}, - {'name': 'mount_specials', 'src': 'boot_mounted', 'dst': 'specials_mounted'}, - {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'boot_mounted'}, - {'name': 'unmount_boot', 'src': 'boot_mounted', 'dst': 'root_mounted'}, - {'name': 'unmount_root', 'src': 'root_mounted', 'dst': 'unmounted'}, - - {'name': 'mount_specials', 'src': 'root_mounted', 'dst': 'specials_mounted_no_boot'}, - {'name': 'mount_boot', 'src': 'specials_mounted_no_boot', 'dst': 'specials_mounted'}, - {'name': 'unmount_specials', 'src': 'specials_mounted_no_boot', 'dst': 'root_mounted'}, - {'name': 'unmount_boot', 'src': 'specials_mounted', 'dst': 'specials_mounted_no_boot'}, - ] - - def __init__(self, partition_map, callbacks={}): + def __init__(self, partition_map): self.device_path = None self.partition_map = partition_map self.size = self.partition_map.get_total_size() - callbacks.update({'onbeforepartition': self._partition, - 'onbeforemap': self._map, - 'onbeforeunmap': self._unmap, - 'onbeforeformat': self._format, - 'onbeforeunmount': self._unmount, - }) + callbacks = {'onbeforedetach': self._check_blocking} + from partitionmaps.none import NoPartitions + if isinstance(self.partition_map, NoPartitions): + callbacks['onafterattach'] = lambda e: self.partition_map.create(self) - mount_callbacks = {'onbeforemount_root': self._mount_root, - 'onbeforemount_boot': self._mount_boot, - 'onbeforemount_specials': self._mount_specials, - 'onbeforeunmount_root': self._unmount_root, - 'onbeforeunmount_boot': self._unmount_boot, - 'onbeforeunmount_specials': self._unmount_specials, - } - self.fsm = Fysom({'initial': 'nonexistent', - 'events': self.events, - 'callbacks': callbacks}) + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks} + super(Volume, self).__init__(cfg) - self.mount_fsm = Fysom({'initial': 'unmounted', - 'events': self.mount_events, - 'callbacks': mount_callbacks}) - - from common.fsm import attach_proxy_methods - attach_proxy_methods(self, self.events, self.fsm) - attach_proxy_methods(self, self.mount_events, self.mount_fsm) - - def state(self): - return self.fsm.current - - def force_state(self, state): - self.fsm.current = state - - def _partition(self, e): - self.partition_map.create(self) - - def _map(self, e): - self.partition_map.map(self) - - def _unmap(self, e): - self.partition_map.unmap(self) - - def _format(self, e): - self.partition_map.format() - - def _unmount(self, e): - if self.mount_fsm.current is 'specials_mounted': - self.unmount_specials() - if self.mount_fsm.current is 'specials_mounted_no_boot': - self.unmount_specials() - if self.mount_fsm.current is 'boot_mounted': - self.unmount_boot() - if self.mount_fsm.current is 'root_mounted': - self.unmount_root() - - def mount_root(self, destination): - self.mount_fsm.mount_root(destination=destination) - - def unmount_root(self): - self.mount_fsm.unmount_root() - self.fsm.unmount() - - def _mount_root(self, e): - self.mount() - self.partition_map.mount_root(e.destination) - - def _unmount_root(self, e): - self.partition_map.unmount_root() - - def _mount_boot(self, e): - self.mount() - self.partition_map.mount_boot() - - def _unmount_boot(self, e): - self.partition_map.unmount_boot() - - def _mount_specials(self, e): - self.mount() - from common.tools import log_check_call + def _before_mount_specials(self, e): root = self.partition_map.root.mount_dir log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) - def _unmount_specials(self, e): - from common.tools import log_check_call + def _before_unmount_specials(self, e): root = self.partition_map.root.mount_dir log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) + + def _check_blocking(self, e): + if self.partition_map.is_blocking(): + raise VolumeError('The partitionmap prevents the detach procedure') diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index 0dcc35b..e670a5e 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -9,45 +9,31 @@ from base.fs.exceptions import VolumeError class LoopbackVolume(Volume): - link_dm_events = [{'name': 'link_dm_node', 'src': 'partitioned', 'dst': 'linked'}, - {'name': 'map', 'src': 'linked', 'dst': 'mapped_lnk'}, - {'name': 'format', 'src': 'mapped_lnk', 'dst': 'formatted_lnk'}, - {'name': 'mount', 'src': ['formatted_lnk', 'mounted_lnk'], 'dst': 'mounted_lnk'}, - {'name': 'unmount', 'src': 'mounted_lnk', 'dst': 'formatted_lnk'}, - {'name': 'unmap', 'src': 'formatted_lnk', 'dst': 'partitioned_fmt_lnk'}, - {'name': 'unlink_dm_node', 'src': 'partitioned_fmt_lnk', 'dst': 'partitioned_fmt'}, - - {'name': 'link_dm_node', 'src': 'partitioned_fmt', 'dst': 'partitioned_fmt_lnk'}, - {'name': 'map', 'src': 'partitioned_fmt_lnk', 'dst': 'formatted_lnk'}, - {'name': 'unmap', 'src': 'mapped_lnk', 'dst': 'linked'}, - {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, - ] + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'}, + {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'attached'}, + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, + ] extension = 'raw' - def __init__(self, partition_map, callbacks={}): - callbacks.update({'onbeforecreate': self._create, - 'onbeforeattach': self._attach, - 'onbeforedetach': self._detach, - 'onbeforedelete': self._delete, - 'onbeforelink_dm_node': self._link_dm_node, - 'onbeforeunlink_dm_node': self._unlink_dm_node, - }) - self.events.extend(self.link_dm_events) - super(LoopbackVolume, self).__init__(partition_map, callbacks=callbacks) + def __init__(self, partition_map): + super(LoopbackVolume, self).__init__(partition_map) def create(self, image_path): self.fsm.create(image_path=image_path) - def _create(self, e): + def _before_create(self, e): self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) - def _attach(self, e): + def _before_attach(self, e): [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) self.device_path = self.loop_device_path - def _link_dm_node(self, e): + def _before_link_dm_node(self, e): import os.path from . import get_partitions proc_partitions = get_partitions() @@ -77,18 +63,18 @@ class LoopbackVolume(Volume): log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) self.device_path = self.dm_node_path - def _unlink_dm_node(self, e): + def _before_unlink_dm_node(self, e): log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) del self.dm_node_name del self.dm_node_path self.device_path = self.loop_device_path - def _detach(self, e): + def _before_detach(self, e): log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) del self.loop_device_path del self.device_path - def _delete(self, e): + def _before_delete(self, e): from os import remove remove(self.image_path) del self.image_path diff --git a/common/fsm.py b/common/fsm.py deleted file mode 100644 index 44286e0..0000000 --- a/common/fsm.py +++ /dev/null @@ -1,7 +0,0 @@ - - -def attach_proxy_methods(obj, events, fsm): - methods = set([e['name'] for e in events]) - for event in methods: - if not hasattr(obj, event): - setattr(obj, event, lambda e=event: getattr(fsm, e)()) diff --git a/common/fsm_proxy.py b/common/fsm_proxy.py new file mode 100644 index 0000000..6235e10 --- /dev/null +++ b/common/fsm_proxy.py @@ -0,0 +1,51 @@ + + +class FSMProxy(object): + + def __init__(self, cfg): + from fysom import Fysom + events = set([event['name'] for event in cfg['events']]) + cfg['callbacks'] = self.collect_event_listeners(events, cfg['callbacks']) + self.fsm = Fysom(cfg) + self.attach_proxy_methods(self.fsm, events) + + def is_state(self, event): + return self.fsm.isstate(event) + + def set_state(self, event): + self.fsm.current = event + + def get_state(self): + return self.fsm.current + + def collect_event_listeners(self, events, callbacks): + callbacks = callbacks.copy() + callback_names = [] + for event in events: + callback_names.append(('_before_' + event, 'onbefore' + event)) + callback_names.append(('_after_' + event, 'onafter' + event)) + for fn_name, listener in callback_names: + fn = getattr(self, fn_name, None) + if callable(fn): + if listener in callbacks: + old_fn = callbacks[listener] + + def wrapper(e, old_fn=old_fn, fn=fn): + old_fn(e) + fn(e) + callbacks[listener] = wrapper + else: + callbacks[listener] = fn + return callbacks + + def attach_proxy_methods(self, fsm, events): + def make_proxy(fsm, event): + fn = getattr(fsm, event) + + def proxy(): + fn() + return proxy + + for event in events: + if not hasattr(self, event): + setattr(self, event, make_proxy(fsm, event)) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 8af2c49..08dd86a 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -10,7 +10,8 @@ class Format(Task): phase = phases.volume_preparation def run(self, info): - info.volume.format() + for partition in info.volume.partition_map.partitions: + partition.format() class TuneVolumeFS(Task): @@ -51,7 +52,7 @@ class MountRoot(Task): after = [CreateMountDir] def run(self, info): - info.volume.mount_root(info.root) + info.volume.partition_map.root.mount(info.root) class MountBoot(Task): @@ -60,7 +61,7 @@ class MountBoot(Task): after = [MountRoot] def run(self, info): - info.volume.mount_boot() + info.volume.partition_map.boot.mount(info.boot_dir) class CreateBootMountDir(Task): @@ -71,8 +72,8 @@ class CreateBootMountDir(Task): def run(self, info): import os - boot_dir = os.path.join(info.root, 'boot') - os.makedirs(boot_dir) + info.boot_dir = os.path.join(info.root, 'boot') + os.makedirs(info.boot_dir) class MountSpecials(Task): @@ -90,7 +91,7 @@ class UnmountRoot(Task): before = [volume.Detach] def run(self, info): - info.volume.unmount_root() + info.volume.partition_map.root.unmount() class UnmountBoot(Task): @@ -99,7 +100,7 @@ class UnmountBoot(Task): before = [UnmountRoot] def run(self, info): - info.volume.unmount_boot() + info.volume.partition_map.boot.unmount() class UnmountSpecials(Task): diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py index 181bb63..eeca38d 100644 --- a/common/tasks/partitioning.py +++ b/common/tasks/partitioning.py @@ -9,7 +9,7 @@ class PartitionVolume(Task): phase = phases.volume_preparation def run(self, info): - info.volume.partition() + info.volume.partition_map.create() class MapPartitions(Task): @@ -19,7 +19,7 @@ class MapPartitions(Task): after = [PartitionVolume] def run(self, info): - info.volume.map() + info.volume.partition_map.map() class UnmapPartitions(Task): @@ -29,4 +29,4 @@ class UnmapPartitions(Task): after = [filesystem.UnmountRoot] def run(self, info): - info.volume.unmap() + info.volume.partition_map.unmap() diff --git a/providers/ec2/volume.py b/providers/ec2/volume.py index bd1983f..16e4e96 100644 --- a/providers/ec2/volume.py +++ b/providers/ec2/volume.py @@ -19,7 +19,10 @@ class EBSVolume(Volume): self.created = True def attach(self, instance_id): - super(EBSVolume, self).attach(self) + self.fsm.attach(instance_id=instance_id) + + def _before_attach(self, e): + instance_id = e.instance_id import os.path import string for letter in string.ascii_lowercase: @@ -37,14 +40,13 @@ class EBSVolume(Volume): time.sleep(2) self.volume.update() - def detach(self): - super(EBSVolume, self).detach(self) + def _before_detach(self, e): self.volume.detach() while self.volume.attachment_state() is not None: time.sleep(2) self.volume.update() - def delete(self): + def _before_delete(self, e): super(EBSVolume, self).delete(self) self.volume.delete() diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index b6d6b0d..6c3d10c 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -8,11 +8,11 @@ class VirtualBoxVolume(LoopbackVolume): extension = 'vdi' - def _create(self, e): + def _before_create(self, e): self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) - def _attach(self, e): + def _before_attach(self, e): num_partitions = len(self.partition_map.partitions) if not self._module_loaded('nbd'): msg = ('The kernel module `nbd\' must be loaded ' @@ -33,7 +33,7 @@ class VirtualBoxVolume(LoopbackVolume): log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path]) self.device_path = self.loop_device_path - def _detach(self, e): + def _before_detach(self, e): log_check_call(['/usr/bin/qemu-nbd', '--disconnect', self.loop_device_path]) del self.loop_device_path del self.device_path From 3c323103462d321c0f5c10fd342fb16f1da28fee Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 25 Sep 2013 23:27:31 +0200 Subject: [PATCH 23/63] Remove mountpoints from partitions --- base/fs/partitionmaps/gpt.py | 11 ++++------- base/fs/partitionmaps/mbr.py | 4 ---- base/fs/partitionmaps/none.py | 1 - common/tasks/filesystem.py | 9 ++++++++- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py index 348fb0c..b884bd9 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/base/fs/partitionmaps/gpt.py @@ -9,20 +9,17 @@ class GPTPartitionMap(AbstractPartitionMap): def __init__(self, data): self.boot = None self.swap = None - self.mount_points = [] if 'boot' in data: self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None) - self.mount_points.append(('/boot', self.boot)) self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', self.boot) - self.mount_points.append(('/', self.root)) if 'swap' in data: self.swap = GPTSwapPartition(data['swap']['size'], self.root) - self.mount_points.append(('none', self.root)) self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) super(GPTPartitionMap, self).__init__() - def _before_create(self, volume): + def _before_create(self, event): + volume = event.volume log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, '--', 'mklabel', 'gpt']) for partition in self.partitions: @@ -31,7 +28,7 @@ class GPTPartitionMap(AbstractPartitionMap): boot_idx = self.root.get_index() if self.boot is not None: boot_idx = self.boot.get_index() - log_check_call(['/sbin/parted', '--script', volume.device_path, - '--', 'set ' + str(boot_idx) + ' bios_grub on']) log_check_call(['/sbin/parted', '--script', volume.device_path, '--', 'set ' + str(boot_idx) + ' boot on']) + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' bios_grub on']) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py index 942f5b4..0bc4cfa 100644 --- a/base/fs/partitionmaps/mbr.py +++ b/base/fs/partitionmaps/mbr.py @@ -9,15 +9,11 @@ class MBRPartitionMap(AbstractPartitionMap): def __init__(self, data): self.boot = None self.swap = None - self.mount_points = [] if 'boot' in data: self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None) - self.mount_points.append(('/boot', self.boot)) self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], self.boot) - self.mount_points.append(('/', self.root)) if 'swap' in data: self.swap = MBRSwapPartition(data['swap']['size'], self.root) - self.mount_points.append(('none', self.root)) self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) super(MBRPartitionMap, self).__init__() diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index fd80f96..cc9a8b0 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -10,7 +10,6 @@ class NoPartitions(FSMProxy): root = data['root'] self.root = SinglePartition(root['size'], root['filesystem']) self.partitions = [self.root] - self.mount_points = [('/', self.root)] cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} super(NoPartitions, self).__init__(cfg) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 08dd86a..2fee046 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -132,8 +132,15 @@ class FStab(Task): # device = '/dev/sda' # if info.manifest.virtualization == 'pvm': # device = '/dev/xvda' + p_map = info.volume.partition_map + mount_points = [('/root', p_map.root)] + if hasattr(p_map, 'boot'): + mount_points.append(('/boot', p_map.boot)) + if hasattr(p_map, 'swap'): + mount_points.append(('none', p_map.swap)) + fstab_lines = [] - for mount_point, partition in info.volume.partition_map.mount_points: + for mount_point, partition in mount_points: mount_opts = ['defaults'] if partition.filesystem in ['ext2', 'ext3', 'ext4']: mount_opts.append('barrier=0') From e740a3b4d89dd74a55ec8c4dd7ebb1bba9acc247 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 25 Sep 2013 23:43:02 +0200 Subject: [PATCH 24/63] Fix grub install --- providers/virtualbox/tasks/boot.py | 45 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 516b6a6..f800f06 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -16,18 +16,29 @@ class ConfigureGrub(Task): boot_dir = os.path.join(info.root, 'boot') grub_dir = os.path.join(boot_dir, 'grub') - # if type(info.volume) is LoopbackVolume: - if isinstance(info.volume, LoopbackVolume): + from partitionmaps.none import NoPartitions + + def remount(volume, fn): # GRUB cannot deal with installing to loopback devices # so we fake a real harddisk with dmsetup. # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ - info.volume.unmount() - info.volume.unmap() - info.volume.link_dm_node() - info.volume.map() - info.volume.mount_root(info.root) - info.volume.mount_boot() - info.volume.mount_specials() + volume.unmount_specials() + p_map = volume.partition_map + if hasattr(p_map, 'boot'): + boot_dir = p_map.boot.mount_dir + p_map.boot.unmount() + p_map.root.unmount() + if isinstance(self.partition_map, NoPartitions): + p_map.unmap() + fn() + p_map.map() + p_map.root.mount(info.root) + if hasattr(p_map, 'boot'): + p_map.boot.mount(boot_dir) + volume.mount_specials() + + if isinstance(info.volume, LoopbackVolume): + remount(info.volume, info.volume.link_dm_node) try: [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) device_map_path = os.path.join(grub_dir, 'device.map') @@ -46,20 +57,8 @@ class ConfigureGrub(Task): log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) except Exception as e: if isinstance(info.volume, LoopbackVolume): - info.volume.unmount() - info.volume.unmap() - info.volume.unlink_dm_node() - info.volume.map() - info.volume.mount_root(info.root) - info.volume.mount_boot() - info.volume.mount_specials() + remount(info.volume, info.volume.unlink_dm_node) raise e if isinstance(info.volume, LoopbackVolume): - info.volume.unmount() - info.volume.unmap() - info.volume.unlink_dm_node() - info.volume.map() - info.volume.mount_root(info.root) - info.volume.mount_boot() - info.volume.mount_specials() + remount(info.volume, info.volume.unlink_dm_node) From 3993ff23a5657d5fbac1f02be4ef79e916b56369 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 25 Sep 2013 23:46:04 +0200 Subject: [PATCH 25/63] Less verbose checking of installed packages --- common/tasks/host.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/tasks/host.py b/common/tasks/host.py index aa76df0..e086fa3 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -14,8 +14,7 @@ class CheckPackages(Task): from subprocess import CalledProcessError for package in info.host_packages: try: - # Use "dpkg-query -W -f='${Status} ${Version}\n' package" instead - log_check_call(['/usr/bin/dpkg', '--status', package]) + log_check_call(['/usr/bin/dpkg-query', '-W', package]) except CalledProcessError: msg = "The package ``{0}\'\' is not installed".format(package) raise TaskError(msg) From fc0471f7801590d97256c557ddf210299a9de5df Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 25 Sep 2013 23:58:05 +0200 Subject: [PATCH 26/63] Remove init from LOVol, add mount_specials --- common/fs/loopbackvolume.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index e670a5e..d33d269 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -12,16 +12,18 @@ class LoopbackVolume(Volume): events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'}, + {'name': 'mount_specials', 'src': 'linked', 'dst': 'lnk_specials_mounted'}, + {'name': 'unmount_specials', 'src': 'lnk_specials_mounted', 'dst': 'linked'}, {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'attached'}, {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, + + {'name': 'mount_specials', 'src': 'linked', 'dst': 'specials_mounted'}, + {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'linked'}, ] extension = 'raw' - def __init__(self, partition_map): - super(LoopbackVolume, self).__init__(partition_map) - def create(self, image_path): self.fsm.create(image_path=image_path) From 8891a18e4b945c807e17e48d62a3a977fbae2d90 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Thu, 26 Sep 2013 22:43:36 +0200 Subject: [PATCH 27/63] Remove mount_specials from FSM It makes modelling the different states more complicated --- base/fs/volume.py | 18 ++++++++++++++---- common/fs/loopbackvolume.py | 8 +++----- plugins/prebootstrapped/tasks.py | 27 ++++++++++++++++----------- providers/virtualbox/tasks/boot.py | 7 +++++-- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/base/fs/volume.py b/base/fs/volume.py index 4a4958e..7df9778 100644 --- a/base/fs/volume.py +++ b/base/fs/volume.py @@ -10,14 +10,13 @@ class Volume(FSMProxy): events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, - {'name': 'mount_specials', 'src': 'attached', 'dst': 'specials_mounted'}, - {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'attached'}, {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, ] def __init__(self, partition_map): self.device_path = None + self.specials_mounted = False self.partition_map = partition_map self.size = self.partition_map.get_total_size() @@ -29,20 +28,31 @@ class Volume(FSMProxy): cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks} super(Volume, self).__init__(cfg) - def _before_mount_specials(self, e): + def can_mount_specials(self): + return self.is_state('attached') + + def mount_specials(self): + if self.specials_mounted: + raise VolumeError('The special devices are already mounted') root = self.partition_map.root.mount_dir log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + self.specials_mounted = True - def _before_unmount_specials(self, e): + def unmount_specials(self): + if not self.specials_mounted: + raise VolumeError('The special devices are not mounted') root = self.partition_map.root.mount_dir log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) + self.specials_mounted = False def _check_blocking(self, e): if self.partition_map.is_blocking(): raise VolumeError('The partitionmap prevents the detach procedure') + if self.specials_mounted: + raise VolumeError('The special devices are mounted and prevent the detaching procedure') diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index d33d269..d2918ff 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -12,18 +12,16 @@ class LoopbackVolume(Volume): events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'}, - {'name': 'mount_specials', 'src': 'linked', 'dst': 'lnk_specials_mounted'}, - {'name': 'unmount_specials', 'src': 'lnk_specials_mounted', 'dst': 'linked'}, {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'attached'}, {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, - - {'name': 'mount_specials', 'src': 'linked', 'dst': 'specials_mounted'}, - {'name': 'unmount_specials', 'src': 'specials_mounted', 'dst': 'linked'}, ] extension = 'raw' + def can_mount_specials(self): + return self.is_state('attached') or self.is_state('linked') + def create(self, image_path): self.fsm.create(image_path=image_path) diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 3899336..38d5275 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -34,12 +34,7 @@ class CreateFromSnapshot(Task): time.sleep(5) info.volume.update() - info.volume.force_state('detached_fmt') - partitions_state = 'formatted' - if 'partitions' in info.manifest.volume: - partitions_state = 'unmapped_fmt' - for partition in info.volume.partition_map.partitions: - partition.force_state(partitions_state) + set_fs_states(info.volume) class CopyImage(Task): @@ -68,10 +63,20 @@ class CreateFromImage(Task): info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] copyfile(loopback_backup_path, info.volume.image_path) + set_fs_states(info.volume) - info.volume.force_state('detached_fmt') - partitions_state = 'formatted' - if 'partitions' in info.manifest.volume: + +def set_fs_states(volume): + volume.set_state('detached') + + p_map = volume.partition_map + partitions_state = 'attached' + from base.fs.partitionmaps.none import NoPartitions + if isinstance(p_map, NoPartitions): + p_map.set_state('created') + partitions_state = 'formatted' + else: + p_map.set_state('unmapped') partitions_state = 'unmapped_fmt' - for partition in info.volume.partition_map.partitions: - partition.force_state(partitions_state) + for partition in p_map.partitions: + partition.set_state(partitions_state) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index f800f06..12694c0 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -16,7 +16,7 @@ class ConfigureGrub(Task): boot_dir = os.path.join(info.root, 'boot') grub_dir = os.path.join(boot_dir, 'grub') - from partitionmaps.none import NoPartitions + from base.fs.partitionmaps.none import NoPartitions def remount(volume, fn): # GRUB cannot deal with installing to loopback devices @@ -28,10 +28,13 @@ class ConfigureGrub(Task): boot_dir = p_map.boot.mount_dir p_map.boot.unmount() p_map.root.unmount() - if isinstance(self.partition_map, NoPartitions): + if not isinstance(p_map, NoPartitions): p_map.unmap() fn() p_map.map() + else: + fn() + p_map.root.device_path = volume.device_path p_map.root.mount(info.root) if hasattr(p_map, 'boot'): p_map.boot.mount(boot_dir) From ba1842ca2b4fe2ef4050c7f4a5461288350bd8d6 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 4 Oct 2013 20:31:51 +0200 Subject: [PATCH 28/63] Move nbd modprobe check into separate function Only require max_part param for partition manifests --- providers/virtualbox/volume.py | 41 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index 6c3d10c..2744233 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -12,23 +12,32 @@ class VirtualBoxVolume(LoopbackVolume): self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + def _check_nbd_module(self): + from base.fs.partitionmaps.none import NoPartitions + if isinstance(self.partition_map, NoPartitions): + if not self._module_loaded('nbd'): + raise VolumeError('The kernel module `nbd\' must be loaded ' + '(`modprobe nbd\') to attach .vdi images') + else: + num_partitions = len(self.partition_map.partitions) + if not self._module_loaded('nbd'): + msg = ('The kernel module `nbd\' must be loaded ' + '(`modprobe nbd max_part={num_partitions}\') to attach .vdi images' + .format(num_partitions=num_partitions)) + raise VolumeError(msg) + nbd_max_part = int(self._module_param('nbd', 'max_part')) + if nbd_max_part < num_partitions: + # Found here: http://bethesignal.org/blog/2011/01/05/how-to-mount-virtualbox-vdi-image/ + msg = ('The kernel module `nbd\' was loaded with the max_part ' + 'parameter set to {max_part}, which is below ' + 'the amount of partitions for this volume ({num_partitions}). ' + 'Reload the nbd kernel module with max_part set to at least {num_partitions} ' + '(`rmmod nbd; modprobe nbd max_part={num_partitions}\').' + .format(max_part=nbd_max_part, num_partitions=num_partitions)) + raise VolumeError(msg) + def _before_attach(self, e): - num_partitions = len(self.partition_map.partitions) - if not self._module_loaded('nbd'): - msg = ('The kernel module `nbd\' must be loaded ' - '(`modprobe nbd max_part={num_partitions}\') to attach .vdi images' - .format(num_partitions=num_partitions)) - raise VolumeError(msg) - nbd_max_part = int(self._module_param('nbd', 'max_part')) - if nbd_max_part < num_partitions: - # Found here: http://bethesignal.org/blog/2011/01/05/how-to-mount-virtualbox-vdi-image/ - msg = ('The kernel module `nbd\' was loaded with the max_part ' - 'parameter set to {max_part}, which is below ' - 'the amount of partitions for this volume ({num_partitions}). ' - 'Reload the nbd kernel module with max_part set to at least {num_partitions} ' - '(`rmmod nbd; modprobe nbd max_part={num_partitions}\').' - .format(max_part=nbd_max_part, num_partitions=num_partitions)) - raise VolumeError(msg) + self._check_nbd_module() self.loop_device_path = self._find_free_nbd_device() log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path]) self.device_path = self.loop_device_path From 387aa62750d9478ae2c03c1fb93bbf71bc2bd91a Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 4 Oct 2013 21:12:48 +0200 Subject: [PATCH 29/63] Remove FSM from NoPartitions p-map Don't set the volume device_path by calling 'create' --- base/fs/partitionmaps/none.py | 14 +------------- base/fs/partitions/single.py | 4 +++- base/fs/volume.py | 10 ++++++++-- plugins/prebootstrapped/tasks.py | 1 - 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index cc9a8b0..1b2cfd8 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -1,27 +1,15 @@ from ..partitions.single import SinglePartition -from common.fsm_proxy import FSMProxy -class NoPartitions(FSMProxy): - - events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}] +class NoPartitions(object): def __init__(self, data): root = data['root'] self.root = SinglePartition(root['size'], root['filesystem']) self.partitions = [self.root] - cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} - super(NoPartitions, self).__init__(cfg) - def is_blocking(self): return self.root.is_blocking() def get_total_size(self): return self.root.size - - def create(self, volume): - self.fsm.create(volume=volume) - - def _before_create(self, event): - self.root.create(event.volume) diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index b7905ea..5ac3ec5 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -3,5 +3,7 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): + initial_state = 'created' + def _before_create(self, e): - self.device_path = e.volume.device_path + pass diff --git a/base/fs/volume.py b/base/fs/volume.py index 7df9778..209fa6d 100644 --- a/base/fs/volume.py +++ b/base/fs/volume.py @@ -2,6 +2,7 @@ from abc import ABCMeta from common.fsm_proxy import FSMProxy from common.tools import log_check_call from exceptions import VolumeError +from partitionmaps.none import NoPartitions class Volume(FSMProxy): @@ -21,13 +22,18 @@ class Volume(FSMProxy): self.size = self.partition_map.get_total_size() callbacks = {'onbeforedetach': self._check_blocking} - from partitionmaps.none import NoPartitions if isinstance(self.partition_map, NoPartitions): - callbacks['onafterattach'] = lambda e: self.partition_map.create(self) + def set_dev_path(e): + self.partition_map.root.device_path = self.device_path + callbacks['onafterattach'] = set_dev_path cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks} super(Volume, self).__init__(cfg) + def _after_create(self, e): + if isinstance(self.partition_map, NoPartitions): + self.partition_map.root.create() + def can_mount_specials(self): return self.is_state('attached') diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 38d5275..1e5f184 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -73,7 +73,6 @@ def set_fs_states(volume): partitions_state = 'attached' from base.fs.partitionmaps.none import NoPartitions if isinstance(p_map, NoPartitions): - p_map.set_state('created') partitions_state = 'formatted' else: p_map.set_state('unmapped') From 977b1f290f9eefcff65129429e428b43a68ae6e8 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 4 Oct 2013 21:13:41 +0200 Subject: [PATCH 30/63] Remove duplicate task addition --- providers/virtualbox/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 11b9b2b..098adff 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -54,8 +54,7 @@ def tasks(tasklist, manifest): cleanup.ClearMOTD, cleanup.CleanTMP, - loopback.MoveImage, - workspace.DeleteWorkspace) + loopback.MoveImage) if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball) From c78866f103c554e5e619ecca3103ada8bd88ef29 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 4 Oct 2013 21:45:19 +0200 Subject: [PATCH 31/63] Take partitionmap into consideration in device.map Simplify boot task a little --- providers/virtualbox/tasks/boot.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 12694c0..4577e92 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -17,13 +17,14 @@ class ConfigureGrub(Task): grub_dir = os.path.join(boot_dir, 'grub') from base.fs.partitionmaps.none import NoPartitions + from base.fs.partitionmaps.gpt import GPTPartitionMap + p_map = info.volume.partition_map - def remount(volume, fn): + def remount(fn): # GRUB cannot deal with installing to loopback devices # so we fake a real harddisk with dmsetup. # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ - volume.unmount_specials() - p_map = volume.partition_map + info.volume.unmount_specials() if hasattr(p_map, 'boot'): boot_dir = p_map.boot.mount_dir p_map.boot.unmount() @@ -34,22 +35,27 @@ class ConfigureGrub(Task): p_map.map() else: fn() - p_map.root.device_path = volume.device_path + p_map.root.device_path = info.volume.device_path p_map.root.mount(info.root) if hasattr(p_map, 'boot'): p_map.boot.mount(boot_dir) - volume.mount_specials() + info.volume.mount_specials() if isinstance(info.volume, LoopbackVolume): - remount(info.volume, info.volume.link_dm_node) + remount(info.volume.link_dm_node) try: [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) device_map_path = os.path.join(grub_dir, 'device.map') + partition_prefix = 'msdos' + if isinstance(p_map, GPTPartitionMap): + partition_prefix = 'gpt' with open(device_map_path, 'w') as device_map: device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) - for idx, partition in enumerate(info.volume.partition_map.partitions): - [partition_path] = log_check_call(['readlink', '-f', partition.device_path]) - device_map.write('(hd0,gpt{idx}) {device_path}\n'.format(device_path=partition_path, idx=idx+1)) + if not isinstance(p_map, NoPartitions): + for idx, partition in enumerate(info.volume.partition_map.partitions): + [partition_path] = log_check_call(['readlink', '-f', partition.device_path]) + device_map.write('(hd0,{prefix}{idx}) {device_path}\n' + .format(device_path=partition_path, prefix=partition_prefix, idx=idx+1)) # Install grub log_check_call(['/usr/sbin/chroot', info.root, @@ -60,7 +66,7 @@ class ConfigureGrub(Task): log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) except Exception as e: if isinstance(info.volume, LoopbackVolume): - remount(info.volume, info.volume.unlink_dm_node) + remount(info.volume.unlink_dm_node) raise e if isinstance(info.volume, LoopbackVolume): From 6ddd8809a33c8922b17372adc1c5e95098e76ba6 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 5 Oct 2013 20:41:05 +0200 Subject: [PATCH 32/63] Partitioning fixes --- base/fs/partitionmaps/abstract.py | 2 +- base/fs/partitionmaps/gpt.py | 9 +++++---- base/fs/partitionmaps/mbr.py | 9 +++++---- base/fs/partitionmaps/none.py | 2 +- base/fs/partitions/abstract.py | 3 --- base/fs/partitions/base.py | 3 --- common/tasks/partitioning.py | 6 +++--- providers/virtualbox/tasks/boot.py | 6 +++--- 8 files changed, 18 insertions(+), 22 deletions(-) diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py index 5871027..a7ea15b 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/base/fs/partitionmaps/abstract.py @@ -70,7 +70,7 @@ class AbstractPartitionMap(FSMProxy): def _before_unmap(self, event): volume = event.volume for partition in self.partitions: - if partition.is_blocking(): + if partition.is_state('mounted'): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) log_check_call(['/sbin/kpartx', '-d', volume.device_path]) diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py index b884bd9..0b2e8bf 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/base/fs/partitionmaps/gpt.py @@ -7,14 +7,15 @@ from common.tools import log_check_call class GPTPartitionMap(AbstractPartitionMap): def __init__(self, data): - self.boot = None - self.swap = None + self.partitions = [] if 'boot' in data: self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None) + self.partitions.append(self.boot) self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', self.boot) + self.partitions.append(self.root) if 'swap' in data: self.swap = GPTSwapPartition(data['swap']['size'], self.root) - self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + self.partitions.append(self.swap) super(GPTPartitionMap, self).__init__() @@ -26,7 +27,7 @@ class GPTPartitionMap(AbstractPartitionMap): partition.create(volume) boot_idx = self.root.get_index() - if self.boot is not None: + if hasattr(self, 'boot'): boot_idx = self.boot.get_index() log_check_call(['/sbin/parted', '--script', volume.device_path, '--', 'set ' + str(boot_idx) + ' boot on']) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py index 0bc4cfa..0d4b50f 100644 --- a/base/fs/partitionmaps/mbr.py +++ b/base/fs/partitionmaps/mbr.py @@ -7,14 +7,15 @@ from common.tools import log_check_call class MBRPartitionMap(AbstractPartitionMap): def __init__(self, data): - self.boot = None - self.swap = None + self.partitions = [] if 'boot' in data: self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None) + self.partitions.append(self.boot) self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], self.boot) + self.partitions.append(self.root) if 'swap' in data: self.swap = MBRSwapPartition(data['swap']['size'], self.root) - self.partitions = filter(lambda p: p is not None, [self.boot, self.root, self.swap]) + self.partitions.append(self.swap) super(MBRPartitionMap, self).__init__() @@ -26,7 +27,7 @@ class MBRPartitionMap(AbstractPartitionMap): partition.create(volume) boot_idx = self.root.get_index() - if self.boot is not None: + if hasattr(self, 'boot'): boot_idx = self.boot.get_index() log_check_call(['/sbin/parted', '--script', volume.device_path, '--', 'set ' + str(boot_idx) + ' boot on']) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index 1b2cfd8..cbca282 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -9,7 +9,7 @@ class NoPartitions(object): self.partitions = [self.root] def is_blocking(self): - return self.root.is_blocking() + return self.root.is_state('mounted') def get_total_size(self): return self.root.size diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py index 14a0b1a..9f80618 100644 --- a/base/fs/partitions/abstract.py +++ b/base/fs/partitions/abstract.py @@ -22,9 +22,6 @@ class AbstractPartition(FSMProxy): cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} super(AbstractPartition, self).__init__(cfg) - def is_blocking(self): - return self.is_state('mounted') - def get_uuid(self): [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) return uuid diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index 65d97ee..8986285 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -18,9 +18,6 @@ class BasePartition(AbstractPartition): self.previous = previous super(BasePartition, self).__init__(size, filesystem) - def is_blocking(self): - return self.get_state() in ['mapped', 'mounted', 'formatted'] - def get_index(self): if self.previous is None: return 1 diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py index eeca38d..a0ec17b 100644 --- a/common/tasks/partitioning.py +++ b/common/tasks/partitioning.py @@ -9,7 +9,7 @@ class PartitionVolume(Task): phase = phases.volume_preparation def run(self, info): - info.volume.partition_map.create() + info.volume.partition_map.create(info.volume) class MapPartitions(Task): @@ -19,7 +19,7 @@ class MapPartitions(Task): after = [PartitionVolume] def run(self, info): - info.volume.partition_map.map() + info.volume.partition_map.map(info.volume) class UnmapPartitions(Task): @@ -29,4 +29,4 @@ class UnmapPartitions(Task): after = [filesystem.UnmountRoot] def run(self, info): - info.volume.partition_map.unmap() + info.volume.partition_map.unmap(info.volume) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 4577e92..8527030 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -30,9 +30,9 @@ class ConfigureGrub(Task): p_map.boot.unmount() p_map.root.unmount() if not isinstance(p_map, NoPartitions): - p_map.unmap() + p_map.unmap(info.volume) fn() - p_map.map() + p_map.map(info.volume) else: fn() p_map.root.device_path = info.volume.device_path @@ -70,4 +70,4 @@ class ConfigureGrub(Task): raise e if isinstance(info.volume, LoopbackVolume): - remount(info.volume, info.volume.unlink_dm_node) + remount(info.volume.unlink_dm_node) From fb0fa1e94e01f3a62a02bb1f4f157f757a1bbb1b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 5 Oct 2013 21:55:15 +0200 Subject: [PATCH 33/63] Unmap partitions --- base/fs/partitionmaps/abstract.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py index a7ea15b..e9bff6f 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/base/fs/partitionmaps/abstract.py @@ -74,3 +74,5 @@ class AbstractPartitionMap(FSMProxy): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + for partition in self.partitions: + partition.unmap() From 1bf61770d9f18366c6d35af35051e65e9f0ad243 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 5 Oct 2013 22:01:05 +0200 Subject: [PATCH 34/63] Access fsm states directly --- base/fs/partitionmaps/abstract.py | 8 ++++---- base/fs/partitionmaps/none.py | 2 +- base/fs/partitions/single.py | 2 -- base/fs/volume.py | 2 +- common/fs/loopbackvolume.py | 2 +- common/fsm_proxy.py | 9 --------- plugins/prebootstrapped/tasks.py | 6 +++--- 7 files changed, 10 insertions(+), 21 deletions(-) diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py index e9bff6f..f26b970 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/base/fs/partitionmaps/abstract.py @@ -19,7 +19,7 @@ class AbstractPartitionMap(FSMProxy): super(AbstractPartitionMap, self).__init__(cfg) def is_blocking(self): - return self.is_state('mapped') + return self.fsm.current == 'mapped' def get_total_size(self): return sum(p.size for p in self.partitions) + 1 @@ -54,12 +54,12 @@ class AbstractPartitionMap(FSMProxy): self.partitions[p_idx].map(partition_path) for idx, partition in enumerate(self.partitions): - if not partition.is_state('mapped'): + if partition.fsm.current not in ['mapped', 'formatted']: raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) except PartitionError as e: for partition in self.partitions: - if not partition.is_state('mapped'): + if not partition.fsm.can('unmap'): partition.unmap() log_check_call(['/sbin/kpartx', '-d', volume.device_path]) raise e @@ -70,7 +70,7 @@ class AbstractPartitionMap(FSMProxy): def _before_unmap(self, event): volume = event.volume for partition in self.partitions: - if partition.is_state('mounted'): + if partition.fsm.cannot('unmap'): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) log_check_call(['/sbin/kpartx', '-d', volume.device_path]) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py index cbca282..d623874 100644 --- a/base/fs/partitionmaps/none.py +++ b/base/fs/partitionmaps/none.py @@ -9,7 +9,7 @@ class NoPartitions(object): self.partitions = [self.root] def is_blocking(self): - return self.root.is_state('mounted') + return self.root.fsm == 'mounted' def get_total_size(self): return self.root.size diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index 5ac3ec5..dd82477 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -3,7 +3,5 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): - initial_state = 'created' - def _before_create(self, e): pass diff --git a/base/fs/volume.py b/base/fs/volume.py index 209fa6d..e83ad5b 100644 --- a/base/fs/volume.py +++ b/base/fs/volume.py @@ -35,7 +35,7 @@ class Volume(FSMProxy): self.partition_map.root.create() def can_mount_specials(self): - return self.is_state('attached') + return self.fsm.current == 'attached' def mount_specials(self): if self.specials_mounted: diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index d2918ff..81269b9 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -20,7 +20,7 @@ class LoopbackVolume(Volume): extension = 'raw' def can_mount_specials(self): - return self.is_state('attached') or self.is_state('linked') + return self.fsm.current in ['attached', 'linked'] def create(self, image_path): self.fsm.create(image_path=image_path) diff --git a/common/fsm_proxy.py b/common/fsm_proxy.py index 6235e10..db3d17a 100644 --- a/common/fsm_proxy.py +++ b/common/fsm_proxy.py @@ -9,15 +9,6 @@ class FSMProxy(object): self.fsm = Fysom(cfg) self.attach_proxy_methods(self.fsm, events) - def is_state(self, event): - return self.fsm.isstate(event) - - def set_state(self, event): - self.fsm.current = event - - def get_state(self): - return self.fsm.current - def collect_event_listeners(self, events, callbacks): callbacks = callbacks.copy() callback_names = [] diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 1e5f184..135cabb 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -67,7 +67,7 @@ class CreateFromImage(Task): def set_fs_states(volume): - volume.set_state('detached') + volume.fsm.current = 'detached' p_map = volume.partition_map partitions_state = 'attached' @@ -75,7 +75,7 @@ def set_fs_states(volume): if isinstance(p_map, NoPartitions): partitions_state = 'formatted' else: - p_map.set_state('unmapped') + p_map.fsm.current = 'unmapped' partitions_state = 'unmapped_fmt' for partition in p_map.partitions: - partition.set_state(partitions_state) + partition.fsm.current = partitions_state From 3dab414561b98c321778c89e64f1f5834e1b50fe Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 00:46:54 +0200 Subject: [PATCH 35/63] Final newline in fstab --- common/tasks/filesystem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 2fee046..39616bb 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -155,3 +155,4 @@ class FStab(Task): fstab_path = os.path.join(info.root, 'etc/fstab') with open(fstab_path, 'w') as fstab: fstab.write('\n'.join(fstab_lines)) + fstab.write('\n') From 54791268e11bb8e5ed551417bf675a04ca5d527c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 01:42:32 +0200 Subject: [PATCH 36/63] Fix TuneFS task --- common/task_sets.py | 18 +++++++++++++----- providers/virtualbox/__init__.py | 7 ++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/common/task_sets.py b/common/task_sets.py index 2d30ba8..f29f0e1 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -55,8 +55,16 @@ locale_set = [locale.GenerateLocale, locale.SetTimezone, ] -fs_specific_set = {'ext2': [filesystem.TuneVolumeFS], - 'ext3': [filesystem.TuneVolumeFS], - 'ext4': [filesystem.TuneVolumeFS], - 'xfs': [filesystem.AddXFSProgs], - } + +def get_fs_specific_set(partitions): + task_set = {'ext2': [filesystem.TuneVolumeFS], + 'ext3': [filesystem.TuneVolumeFS], + 'ext4': [filesystem.TuneVolumeFS], + 'xfs': [filesystem.AddXFSProgs], + } + tasks = set() + if 'boot' in partitions: + tasks.update(task_set.get(partitions['boot']['filesystem'], [])) + if 'root' in partitions: + tasks.update(task_set.get(partitions['root']['filesystem'], [])) + return tasks diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 098adff..175ca84 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -59,11 +59,8 @@ def tasks(tasklist, manifest): if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball) - from common.task_sets import fs_specific_set - for partition in manifest.volume['partitions']: - if 'filesystem' in partition: - fs_tasks = fs_specific_set.get(partition['filesystem'], []) - tasklist.add(fs_tasks) + from common.task_sets import get_fs_specific_set + tasklist.add(*get_fs_specific_set(manifest.volume['partitions'])) if 'boot' in manifest.volume['partitions']: from common.task_sets import boot_partition_set From d4601f08af616c041ae8c66dec22c42286be4205 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 13:20:39 +0200 Subject: [PATCH 37/63] Only install gen-hostkeys when sshd is installed --- common/tasks/initd.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/common/tasks/initd.py b/common/tasks/initd.py index 377a3d9..8d77059 100644 --- a/common/tasks/initd.py +++ b/common/tasks/initd.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.tools import log_check_call import os.path @@ -10,9 +11,15 @@ class ResolveInitScripts(Task): def run(self, info): init_scripts = {'expand-volume': 'expand-volume'} - init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' - if info.manifest.system['release'] == 'squeeze': - init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + from subprocess import CalledProcessError + try: + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/bin/dpkg-query', '-W', 'openssh-server']) + init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' + if info.manifest.system['release'] == 'squeeze': + init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + except CalledProcessError: + pass disable_scripts = ['hwclock.sh'] if info.manifest.system['release'] == 'squeeze': @@ -37,7 +44,6 @@ class InstallInitScripts(Task): stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) from shutil import copy - from common.tools import log_check_call for name, src in info.initd['install'].iteritems(): dst = os.path.join(info.root, 'etc/init.d', name) copy(src, dst) From a3006fbe472fd97303e9dc0228a35856c58ac018 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 13:25:00 +0200 Subject: [PATCH 38/63] Fix humongous brainfart --- common/tasks/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 39616bb..76e7604 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -133,7 +133,7 @@ class FStab(Task): # if info.manifest.virtualization == 'pvm': # device = '/dev/xvda' p_map = info.volume.partition_map - mount_points = [('/root', p_map.root)] + mount_points = [('/', p_map.root)] if hasattr(p_map, 'boot'): mount_points.append(('/boot', p_map.boot)) if hasattr(p_map, 'swap'): From 854ab4f20230aa386e6939e2d1d832012e4860af Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 13:40:27 +0200 Subject: [PATCH 39/63] Make fstab more generic --- common/tasks/filesystem.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 76e7604..80288a7 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -129,28 +129,36 @@ class FStab(Task): def run(self, info): import os.path - # device = '/dev/sda' - # if info.manifest.virtualization == 'pvm': - # device = '/dev/xvda' p_map = info.volume.partition_map - mount_points = [('/', p_map.root)] + mount_points = [{'path': '/', + 'partition': p_map.root, + 'dump': '1', + 'pass_num': '1', + }] if hasattr(p_map, 'boot'): - mount_points.append(('/boot', p_map.boot)) + mount_points.append({'path': '/boot', + 'partition': p_map.boot, + 'dump': '1', + 'pass_num': '2', + }) if hasattr(p_map, 'swap'): - mount_points.append(('none', p_map.swap)) + mount_points.append({'path': 'none', + 'partition': p_map.swap, + 'dump': '1', + 'pass_num': '0', + }) fstab_lines = [] - for mount_point, partition in mount_points: + for mount_point in mount_points: + partition = mount_point['partition'] mount_opts = ['defaults'] - if partition.filesystem in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if partition.filesystem == 'xfs': - mount_opts.append('nobarrier') - fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} 1 1' + fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}' .format(uuid=partition.get_uuid(), - mountpoint=mount_point, + mountpoint=mount_point['path'], filesystem=partition.filesystem, - mount_opts=','.join(mount_opts))) + mount_opts=','.join(mount_opts), + dump=mount_point['dump'], + pass_num=mount_point['pass_num'])) fstab_path = os.path.join(info.root, 'etc/fstab') with open(fstab_path, 'w') as fstab: From 38bcb12c7201bf310f3b9175a24fbc8d62912e54 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 14:06:01 +0200 Subject: [PATCH 40/63] SetBootMountDir task in prebootstrapped plugin --- plugins/prebootstrapped/__init__.py | 5 +++++ plugins/prebootstrapped/tasks.py | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index ff6aec9..b0ee99f 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -2,6 +2,7 @@ from tasks import Snapshot from tasks import CopyImage from tasks import CreateFromSnapshot from tasks import CreateFromImage +from tasks import SetBootMountDir from providers.ec2.tasks import ebs from common.tasks import loopback from common.tasks import volume @@ -23,12 +24,16 @@ def tasks(tasklist, manifest): if 'snapshot' in settings and settings['snapshot'] is not None: tasklist.replace(ebs.Create, CreateFromSnapshot) tasklist.remove(*skip_tasks) + if 'boot' in manifest.volume['partitions']: + tasklist.add(SetBootMountDir) else: tasklist.add(Snapshot) else: if 'image' in settings and settings['image'] is not None: tasklist.replace(loopback.Create, CreateFromImage) tasklist.remove(*skip_tasks) + if 'boot' in manifest.volume['partitions']: + tasklist.add(SetBootMountDir) else: tasklist.add(CopyImage) diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 135cabb..4bbd884 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -3,6 +3,9 @@ from common import phases from providers.ec2.tasks import ebs from common.tasks import volume from common.tasks import bootstrap +from common.tasks import filesystem +from shutil import copyfile +import os.path import time import logging log = logging.getLogger(__name__) @@ -43,8 +46,6 @@ class CopyImage(Task): after = [bootstrap.Bootstrap] def run(self, info): - import os.path - from shutil import copyfile loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension) destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) copyfile(info.volume.image_path, destination) @@ -58,14 +59,23 @@ class CreateFromImage(Task): before = [volume.Attach] def run(self, info): - import os.path - from shutil import copyfile info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] copyfile(loopback_backup_path, info.volume.image_path) + set_fs_states(info.volume) +class SetBootMountDir(Task): + description = 'Setting mountpoint for the boot partition' + phase = phases.volume_mounting + after = [filesystem.MountRoot] + before = [filesystem.MountBoot] + + def run(self, info): + info.boot_dir = os.path.join(info.root, 'boot') + + def set_fs_states(volume): volume.fsm.current = 'detached' From aa91bc701f1e39fd54b8a240ebba9cae3324171b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 14:06:32 +0200 Subject: [PATCH 41/63] Use add/remove instead of replace in rootpw plugin --- plugins/root_password/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/root_password/__init__.py b/plugins/root_password/__init__.py index 34b1248..91587b8 100644 --- a/plugins/root_password/__init__.py +++ b/plugins/root_password/__init__.py @@ -3,7 +3,8 @@ def tasks(tasklist, manifest): from common.tasks.security import DisableSSHPasswordAuthentication from tasks import SetRootPassword - tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword) + tasklist.remove(DisableSSHPasswordAuthentication) + tasklist.add(SetRootPassword) def validate_manifest(data, schema_validate): From f075d1f2b9d42d83032e67ad8d84687eeffa1901 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 14:08:53 +0200 Subject: [PATCH 42/63] Remove ambiguous tasklist function replace() --- base/tasklist.py | 4 ---- plugins/prebootstrapped/__init__.py | 9 ++++++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/base/tasklist.py b/base/tasklist.py index 68b1427..ec530ef 100644 --- a/base/tasklist.py +++ b/base/tasklist.py @@ -16,10 +16,6 @@ class TaskList(object): for task in args: self.tasks.discard(task) - def replace(self, task, replacement): - self.remove(task) - self.add(replacement) - def run(self, bootstrap_info): task_list = self.create_list(self.tasks) log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list))) diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index b0ee99f..35e7518 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -13,7 +13,10 @@ from common.tasks import partitioning def tasks(tasklist, manifest): settings = manifest.plugins['prebootstrapped'] - skip_tasks = [filesystem.Format, + skip_tasks = [ebs.Create, + loopback.Create, + + filesystem.Format, partitioning.PartitionVolume, filesystem.TuneVolumeFS, filesystem.AddXFSProgs, @@ -22,7 +25,7 @@ def tasks(tasklist, manifest): bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: - tasklist.replace(ebs.Create, CreateFromSnapshot) + tasklist.add(CreateFromSnapshot) tasklist.remove(*skip_tasks) if 'boot' in manifest.volume['partitions']: tasklist.add(SetBootMountDir) @@ -30,7 +33,7 @@ def tasks(tasklist, manifest): tasklist.add(Snapshot) else: if 'image' in settings and settings['image'] is not None: - tasklist.replace(loopback.Create, CreateFromImage) + tasklist.add(CreateFromImage) tasklist.remove(*skip_tasks) if 'boot' in manifest.volume['partitions']: tasklist.add(SetBootMountDir) From b025f6ed35f83c433ea2b3a698e4a25cd018c2f1 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 14:34:03 +0200 Subject: [PATCH 43/63] Refactor remount() into common function --- common/fs/__init__.py | 22 ++++++++++++++++++ plugins/prebootstrapped/tasks.py | 5 ++++- providers/virtualbox/tasks/boot.py | 36 ++++++++++++------------------ 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/common/fs/__init__.py b/common/fs/__init__.py index a609034..314e008 100644 --- a/common/fs/__init__.py +++ b/common/fs/__init__.py @@ -14,3 +14,25 @@ def get_partitions(): raise RuntimeError('Unable to parse {line} in {path}'.format(line=line, path=path)) matches[match.group('dev_name')] = match.groupdict() return matches + + +def remount(volume, fn): + from base.fs.partitionmaps.none import NoPartitions + + p_map = volume.partition_map + volume.unmount_specials() + if hasattr(p_map, 'boot'): + boot_dir = p_map.boot.mount_dir + p_map.boot.unmount() + root_dir = p_map.root.mount_dir + p_map.root.unmount() + if not isinstance(p_map, NoPartitions): + p_map.unmap(volume) + fn() + p_map.map(volume) + else: + fn() + p_map.root.mount(root_dir) + if hasattr(p_map, 'boot'): + p_map.boot.mount(boot_dir) + volume.mount_specials() diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 4bbd884..ab7acee 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -48,7 +48,10 @@ class CopyImage(Task): def run(self, info): loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension) destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) - copyfile(info.volume.image_path, destination) + + def mk_snapshot(): + copyfile(info.volume.image_path, destination) + remount(info.volume, mk_snapshot) msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination) log.info(msg) diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 8527030..bddb860 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -18,31 +18,23 @@ class ConfigureGrub(Task): from base.fs.partitionmaps.none import NoPartitions from base.fs.partitionmaps.gpt import GPTPartitionMap + from common.fs import remount p_map = info.volume.partition_map - def remount(fn): - # GRUB cannot deal with installing to loopback devices - # so we fake a real harddisk with dmsetup. - # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ - info.volume.unmount_specials() - if hasattr(p_map, 'boot'): - boot_dir = p_map.boot.mount_dir - p_map.boot.unmount() - p_map.root.unmount() - if not isinstance(p_map, NoPartitions): - p_map.unmap(info.volume) + def mk_remount_fn(fn): + def set_device_path(): fn() - p_map.map(info.volume) - else: - fn() - p_map.root.device_path = info.volume.device_path - p_map.root.mount(info.root) - if hasattr(p_map, 'boot'): - p_map.boot.mount(boot_dir) - info.volume.mount_specials() + if isinstance(p_map, NoPartitions): + p_map.root.device_path = info.volume.device_path + return set_device_path + link_fn = mk_remount_fn(info.volume.link_dm_node) + unlink_fn = mk_remount_fn(info.volume.unlink_dm_node) + # GRUB cannot deal with installing to loopback devices + # so we fake a real harddisk with dmsetup. + # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ if isinstance(info.volume, LoopbackVolume): - remount(info.volume.link_dm_node) + remount(info.volume, link_fn) try: [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) device_map_path = os.path.join(grub_dir, 'device.map') @@ -66,8 +58,8 @@ class ConfigureGrub(Task): log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) except Exception as e: if isinstance(info.volume, LoopbackVolume): - remount(info.volume.unlink_dm_node) + remount(info.volume, unlink_fn) raise e if isinstance(info.volume, LoopbackVolume): - remount(info.volume.unlink_dm_node) + remount(info.volume, unlink_fn) From ceae48921008afc9615c73f6f118d960772f82de Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 6 Oct 2013 14:44:23 +0200 Subject: [PATCH 44/63] Use remount() to create a clean snapshot --- common/fs/__init__.py | 5 +++-- plugins/prebootstrapped/tasks.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/common/fs/__init__.py b/common/fs/__init__.py index 314e008..3cde7cf 100644 --- a/common/fs/__init__.py +++ b/common/fs/__init__.py @@ -28,11 +28,12 @@ def remount(volume, fn): p_map.root.unmount() if not isinstance(p_map, NoPartitions): p_map.unmap(volume) - fn() + result = fn() p_map.map(volume) else: - fn() + result = fn() p_map.root.mount(root_dir) if hasattr(p_map, 'boot'): p_map.boot.mount(boot_dir) volume.mount_specials() + return result diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index ab7acee..d5fcb57 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,9 +1,9 @@ from base import Task from common import phases -from providers.ec2.tasks import ebs from common.tasks import volume from common.tasks import bootstrap from common.tasks import filesystem +from common.fs import remount from shutil import copyfile import os.path import time @@ -11,14 +11,16 @@ import logging log = logging.getLogger(__name__) -class Snapshot(ebs.Snapshot): +class Snapshot(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.os_installation - after = [bootstrap.Bootstrap] + after = [bootstrap.Bootstrap, filesystem.MountSpecials] def run(self, info): - super(Snapshot, self).run(info) - msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id) + def mk_snapshot(): + return info.volume.snapshot() + snapshot = remount(info.volume, mk_snapshot) + msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=snapshot.id) log.info(msg) @@ -43,7 +45,7 @@ class CreateFromSnapshot(Task): class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.os_installation - after = [bootstrap.Bootstrap] + after = [bootstrap.Bootstrap, filesystem.MountSpecials] def run(self, info): loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension) From 754e414742e0d68d8afec6de143e7bbb0a2908e3 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Tue, 8 Oct 2013 23:12:03 +0200 Subject: [PATCH 45/63] Document post-mbr gap and make it MBR-only --- base/fs/partitionmaps/abstract.py | 2 +- base/fs/partitionmaps/mbr.py | 3 +++ base/fs/partitions/base.py | 2 +- base/fs/partitions/mbr.py | 6 ++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py index f26b970..bac76c4 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/base/fs/partitionmaps/abstract.py @@ -22,7 +22,7 @@ class AbstractPartitionMap(FSMProxy): return self.fsm.current == 'mapped' def get_total_size(self): - return sum(p.size for p in self.partitions) + 1 + return sum(p.size for p in self.partitions) def create(self, volume): self.fsm.create(volume=volume) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py index 0d4b50f..d2d8a8d 100644 --- a/base/fs/partitionmaps/mbr.py +++ b/base/fs/partitionmaps/mbr.py @@ -19,6 +19,9 @@ class MBRPartitionMap(AbstractPartitionMap): super(MBRPartitionMap, self).__init__() + def get_total_size(self): + return sum(p.size for p in self.partitions) + 1 # Post-MBR gap for embedding grub + def _before_create(self, event): volume = event.volume log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index 8986285..0769de5 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -26,7 +26,7 @@ class BasePartition(AbstractPartition): def get_start(self): if self.previous is None: - return 1 + return 0 else: return self.previous.get_start() + self.previous.size diff --git a/base/fs/partitions/mbr.py b/base/fs/partitions/mbr.py index 511cd10..d10c2d8 100644 --- a/base/fs/partitions/mbr.py +++ b/base/fs/partitions/mbr.py @@ -4,6 +4,12 @@ from base import BasePartition class MBRPartition(BasePartition): + def get_start(self): + if self.previous is None: + return 1 # Post-MBR gap for embedding grub + else: + return self.previous.get_start() + self.previous.size + def _before_create(self, e): start = self.get_start() parted_command = ('mkpart primary {start}MiB {end}MiB' From e1ab4dc1aeadee6d7c61b1084f8dab473245415e Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 9 Oct 2013 00:09:34 +0200 Subject: [PATCH 46/63] EC2 provider can now bootstrap EBS volumes again Use tasksets in EC2 provider --- base/fs/__init__.py | 2 +- base/fs/partitions/abstract.py | 8 -- base/fs/partitions/base.py | 3 + base/fs/partitions/single.py | 2 - common/task_sets.py | 4 + providers/ec2/__init__.py | 143 +++++++++++----------- providers/ec2/{volume.py => ebsvolume.py} | 15 ++- providers/ec2/manifest-schema.json | 6 +- providers/ec2/manifest.py | 17 ++- providers/ec2/tasks/ami.py | 9 +- providers/ec2/tasks/ebs.py | 11 +- providers/ec2/tasks/packages.py | 7 +- providers/virtualbox/__init__.py | 3 - 13 files changed, 119 insertions(+), 111 deletions(-) rename providers/ec2/{volume.py => ebsvolume.py} (82%) diff --git a/base/fs/__init__.py b/base/fs/__init__.py index 6b4763b..5817220 100644 --- a/base/fs/__init__.py +++ b/base/fs/__init__.py @@ -2,7 +2,7 @@ def load_volume(data): from common.fs.loopbackvolume import LoopbackVolume - from providers.ec2.volume import EBSVolume + from providers.ec2.ebsvolume import EBSVolume from providers.virtualbox.volume import VirtualBoxVolume from partitionmaps.gpt import GPTPartitionMap from partitionmaps.mbr import MBRPartitionMap diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py index 9f80618..bb2354c 100644 --- a/base/fs/partitions/abstract.py +++ b/base/fs/partitions/abstract.py @@ -1,5 +1,4 @@ from abc import ABCMeta -from abc import abstractmethod from common.tools import log_check_call from common.fsm_proxy import FSMProxy @@ -26,13 +25,6 @@ class AbstractPartition(FSMProxy): [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) return uuid - def create(self, volume): - self.fsm.create(volume=volume) - - @abstractmethod - def _before_create(self, e): - pass - def _before_format(self, e): mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) log_check_call([mkfs, self.device_path]) diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index 0769de5..3f5792f 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -18,6 +18,9 @@ class BasePartition(AbstractPartition): self.previous = previous super(BasePartition, self).__init__(size, filesystem) + def create(self, volume): + self.fsm.create(volume=volume) + def get_index(self): if self.previous is None: return 1 diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index dd82477..2017c91 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -2,6 +2,4 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): - - def _before_create(self, e): pass diff --git a/common/task_sets.py b/common/task_sets.py index f29f0e1..07e7a31 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -1,6 +1,7 @@ from common.tasks import workspace from common.tasks import packages from common.tasks import host +from common.tasks import bootstrap from common.tasks import volume from common.tasks import filesystem from common.tasks import partitioning @@ -13,11 +14,14 @@ base_set = [workspace.CreateWorkspace, packages.HostPackages, packages.ImagePackages, host.CheckPackages, + bootstrap.Bootstrap, workspace.DeleteWorkspace, ] volume_set = [volume.Attach, volume.Detach, + filesystem.Format, + filesystem.FStab, ] partitioning_set = [partitioning.PartitionVolume, diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 68c47c5..9741c7a 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -1,18 +1,15 @@ from manifest import Manifest import logging from tasks import packages -from common.tasks import packages as common_packages from tasks import connection from tasks import host -from common.tasks import host as common_host from tasks import ami -from common.tasks import volume +from common.tasks import volume as volume_tasks from tasks import ebs +from common.tasks import partitioning from common.tasks import loopback from common.tasks import filesystem from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security @@ -21,6 +18,7 @@ from common.tasks import network as common_network from tasks import initd from common.tasks import initd as common_initd from common.tasks import cleanup +from common.tasks import workspace def initialize(): @@ -29,74 +27,66 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - common_host.CheckPackages(), - connection.GetCredentials(), - host.GetInfo(), - ami.AMIName(), - connection.Connect(), + from common.task_sets import base_set + from common.task_sets import mounting_set + from common.task_sets import apt_set + from common.task_sets import locale_set + from common.task_sets import ssh_set + tasklist.add(*base_set) + tasklist.add(*mounting_set) + tasklist.add(*apt_set) + tasklist.add(*locale_set) + tasklist.add(*ssh_set) - filesystem.FormatVolume(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), + if manifest.volume['partitions']['type'] != 'none': + from common.task_sets import partitioning_set + tasklist.add(*partitioning_set) - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.FStab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - common_network.RemoveDNSInfo(), - common_network.ConfigureNetworkIF(), - network.EnableDHCPCDDNS(), - common_initd.ResolveInitScripts(), - initd.AddEC2InitScripts(), - common_initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), + tasklist.add(packages.HostPackages, + packages.ImagePackages, + connection.GetCredentials, + host.GetInfo, + ami.AMIName, + connection.Connect, - filesystem.UnmountVolume(), - filesystem.DeleteMountDir(), - ami.RegisterAMI()) + boot.ConfigureGrub, + common_boot.BlackListModules, + common_boot.DisableGetTTYs, + security.EnableShadowConfig, + common_network.RemoveDNSInfo, + common_network.ConfigureNetworkIF, + network.EnableDHCPCDDNS, + common_initd.ResolveInitScripts, + initd.AddEC2InitScripts, + common_initd.InstallInitScripts, + cleanup.ClearMOTD, + cleanup.CleanTMP, + + ami.RegisterAMI) + + backing_specific_tasks = {'ebs': [ebs.Create, + ebs.Attach, + ebs.Snapshot], + 's3': [loopback.Create, + volume_tasks.Attach, + ami.BundleImage, + ami.UploadImage, + ami.RemoveBundle]} + tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + tasklist.add(filesystem.Format, + filesystem.FStab, + volume_tasks.Detach, + volume_tasks.Delete) if manifest.bootstrapper.get('tarball', False): - tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.MakeTarball) - backing_specific_tasks = {'ebs': [ebs.Create(), - volume.Attach(), - volume.Detach(), - ebs.Snapshot(), - volume.Delete()], - 's3': [loopback.Create(), - volume.Attach(), - volume.Detach(), - ami.BundleImage(), - ami.UploadImage(), - volume.Delete(), - ami.RemoveBundle()]} - tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + from common.task_sets import get_fs_specific_set + tasklist.add(*get_fs_specific_set(manifest.volume['partitions'])) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + if 'boot' in manifest.volume['partitions']: + from common.task_sets import boot_partition_set + tasklist.add(*boot_partition_set) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -104,14 +94,19 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) - if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, volume.Delete) - counter_task(volume.Attach, volume.Detach) - if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, volume.Delete) - counter_task(volume.Attach, volume.Detach) + counter_task(ebs.Create, volume_tasks.Delete) + counter_task(ebs.Attach, volume_tasks.Detach) + + counter_task(loopback.Create, volume_tasks.Delete) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/ec2/volume.py b/providers/ec2/ebsvolume.py similarity index 82% rename from providers/ec2/volume.py rename to providers/ec2/ebsvolume.py index 16e4e96..29e9e41 100644 --- a/providers/ec2/volume.py +++ b/providers/ec2/ebsvolume.py @@ -5,18 +5,18 @@ import time class EBSVolume(Volume): - volume = None - def create(self, conn, zone): - super(EBSVolume, self).create(self) + self.fsm.create(connection=conn, zone=zone) + + def _before_create(self, e): + conn = e.connection + zone = e.zone import math - # TODO: Warn if volume size is not a multiple of 1024 - size = int(math.ceil(self.partition_map.get_volume_size() / 1024)) + size = int(math.ceil(self.partition_map.get_total_size() / 1024)) self.volume = conn.create_volume(size, zone) while self.volume.volume_state() != 'available': time.sleep(5) self.volume.update() - self.created = True def attach(self, instance_id): self.fsm.attach(instance_id=instance_id) @@ -25,7 +25,7 @@ class EBSVolume(Volume): instance_id = e.instance_id import os.path import string - for letter in string.ascii_lowercase: + for letter in string.ascii_lowercase[5:]: dev_path = os.path.join('/dev', 'xvd' + letter) if not os.path.exists(dev_path): self.device_path = dev_path @@ -47,7 +47,6 @@ class EBSVolume(Volume): self.volume.update() def _before_delete(self, e): - super(EBSVolume, self).delete(self) self.volume.delete() def snapshot(self): diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index ffa360d..43fda5d 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -20,13 +20,9 @@ "backing": { "type": "string", "enum": ["ebs", "s3"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, "required": ["volume"] diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index abab855..9bfea5c 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -9,7 +9,8 @@ class Manifest(base.Manifest): schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') self.schema_validate(data, schema_path) if data['volume']['backing'] == 'ebs': - if data['volume']['size'] % 1024 != 0: + volume_size = self._calculate_volume_size(data['volume']['partitions']) + if volume_size % 1024 != 0: msg = 'The volume size must be a multiple of 1024 when using EBS backing' raise ManifestError(msg, self) else: @@ -21,5 +22,15 @@ class Manifest(base.Manifest): self.credentials = data['credentials'] self.virtualization = data['virtualization'] self.image = data['image'] - if data['volume']['backing'] == 'ebs': - self.ebs_volume_size = data['volume']['size'] / 1024 + + def _calculate_volume_size(self, partitions): + if partitions['type'] == 'mbr': + size = 1 + else: + size = 0 + if 'boot' in partitions: + size += partitions['boot']['size'] + size += partitions['root']['size'] + if 'swap' in partitions: + size += partitions['swap']['size'] + return size diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 81c99ed..4cb4577 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -117,11 +117,14 @@ class RegisterAMI(Task): arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - if info.manifest.volume['backing'] == 'ebs': + from providers.ec2.ebsvolume import EBSVolume + from common.fs.loopbackvolume import LoopbackVolume + + if isinstance(info.volume, EBSVolume): from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.manifest.ebs_volume_size) + size=info.volume.partition_map.get_total_size()/1024) block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = block_device @@ -129,7 +132,7 @@ class RegisterAMI(Task): architecture=arch, kernel_id=kernel_id, root_device_name='/dev/sda1', block_device_map=block_device_map) - if info.manifest.volume['backing'] == 's3': + if isinstance(info.volume, LoopbackVolume): image_location = ('{bucket}/{ami_name}.manifest.xml' .format(bucket=info.manifest.image['bucket'], ami_name=info.ami_name)) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index bb84a44..e8c363e 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,17 +1,24 @@ from base import Task from common import phases -from common.tasks import volume class Create(Task): description = 'Creating the EBS volume' phase = phases.volume_creation - before = [volume.Attach] def run(self, info): info.volume.create(info.connection, info.host['availabilityZone']) +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + after = [Create] + + def run(self, info): + info.volume.attach(info.host['instanceId']) + + class Snapshot(Task): description = 'Creating a snapshot of the EBS volume' phase = phases.image_registration diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index a884b7f..46f1222 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -11,8 +11,10 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') + for partition in info.volume.partition_map.partitions: + if partition.filesystem == 'xfs': + info.host_packages.add('xfsprogs') + break if info.manifest.volume['backing'] == 's3': info.host_packages.add('euca2ools') @@ -25,6 +27,7 @@ class ImagePackages(Task): def run(self, info): manifest = info.manifest include, exclude = info.img_packages + include.add('openssh-server') include.add('file') # Needed for the init scripts include.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2 if manifest.virtualization == 'pvm': diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 175ca84..70412be 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -38,11 +38,8 @@ def tasks(tasklist, manifest): packages.ImagePackages, loopback.Create, - filesystem.Format, - bootstrap.Bootstrap, boot.ConfigureGrub, - filesystem.FStab, common_boot.BlackListModules, common_boot.DisableGetTTYs, security.EnableShadowConfig, From 0fdcc2e27b744fae4c5b8ad7df10e3229e96ca29 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 08:52:26 +0100 Subject: [PATCH 47/63] Fix bug when no boot partition is specified --- base/fs/partitionmaps/gpt.py | 3 ++- base/fs/partitionmaps/mbr.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py index 0b2e8bf..7a95943 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/base/fs/partitionmaps/gpt.py @@ -11,7 +11,8 @@ class GPTPartitionMap(AbstractPartitionMap): if 'boot' in data: self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None) self.partitions.append(self.boot) - self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', self.boot) + self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', + getattr(self, 'boot', None)) self.partitions.append(self.root) if 'swap' in data: self.swap = GPTSwapPartition(data['swap']['size'], self.root) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py index d2d8a8d..67c40cb 100644 --- a/base/fs/partitionmaps/mbr.py +++ b/base/fs/partitionmaps/mbr.py @@ -11,7 +11,8 @@ class MBRPartitionMap(AbstractPartitionMap): if 'boot' in data: self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None) self.partitions.append(self.boot) - self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], self.boot) + self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], + getattr(self, 'boot', None)) self.partitions.append(self.root) if 'swap' in data: self.swap = MBRSwapPartition(data['swap']['size'], self.root) From a07e02cfcc34c165b2cffb491f29dc43d94ff352 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 09:10:37 +0100 Subject: [PATCH 48/63] Add required packages in the right places --- common/tasks/packages.py | 16 +++++++++------- providers/virtualbox/tasks/packages.py | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/common/tasks/packages.py b/common/tasks/packages.py index abccdf5..cdc9497 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -7,8 +7,12 @@ class HostPackages(Task): phase = phases.preparation def run(self, info): - packages = set(['debootstrap']) - info.host_packages = packages + info.host_packages = set() + info.host_packages.add('debootstrap') + + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + info.host_packages.update(['parted', 'kpartx']) class ImagePackages(Task): @@ -16,9 +20,7 @@ class ImagePackages(Task): phase = phases.preparation def run(self, info): - # Add some basic packages we are going to need + info.img_packages = set(), set() + include, exclude = info.img_packages # We could bootstrap without locales, but things just suck without them, error messages etc. - include = set(['locales']) - exclude = set() - - info.img_packages = include, exclude + include.add('locales') diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 871152a..228ad59 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -11,7 +11,7 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'kpartx']) + info.host_packages.add('qemu-utils') if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): info.host_packages.add('xfsprogs') @@ -25,7 +25,7 @@ class ImagePackages(Task): manifest = info.manifest include, exclude = info.img_packages # Add some basic packages we are going to need - include.update(['grub2']) + include.add('grub2') # In squeeze, we need a special kernel flavor for xen kernels = {'squeeze': {'amd64': 'linux-image-amd64', From 1b5a1e0579ba83f3b4cc06522caff1ab339680c9 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 09:24:07 +0100 Subject: [PATCH 49/63] Fix missing package check `dpkg-query -W parted` exits with 0 even if it is not installed --- common/tasks/host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/tasks/host.py b/common/tasks/host.py index e086fa3..f62f6e0 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -14,7 +14,7 @@ class CheckPackages(Task): from subprocess import CalledProcessError for package in info.host_packages: try: - log_check_call(['/usr/bin/dpkg-query', '-W', package]) + log_check_call(['/usr/bin/dpkg-query', '-s', package]) except CalledProcessError: msg = "The package ``{0}\'\' is not installed".format(package) raise TaskError(msg) From c95b9077e59fe531d236a12e26dbcb73b13b8fd4 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 10:02:11 +0100 Subject: [PATCH 50/63] Add helpful message to volsize%1024 error --- providers/ec2/manifest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index 9bfea5c..10d816f 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -11,7 +11,8 @@ class Manifest(base.Manifest): if data['volume']['backing'] == 'ebs': volume_size = self._calculate_volume_size(data['volume']['partitions']) if volume_size % 1024 != 0: - msg = 'The volume size must be a multiple of 1024 when using EBS backing' + msg = ('The volume size must be a multiple of 1024 when using EBS backing ' + '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)') raise ManifestError(msg, self) else: schema_path = path.join(path.dirname(__file__), 'manifest-schema-s3.json') From 6c800145dc8c6b8f9da1a364c762c7f6594db3fc Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 12:55:03 +0100 Subject: [PATCH 51/63] Alias s3 backing to raw --- base/fs/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/base/fs/__init__.py b/base/fs/__init__.py index 5817220..5f1689c 100644 --- a/base/fs/__init__.py +++ b/base/fs/__init__.py @@ -13,6 +13,7 @@ def load_volume(data): } partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) volume_backings = {'raw': LoopbackVolume, + 's3': LoopbackVolume, 'vdi': VirtualBoxVolume, 'ebs': EBSVolume } From a397091e460f74e57bff548631f3fb1fe6c2480b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 13:01:01 +0100 Subject: [PATCH 52/63] Generalize req. host package additions --- common/tasks/packages.py | 7 +++++++ providers/ec2/tasks/packages.py | 4 ---- providers/virtualbox/__init__.py | 3 +-- providers/virtualbox/tasks/packages.py | 13 ------------- 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/common/tasks/packages.py b/common/tasks/packages.py index cdc9497..41a3774 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -10,6 +10,13 @@ class HostPackages(Task): info.host_packages = set() info.host_packages.add('debootstrap') + from common.fs.loopbackvolume import LoopbackVolume + if isinstance(info.volume, LoopbackVolume): + info.host_packages.add('qemu-utils') + + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): + info.host_packages.add('xfsprogs') + from base.fs.partitionmaps.none import NoPartitions if not isinstance(info.volume.partition_map, NoPartitions): info.host_packages.update(['parted', 'kpartx']) diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index 46f1222..814a6f6 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -11,10 +11,6 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - for partition in info.volume.partition_map.partitions: - if partition.filesystem == 'xfs': - info.host_packages.add('xfsprogs') - break if info.manifest.volume['backing'] == 's3': info.host_packages.add('euca2ools') diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 70412be..8e31b1b 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -34,8 +34,7 @@ def tasks(tasklist, manifest): from common.task_sets import partitioning_set tasklist.add(*partitioning_set) - tasklist.add(packages.HostPackages, - packages.ImagePackages, + tasklist.add(packages.ImagePackages, loopback.Create, diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 228ad59..58b1d7f 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -1,19 +1,6 @@ from base import Task from common import phases from common.tasks import packages -from common.tasks.host import CheckPackages - - -class HostPackages(Task): - description = 'Determining required host packages' - phase = phases.preparation - before = [CheckPackages] - after = [packages.HostPackages] - - def run(self, info): - info.host_packages.add('qemu-utils') - if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): - info.host_packages.add('xfsprogs') class ImagePackages(Task): From 6eb56721dca0f6be07f474fd23e702c607e7f940 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 17:47:39 +0100 Subject: [PATCH 53/63] AWS provider fully working again (S3 & EBS) EBS booted AMIs now also support partitions --- providers/ec2/__init__.py | 17 +-- providers/ec2/assets/grub.d/40_custom | 2 +- providers/ec2/tasks/ami.py | 142 ++++++++++++++++++-------- providers/ec2/tasks/boot.py | 13 +++ providers/ec2/tasks/filesystem.py | 27 +++++ 5 files changed, 152 insertions(+), 49 deletions(-) create mode 100644 providers/ec2/tasks/filesystem.py diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 9741c7a..98cef05 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -8,7 +8,8 @@ from common.tasks import volume as volume_tasks from tasks import ebs from common.tasks import partitioning from common.tasks import loopback -from common.tasks import filesystem +from common.tasks import filesystem as common_filesystem +from tasks import filesystem from common.tasks import bootstrap from tasks import boot from common.tasks import boot as common_boot @@ -66,15 +67,16 @@ def tasks(tasklist, manifest): backing_specific_tasks = {'ebs': [ebs.Create, ebs.Attach, + common_filesystem.FStab, ebs.Snapshot], 's3': [loopback.Create, volume_tasks.Attach, + filesystem.S3FStab, ami.BundleImage, ami.UploadImage, ami.RemoveBundle]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) - tasklist.add(filesystem.Format, - filesystem.FStab, + tasklist.add(common_filesystem.Format, volume_tasks.Detach, volume_tasks.Delete) @@ -103,10 +105,11 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(volume_tasks.Attach, volume_tasks.Detach) counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) - counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir) + counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials) - counter_task(filesystem.MountRoot, filesystem.UnmountRoot) - counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(common_filesystem.MountRoot, common_filesystem.UnmountRoot) + counter_task(common_filesystem.MountBoot, common_filesystem.UnmountBoot) counter_task(volume_tasks.Attach, volume_tasks.Detach) counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) + counter_task(ami.BundleImage, ami.RemoveBundle) diff --git a/providers/ec2/assets/grub.d/40_custom b/providers/ec2/assets/grub.d/40_custom index 799e887..0946bbd 100644 --- a/providers/ec2/assets/grub.d/40_custom +++ b/providers/ec2/assets/grub.d/40_custom @@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib export TEXTDOMAIN=grub export TEXTDOMAINDIR=${prefix}/share/locale -GRUB_DEVICE=/dev/xvda1 +GRUB_DEVICE=/dev/xvda cat << EOF diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 4cb4577..11e3fd8 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -48,7 +48,7 @@ class BundleImage(Task): bundle_name = 'bundle-{id:x}'.format(id=info.run_id) info.bundle_path = os.path.join(info.workspace, bundle_name) log_check_call(['/usr/bin/euca-bundle-image', - '--image', info.loopback_file, + '--image', info.volume.image_path, '--user', info.credentials['user-id'], '--privatekey', info.credentials['private-key'], '--cert', info.credentials['certificate'], @@ -94,49 +94,109 @@ class RegisterAMI(Task): phase = phases.image_registration after = [Snapshot, UploadImage] - kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', - 'i386': 'aki-b6aa75df'}, - 'us-west-1': {'amd64': 'aki-f77e26b2', - 'i386': 'aki-f57e26b0'}, - 'us-west-2': {'amd64': 'aki-fc37bacc', - 'i386': 'aki-fa37baca'}, - 'eu-west-1': {'amd64': 'aki-71665e05', - 'i386': 'aki-75665e01'}, - 'ap-southeast-1': {'amd64': 'aki-fe1354ac', - 'i386': 'aki-f81354aa'}, - 'ap-southeast-2': {'amd64': 'aki-31990e0b', - 'i386': 'aki-33990e09'}, - 'ap-northeast-1': {'amd64': 'aki-44992845', - 'i386': 'aki-42992843'}, - 'sa-east-1': {'amd64': 'aki-c48f51d9', - 'i386': 'aki-ca8f51d7'}, - 'us-gov-west-1': {'amd64': 'aki-79a4c05a', - 'i386': 'aki-7ba4c058'}} + # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs + kernel_mapping = {'ap-northeast-1': { # Asia Pacific (Tokyo) Region + 'hd0': {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-176bf516'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-196bf518', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-1f6bf51e'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-1': { # Asia Pacific (Singapore) Region + 'hd0': {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-503e7402'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-563e7404', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5e3e740c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-2': { # Asia Pacific (Sydney) Region + 'hd0': {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-c362fff9'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-c162fffb', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-3b1d8001'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'eu-west-1': { # EU (Ireland) Region + 'hd0': {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-52a34525'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5ea34529', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-58a3452f'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'sa-east-1': { # South America (Sao Paulo) Region + 'hd0': {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-5553f448'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5753f44a', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5153f44c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-east-1': { # US East (Northern Virginia) Region + 'hd0': {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-919dcaf8'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-659ccb0c', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-499ccb20'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-gov-west-1': { # AWS GovCloud (US) + 'hd0': {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-1de98d3e'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-63e98d40', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-61e98d42'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-1': { # US West (Northern California) Region + 'hd0': {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-880531cd'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-960531d3', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-920531d7'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-2': { # US West (Oregon) Region + 'hd0': {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-fc8f11cc'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-e28f11d2', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-e68f11d6'} # pv-grub-hd00_1.04-x86_64.gz + }} def run(self, info): + if info.manifest.volume['backing'] == 'ebs': + self.run_ebs(info) + if info.manifest.volume['backing'] == 's3': + self.run_s3(info) + + def run_ebs(self, info): arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) - kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - from providers.ec2.ebsvolume import EBSVolume - from common.fs.loopbackvolume import LoopbackVolume + from base.fs.partitionmaps.none import NoPartitions + if isinstance(info.volume.partition_map, NoPartitions): + grub_boot_device = 'hd0' + root_device_name = '/dev/sda' + else: + grub_boot_device = 'hd00' + root_idx = info.volume.partition_map.root.get_index() + root_device_name = '/dev/sda{idx}'.format(idx=root_idx) - if isinstance(info.volume, EBSVolume): - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.volume.partition_map.get_total_size()/1024) - block_device_map = BlockDeviceMapping() - block_device_map['/dev/sda1'] = block_device + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get(grub_boot_device) + .get(info.manifest.system['architecture'])) - info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - block_device_map=block_device_map) - if isinstance(info.volume, LoopbackVolume): - image_location = ('{bucket}/{ami_name}.manifest.xml' - .format(bucket=info.manifest.image['bucket'], - ami_name=info.ami_name)) - info.image = info.connection.register_image(description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - image_location=image_location) + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=info.volume.partition_map.get_total_size()/1024) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda'] = block_device + + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name=root_device_name, + block_device_map=block_device_map) + + def run_s3(self, info): + arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get('hd0') + .get(info.manifest.system['architecture'])) + + image_manifest = ('{bucket}/{ami_name}.manifest.xml' + .format(bucket=info.manifest.image['bucket'], + ami_name=info.ami_name)) + info.image = info.connection.register_image(description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='dev/sda1', + image_location=image_manifest) diff --git a/providers/ec2/tasks/boot.py b/providers/ec2/tasks/boot.py index cee03e8..48f8d27 100644 --- a/providers/ec2/tasks/boot.py +++ b/providers/ec2/tasks/boot.py @@ -24,6 +24,19 @@ class ConfigureGrub(Task): copy(script_src, script_dst) os.chmod(script_dst, rwxr_xr_x) + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + from common.tools import sed_i + root_idx = info.volume.partition_map.root.get_index() + grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx) + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device) + grub_root = '\troot (hd0,{idx})'.format(idx=root_idx-1) + sed_i(script_dst, '^\troot \(hd0\)$', grub_root) + + if info.manifest.volume['backing'] == 's3': + from common.tools import sed_i + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1') + from common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' diff --git a/providers/ec2/tasks/filesystem.py b/providers/ec2/tasks/filesystem.py new file mode 100644 index 0000000..c2de2a1 --- /dev/null +++ b/providers/ec2/tasks/filesystem.py @@ -0,0 +1,27 @@ +from base import Task +from common import phases + + +class S3FStab(Task): + description = 'Adding the S3 root partition to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + root = info.volume.partition_map.root + + fstab_lines = [] + mount_opts = ['defaults'] + fstab_lines.append('{device_path}{idx} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}' + .format(device_path='/dev/xvda', + idx=1, + mountpoint='/', + filesystem=root.filesystem, + mount_opts=','.join(mount_opts), + dump='1', + pass_num='1')) + + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) + fstab.write('\n') From 1db38ec7dd2b52aaf7d04eb5ca85ec97e58c40e8 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 17:49:39 +0100 Subject: [PATCH 54/63] New task for adjusting the expand-volume script By default it only expands /dev/xvda --- common/assets/init.d/expand-volume | 2 +- common/tasks/initd.py | 3 ++- providers/ec2/__init__.py | 1 + providers/ec2/tasks/initd.py | 20 ++++++++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/common/assets/init.d/expand-volume b/common/assets/init.d/expand-volume index 3b2d2a6..cfee83c 100644 --- a/common/assets/init.d/expand-volume +++ b/common/assets/init.d/expand-volume @@ -13,7 +13,7 @@ prog=$(basename $0) logger="logger -t $prog" -device_path="/dev/xvda1" +device_path="/dev/xvda" filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'` diff --git a/common/tasks/initd.py b/common/tasks/initd.py index 8d77059..a0df088 100644 --- a/common/tasks/initd.py +++ b/common/tasks/initd.py @@ -9,7 +9,8 @@ class ResolveInitScripts(Task): phase = phases.system_modification def run(self, info): - init_scripts = {'expand-volume': 'expand-volume'} + init_scripts = {} + init_scripts['expand-volume'] = 'expand-volume' from subprocess import CalledProcessError try: diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 98cef05..a383f7b 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -60,6 +60,7 @@ def tasks(tasklist, manifest): common_initd.ResolveInitScripts, initd.AddEC2InitScripts, common_initd.InstallInitScripts, + initd.AdjustExpandVolumeScript, cleanup.ClearMOTD, cleanup.CleanTMP, diff --git a/providers/ec2/tasks/initd.py b/providers/ec2/tasks/initd.py index 6f1d394..7be1667 100644 --- a/providers/ec2/tasks/initd.py +++ b/providers/ec2/tasks/initd.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.exceptions import TaskError from common.tasks import initd import os.path @@ -17,3 +18,22 @@ class AddEC2InitScripts(Task): init_scripts_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d')) for name, path in init_scripts.iteritems(): info.initd['install'][name] = os.path.join(init_scripts_dir, path) + + +class AdjustExpandVolumeScript(Task): + description = 'Adjusting the expand-volume script' + phase = phases.system_modification + after = [initd.InstallInitScripts] + + def run(self, info): + if 'expand-volume' not in info.initd['install']: + raise TaskError('The expand-volume script was not installed') + + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + import os.path + from common.tools import sed_i + script = os.path.join(info.root, 'etc/init.d.expand-volume') + root_idx = info.volume.partition_map.root.get_index() + device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx) + sed_i(script, '^device_path="/dev/xvda$', device_path) From cf3c861a275ace8b6002e0cf3753552393a9ebc4 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 27 Oct 2013 18:37:43 +0100 Subject: [PATCH 55/63] Added the dry-run option --- base/main.py | 6 ++++-- base/tasklist.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/base/main.py b/base/main.py index 398adc2..32b9aba 100644 --- a/base/main.py +++ b/base/main.py @@ -17,6 +17,8 @@ def get_args(): help='Print debugging information') parser.add_argument('--pause-on-error', action='store_true', help='Pause on error, before rollback') + parser.add_argument('--dry-run', action='store_true', + help='Dont\'t actually run the tasks') parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') return parser.parse_args() @@ -35,7 +37,7 @@ def run(args): bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug) try: - tasklist.run(bootstrap_info) + tasklist.run(info=bootstrap_info, dry_run=args.dry_run) log.info('Successfully completed bootstrapping') except (Exception, KeyboardInterrupt) as e: log.exception(e) @@ -48,5 +50,5 @@ def run(args): rollback_tasks = getattr(plugin, 'rollback_tasks', None) if callable(rollback_tasks): plugin.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) - rollback_tasklist.run(bootstrap_info) + rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run) log.info('Successfully completed rollback') diff --git a/base/tasklist.py b/base/tasklist.py index ec530ef..fe07755 100644 --- a/base/tasklist.py +++ b/base/tasklist.py @@ -16,7 +16,7 @@ class TaskList(object): for task in args: self.tasks.discard(task) - def run(self, bootstrap_info): + def run(self, info={}, dry_run=False): task_list = self.create_list(self.tasks) log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list))) @@ -26,7 +26,8 @@ class TaskList(object): log.info(task.description) else: log.info('Running {task}'.format(task=task)) - task.run(bootstrap_info) + if not dry_run: + task.run(info) self.tasks_completed.append(task) def create_list(self, tasks): From 71adb3d04e6ca271c6263493ca50e129470a01df Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Tue, 29 Oct 2013 22:41:29 +0100 Subject: [PATCH 56/63] Manifest schema now validates partition definition S3 partitioning is disallowed for now --- base/manifest-schema.json | 82 +++++++++++++++++------ providers/ec2/manifest-schema-s3.json | 11 +++ providers/ec2/manifest-schema.json | 18 +++-- providers/virtualbox/manifest-schema.json | 7 +- 4 files changed, 85 insertions(+), 33 deletions(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 1d751b3..0db594d 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -9,37 +9,36 @@ "bootstrapper": { "type": "object", "properties": { - "workspace": { "type": "string" }, - "mirror": { "type": "string" }, - "tarball": { "type": "boolean" } + "workspace": { "$ref": "#/definitions/path" }, + "mirror": { "type": "string", "format": "uri" }, + "tarball": { "type": "boolean" } }, "required": ["workspace"] }, "system": { "type": "object", "properties": { - "release": { - "type": "string", - "enum": ["wheezy"] - }, - "architecture": { - "type": "string", - "enum": ["i386", "amd64"] - }, - "timezone": { "type": "string" }, - "locale": { "type": "string" }, - "charmap": { "type": "string" } + "release": { "enum": ["wheezy"] }, + "architecture": { "enum": ["i386", "amd64"] }, + "timezone": { "type": "string" }, + "locale": { "type": "string" }, + "charmap": { "type": "string" } }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { - "type": "object" - // "properties": { - // "backing": { - // "type": "string" - // } - // }, - // "required": ["backing"] + "type": "object", + "properties": { + "backing": { "type": "string" }, + "partitions": { + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/no_partitions" }, + { "$ref": "#/definitions/partition_table" } + ] + } + }, + "required": ["partitions"] }, "plugins": { "type": "object", @@ -57,5 +56,44 @@ "additionalProperties": false } }, - "required": ["provider", "bootstrapper", "volume", "system"] + "required": ["provider", "bootstrapper", "volume", "system"], + "definitions": { + "path": { + "type": "string", + "pattern": "^[^\\0]+$" + }, + "no_partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none"] }, + "root": { "$ref": "#/definitions/partition" } + }, + "required": ["root"], + "additionalProperties": false + }, + "partition_table": { + "type": "object", + "properties": { + "type": { "enum": ["mbr", "gpt"] }, + "boot": { "$ref": "#/definitions/partition" }, + "root": { "$ref": "#/definitions/partition" }, + "swap": { + "type": "object", + "properties": { "size": { "type": "integer", "minimum": 1 } }, + "required": ["size"] + } + }, + "required": ["root"], + "additionalProperties": false + }, + "partition": { + "type": "object", + "properties": { + "size": { "type": "integer", "minimum": 1 }, + "filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] } + }, + "required": ["size", "filesystem"] + } + }, + "required": ["provider", "bootstrapper", "system", "volume"] } diff --git a/providers/ec2/manifest-schema-s3.json b/providers/ec2/manifest-schema-s3.json index 654629e..f9cf5d2 100644 --- a/providers/ec2/manifest-schema-s3.json +++ b/providers/ec2/manifest-schema-s3.json @@ -25,6 +25,17 @@ } }, "required": ["bucket"] + }, + "volume": { + "type": "object", + "properties": { + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none"] } + } + } + } } }, "required": ["image"] diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index 43fda5d..d56bf25 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -3,6 +3,17 @@ "title": "EC2 manifest", "type": "object", "properties": { + "image": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + } + } + }, "credentials": { "type": "object", "properties": { @@ -17,13 +28,10 @@ "volume": { "type": "object", "properties": { - "backing": { - "type": "string", - "enum": ["ebs", "s3"] - } + "backing": { "enum": ["ebs", "s3"] } }, "required": ["backing"] } }, - "required": ["volume"] + "required": ["image"] } diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 5e2b58b..476dede 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -10,13 +10,8 @@ "type": "string", "enum": ["raw", "vdi", "qcow2"] } - // "filesystem": { - // "type": "string", - // "enum": ["ext2", "ext3", "ext4", "xfs"] - // } }, "required": ["backing"] } - }, - "required": ["volume"] + } } From c62dcccd8dad7fab3e2dcda99db7937238890ed7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Mon, 4 Nov 2013 17:31:32 +0100 Subject: [PATCH 57/63] credential keys now actually work when set via env all credential keys are converted to uppercase, dashes replaced with underscores and prefixed with 'AWS_' --- providers/ec2/tasks/connection.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/providers/ec2/tasks/connection.py b/providers/ec2/tasks/connection.py index c6d47df..65d9871 100644 --- a/providers/ec2/tasks/connection.py +++ b/providers/ec2/tasks/connection.py @@ -20,9 +20,12 @@ class GetCredentials(Task): for key in keys: creds[key] = manifest.credentials[key] return creds - if all(getenv(key) is not None for key in keys): + + def env_key(key): + return ('aws-'+key).upper().replace('-', '_') + if all(getenv(env_key(key)) is not None for key in keys): for key in keys: - creds[key] = getenv(key) + creds[key] = getenv(env_key(key)) return creds raise RuntimeError(('No ec2 credentials found, they must all be specified ' 'exclusively via environment variables or through the manifest.')) From ae1327811bfe565871ae453456ec42d57a12913d Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:17:55 +0100 Subject: [PATCH 58/63] Manifest cleanup, begin with simple manifests only No plugins. They should be explained in the documentation --- manifests/ec2-ebs-partitioned.manifest.json | 33 +++++++++++++ manifests/ec2-ebs-pvm.manifest.json | 53 --------------------- manifests/ec2-ebs.manifest.json | 33 +++++++++++++ manifests/ec2-s3-pvm.manifest.json | 43 ----------------- manifests/ec2-s3.manifest.json | 37 ++++++++++++++ manifests/kvm-virtio.manifest.json | 39 --------------- manifests/one-ide.manifest.json | 31 ------------ manifests/one-virtio.manifest.json | 31 ------------ manifests/virtualbox.manifest.json | 6 +-- 9 files changed, 106 insertions(+), 200 deletions(-) create mode 100644 manifests/ec2-ebs-partitioned.manifest.json delete mode 100644 manifests/ec2-ebs-pvm.manifest.json create mode 100644 manifests/ec2-ebs.manifest.json delete mode 100644 manifests/ec2-s3-pvm.manifest.json create mode 100644 manifests/ec2-s3.manifest.json delete mode 100644 manifests/kvm-virtio.manifest.json delete mode 100644 manifests/one-ide.manifest.json delete mode 100644 manifests/one-virtio.manifest.json diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json new file mode 100644 index 0000000..41666dc --- /dev/null +++ b/manifests/ec2-ebs-partitioned.manifest.json @@ -0,0 +1,33 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "mbr", + "root": { + "size": 1023, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json deleted file mode 100644 index d932077..0000000 --- a/manifests/ec2-ebs-pvm.manifest.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "provider": "ec2", - "virtualization": "pvm", - "credentials": { - "access-key": null, - "secret-key": null - }, - - "bootstrapper": { - "workspace": "/target", - "mirror": "http://http.debian.net/debian" - }, - "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" - }, - "system": { - "release": "wheezy", - "architecture": "amd64", - "timezone": "UTC", - "locale": "en_US", - "charmap": "UTF-8" - }, - "volume": { - "backing": "ebs", - "partition_table": { - "type": null, - "partitions": [ - {"size": 8192, - "filesystem": "ext4", - "flags": ["root"]} - ] - } - }, - "plugins": { - "admin_user": { - "enabled": true, - "username": "admin" - }, - "backports": { - "enabled": true, - "packages": [ "cloud-init" ] - }, - "build_metadata": { - "enabled": false, - "path": "/root/build-metadata-{ami_name}" - }, - "prebootstrapped": { - "enabled": false, - "snapshot": "" - } - } -} diff --git a/manifests/ec2-ebs.manifest.json b/manifests/ec2-ebs.manifest.json new file mode 100644 index 0000000..b8c6456 --- /dev/null +++ b/manifests/ec2-ebs.manifest.json @@ -0,0 +1,33 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/ec2-s3-pvm.manifest.json b/manifests/ec2-s3-pvm.manifest.json deleted file mode 100644 index 9c8a5a5..0000000 --- a/manifests/ec2-s3-pvm.manifest.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "provider": "ec2", - "virtualization": "pvm", - "credentials": { - "access-key": null, - "secret-key": null, - "certificate": null, - "private-key": null, - "user-id": null - }, - - "bootstrapper": { - "workspace": "/target", - "mirror": "http://http.debian.net/debian" - }, - "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" - "bucket": "" - }, - "system": { - "release": "wheezy", - "architecture": "amd64", - "timezone": "UTC", - "locale": "en_US", - "charmap": "UTF-8" - }, - "volume": { - "backing": "s3", - "filesystem": "ext4", - "size": 1024 - }, - "plugins": { - "admin_user": { - "enabled": true, - "username": "admin" - }, - "prebootstrapped": { - "enabled": false, - "image": null - } - } -} diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json new file mode 100644 index 0000000..93100d0 --- /dev/null +++ b/manifests/ec2-s3.manifest.json @@ -0,0 +1,37 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null, + // "certificate": null, + // "private-key": null, + // "user-id": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI", + "bucket": "debian-amis" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "s3", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json deleted file mode 100644 index 7076d92..0000000 --- a/manifests/kvm-virtio.manifest.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "provider" : "kvm", - "virtualization": "virtio", - - "bootstrapper": { - "workspace": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024 - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "opennebula": { - "enabled": true - } - } -} diff --git a/manifests/one-ide.manifest.json b/manifests/one-ide.manifest.json deleted file mode 100644 index 05264f4..0000000 --- a/manifests/one-ide.manifest.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "provider" : "one", - "virtualization": "ide", - - "bootstrapper": { - "workspace": "/target" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024 - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - } - } -} diff --git a/manifests/one-virtio.manifest.json b/manifests/one-virtio.manifest.json deleted file mode 100644 index 930a16d..0000000 --- a/manifests/one-virtio.manifest.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "provider" : "one", - "virtualization": "virtio", - - "bootstrapper": { - "workspace": "/target" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024 - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - } - } -} diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index d3ce81f..71913ec 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -16,7 +16,7 @@ "charmap" : "UTF-8" }, "volume": { - "backing": "vdi", + "backing": "vdi", "partitions": { "type": "mbr", "boot": { @@ -26,8 +26,8 @@ "root": { "size": 991, "filesystem": "ext4" - } - // ,"swap": {"size": 128} + }, + "swap": {"size": 128} } } } From 1edf89df62331041f81d5b4cfe723a946ea2de60 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:22:24 +0100 Subject: [PATCH 59/63] First stab at official debian manifest It will probably need some adjustment --- .../ec2-ebs-debian-official.manifest.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 manifests/ec2-ebs-debian-official.manifest.json diff --git a/manifests/ec2-ebs-debian-official.manifest.json b/manifests/ec2-ebs-debian-official.manifest.json new file mode 100644 index 0000000..6292fe5 --- /dev/null +++ b/manifests/ec2-ebs-debian-official.manifest.json @@ -0,0 +1,39 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } + }, + "plugins": { + "admin_user": { + "enabled": true, + "username": "admin" + } + } +} From c165bab68ea02e06572305a6d6ba4dfd86122d31 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:23:58 +0100 Subject: [PATCH 60/63] Delete incomplete kvm provider Not sure if it ever worked, it was basically a copy of the virtualbox plugin --- providers/kvm/README.md | 9 --- providers/kvm/__init__.py | 92 -------------------------- providers/kvm/assets/grub.d/00_header | 4 -- providers/kvm/assets/grub.d/10_linux | 93 --------------------------- providers/kvm/manifest-schema.json | 22 ------- providers/kvm/manifest.py | 14 ---- providers/kvm/tasks/__init__.py | 0 providers/kvm/tasks/boot.py | 55 ---------------- providers/kvm/tasks/packages.py | 48 -------------- 9 files changed, 337 deletions(-) delete mode 100644 providers/kvm/README.md delete mode 100644 providers/kvm/__init__.py delete mode 100644 providers/kvm/assets/grub.d/00_header delete mode 100644 providers/kvm/assets/grub.d/10_linux delete mode 100644 providers/kvm/manifest-schema.json delete mode 100644 providers/kvm/manifest.py delete mode 100644 providers/kvm/tasks/__init__.py delete mode 100644 providers/kvm/tasks/boot.py delete mode 100644 providers/kvm/tasks/packages.py diff --git a/providers/kvm/README.md b/providers/kvm/README.md deleted file mode 100644 index 87e28a3..0000000 --- a/providers/kvm/README.md +++ /dev/null @@ -1,9 +0,0 @@ -this provider creates images for KVM. - -It is possible to add opennebula plugin for OpenNebula images. - -Disk virtuzalition is specified by the virtualization field in the manifest. -Allowed values are: - -* ide: basic disk emulation on /dev/sda1 -* virtio: enhanced performance on /dev/vda1. It adds virtio modules in the kernel and needs configuration update in VM template: diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py deleted file mode 100644 index 8337e7b..0000000 --- a/providers/kvm/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -from manifest import Manifest -from tasks import packages -from common.tasks import packages as common_packages -from common.tasks import host -from common.tasks import loopback -from common.tasks import parted -from common.tasks import filesystem -from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt -from tasks import boot -from common.tasks import boot as common_boot -from common.tasks import security -from common.tasks import network -from common.tasks import initd -from common.tasks import cleanup -from common.tasks import loopback - - -def initialize(): - pass - - -def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - host.CheckPackages(), - - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), - - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.FStab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.ConfigureDHCP(), - initd.ResolveInitScripts(), - initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), - - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) - - if manifest.bootstrapper.get('tarball', False): - tasklist.add(bootstrap.MakeTarball()) - - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) - - -def rollback_tasks(tasklist, tasks_completed, manifest): - completed = [type(task) for task in tasks_completed] - - def counter_task(task, counter): - if task in completed and counter not in completed: - tasklist.add(counter()) - - counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) diff --git a/providers/kvm/assets/grub.d/00_header b/providers/kvm/assets/grub.d/00_header deleted file mode 100644 index 43927bc..0000000 --- a/providers/kvm/assets/grub.d/00_header +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -set -e - -# nothing to do, skip grub mkconfig for this diff --git a/providers/kvm/assets/grub.d/10_linux b/providers/kvm/assets/grub.d/10_linux deleted file mode 100644 index 0b08346..0000000 --- a/providers/kvm/assets/grub.d/10_linux +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# This file generates the old menu.lst configuration with grub2 -# It was copied from tomheadys github repo: -# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom - -prefix=/usr -exec_prefix=${prefix} -bindir=${exec_prefix}/bin -libdir=${exec_prefix}/lib -. ${libdir}/grub/grub-mkconfig_lib - -export TEXTDOMAIN=grub -export TEXTDOMAINDIR=${prefix}/share/locale - -GRUB_DEVICE=/dev/sda1 - - -cat << EOF -set default=${GRUB_DEFAULT} -set timeout=${GRUB_TIMEOUT} -insmod part_msdos -insmod ext2 -insmod gettext -set menu_color_normal=cyan/blue -set menu_color_highlight=white/blue -set root='(hd0,msdos1)' -EOF - -if ${GRUB_HIDDEN_TIMEOUT:-false}; then - printf "hiddenmenu\n" -fi - -linux_entry () -{ - os="$1" - version="$2" - args="$4" - - title="$(gettext_quoted "%s, with Linux %s")" - - cat << EOF -menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { - insmod part_msdos - insmod ext2 - set timeout=${GRUB_TIMEOUT} - set root='(hd0,msdos1)' - echo 'Loading Linux ${version}' - linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} - echo 'Loading initial ramdisk ...' - initrd ${rel_dirname}/${initrd} -} -EOF -} - -list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do - if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi - done` -prepare_boot_cache= - -while [ "x$list" != "x" ] ; do - linux=`version_find_latest $list` - basename=`basename $linux` - dirname=`dirname $linux` - rel_dirname=`make_system_path_relative_to_its_root $dirname` - version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` - alt_version=`echo $version | sed -e "s,\.old$,,g"` - linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" - - initrd= - for i in "initrd.img-${version}" "initrd-${version}.img" \ - "initrd-${version}" "initramfs-${version}.img" \ - "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ - "initrd-${alt_version}" "initramfs-${alt_version}.img"; do - if test -e "${dirname}/${i}" ; then - initrd="$i" - break - fi - done - - initramfs= - for i in "config-${version}" "config-${alt_version}"; do - if test -e "${dirname}/${i}" ; then - initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` - break - fi - done - - linux_entry "${OS}" "${version}" \ - "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" - - list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` -done diff --git a/providers/kvm/manifest-schema.json b/providers/kvm/manifest-schema.json deleted file mode 100644 index c7fbd83..0000000 --- a/providers/kvm/manifest-schema.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Kvm manifest", - "type": "object", - "properties": { - "volume": { - "type": "object", - "properties": { - "backing": { - "type": "string", - "enum": ["raw", "qcow2"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] - } - }, - "required": ["backing", "filesystem"] - } - }, - "required": ["volume"] -} diff --git a/providers/kvm/manifest.py b/providers/kvm/manifest.py deleted file mode 100644 index 8ae196b..0000000 --- a/providers/kvm/manifest.py +++ /dev/null @@ -1,14 +0,0 @@ -import base - - -class Manifest(base.Manifest): - def validate(self, data): - super(Manifest, self).validate(data) - from os import path - schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') - self.schema_validate(data, schema_path) - - def parse(self, data): - super(Manifest, self).parse(data) - self.image = data['image'] - self.virtualization = data['virtualization'] diff --git a/providers/kvm/tasks/__init__.py b/providers/kvm/tasks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/providers/kvm/tasks/boot.py b/providers/kvm/tasks/boot.py deleted file mode 100644 index 73a6ea0..0000000 --- a/providers/kvm/tasks/boot.py +++ /dev/null @@ -1,55 +0,0 @@ -from base import Task -from common import phases - - -class ConfigureGrub(Task): - description = 'Configuring grub for KVM' - phase = phases.system_modification - - def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - - from common.tools import log_check_call - - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - - if info.manifest.virtualization == 'virtio': - print "Using virtio" - modules_path = os.path.join(info.root, - 'etc/initramfs-tools/modules') - with open(modules_path, 'a') as modules: - modules.write("\nvirtio_pci\nvirtio_blk\n") - - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']]) - - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - - if info.manifest.virtualization == 'virtio': - from common.tools import sed_i - grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg') - sed_i(grub_cfg, 'sda', 'vda') - device_map = os.path.join(info.root, - 'boot/grub/device.map') - sed_i(device_map, 'sda', 'vda') - fstab_file = os.path.join(info.root, - 'etc/fstab') - sed_i(fstab_file, 'sda', 'vda') - diff --git a/providers/kvm/tasks/packages.py b/providers/kvm/tasks/packages.py deleted file mode 100644 index c04bce0..0000000 --- a/providers/kvm/tasks/packages.py +++ /dev/null @@ -1,48 +0,0 @@ -from base import Task -from common import phases -from common.tasks import packages -from common.tasks.host import CheckPackages - - -class HostPackages(Task): - description = 'Determining required host packages' - phase = phases.preparation - before = [CheckPackages] - after = [packages.HostPackages] - - def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') - - -class ImagePackages(Task): - description = 'Determining required image packages' - phase = phases.preparation - after = [packages.ImagePackages] - - def run(self, info): - manifest = info.manifest - include, exclude = info.img_packages - # Add some basic packages we are going to need - include.update(['parted', - 'kpartx', - # Needed for the init scripts - 'file', - # isc-dhcp-client doesn't work properly with ec2 - 'dhcpcd', - 'chkconfig', - 'openssh-client', - 'grub2' - ]) - - exclude.update(['isc-dhcp-client', - 'isc-dhcp-common', - ]) - - # In squeeze, we need a special kernel flavor for xen - kernels = {'squeeze': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, - 'wheezy': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, } - include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) From 90826662acabc27335dbacacd2b0eccb0a9d9068 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:33:18 +0100 Subject: [PATCH 61/63] Disallow gpt for ebs AMIs --- providers/ec2/manifest-schema.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index d56bf25..e937744 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -28,7 +28,13 @@ "volume": { "type": "object", "properties": { - "backing": { "enum": ["ebs", "s3"] } + "backing": { "enum": ["ebs", "s3"] }, + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none", "mbr"] } + } + } }, "required": ["backing"] } From 9b906aeaf8e745f5947fe64461f4e520456e74be Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:34:49 +0100 Subject: [PATCH 62/63] Disallow gpt for virtualbox --- providers/virtualbox/manifest-schema.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 476dede..e9b1474 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -9,6 +9,12 @@ "backing": { "type": "string", "enum": ["raw", "vdi", "qcow2"] + }, + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none", "mbr"] } + } } }, "required": ["backing"] From bcd1735ebd9e9e8526024741f6189c087df9fabe Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 6 Nov 2013 22:35:07 +0100 Subject: [PATCH 63/63] Remove unneeded virtualbox assets folder --- providers/virtualbox/assets/grub.d/00_header | 4 - providers/virtualbox/assets/grub.d/10_linux | 93 -------------------- 2 files changed, 97 deletions(-) delete mode 100644 providers/virtualbox/assets/grub.d/00_header delete mode 100644 providers/virtualbox/assets/grub.d/10_linux diff --git a/providers/virtualbox/assets/grub.d/00_header b/providers/virtualbox/assets/grub.d/00_header deleted file mode 100644 index 43927bc..0000000 --- a/providers/virtualbox/assets/grub.d/00_header +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -set -e - -# nothing to do, skip grub mkconfig for this diff --git a/providers/virtualbox/assets/grub.d/10_linux b/providers/virtualbox/assets/grub.d/10_linux deleted file mode 100644 index 0b08346..0000000 --- a/providers/virtualbox/assets/grub.d/10_linux +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# This file generates the old menu.lst configuration with grub2 -# It was copied from tomheadys github repo: -# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom - -prefix=/usr -exec_prefix=${prefix} -bindir=${exec_prefix}/bin -libdir=${exec_prefix}/lib -. ${libdir}/grub/grub-mkconfig_lib - -export TEXTDOMAIN=grub -export TEXTDOMAINDIR=${prefix}/share/locale - -GRUB_DEVICE=/dev/sda1 - - -cat << EOF -set default=${GRUB_DEFAULT} -set timeout=${GRUB_TIMEOUT} -insmod part_msdos -insmod ext2 -insmod gettext -set menu_color_normal=cyan/blue -set menu_color_highlight=white/blue -set root='(hd0,msdos1)' -EOF - -if ${GRUB_HIDDEN_TIMEOUT:-false}; then - printf "hiddenmenu\n" -fi - -linux_entry () -{ - os="$1" - version="$2" - args="$4" - - title="$(gettext_quoted "%s, with Linux %s")" - - cat << EOF -menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { - insmod part_msdos - insmod ext2 - set timeout=${GRUB_TIMEOUT} - set root='(hd0,msdos1)' - echo 'Loading Linux ${version}' - linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} - echo 'Loading initial ramdisk ...' - initrd ${rel_dirname}/${initrd} -} -EOF -} - -list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do - if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi - done` -prepare_boot_cache= - -while [ "x$list" != "x" ] ; do - linux=`version_find_latest $list` - basename=`basename $linux` - dirname=`dirname $linux` - rel_dirname=`make_system_path_relative_to_its_root $dirname` - version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` - alt_version=`echo $version | sed -e "s,\.old$,,g"` - linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" - - initrd= - for i in "initrd.img-${version}" "initrd-${version}.img" \ - "initrd-${version}" "initramfs-${version}.img" \ - "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ - "initrd-${alt_version}" "initramfs-${alt_version}.img"; do - if test -e "${dirname}/${i}" ; then - initrd="$i" - break - fi - done - - initramfs= - for i in "config-${version}" "config-${alt_version}"; do - if test -e "${dirname}/${i}" ; then - initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` - break - fi - done - - linux_entry "${OS}" "${version}" \ - "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" - - list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` -done