diff --git a/README.md b/README.md index ea4d99d..67db323 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,17 @@ Pull requests are also welcome! Dependencies ------------ You will need to run debian wheezy with **python 2.7** and **debootstrap** installed. +Other depencies include: +* qemu-utils +* parted +* grub2 +* euca2ools +* xfsprogs (If you want to use XFS as a filesystem) Also the following python libraries are required: * **boto** * **jsomschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip) * **termcolor** +* **fysom** Bootstrapping instance store AMIs requires **euca2ools** to be installed. diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index d1c03ec..ff425b9 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -3,6 +3,11 @@ class BootstrapInformation(object): def __init__(self, manifest=None, debug=False): self.manifest = manifest + from fs import load_volume + self.volume = load_volume(self.manifest.volume) self.debug = debug import random self.run_id = random.randrange(16 ** 8) + import os.path + workspace_dirname = '{id:x}'.format(id=self.run_id) + self.workspace = os.path.join(manifest.bootstrapper['workspace'], workspace_dirname) diff --git a/base/fs/__init__.py b/base/fs/__init__.py new file mode 100644 index 0000000..5f1689c --- /dev/null +++ b/base/fs/__init__.py @@ -0,0 +1,20 @@ + + +def load_volume(data): + from common.fs.loopbackvolume import LoopbackVolume + from providers.ec2.ebsvolume import EBSVolume + from providers.virtualbox.volume import VirtualBoxVolume + from partitionmaps.gpt import GPTPartitionMap + from partitionmaps.mbr import MBRPartitionMap + from partitionmaps.none import NoPartitions + partition_maps = {'none': NoPartitions, + 'gpt': GPTPartitionMap, + 'mbr': MBRPartitionMap, + } + partition_map = partition_maps.get(data['partitions']['type'])(data['partitions']) + volume_backings = {'raw': LoopbackVolume, + 's3': LoopbackVolume, + 'vdi': VirtualBoxVolume, + 'ebs': EBSVolume + } + return volume_backings.get(data['backing'])(partition_map) diff --git a/base/fs/exceptions.py b/base/fs/exceptions.py new file mode 100644 index 0000000..bc38490 --- /dev/null +++ b/base/fs/exceptions.py @@ -0,0 +1,8 @@ + + +class VolumeError(Exception): + pass + + +class PartitionError(Exception): + pass diff --git a/providers/kvm/tasks/__init__.py b/base/fs/partitionmaps/__init__.py similarity index 100% rename from providers/kvm/tasks/__init__.py rename to base/fs/partitionmaps/__init__.py diff --git a/base/fs/partitionmaps/abstract.py b/base/fs/partitionmaps/abstract.py new file mode 100644 index 0000000..bac76c4 --- /dev/null +++ b/base/fs/partitionmaps/abstract.py @@ -0,0 +1,78 @@ +from abc import ABCMeta +from abc import abstractmethod +from common.tools import log_check_call +from common.fsm_proxy import FSMProxy +from ..exceptions import PartitionError + + +class AbstractPartitionMap(FSMProxy): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self): + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(AbstractPartitionMap, self).__init__(cfg) + + def is_blocking(self): + return self.fsm.current == 'mapped' + + def get_total_size(self): + return sum(p.size for p in self.partitions) + + def create(self, volume): + self.fsm.create(volume=volume) + + @abstractmethod + def _before_create(self, event): + pass + + def map(self, volume): + self.fsm.map(volume=volume) + + def _before_map(self, event): + volume = event.volume + try: + mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) + import re + regexp = re.compile('^(?P.+[^\d](?P\d+)) : ' + '(?P\d) (?P\d+) ' + '{device_path} (?P\d+)$' + .format(device_path=volume.device_path)) + log_check_call(['/sbin/kpartx', '-a', volume.device_path]) + import os.path + for mapping in mappings: + match = regexp.match(mapping) + if match is None: + raise PartitionError('Unable to parse kpartx output: {line}'.format(line=mapping)) + partition_path = os.path.join('/dev/mapper', match.group('name')) + p_idx = int(match.group('p_idx'))-1 + self.partitions[p_idx].map(partition_path) + + for idx, partition in enumerate(self.partitions): + if partition.fsm.current not in ['mapped', 'formatted']: + raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx+1)) + + except PartitionError as e: + for partition in self.partitions: + if not partition.fsm.can('unmap'): + partition.unmap() + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + raise e + + def unmap(self, volume): + self.fsm.unmap(volume=volume) + + def _before_unmap(self, event): + volume = event.volume + for partition in self.partitions: + if partition.fsm.cannot('unmap'): + msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) + raise PartitionError(msg) + log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + for partition in self.partitions: + partition.unmap() diff --git a/base/fs/partitionmaps/gpt.py b/base/fs/partitionmaps/gpt.py new file mode 100644 index 0000000..7a95943 --- /dev/null +++ b/base/fs/partitionmaps/gpt.py @@ -0,0 +1,36 @@ +from abstract import AbstractPartitionMap +from ..partitions.gpt import GPTPartition +from ..partitions.gpt_swap import GPTSwapPartition +from common.tools import log_check_call + + +class GPTPartitionMap(AbstractPartitionMap): + + def __init__(self, data): + self.partitions = [] + if 'boot' in data: + self.boot = GPTPartition(data['boot']['size'], data['boot']['filesystem'], 'boot', None) + self.partitions.append(self.boot) + self.root = GPTPartition(data['root']['size'], data['root']['filesystem'], 'root', + getattr(self, 'boot', None)) + self.partitions.append(self.root) + if 'swap' in data: + self.swap = GPTSwapPartition(data['swap']['size'], self.root) + self.partitions.append(self.swap) + + super(GPTPartitionMap, self).__init__() + + def _before_create(self, event): + volume = event.volume + log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, + '--', 'mklabel', 'gpt']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if hasattr(self, 'boot'): + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' boot on']) + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' bios_grub on']) diff --git a/base/fs/partitionmaps/mbr.py b/base/fs/partitionmaps/mbr.py new file mode 100644 index 0000000..67c40cb --- /dev/null +++ b/base/fs/partitionmaps/mbr.py @@ -0,0 +1,37 @@ +from abstract import AbstractPartitionMap +from ..partitions.mbr import MBRPartition +from ..partitions.mbr_swap import MBRSwapPartition +from common.tools import log_check_call + + +class MBRPartitionMap(AbstractPartitionMap): + + def __init__(self, data): + self.partitions = [] + if 'boot' in data: + self.boot = MBRPartition(data['boot']['size'], data['boot']['filesystem'], None) + self.partitions.append(self.boot) + self.root = MBRPartition(data['root']['size'], data['root']['filesystem'], + getattr(self, 'boot', None)) + self.partitions.append(self.root) + if 'swap' in data: + self.swap = MBRSwapPartition(data['swap']['size'], self.root) + self.partitions.append(self.swap) + + super(MBRPartitionMap, self).__init__() + + def get_total_size(self): + return sum(p.size for p in self.partitions) + 1 # Post-MBR gap for embedding grub + + def _before_create(self, event): + volume = event.volume + log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, + '--', 'mklabel', 'msdos']) + for partition in self.partitions: + partition.create(volume) + + boot_idx = self.root.get_index() + if hasattr(self, 'boot'): + boot_idx = self.boot.get_index() + log_check_call(['/sbin/parted', '--script', volume.device_path, + '--', 'set ' + str(boot_idx) + ' boot on']) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py new file mode 100644 index 0000000..d623874 --- /dev/null +++ b/base/fs/partitionmaps/none.py @@ -0,0 +1,15 @@ +from ..partitions.single import SinglePartition + + +class NoPartitions(object): + + def __init__(self, data): + root = data['root'] + self.root = SinglePartition(root['size'], root['filesystem']) + self.partitions = [self.root] + + def is_blocking(self): + return self.root.fsm == 'mounted' + + def get_total_size(self): + return self.root.size diff --git a/base/fs/partitions/__init__.py b/base/fs/partitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py new file mode 100644 index 0000000..bb2354c --- /dev/null +++ b/base/fs/partitions/abstract.py @@ -0,0 +1,41 @@ +from abc import ABCMeta +from common.tools import log_check_call +from common.fsm_proxy import FSMProxy + + +class AbstractPartition(FSMProxy): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + def __init__(self, size, filesystem): + self.size = size + self.filesystem = filesystem + self.device_path = None + + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(AbstractPartition, self).__init__(cfg) + + def get_uuid(self): + [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) + return uuid + + def _before_format(self, e): + mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) + log_check_call([mkfs, self.device_path]) + + def mount(self, destination): + self.fsm.mount(destination=destination) + + def _before_mount(self, e): + log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) + self.mount_dir = e.destination + + def _before_unmount(self, e): + log_check_call(['/bin/umount', self.mount_dir]) + del self.mount_dir diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py new file mode 100644 index 0000000..3f5792f --- /dev/null +++ b/base/fs/partitions/base.py @@ -0,0 +1,43 @@ +from abstract import AbstractPartition + + +class BasePartition(AbstractPartition): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, filesystem, previous): + self.previous = previous + super(BasePartition, self).__init__(size, filesystem) + + def create(self, volume): + self.fsm.create(volume=volume) + + def get_index(self): + if self.previous is None: + return 1 + else: + return self.previous.get_index()+1 + + def get_start(self): + if self.previous is None: + return 0 + else: + return self.previous.get_start() + self.previous.size + + def map(self, device_path): + self.fsm.map(device_path=device_path) + + def _before_map(self, e): + self.device_path = e.device_path + + def _before_unmap(self, e): + self.device_path = None diff --git a/base/fs/partitions/gpt.py b/base/fs/partitions/gpt.py new file mode 100644 index 0000000..a50f76d --- /dev/null +++ b/base/fs/partitions/gpt.py @@ -0,0 +1,19 @@ +from common.tools import log_check_call +from base import BasePartition + + +class GPTPartition(BasePartition): + + def __init__(self, size, filesystem, name, previous): + self.name = name + super(GPTPartition, self).__init__(size, filesystem, previous) + + def _before_create(self, e): + start = self.get_start() + # {name} only works for gpt, for msdos that becomes the part-type (primary, extended, logical) + parted_command = ('mkpart primary {start}MiB {end}MiB' + .format(name=self.name, + start=str(start), + end=str(start + self.size))) + log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, + '--', parted_command]) diff --git a/base/fs/partitions/gpt_swap.py b/base/fs/partitions/gpt_swap.py new file mode 100644 index 0000000..0217770 --- /dev/null +++ b/base/fs/partitions/gpt_swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from gpt import GPTPartition + + +class GPTSwapPartition(GPTPartition): + + def __init__(self, size, previous): + super(GPTSwapPartition, self).__init__(size, 'swap', 'swap', previous) + + def _before_format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/mbr.py b/base/fs/partitions/mbr.py new file mode 100644 index 0000000..d10c2d8 --- /dev/null +++ b/base/fs/partitions/mbr.py @@ -0,0 +1,19 @@ +from common.tools import log_check_call +from base import BasePartition + + +class MBRPartition(BasePartition): + + def get_start(self): + if self.previous is None: + return 1 # Post-MBR gap for embedding grub + else: + return self.previous.get_start() + self.previous.size + + def _before_create(self, e): + start = self.get_start() + parted_command = ('mkpart primary {start}MiB {end}MiB' + .format(start=str(start), + end=str(start + self.size))) + log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, + '--', parted_command]) diff --git a/base/fs/partitions/mbr_swap.py b/base/fs/partitions/mbr_swap.py new file mode 100644 index 0000000..4e62e4b --- /dev/null +++ b/base/fs/partitions/mbr_swap.py @@ -0,0 +1,11 @@ +from common.tools import log_check_call +from mbr import MBRPartition + + +class MBRSwapPartition(MBRPartition): + + def __init__(self, size, previous): + super(MBRSwapPartition, self).__init__(size, 'swap', previous) + + def _before_format(self, e): + log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py new file mode 100644 index 0000000..2017c91 --- /dev/null +++ b/base/fs/partitions/single.py @@ -0,0 +1,5 @@ +from abstract import AbstractPartition + + +class SinglePartition(AbstractPartition): + pass diff --git a/base/fs/volume.py b/base/fs/volume.py new file mode 100644 index 0000000..e83ad5b --- /dev/null +++ b/base/fs/volume.py @@ -0,0 +1,64 @@ +from abc import ABCMeta +from common.fsm_proxy import FSMProxy +from common.tools import log_check_call +from exceptions import VolumeError +from partitionmaps.none import NoPartitions + + +class Volume(FSMProxy): + + __metaclass__ = ABCMeta + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, + ] + + def __init__(self, partition_map): + self.device_path = None + self.specials_mounted = False + self.partition_map = partition_map + self.size = self.partition_map.get_total_size() + + callbacks = {'onbeforedetach': self._check_blocking} + if isinstance(self.partition_map, NoPartitions): + def set_dev_path(e): + self.partition_map.root.device_path = self.device_path + callbacks['onafterattach'] = set_dev_path + + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks} + super(Volume, self).__init__(cfg) + + def _after_create(self, e): + if isinstance(self.partition_map, NoPartitions): + self.partition_map.root.create() + + def can_mount_specials(self): + return self.fsm.current == 'attached' + + def mount_specials(self): + if self.specials_mounted: + raise VolumeError('The special devices are already mounted') + root = self.partition_map.root.mount_dir + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=root)]) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + self.specials_mounted = True + + def unmount_specials(self): + if not self.specials_mounted: + raise VolumeError('The special devices are not mounted') + root = self.partition_map.root.mount_dir + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=root)]) + self.specials_mounted = False + + def _check_blocking(self, e): + if self.partition_map.is_blocking(): + raise VolumeError('The partitionmap prevents the detach procedure') + if self.specials_mounted: + raise VolumeError('The special devices are mounted and prevent the detaching procedure') diff --git a/base/main.py b/base/main.py index e5dcd64..32b9aba 100644 --- a/base/main.py +++ b/base/main.py @@ -15,6 +15,10 @@ def get_args(): parser = ArgumentParser(description='Bootstrap Debian for the cloud.') parser.add_argument('--debug', action='store_true', help='Print debugging information') + parser.add_argument('--pause-on-error', action='store_true', + help='Pause on error, before rollback') + parser.add_argument('--dry-run', action='store_true', + help='Dont\'t actually run the tasks') parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') return parser.parse_args() @@ -33,10 +37,12 @@ def run(args): bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug) try: - tasklist.run(bootstrap_info) + tasklist.run(info=bootstrap_info, dry_run=args.dry_run) log.info('Successfully completed bootstrapping') except (Exception, KeyboardInterrupt) as e: log.exception(e) + if args.pause_on_error: + raw_input("Press Enter to commence rollback") log.error('Rolling back') rollback_tasklist = TaskList() provider.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) @@ -44,5 +50,5 @@ def run(args): rollback_tasks = getattr(plugin, 'rollback_tasks', None) if callable(rollback_tasks): plugin.rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) - rollback_tasklist.run(bootstrap_info) + rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run) log.info('Successfully completed rollback') diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 6c5627d..0db594d 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -9,39 +9,36 @@ "bootstrapper": { "type": "object", "properties": { - "mount_dir": { "type": "string" }, - "mirror": { "type": "string" }, - "tarball": { "type": "boolean" }, - "tarball_dir": { "type": "string" } + "workspace": { "$ref": "#/definitions/path" }, + "mirror": { "type": "string", "format": "uri" }, + "tarball": { "type": "boolean" } }, - "required": ["mount_dir"] + "required": ["workspace"] }, "system": { "type": "object", "properties": { - "release": { - "type": "string", - "enum": ["wheezy"] - }, - "architecture": { - "type": "string", - "enum": ["i386", "amd64"] - }, - "timezone": { "type": "string" }, - "locale": { "type": "string" }, - "charmap": { "type": "string" } + "release": { "enum": ["wheezy"] }, + "architecture": { "enum": ["i386", "amd64"] }, + "timezone": { "type": "string" }, + "locale": { "type": "string" }, + "charmap": { "type": "string" } }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { "type": "object", "properties": { - "size": { - "type": "integer", - "minimum": 1 + "backing": { "type": "string" }, + "partitions": { + "type": "object", + "oneOf": [ + { "$ref": "#/definitions/no_partitions" }, + { "$ref": "#/definitions/partition_table" } + ] } }, - "required": ["size"] + "required": ["partitions"] }, "plugins": { "type": "object", @@ -59,5 +56,44 @@ "additionalProperties": false } }, - "required": ["provider", "bootstrapper", "volume", "system"] + "required": ["provider", "bootstrapper", "volume", "system"], + "definitions": { + "path": { + "type": "string", + "pattern": "^[^\\0]+$" + }, + "no_partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none"] }, + "root": { "$ref": "#/definitions/partition" } + }, + "required": ["root"], + "additionalProperties": false + }, + "partition_table": { + "type": "object", + "properties": { + "type": { "enum": ["mbr", "gpt"] }, + "boot": { "$ref": "#/definitions/partition" }, + "root": { "$ref": "#/definitions/partition" }, + "swap": { + "type": "object", + "properties": { "size": { "type": "integer", "minimum": 1 } }, + "required": ["size"] + } + }, + "required": ["root"], + "additionalProperties": false + }, + "partition": { + "type": "object", + "properties": { + "size": { "type": "integer", "minimum": 1 }, + "filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] } + }, + "required": ["size", "filesystem"] + } + }, + "required": ["provider", "bootstrapper", "system", "volume"] } diff --git a/base/manifest.py b/base/manifest.py index 7aebdae..df17372 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -49,10 +49,6 @@ class Manifest(object): self.bootstrapper = data['bootstrapper'] if 'mirror' not in self.bootstrapper: self.bootstrapper['mirror'] = 'http://http.debian.net/debian' - if 'tarball' not in self.bootstrapper: - self.bootstrapper['tarball'] = False - if 'tarball_dir' not in self.bootstrapper and self.bootstrapper['tarball']: - self.bootstrapper['tarball_dir'] = '/tmp' self.volume = data['volume'] self.system = data['system'] self.plugins = data['plugins'] if 'plugins' in data else {} diff --git a/base/task.py b/base/task.py index c3841a6..8a10462 100644 --- a/base/task.py +++ b/base/task.py @@ -1,4 +1,3 @@ -from common.exceptions import TaskListError class Task(object): @@ -6,27 +5,8 @@ class Task(object): before = [] after = [] - def __init__(self): - self._check_ordering() - def __str__(self): return '{module}.{task}'.format(module=self.__module__, task=self.__class__.__name__) def __repr__(self): return self.__str__() - - def _check_ordering(self): - def name(ref): - return '{module}.{task}'.format(module=ref.__module__, task=ref.__class__.__name__) - for task in self.before: - if self.phase > task.phase: - msg = ("The task {self} is specified as running before {other}, " - "but its phase '{phase}' lies after the phase '{other_phase}'" - .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase)) - raise TaskListError(msg) - for task in self.after: - if self.phase < task.phase: - msg = ("The task {self} is specified as running after {other}, " - "but its phase '{phase}' lies before the phase '{other_phase}'" - .format(self=type(self), other=task, phase=self.phase, other_phase=task.phase)) - raise TaskListError(msg) diff --git a/base/tasklist.py b/base/tasklist.py index d786abd..fe07755 100644 --- a/base/tasklist.py +++ b/base/tasklist.py @@ -13,40 +13,34 @@ class TaskList(object): self.tasks.update(args) def remove(self, *args): - for task_type in args: - task = self.get(task_type) - if task is not None: - self.tasks.discard(task) + for task in args: + self.tasks.discard(task) - def replace(self, task, replacement): - self.remove(task) - self.add(replacement) - - def get(self, ref): - return next((task for task in self.tasks if type(task) is ref), None) - - def run(self, bootstrap_info): + def run(self, info={}, dry_run=False): task_list = self.create_list(self.tasks) log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list))) - for task in task_list: + for task_type in task_list: + task = task_type() if hasattr(task, 'description'): log.info(task.description) else: log.info('Running {task}'.format(task=task)) - task.run(bootstrap_info) + if not dry_run: + task.run(info) self.tasks_completed.append(task) def create_list(self, tasks): from common.phases import order graph = {} for task in tasks: - successors = [] - successors.extend([self.get(succ) for succ in task.before]) - successors.extend(filter(lambda succ: type(task) in succ.after, tasks)) + self.check_ordering(task) + successors = set() + successors.update(task.before) + successors.update(filter(lambda succ: task in succ.after, tasks)) succeeding_phases = order[order.index(task.phase) + 1:] - successors.extend(filter(lambda succ: succ.phase in succeeding_phases, tasks)) - graph[task] = filter(lambda succ: succ in self.tasks, successors) + successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks)) + graph[task] = filter(lambda succ: succ in tasks, successors) components = self.strongly_connected_components(graph) cycles_found = 0 @@ -63,6 +57,20 @@ class TaskList(object): return sorted_tasks + def check_ordering(self, task): + for successor in task.before: + if successor.phase > successor.phase: + msg = ("The task {task} is specified as running before {other}, " + "but its phase '{phase}' lies after the phase '{other_phase}'" + .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase)) + raise TaskListError(msg) + for predecessor in task.after: + if task.phase < predecessor.phase: + msg = ("The task {task} is specified as running after {other}, " + "but its phase '{phase}' lies before the phase '{other_phase}'" + .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase)) + raise TaskListError(msg) + def strongly_connected_components(self, graph): # Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py # Find the strongly connected components in a graph using Tarjan's algorithm. diff --git a/common/assets/init.d/expand-volume b/common/assets/init.d/expand-volume index 3b2d2a6..cfee83c 100644 --- a/common/assets/init.d/expand-volume +++ b/common/assets/init.d/expand-volume @@ -13,7 +13,7 @@ prog=$(basename $0) logger="logger -t $prog" -device_path="/dev/xvda1" +device_path="/dev/xvda" filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'` diff --git a/common/fs/__init__.py b/common/fs/__init__.py new file mode 100644 index 0000000..3cde7cf --- /dev/null +++ b/common/fs/__init__.py @@ -0,0 +1,39 @@ + + +def get_partitions(): + import re + regexp = re.compile('^ *(?P\d+) *(?P\d+) *(?P\d+) (?P\S+)$') + matches = {} + path = '/proc/partitions' + with open(path) as partitions: + next(partitions) + next(partitions) + for line in partitions: + match = regexp.match(line) + if match is None: + raise RuntimeError('Unable to parse {line} in {path}'.format(line=line, path=path)) + matches[match.group('dev_name')] = match.groupdict() + return matches + + +def remount(volume, fn): + from base.fs.partitionmaps.none import NoPartitions + + p_map = volume.partition_map + volume.unmount_specials() + if hasattr(p_map, 'boot'): + boot_dir = p_map.boot.mount_dir + p_map.boot.unmount() + root_dir = p_map.root.mount_dir + p_map.root.unmount() + if not isinstance(p_map, NoPartitions): + p_map.unmap(volume) + result = fn() + p_map.map(volume) + else: + result = fn() + p_map.root.mount(root_dir) + if hasattr(p_map, 'boot'): + p_map.boot.mount(boot_dir) + volume.mount_specials() + return result diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py new file mode 100644 index 0000000..81269b9 --- /dev/null +++ b/common/fs/loopbackvolume.py @@ -0,0 +1,80 @@ +from base.fs.volume import Volume +from common.tools import log_check_call +from base.fs.exceptions import VolumeError + + +# QEMU formats: +# raw, host_device, qcow2, qcow, cow, vdi, vmdk, vpc, cloop + + +class LoopbackVolume(Volume): + + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, + {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, + {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'}, + {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'attached'}, + {'name': 'detach', 'src': 'attached', 'dst': 'detached'}, + {'name': 'delete', 'src': 'detached', 'dst': 'deleted'}, + ] + + extension = 'raw' + + def can_mount_specials(self): + return self.fsm.current in ['attached', 'linked'] + + def create(self, image_path): + self.fsm.create(image_path=image_path) + + def _before_create(self, e): + self.image_path = e.image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, str(self.size) + 'M']) + + def _before_attach(self, e): + [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) + self.device_path = self.loop_device_path + + def _before_link_dm_node(self, e): + import os.path + from . import get_partitions + proc_partitions = get_partitions() + loop_device_name = os.path.basename(self.loop_device_path) + loop_device_partition = proc_partitions[loop_device_name] + + sectors = self.size*1024*1024/512 + table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' + .format(log_start_sec=0, + sectors=sectors, + major=loop_device_partition['major'], + minor=loop_device_partition['minor'], + start_sec=0)) + import string + import os.path + for letter in string.ascii_lowercase: + dev_name = 'vd' + letter + dev_path = os.path.join('/dev/mapper', dev_name) + if not os.path.exists(dev_path): + self.dm_node_name = dev_name + self.dm_node_path = dev_path + break + + if not hasattr(self, 'dm_node_name'): + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) + self.device_path = self.dm_node_path + + def _before_unlink_dm_node(self, e): + log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) + del self.dm_node_name + del self.dm_node_path + self.device_path = self.loop_device_path + + def _before_detach(self, e): + log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _before_delete(self, e): + from os import remove + remove(self.image_path) + del self.image_path diff --git a/common/fsm_proxy.py b/common/fsm_proxy.py new file mode 100644 index 0000000..db3d17a --- /dev/null +++ b/common/fsm_proxy.py @@ -0,0 +1,42 @@ + + +class FSMProxy(object): + + def __init__(self, cfg): + from fysom import Fysom + events = set([event['name'] for event in cfg['events']]) + cfg['callbacks'] = self.collect_event_listeners(events, cfg['callbacks']) + self.fsm = Fysom(cfg) + self.attach_proxy_methods(self.fsm, events) + + def collect_event_listeners(self, events, callbacks): + callbacks = callbacks.copy() + callback_names = [] + for event in events: + callback_names.append(('_before_' + event, 'onbefore' + event)) + callback_names.append(('_after_' + event, 'onafter' + event)) + for fn_name, listener in callback_names: + fn = getattr(self, fn_name, None) + if callable(fn): + if listener in callbacks: + old_fn = callbacks[listener] + + def wrapper(e, old_fn=old_fn, fn=fn): + old_fn(e) + fn(e) + callbacks[listener] = wrapper + else: + callbacks[listener] = fn + return callbacks + + def attach_proxy_methods(self, fsm, events): + def make_proxy(fsm, event): + fn = getattr(fsm, event) + + def proxy(): + fn() + return proxy + + for event in events: + if not hasattr(self, event): + setattr(self, event, make_proxy(fsm, event)) diff --git a/common/task_sets.py b/common/task_sets.py new file mode 100644 index 0000000..07e7a31 --- /dev/null +++ b/common/task_sets.py @@ -0,0 +1,74 @@ +from common.tasks import workspace +from common.tasks import packages +from common.tasks import host +from common.tasks import bootstrap +from common.tasks import volume +from common.tasks import filesystem +from common.tasks import partitioning +from common.tasks import cleanup +from common.tasks import apt +from common.tasks import security +from common.tasks import locale + +base_set = [workspace.CreateWorkspace, + packages.HostPackages, + packages.ImagePackages, + host.CheckPackages, + bootstrap.Bootstrap, + workspace.DeleteWorkspace, + ] + +volume_set = [volume.Attach, + volume.Detach, + filesystem.Format, + filesystem.FStab, + ] + +partitioning_set = [partitioning.PartitionVolume, + partitioning.MapPartitions, + partitioning.UnmapPartitions, + ] + +boot_partition_set = [filesystem.CreateBootMountDir, + filesystem.MountBoot, + filesystem.UnmountBoot, + ] + +mounting_set = [filesystem.CreateMountDir, + filesystem.MountRoot, + filesystem.MountSpecials, + filesystem.UnmountSpecials, + filesystem.UnmountRoot, + filesystem.DeleteMountDir, + ] + +ssh_set = [security.DisableSSHPasswordAuthentication, + security.DisableSSHDNSLookup, + cleanup.ShredHostkeys, + ] + +apt_set = [apt.DisableDaemonAutostart, + apt.AptSources, + apt.AptUpgrade, + apt.PurgeUnusedPackages, + apt.AptClean, + apt.EnableDaemonAutostart, + ] + +locale_set = [locale.GenerateLocale, + locale.SetTimezone, + ] + + +def get_fs_specific_set(partitions): + task_set = {'ext2': [filesystem.TuneVolumeFS], + 'ext3': [filesystem.TuneVolumeFS], + 'ext4': [filesystem.TuneVolumeFS], + 'xfs': [filesystem.AddXFSProgs], + } + tasks = set() + if 'boot' in partitions: + tasks.update(task_set.get(partitions['boot']['filesystem'], [])) + if 'root' in partitions: + tasks.update(task_set.get(partitions['root']['filesystem'], [])) + return tasks diff --git a/common/tasks/bootstrap.py b/common/tasks/bootstrap.py index 68ffc5f..42e1da9 100644 --- a/common/tasks/bootstrap.py +++ b/common/tasks/bootstrap.py @@ -29,7 +29,7 @@ class MakeTarball(Task): hash_args = [arg for arg in arguments if arg != info.root] tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) - info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + info.tarball = os.path.join(info.manifest.bootstrapper['workspace'], tarball_filename) if os.path.isfile(info.tarball): log.debug('Found matching tarball, skipping download') else: diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index c0b5a75..80288a7 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -1,29 +1,30 @@ from base import Task from common import phases -from common.exceptions import TaskError from common.tools import log_check_call from bootstrap import Bootstrap +import volume -class FormatVolume(Task): +class Format(Task): description = 'Formatting the volume' phase = phases.volume_preparation def run(self, info): - dev_path = info.bootstrap_device['path'] - mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) - log_check_call([mkfs, dev_path]) - info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']} + for partition in info.volume.partition_map.partitions: + partition.format() class TuneVolumeFS(Task): description = 'Tuning the bootstrap volume filesystem' phase = phases.volume_preparation - after = [FormatVolume] + after = [Format] def run(self, info): + import re # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']]) + for partition in info.volume.partition_map.partitions: + if re.match('^ext[2-4]$', partition.filesystem) is not None: + log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): @@ -36,33 +37,43 @@ class AddXFSProgs(Task): class CreateMountDir(Task): - description = 'Creating mountpoint for the bootstrap volume' + description = 'Creating mountpoint for the root partition' phase = phases.volume_mounting def run(self, info): import os - mount_dir = info.manifest.bootstrapper['mount_dir'] - info.root = '{mount_dir}/{id:x}'.format(mount_dir=mount_dir, id=info.run_id) - # Works recursively, fails if last part exists, which is exaclty what we want. + info.root = os.path.join(info.workspace, 'root') os.makedirs(info.root) -class MountVolume(Task): - description = 'Mounting the bootstrap volume' +class MountRoot(Task): + description = 'Mounting the root partition' phase = phases.volume_mounting after = [CreateMountDir] def run(self, info): - with open('/proc/mounts') as mounts: - for mount in mounts: - if info.root in mount: - msg = 'Something is already mounted at {root}'.format(root=info.root) - raise TaskError(msg) + info.volume.partition_map.root.mount(info.root) - log_check_call(['/bin/mount', - '--types', info.manifest.volume['filesystem'], - info.bootstrap_device['partitions']['root_path'], - info.root]) + +class MountBoot(Task): + description = 'Mounting the boot partition' + phase = phases.volume_mounting + after = [MountRoot] + + def run(self, info): + info.volume.partition_map.boot.mount(info.boot_dir) + + +class CreateBootMountDir(Task): + description = 'Creating mountpoint for the boot partition' + phase = phases.volume_mounting + after = [MountRoot] + before = [MountBoot] + + def run(self, info): + import os + info.boot_dir = os.path.join(info.root, 'boot') + os.makedirs(info.boot_dir) class MountSpecials(Task): @@ -71,36 +82,40 @@ class MountSpecials(Task): after = [Bootstrap] def run(self, info): - log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'proc', 'none', '/proc']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'sysfs', 'none', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '--types', 'devpts', 'none', '/dev/pts']) + info.volume.mount_specials() + + +class UnmountRoot(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + before = [volume.Detach] + + def run(self, info): + info.volume.partition_map.root.unmount() + + +class UnmountBoot(Task): + description = 'Unmounting the boot partition' + phase = phases.volume_unmounting + before = [UnmountRoot] + + def run(self, info): + info.volume.partition_map.boot.unmount() class UnmountSpecials(Task): description = 'Unmunting special block devices' phase = phases.volume_unmounting + before = [UnmountRoot] def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) - log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) - log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) - - -class UnmountVolume(Task): - description = 'Unmounting the bootstrap volume' - phase = phases.volume_unmounting - after = [UnmountSpecials] - - def run(self, info): - log_check_call(['/bin/umount', info.root]) + info.volume.unmount_specials() class DeleteMountDir(Task): description = 'Deleting mountpoint for the bootstrap volume' phase = phases.volume_unmounting - after = [UnmountVolume] + after = [UnmountRoot] def run(self, info): import os @@ -108,24 +123,44 @@ class DeleteMountDir(Task): del info.root -class ModifyFstab(Task): - description = 'Adding root volume to the fstab' +class FStab(Task): + description = 'Adding partitions to the fstab' phase = phases.system_modification def run(self, info): import os.path - mount_opts = ['defaults'] - if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if info.manifest.volume['filesystem'].lower() == 'xfs': - mount_opts.append('nobarrier') - fstab_path = os.path.join(info.root, 'etc/fstab') + p_map = info.volume.partition_map + mount_points = [{'path': '/', + 'partition': p_map.root, + 'dump': '1', + 'pass_num': '1', + }] + if hasattr(p_map, 'boot'): + mount_points.append({'path': '/boot', + 'partition': p_map.boot, + 'dump': '1', + 'pass_num': '2', + }) + if hasattr(p_map, 'swap'): + mount_points.append({'path': 'none', + 'partition': p_map.swap, + 'dump': '1', + 'pass_num': '0', + }) - device = '/dev/sda1' - if info.manifest.virtualization == 'pvm': - device = '/dev/xvda1' - with open(fstab_path, 'a') as fstab: - fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' - .format(device=device, - filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts))) + fstab_lines = [] + for mount_point in mount_points: + partition = mount_point['partition'] + mount_opts = ['defaults'] + fstab_lines.append('UUID={uuid} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}' + .format(uuid=partition.get_uuid(), + mountpoint=mount_point['path'], + filesystem=partition.filesystem, + mount_opts=','.join(mount_opts), + dump=mount_point['dump'], + pass_num=mount_point['pass_num'])) + + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) + fstab.write('\n') diff --git a/common/tasks/host.py b/common/tasks/host.py index c377ecc..f62f6e0 100644 --- a/common/tasks/host.py +++ b/common/tasks/host.py @@ -14,7 +14,7 @@ class CheckPackages(Task): from subprocess import CalledProcessError for package in info.host_packages: try: - log_check_call(['/usr/bin/dpkg', '--status', package]) + log_check_call(['/usr/bin/dpkg-query', '-s', package]) except CalledProcessError: msg = "The package ``{0}\'\' is not installed".format(package) raise TaskError(msg) diff --git a/common/tasks/initd.py b/common/tasks/initd.py index 377a3d9..a0df088 100644 --- a/common/tasks/initd.py +++ b/common/tasks/initd.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.tools import log_check_call import os.path @@ -8,11 +9,18 @@ class ResolveInitScripts(Task): phase = phases.system_modification def run(self, info): - init_scripts = {'expand-volume': 'expand-volume'} + init_scripts = {} + init_scripts['expand-volume'] = 'expand-volume' - init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' - if info.manifest.system['release'] == 'squeeze': - init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + from subprocess import CalledProcessError + try: + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/bin/dpkg-query', '-W', 'openssh-server']) + init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' + if info.manifest.system['release'] == 'squeeze': + init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + except CalledProcessError: + pass disable_scripts = ['hwclock.sh'] if info.manifest.system['release'] == 'squeeze': @@ -37,7 +45,6 @@ class InstallInitScripts(Task): stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) from shutil import copy - from common.tools import log_check_call for name, src in info.initd['install'].iteritems(): dst = os.path.join(info.root, 'etc/init.d', name) copy(src, dst) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index b801fc1..8a0b2c9 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -1,60 +1,29 @@ from base import Task from common import phases -from filesystem import UnmountVolume -from common.tools import log_check_call +import volume class Create(Task): description = 'Creating a loopback volume' phase = phases.volume_creation + before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/bin/dd', - 'if=/dev/zero', 'of=' + info.loopback_file, - 'bs=1M', 'seek=' + str(info.manifest.volume['size']), 'count=0']) + image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) + info.volume.create(image_path) -class CreateQemuImg(Task): - description = 'Creating a loopback volume with qemu' - phase = phases.volume_creation +class MoveImage(Task): + description = 'Moving volume image' + phase = phases.image_registration def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', - info.loopback_file, str(info.manifest.volume['size']) + 'M']) - - -class Attach(Task): - description = 'Attaching the loopback volume' - phase = phases.volume_creation - after = [Create, CreateQemuImg] - - def run(self, info): - info.bootstrap_device = {} - command = ['/sbin/losetup', '--show', '--find', info.loopback_file] - [info.bootstrap_device['path']] = log_check_call(command) - - -class Detach(Task): - description = 'Detaching the loopback volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - log_check_call(['/sbin/losetup', '--detach', info.bootstrap_device['path']]) - del info.bootstrap_device - - -class Delete(Task): - description = 'Deleting the loopback volume' - phase = phases.cleaning - - def run(self, info): - from os import remove - remove(info.loopback_file) - del info.loopback_file + filename = 'volume-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) + destination = os.path.join(info.manifest.bootstrapper['workspace'], filename) + import shutil + shutil.move(info.volume.image_path, destination) + import logging + log = logging.getLogger(__name__) + log.info('The volume image has been moved to {image_path}'.format(image_path=destination)) diff --git a/common/tasks/network.py b/common/tasks/network.py index 0910608..e3129d6 100644 --- a/common/tasks/network.py +++ b/common/tasks/network.py @@ -12,6 +12,15 @@ class RemoveDNSInfo(Task): remove(os.path.join(info.root, 'etc/resolv.conf')) +class RemoveHostname(Task): + description = 'Removing the hostname file' + phase = phases.system_modification + + def run(self, info): + from os import remove + remove(os.path.join(info.root, 'etc/hostname')) + + class ConfigureNetworkIF(Task): description = 'Configuring network interfaces' phase = phases.system_modification @@ -26,13 +35,3 @@ class ConfigureNetworkIF(Task): 'iface eth0 inet dhcp\n'} with open(interfaces_path, 'a') as interfaces: interfaces.write(if_config.get(info.manifest.system['release'])) - - -class ConfigureDHCP(Task): - description = 'Configuring the DHCP client' - phase = phases.system_modification - - def run(self, info): - from common.tools import sed_i - dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') - sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/common/tasks/packages.py b/common/tasks/packages.py index 2d868d9..41a3774 100644 --- a/common/tasks/packages.py +++ b/common/tasks/packages.py @@ -7,8 +7,19 @@ class HostPackages(Task): phase = phases.preparation def run(self, info): - packages = set(['debootstrap']) - info.host_packages = packages + info.host_packages = set() + info.host_packages.add('debootstrap') + + from common.fs.loopbackvolume import LoopbackVolume + if isinstance(info.volume, LoopbackVolume): + info.host_packages.add('qemu-utils') + + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): + info.host_packages.add('xfsprogs') + + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + info.host_packages.update(['parted', 'kpartx']) class ImagePackages(Task): @@ -16,12 +27,7 @@ class ImagePackages(Task): phase = phases.preparation def run(self, info): - # Add some basic packages we are going to need - include = set(['udev', - 'openssh-server', - # We could bootstrap without locales, but things just suck without them, error messages etc. - 'locales', - ]) - exclude = set() - - info.img_packages = include, exclude + info.img_packages = set(), set() + include, exclude = info.img_packages + # We could bootstrap without locales, but things just suck without them, error messages etc. + include.add('locales') diff --git a/common/tasks/parted.py b/common/tasks/parted.py deleted file mode 100644 index e242765..0000000 --- a/common/tasks/parted.py +++ /dev/null @@ -1,53 +0,0 @@ -from base import Task -from common import phases -from common.tools import log_check_call -import filesystem -import loopback - - -class PartitionVolume(Task): - description = 'Partitioning the volume' - phase = phases.volume_preparation - - def run(self, info): - # parted - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mklabel', 'msdos']) - log_check_call(['parted', '--align', 'optimal', '--script', info.bootstrap_device['path'], - '--', 'mkpart', 'primary', 'ext4', '32k', '-1']) - log_check_call(['parted', '--script', info.bootstrap_device['path'], - '--', 'set', '1', 'boot', 'on']) - - -class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - after = [PartitionVolume] - - def run(self, info): - root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper') + 'p1' - log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) - info.bootstrap_device['partitions'] = {'root_path': root_partition_path} - - -class FormatPartitions(Task): - description = 'Formatting the partitions' - phase = phases.volume_preparation - before = [filesystem.TuneVolumeFS] - after = [MapPartitions] - - def run(self, info): - # These params will fail for mkfs.xfs - log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), - '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']]) - - -class UnmapPartitions(Task): - description = 'Removing volume partitions mapping' - phase = phases.volume_unmounting - before = [loopback.Detach] - after = [filesystem.UnmountVolume] - - def run(self, info): - log_check_call(['kpartx', '-d', info.bootstrap_device['path']]) - del info.bootstrap_device['partitions']['root_path'] diff --git a/common/tasks/partitioning.py b/common/tasks/partitioning.py new file mode 100644 index 0000000..a0ec17b --- /dev/null +++ b/common/tasks/partitioning.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import filesystem +import volume + + +class PartitionVolume(Task): + description = 'Partitioning the volume' + phase = phases.volume_preparation + + def run(self, info): + info.volume.partition_map.create(info.volume) + + +class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + before = [filesystem.Format] + after = [PartitionVolume] + + def run(self, info): + info.volume.partition_map.map(info.volume) + + +class UnmapPartitions(Task): + description = 'Removing volume partitions mapping' + phase = phases.volume_unmounting + before = [volume.Detach] + after = [filesystem.UnmountRoot] + + def run(self, info): + info.volume.partition_map.unmap(info.volume) diff --git a/common/tasks/volume.py b/common/tasks/volume.py new file mode 100644 index 0000000..64be8eb --- /dev/null +++ b/common/tasks/volume.py @@ -0,0 +1,28 @@ +from base import Task +from common import phases +from common.tasks import workspace + + +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + + def run(self, info): + info.volume.attach() + + +class Detach(Task): + description = 'Detaching the volume' + phase = phases.volume_unmounting + + def run(self, info): + info.volume.detach() + + +class Delete(Task): + description = 'Deleting the volume' + phase = phases.cleaning + before = [workspace.DeleteWorkspace] + + def run(self, info): + info.volume.delete() diff --git a/common/tasks/workspace.py b/common/tasks/workspace.py new file mode 100644 index 0000000..836443b --- /dev/null +++ b/common/tasks/workspace.py @@ -0,0 +1,20 @@ +from base import Task +from common import phases + + +class CreateWorkspace(Task): + description = 'Creating workspace' + phase = phases.preparation + + def run(self, info): + import os + os.makedirs(info.workspace) + + +class DeleteWorkspace(Task): + description = 'Deleting workspace' + phase = phases.cleaning + + def run(self, info): + import os + os.rmdir(info.workspace) diff --git a/common/tools.py b/common/tools.py index 468394c..b6b2547 100644 --- a/common/tools.py +++ b/common/tools.py @@ -16,6 +16,7 @@ def log_call(command, stdin=None): from os.path import realpath command_log = realpath(command[0]).replace('/', '.') log = logging.getLogger(__name__ + command_log) + log.debug('Executing: {command}'.format(command=' '.join(command))) if stdin is not None: process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -31,13 +32,11 @@ def log_call(command, stdin=None): ret = select.select(reads, [], []) for fd in ret[0]: if fd == process.stdout.fileno(): - line = process.stdout.readline() - if line != '': + for line in iter(process.stdout.readline, ''): log.debug(line.strip()) stdout.append(line.strip()) if fd == process.stderr.fileno(): - line = process.stderr.readline() - if line != '': + for line in iter(process.stderr.readline, ''): log.error(line.strip()) stderr.append(line.strip()) if process.poll() is not None: diff --git a/manifests/ec2-s3-pvm.manifest.json b/manifests/ec2-ebs-debian-official.manifest.json similarity index 58% rename from manifests/ec2-s3-pvm.manifest.json rename to manifests/ec2-ebs-debian-official.manifest.json index 006de3b..6292fe5 100644 --- a/manifests/ec2-s3-pvm.manifest.json +++ b/manifests/ec2-ebs-debian-official.manifest.json @@ -2,21 +2,16 @@ "provider": "ec2", "virtualization": "pvm", "credentials": { - "access-key": null, - "secret-key": null, - "certificate": null, - "private-key": null, - "user-id": null + // "access-key": null, + // "secret-key": null }, "bootstrapper": { - "mount_dir": "/target", - "mirror": "http://http.debian.net/debian" + "workspace": "/target" }, "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}", + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", "description": "Debian {release} {architecture} AMI ({virtualization})" - "bucket": "" }, "system": { "release": "wheezy", @@ -26,18 +21,19 @@ "charmap": "UTF-8" }, "volume": { - "backing": "s3", - "filesystem": "ext4", - "size": 1024 + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } }, "plugins": { "admin_user": { "enabled": true, "username": "admin" - }, - "prebootstrapped": { - "enabled": false, - "image": null } } } diff --git a/manifests/ec2-ebs-partitioned.manifest.json b/manifests/ec2-ebs-partitioned.manifest.json new file mode 100644 index 0000000..41666dc --- /dev/null +++ b/manifests/ec2-ebs-partitioned.manifest.json @@ -0,0 +1,33 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "mbr", + "root": { + "size": 1023, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json deleted file mode 100644 index bb63082..0000000 --- a/manifests/ec2-ebs-pvm.manifest.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "provider": "ec2", - "virtualization": "pvm", - "credentials": { - "access-key": null, - "secret-key": null - }, - - "bootstrapper": { - "mount_dir": "/target", - "mirror": "http://http.debian.net/debian" - }, - "image": { - "name": "debian-{release}-{architecture}-{virtualization}-{%Y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" - }, - "system": { - "release": "wheezy", - "architecture": "amd64", - "timezone": "UTC", - "locale": "en_US", - "charmap": "UTF-8" - }, - "volume": { - "backing": "ebs", - "filesystem": "ext4", - "size": 8192 - }, - "plugins": { - "admin_user": { - "enabled": true, - "username": "admin" - }, - "backports": { - "enabled": true, - "packages": [ "cloud-init" ] - }, - "build_metadata": { - "enabled": false, - "path": "/root/build-metadata-{ami_name}" - }, - "prebootstrapped": { - "enabled": false, - "snapshot": "" - } - } -} diff --git a/manifests/ec2-ebs.manifest.json b/manifests/ec2-ebs.manifest.json new file mode 100644 index 0000000..b8c6456 --- /dev/null +++ b/manifests/ec2-ebs.manifest.json @@ -0,0 +1,33 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "ebs", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/ec2-s3.manifest.json b/manifests/ec2-s3.manifest.json new file mode 100644 index 0000000..93100d0 --- /dev/null +++ b/manifests/ec2-s3.manifest.json @@ -0,0 +1,37 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + // "access-key": null, + // "secret-key": null, + // "certificate": null, + // "private-key": null, + // "user-id": null + }, + + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI", + "bucket": "debian-amis" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "s3", + "partitions": { + "type": "none", + "root": { + "size": 1024, + "filesystem": "ext4" + } + } + } +} diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json deleted file mode 100644 index c4b828a..0000000 --- a/manifests/kvm-virtio.manifest.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "provider" : "kvm", - "virtualization": "virtio", - - "bootstrapper": { - "mount_dir": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "opennebula": { - "enabled": true - } - } -} diff --git a/manifests/one-ide.manifest.json b/manifests/one-ide.manifest.json deleted file mode 100644 index 1593005..0000000 --- a/manifests/one-ide.manifest.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "provider" : "one", - "virtualization": "ide", - - "bootstrapper": { - "mount_dir": "/target" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024 - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - } - } -} diff --git a/manifests/one-virtio.manifest.json b/manifests/one-virtio.manifest.json deleted file mode 100644 index 62a81f8..0000000 --- a/manifests/one-virtio.manifest.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "provider" : "one", - "virtualization": "virtio", - - "bootstrapper": { - "mount_dir": "/target" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024 - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - } - } -} diff --git a/manifests/virtualbox.json b/manifests/virtualbox.json deleted file mode 100644 index 238ef98..0000000 --- a/manifests/virtualbox.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "provider" : "virtualbox", - "virtualization": "ide", - - "bootstrapper": { - "mount_dir": "/mnt/target", - "mirror" : "http://ftp.fr.debian.org/debian/" - }, - "image": { - "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} ({virtualization})" - }, - "system": { - "release" : "wheezy", - "architecture": "amd64", - "timezone" : "UTC", - "locale" : "en_US", - "charmap" : "UTF-8" - }, - "volume": { - "backing" : "raw", - "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" - }, - "plugins": { - "user_packages": { - "enabled": true, - "repo": [ "apache2" ], - "local": [] - }, - "root_password": { - "enabled": true, - "password": "test" - }, - "convert_image": { - "enabled": true, - "format": "vdi" - } - } -} diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json new file mode 100644 index 0000000..71913ec --- /dev/null +++ b/manifests/virtualbox.manifest.json @@ -0,0 +1,33 @@ +{ + "provider" : "virtualbox", + "bootstrapper": { + "workspace": "/target", + "tarball": true + }, + "image": { + "name" : "debian-{release}-{architecture}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture}" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8" + }, + "volume": { + "backing": "vdi", + "partitions": { + "type": "mbr", + "boot": { + "size": 32, + "filesystem": "ext2" + }, + "root": { + "size": 991, + "filesystem": "ext4" + }, + "swap": {"size": 128} + } + } +} diff --git a/plugins/admin_user/__init__.py b/plugins/admin_user/__init__.py index 8c396c9..2916bd0 100644 --- a/plugins/admin_user/__init__.py +++ b/plugins/admin_user/__init__.py @@ -2,11 +2,11 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AddSudoPackage()) - tasklist.add(tasks.CreateAdminUser()) - tasklist.add(tasks.PasswordlessSudo()) - tasklist.add(tasks.AdminUserCredentials()) - tasklist.add(tasks.DisableRootLogin()) + tasklist.add(tasks.AddSudoPackage. + tasks.CreateAdminUser, + tasks.PasswordlessSudo, + tasks.AdminUserCredentials, + tasks.DisableRootLogin) def validate_manifest(data, schema_validate): diff --git a/plugins/backports/__init__.py b/plugins/backports/__init__.py index bdeae13..834c2d8 100644 --- a/plugins/backports/__init__.py +++ b/plugins/backports/__init__.py @@ -2,8 +2,8 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AptSourcesBackports()) - tasklist.add(tasks.AddBackportsPackages()) + tasklist.add(tasks.AptSourcesBackports, + tasks.AddBackportsPackages) def validate_manifest(data, schema_validate): diff --git a/plugins/build_metadata/__init__.py b/plugins/build_metadata/__init__.py index a8db8ba..71d1abf 100644 --- a/plugins/build_metadata/__init__.py +++ b/plugins/build_metadata/__init__.py @@ -2,4 +2,4 @@ def tasks(tasklist, manifest): from tasks import WriteMetadata - tasklist.add(WriteMetadata()) + tasklist.add(WriteMetadata diff --git a/plugins/convert_image/__init__.py b/plugins/convert_image/__init__.py index a36d8f4..f37792b 100644 --- a/plugins/convert_image/__init__.py +++ b/plugins/convert_image/__init__.py @@ -2,7 +2,7 @@ def tasks(tasklist, manifest): from tasks import ConvertImage - tasklist.add(ConvertImage()) + tasklist.add(ConvertImage) def validate_manifest(data, schema_validate): diff --git a/plugins/convert_image/tasks.py b/plugins/convert_image/tasks.py index c547417..c573d08 100644 --- a/plugins/convert_image/tasks.py +++ b/plugins/convert_image/tasks.py @@ -1,10 +1,12 @@ from base import Task from common import phases +from common.tasks import loopback class ConvertImage(Task): description = 'Converting raw image' phase = phases.image_registration + before = [loopback.MoveImage] def run(self, info): from common.tools import log_check_call diff --git a/plugins/opennebula/__init__.py b/plugins/opennebula/__init__.py index a9ae730..6ffcec9 100644 --- a/plugins/opennebula/__init__.py +++ b/plugins/opennebula/__init__.py @@ -2,4 +2,4 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.OpenNebulaContext()) + tasklist.add(tasks.OpenNebulaContext) diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index 8fd94ea..35e7518 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -2,39 +2,43 @@ from tasks import Snapshot from tasks import CopyImage from tasks import CreateFromSnapshot from tasks import CreateFromImage +from tasks import SetBootMountDir from providers.ec2.tasks import ebs from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap from common.tasks import filesystem -from common.tasks import parted +from common.tasks import partitioning def tasks(tasklist, manifest): settings = manifest.plugins['prebootstrapped'] + skip_tasks = [ebs.Create, + loopback.Create, + + filesystem.Format, + partitioning.PartitionVolume, + filesystem.TuneVolumeFS, + filesystem.AddXFSProgs, + filesystem.CreateBootMountDir, + bootstrap.MakeTarball, + bootstrap.Bootstrap] if manifest.volume['backing'] == 'ebs': if 'snapshot' in settings and settings['snapshot'] is not None: - tasklist.replace(ebs.Create, CreateFromSnapshot()) - tasklist.remove(filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.add(CreateFromSnapshot) + tasklist.remove(*skip_tasks) + if 'boot' in manifest.volume['partitions']: + tasklist.add(SetBootMountDir) else: - tasklist.add(Snapshot()) + tasklist.add(Snapshot) else: if 'image' in settings and settings['image'] is not None: - tasklist.add(CreateFromImage()) - tasklist.remove(loopback.Create, - loopback.CreateQemuImg, - parted.PartitionVolume, - parted.FormatPartitions, - filesystem.FormatVolume, - filesystem.TuneVolumeFS, - filesystem.AddXFSProgs, - bootstrap.MakeTarball, - bootstrap.Bootstrap) + tasklist.add(CreateFromImage) + tasklist.remove(*skip_tasks) + if 'boot' in manifest.volume['partitions']: + tasklist.add(SetBootMountDir) else: - tasklist.add(CopyImage()) + tasklist.add(CopyImage) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -42,12 +46,12 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) if manifest.volume['backing'] == 'ebs': - counter_task(CreateFromSnapshot, ebs.Delete) + counter_task(CreateFromSnapshot, volume.Delete) else: - counter_task(CreateFromImage, loopback.Delete) + counter_task(CreateFromImage, volume.Delete) def validate_manifest(data, schema_validate): diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index bcaebc8..d5fcb57 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -1,64 +1,96 @@ from base import Task from common import phases -from providers.ec2.tasks import ebs -from common.tasks import loopback +from common.tasks import volume from common.tasks import bootstrap +from common.tasks import filesystem +from common.fs import remount +from shutil import copyfile +import os.path import time import logging log = logging.getLogger(__name__) -class Snapshot(ebs.Snapshot): +class Snapshot(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.os_installation - after = [bootstrap.Bootstrap] + after = [bootstrap.Bootstrap, filesystem.MountSpecials] def run(self, info): - super(Snapshot, self).run(info) - msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=info.snapshot.id) + def mk_snapshot(): + return info.volume.snapshot() + snapshot = remount(info.volume, mk_snapshot) + msg = 'A snapshot of the bootstrapped volume was created. ID: {id}'.format(id=snapshot.id) log.info(msg) class CreateFromSnapshot(Task): description = 'Creating EBS volume from a snapshot' phase = phases.volume_creation - before = [ebs.Attach] + before = [volume.Attach] def run(self, info): volume_size = int(info.manifest.volume['size'] / 1024) snapshot = info.manifest.plugins['prebootstrapped']['snapshot'] - info.volume = info.connection.create_volume(volume_size, - info.host['availabilityZone'], - snapshot=snapshot) + info.volume.volume = info.connection.create_volume(volume_size, + info.host['availabilityZone'], + snapshot=snapshot) while info.volume.volume_state() != 'available': time.sleep(5) info.volume.update() + set_fs_states(info.volume) + class CopyImage(Task): description = 'Creating a snapshot of the bootstrapped volume' phase = phases.os_installation - after = [bootstrap.Bootstrap] + after = [bootstrap.Bootstrap, filesystem.MountSpecials] def run(self, info): - import os.path - from shutil import copyfile - loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) - image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.loopback_file, image_copy_path) - msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) + loopback_backup_name = 'volume-{id:x}.{ext}.backup'.format(id=info.run_id, ext=info.volume.extension) + destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) + + def mk_snapshot(): + copyfile(info.volume.image_path, destination) + remount(info.volume, mk_snapshot) + msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination) log.info(msg) class CreateFromImage(Task): description = 'Creating loopback image from a copy' phase = phases.volume_creation - before = [loopback.Attach] + before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - import os.path - info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] - from shutil import copyfile - copyfile(loopback_backup_path, info.loopback_file) + copyfile(loopback_backup_path, info.volume.image_path) + + set_fs_states(info.volume) + + +class SetBootMountDir(Task): + description = 'Setting mountpoint for the boot partition' + phase = phases.volume_mounting + after = [filesystem.MountRoot] + before = [filesystem.MountBoot] + + def run(self, info): + info.boot_dir = os.path.join(info.root, 'boot') + + +def set_fs_states(volume): + volume.fsm.current = 'detached' + + p_map = volume.partition_map + partitions_state = 'attached' + from base.fs.partitionmaps.none import NoPartitions + if isinstance(p_map, NoPartitions): + partitions_state = 'formatted' + else: + p_map.fsm.current = 'unmapped' + partitions_state = 'unmapped_fmt' + for partition in p_map.partitions: + partition.fsm.current = partitions_state diff --git a/plugins/root_password/__init__.py b/plugins/root_password/__init__.py index f3d4e87..91587b8 100644 --- a/plugins/root_password/__init__.py +++ b/plugins/root_password/__init__.py @@ -3,7 +3,8 @@ def tasks(tasklist, manifest): from common.tasks.security import DisableSSHPasswordAuthentication from tasks import SetRootPassword - tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword()) + tasklist.remove(DisableSSHPasswordAuthentication) + tasklist.add(SetRootPassword) def validate_manifest(data, schema_validate): diff --git a/plugins/unattended_upgrades/__init__.py b/plugins/unattended_upgrades/__init__.py index aa1d239..8025744 100644 --- a/plugins/unattended_upgrades/__init__.py +++ b/plugins/unattended_upgrades/__init__.py @@ -2,8 +2,8 @@ def tasks(tasklist, manifest): import tasks - tasklist.add(tasks.AddUnattendedUpgradesPackage()) - tasklist.add(tasks.EnablePeriodicUpgrades()) + tasklist.add(tasks.AddUnattendedUpgradesPackage, + tasks.EnablePeriodicUpgrades) def validate_manifest(data, schema_validate): diff --git a/plugins/user_packages/__init__.py b/plugins/user_packages/__init__.py index e223e0a..218975c 100644 --- a/plugins/user_packages/__init__.py +++ b/plugins/user_packages/__init__.py @@ -2,5 +2,5 @@ def tasks(tasklist, manifest): from user_packages import AddUserPackages, AddLocalUserPackages - tasklist.add(AddUserPackages()) - tasklist.add(AddLocalUserPackages()) + tasklist.add(AddUserPackages, + AddLocalUserPackages) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 04be78b..a383f7b 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -1,24 +1,25 @@ from manifest import Manifest import logging from tasks import packages -from common.tasks import packages as common_packages from tasks import connection from tasks import host -from common.tasks import host as common_host from tasks import ami +from common.tasks import volume as volume_tasks from tasks import ebs +from common.tasks import partitioning from common.tasks import loopback -from common.tasks import filesystem +from common.tasks import filesystem as common_filesystem +from tasks import filesystem from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security -from common.tasks import network +from tasks import network +from common.tasks import network as common_network from tasks import initd from common.tasks import initd as common_initd from common.tasks import cleanup +from common.tasks import workspace def initialize(): @@ -27,74 +28,68 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - common_host.CheckPackages(), - connection.GetCredentials(), - host.GetInfo(), - ami.AMIName(), - connection.Connect(), + from common.task_sets import base_set + from common.task_sets import mounting_set + from common.task_sets import apt_set + from common.task_sets import locale_set + from common.task_sets import ssh_set + tasklist.add(*base_set) + tasklist.add(*mounting_set) + tasklist.add(*apt_set) + tasklist.add(*locale_set) + tasklist.add(*ssh_set) - filesystem.FormatVolume(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), + if manifest.volume['partitions']['type'] != 'none': + from common.task_sets import partitioning_set + tasklist.add(*partitioning_set) - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.ModifyFstab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.ConfigureDHCP(), - common_initd.ResolveInitScripts(), - initd.AddEC2InitScripts(), - common_initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), + tasklist.add(packages.HostPackages, + packages.ImagePackages, + connection.GetCredentials, + host.GetInfo, + ami.AMIName, + connection.Connect, - filesystem.UnmountVolume(), - filesystem.DeleteMountDir(), - ami.RegisterAMI()) + boot.ConfigureGrub, + common_boot.BlackListModules, + common_boot.DisableGetTTYs, + security.EnableShadowConfig, + common_network.RemoveDNSInfo, + common_network.ConfigureNetworkIF, + network.EnableDHCPCDDNS, + common_initd.ResolveInitScripts, + initd.AddEC2InitScripts, + common_initd.InstallInitScripts, + initd.AdjustExpandVolumeScript, + cleanup.ClearMOTD, + cleanup.CleanTMP, - if manifest.bootstrapper['tarball']: - tasklist.add(bootstrap.MakeTarball()) + ami.RegisterAMI) - backing_specific_tasks = {'ebs': [ebs.Create(), - ebs.Attach(), - ebs.Detach(), - ebs.Snapshot(), - ebs.Delete()], - 's3': [loopback.Create(), - loopback.Attach(), - loopback.Detach(), - ami.BundleImage(), - ami.UploadImage(), - loopback.Delete(), - ami.RemoveBundle()]} + backing_specific_tasks = {'ebs': [ebs.Create, + ebs.Attach, + common_filesystem.FStab, + ebs.Snapshot], + 's3': [loopback.Create, + volume_tasks.Attach, + filesystem.S3FStab, + ami.BundleImage, + ami.UploadImage, + ami.RemoveBundle]} tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + tasklist.add(common_filesystem.Format, + volume_tasks.Detach, + volume_tasks.Delete) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + if manifest.bootstrapper.get('tarball', False): + tasklist.add(bootstrap.MakeTarball) + + from common.task_sets import get_fs_specific_set + tasklist.add(*get_fs_specific_set(manifest.volume['partitions'])) + + if 'boot' in manifest.volume['partitions']: + from common.task_sets import boot_partition_set + tasklist.add(*boot_partition_set) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -102,14 +97,20 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) - if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, ebs.Delete) - counter_task(ebs.Attach, ebs.Detach) - if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, loopback.Delete) - counter_task(loopback.Attach, loopback.Detach) - counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + counter_task(ebs.Create, volume_tasks.Delete) + counter_task(ebs.Attach, volume_tasks.Detach) + + counter_task(loopback.Create, volume_tasks.Delete) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir) + counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials) + + counter_task(common_filesystem.MountRoot, common_filesystem.UnmountRoot) + counter_task(common_filesystem.MountBoot, common_filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) + counter_task(ami.BundleImage, ami.RemoveBundle) diff --git a/providers/ec2/assets/grub.d/40_custom b/providers/ec2/assets/grub.d/40_custom index 799e887..0946bbd 100644 --- a/providers/ec2/assets/grub.d/40_custom +++ b/providers/ec2/assets/grub.d/40_custom @@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib export TEXTDOMAIN=grub export TEXTDOMAINDIR=${prefix}/share/locale -GRUB_DEVICE=/dev/xvda1 +GRUB_DEVICE=/dev/xvda cat << EOF diff --git a/providers/ec2/ebsvolume.py b/providers/ec2/ebsvolume.py new file mode 100644 index 0000000..29e9e41 --- /dev/null +++ b/providers/ec2/ebsvolume.py @@ -0,0 +1,57 @@ +from base.fs.volume import Volume +from base.fs.exceptions import VolumeError +import time + + +class EBSVolume(Volume): + + def create(self, conn, zone): + self.fsm.create(connection=conn, zone=zone) + + def _before_create(self, e): + conn = e.connection + zone = e.zone + import math + size = int(math.ceil(self.partition_map.get_total_size() / 1024)) + self.volume = conn.create_volume(size, zone) + while self.volume.volume_state() != 'available': + time.sleep(5) + self.volume.update() + + def attach(self, instance_id): + self.fsm.attach(instance_id=instance_id) + + def _before_attach(self, e): + instance_id = e.instance_id + import os.path + import string + for letter in string.ascii_lowercase[5:]: + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + self.device_path = dev_path + self.ec2_device_path = os.path.join('/dev', 'sd' + letter) + break + + if self.device_path is None: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + self.volume.attach(instance_id, self.ec2_device_path) + while self.volume.attachment_state() != 'attached': + time.sleep(2) + self.volume.update() + + def _before_detach(self, e): + self.volume.detach() + while self.volume.attachment_state() is not None: + time.sleep(2) + self.volume.update() + + def _before_delete(self, e): + self.volume.delete() + + def snapshot(self): + snapshot = self.volume.create_snapshot() + while snapshot.status != 'completed': + time.sleep(2) + snapshot.update() + return snapshot diff --git a/providers/ec2/manifest-schema-s3.json b/providers/ec2/manifest-schema-s3.json index 654629e..f9cf5d2 100644 --- a/providers/ec2/manifest-schema-s3.json +++ b/providers/ec2/manifest-schema-s3.json @@ -25,6 +25,17 @@ } }, "required": ["bucket"] + }, + "volume": { + "type": "object", + "properties": { + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none"] } + } + } + } } }, "required": ["image"] diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index ffa360d..e937744 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -3,6 +3,17 @@ "title": "EC2 manifest", "type": "object", "properties": { + "image": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + } + } + }, "credentials": { "type": "object", "properties": { @@ -17,17 +28,16 @@ "volume": { "type": "object", "properties": { - "backing": { - "type": "string", - "enum": ["ebs", "s3"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] + "backing": { "enum": ["ebs", "s3"] }, + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none", "mbr"] } + } } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, - "required": ["volume"] + "required": ["image"] } diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index 1d35897..10d816f 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -9,8 +9,10 @@ class Manifest(base.Manifest): schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') self.schema_validate(data, schema_path) if data['volume']['backing'] == 'ebs': - if data['volume']['size'] % 1024 != 0: - msg = 'The volume size must be a multiple of 1024 when using EBS backing' + volume_size = self._calculate_volume_size(data['volume']['partitions']) + if volume_size % 1024 != 0: + msg = ('The volume size must be a multiple of 1024 when using EBS backing ' + '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)') raise ManifestError(msg, self) else: schema_path = path.join(path.dirname(__file__), 'manifest-schema-s3.json') @@ -21,9 +23,15 @@ class Manifest(base.Manifest): self.credentials = data['credentials'] self.virtualization = data['virtualization'] self.image = data['image'] - if data['volume']['backing'] == 'ebs': - self.ebs_volume_size = data['volume']['size'] / 1024 - if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3': - self.volume['loopback_dir'] = '/tmp' - if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3': - self.image['bundle_dir'] = '/tmp' + + def _calculate_volume_size(self, partitions): + if partitions['type'] == 'mbr': + size = 1 + else: + size = 0 + if 'boot' in partitions: + size += partitions['boot']['size'] + size += partitions['root']['size'] + if 'swap' in partitions: + size += partitions['swap']['size'] + return size diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index be82f43..11e3fd8 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -3,6 +3,7 @@ from common import phases from common.exceptions import TaskError from common.tools import log_check_call from ebs import Snapshot +from common.tasks import workspace from connection import Connect import os.path @@ -45,9 +46,9 @@ class BundleImage(Task): def run(self, info): bundle_name = 'bundle-{id:x}'.format(id=info.run_id) - info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name) + info.bundle_path = os.path.join(info.workspace, bundle_name) log_check_call(['/usr/bin/euca-bundle-image', - '--image', info.loopback_file, + '--image', info.volume.image_path, '--user', info.credentials['user-id'], '--privatekey', info.credentials['private-key'], '--cert', info.credentials['certificate'], @@ -80,6 +81,7 @@ class UploadImage(Task): class RemoveBundle(Task): description = 'Removing the bundle files' phase = phases.cleaning + before = [workspace.DeleteWorkspace] def run(self, info): from shutil import rmtree @@ -92,46 +94,109 @@ class RegisterAMI(Task): phase = phases.image_registration after = [Snapshot, UploadImage] - kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', - 'i386': 'aki-b6aa75df'}, - 'us-west-1': {'amd64': 'aki-f77e26b2', - 'i386': 'aki-f57e26b0'}, - 'us-west-2': {'amd64': 'aki-fc37bacc', - 'i386': 'aki-fa37baca'}, - 'eu-west-1': {'amd64': 'aki-71665e05', - 'i386': 'aki-75665e01'}, - 'ap-southeast-1': {'amd64': 'aki-fe1354ac', - 'i386': 'aki-f81354aa'}, - 'ap-southeast-2': {'amd64': 'aki-31990e0b', - 'i386': 'aki-33990e09'}, - 'ap-northeast-1': {'amd64': 'aki-44992845', - 'i386': 'aki-42992843'}, - 'sa-east-1': {'amd64': 'aki-c48f51d9', - 'i386': 'aki-ca8f51d7'}, - 'us-gov-west-1': {'amd64': 'aki-79a4c05a', - 'i386': 'aki-7ba4c058'}} + # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs + kernel_mapping = {'ap-northeast-1': { # Asia Pacific (Tokyo) Region + 'hd0': {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-176bf516'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-196bf518', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-1f6bf51e'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-1': { # Asia Pacific (Singapore) Region + 'hd0': {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-503e7402'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-563e7404', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5e3e740c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'ap-southeast-2': { # Asia Pacific (Sydney) Region + 'hd0': {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-c362fff9'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-c162fffb', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-3b1d8001'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'eu-west-1': { # EU (Ireland) Region + 'hd0': {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-52a34525'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5ea34529', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-58a3452f'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'sa-east-1': { # South America (Sao Paulo) Region + 'hd0': {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-5553f448'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-5753f44a', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-5153f44c'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-east-1': { # US East (Northern Virginia) Region + 'hd0': {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-919dcaf8'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-659ccb0c', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-499ccb20'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-gov-west-1': { # AWS GovCloud (US) + 'hd0': {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-1de98d3e'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-63e98d40', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-61e98d42'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-1': { # US West (Northern California) Region + 'hd0': {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-880531cd'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-960531d3', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-920531d7'} # pv-grub-hd00_1.04-x86_64.gz + }, + 'us-west-2': { # US West (Oregon) Region + 'hd0': {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz + 'amd64': 'aki-fc8f11cc'}, # pv-grub-hd0_1.04-x86_64.gz + 'hd00': {'i386': 'aki-e28f11d2', # pv-grub-hd00_1.04-i386.gz + 'amd64': 'aki-e68f11d6'} # pv-grub-hd00_1.04-x86_64.gz + }} def run(self, info): - arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) - kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - if info.manifest.volume['backing'] == 'ebs': - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.manifest.ebs_volume_size) - block_device_map = BlockDeviceMapping() - block_device_map['/dev/sda1'] = block_device - - info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - block_device_map=block_device_map) + self.run_ebs(info) if info.manifest.volume['backing'] == 's3': - image_location = ('{bucket}/{ami_name}.manifest.xml' - .format(bucket=info.manifest.image['bucket'], - ami_name=info.ami_name)) - info.image = info.connection.register_image(description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - image_location=image_location) + self.run_s3(info) + + def run_ebs(self, info): + arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + + from base.fs.partitionmaps.none import NoPartitions + if isinstance(info.volume.partition_map, NoPartitions): + grub_boot_device = 'hd0' + root_device_name = '/dev/sda' + else: + grub_boot_device = 'hd00' + root_idx = info.volume.partition_map.root.get_index() + root_device_name = '/dev/sda{idx}'.format(idx=root_idx) + + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get(grub_boot_device) + .get(info.manifest.system['architecture'])) + + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=info.volume.partition_map.get_total_size()/1024) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda'] = block_device + + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name=root_device_name, + block_device_map=block_device_map) + + def run_s3(self, info): + arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + + kernel_id = (self.kernel_mapping + .get(info.host['region']) + .get('hd0') + .get(info.manifest.system['architecture'])) + + image_manifest = ('{bucket}/{ami_name}.manifest.xml' + .format(bucket=info.manifest.image['bucket'], + ami_name=info.ami_name)) + info.image = info.connection.register_image(description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='dev/sda1', + image_location=image_manifest) diff --git a/providers/ec2/tasks/boot.py b/providers/ec2/tasks/boot.py index cee03e8..48f8d27 100644 --- a/providers/ec2/tasks/boot.py +++ b/providers/ec2/tasks/boot.py @@ -24,6 +24,19 @@ class ConfigureGrub(Task): copy(script_src, script_dst) os.chmod(script_dst, rwxr_xr_x) + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + from common.tools import sed_i + root_idx = info.volume.partition_map.root.get_index() + grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx) + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device) + grub_root = '\troot (hd0,{idx})'.format(idx=root_idx-1) + sed_i(script_dst, '^\troot \(hd0\)$', grub_root) + + if info.manifest.volume['backing'] == 's3': + from common.tools import sed_i + sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1') + from common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' diff --git a/providers/ec2/tasks/connection.py b/providers/ec2/tasks/connection.py index c6d47df..65d9871 100644 --- a/providers/ec2/tasks/connection.py +++ b/providers/ec2/tasks/connection.py @@ -20,9 +20,12 @@ class GetCredentials(Task): for key in keys: creds[key] = manifest.credentials[key] return creds - if all(getenv(key) is not None for key in keys): + + def env_key(key): + return ('aws-'+key).upper().replace('-', '_') + if all(getenv(env_key(key)) is not None for key in keys): for key in keys: - creds[key] = getenv(key) + creds[key] = getenv(env_key(key)) return creds raise RuntimeError(('No ec2 credentials found, they must all be specified ' 'exclusively via environment variables or through the manifest.')) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index 85ad4b6..e8c363e 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,60 +1,22 @@ from base import Task from common import phases -from common.exceptions import TaskError -from common.tasks.filesystem import UnmountVolume -import time class Create(Task): - description = 'Creating an EBS volume for bootstrapping' + description = 'Creating the EBS volume' phase = phases.volume_creation def run(self, info): - info.volume = info.connection.create_volume(info.manifest.ebs_volume_size, - info.host['availabilityZone']) - while info.volume.volume_state() != 'available': - time.sleep(5) - info.volume.update() + info.volume.create(info.connection, info.host['availabilityZone']) class Attach(Task): - description = 'Attaching the EBS volume' + description = 'Attaching the volume' phase = phases.volume_creation after = [Create] def run(self, info): - def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" - for c in xrange(ord(c1), ord(c2) + 1): - yield chr(c) - - import os.path - info.bootstrap_device = {} - for letter in char_range('f', 'z'): - dev_path = os.path.join('/dev', 'xvd' + letter) - if not os.path.exists(dev_path): - info.bootstrap_device['path'] = dev_path - info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) - break - if 'path' not in info.bootstrap_device: - raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - - info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) - while info.volume.attachment_state() != 'attached': - time.sleep(2) - info.volume.update() - - -class Detach(Task): - description = 'Detaching the EBS volume' - phase = phases.volume_unmounting - after = [UnmountVolume] - - def run(self, info): - info.volume.detach() - while info.volume.attachment_state() is not None: - time.sleep(2) - info.volume.update() + info.volume.attach(info.host['instanceId']) class Snapshot(Task): @@ -62,20 +24,4 @@ class Snapshot(Task): phase = phases.image_registration def run(self, info): - info.snapshot = info.volume.create_snapshot() - while info.snapshot.status != 'completed': - time.sleep(2) - info.snapshot.update() - - -class Delete(Task): - description = 'Deleting the EBS volume' - phase = phases.cleaning - - def run(self, info): - info.volume.delete() - del info.volume - - -class VolumeError(TaskError): - pass + info.snapshot = info.volume.snapshot() diff --git a/providers/ec2/tasks/filesystem.py b/providers/ec2/tasks/filesystem.py new file mode 100644 index 0000000..c2de2a1 --- /dev/null +++ b/providers/ec2/tasks/filesystem.py @@ -0,0 +1,27 @@ +from base import Task +from common import phases + + +class S3FStab(Task): + description = 'Adding the S3 root partition to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + root = info.volume.partition_map.root + + fstab_lines = [] + mount_opts = ['defaults'] + fstab_lines.append('{device_path}{idx} {mountpoint} {filesystem} {mount_opts} {dump} {pass_num}' + .format(device_path='/dev/xvda', + idx=1, + mountpoint='/', + filesystem=root.filesystem, + mount_opts=','.join(mount_opts), + dump='1', + pass_num='1')) + + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'w') as fstab: + fstab.write('\n'.join(fstab_lines)) + fstab.write('\n') diff --git a/providers/ec2/tasks/initd.py b/providers/ec2/tasks/initd.py index 6f1d394..7be1667 100644 --- a/providers/ec2/tasks/initd.py +++ b/providers/ec2/tasks/initd.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.exceptions import TaskError from common.tasks import initd import os.path @@ -17,3 +18,22 @@ class AddEC2InitScripts(Task): init_scripts_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d')) for name, path in init_scripts.iteritems(): info.initd['install'][name] = os.path.join(init_scripts_dir, path) + + +class AdjustExpandVolumeScript(Task): + description = 'Adjusting the expand-volume script' + phase = phases.system_modification + after = [initd.InstallInitScripts] + + def run(self, info): + if 'expand-volume' not in info.initd['install']: + raise TaskError('The expand-volume script was not installed') + + from base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + import os.path + from common.tools import sed_i + script = os.path.join(info.root, 'etc/init.d.expand-volume') + root_idx = info.volume.partition_map.root.get_index() + device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx) + sed_i(script, '^device_path="/dev/xvda$', device_path) diff --git a/providers/ec2/tasks/network.py b/providers/ec2/tasks/network.py new file mode 100644 index 0000000..3f00fb2 --- /dev/null +++ b/providers/ec2/tasks/network.py @@ -0,0 +1,15 @@ +from base import Task +from common import phases +import os.path + + +class EnableDHCPCDDNS(Task): + description = 'Configuring the DHCP client to set the nameservers' + phase = phases.system_modification + + def run(self, info): + # The dhcp client that ships with debian sets the DNS servers per default. + # For dhcpcd we need to configure it to do that. + from common.tools import sed_i + dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') + sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index a884b7f..814a6f6 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -11,8 +11,6 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') if info.manifest.volume['backing'] == 's3': info.host_packages.add('euca2ools') @@ -25,6 +23,7 @@ class ImagePackages(Task): def run(self, info): manifest = info.manifest include, exclude = info.img_packages + include.add('openssh-server') include.add('file') # Needed for the init scripts include.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2 if manifest.virtualization == 'pvm': diff --git a/providers/kvm/README.md b/providers/kvm/README.md deleted file mode 100644 index 87e28a3..0000000 --- a/providers/kvm/README.md +++ /dev/null @@ -1,9 +0,0 @@ -this provider creates images for KVM. - -It is possible to add opennebula plugin for OpenNebula images. - -Disk virtuzalition is specified by the virtualization field in the manifest. -Allowed values are: - -* ide: basic disk emulation on /dev/sda1 -* virtio: enhanced performance on /dev/vda1. It adds virtio modules in the kernel and needs configuration update in VM template: diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py deleted file mode 100644 index 6a2603c..0000000 --- a/providers/kvm/__init__.py +++ /dev/null @@ -1,92 +0,0 @@ -from manifest import Manifest -from tasks import packages -from common.tasks import packages as common_packages -from common.tasks import host -from common.tasks import loopback -from common.tasks import parted -from common.tasks import filesystem -from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt -from tasks import boot -from common.tasks import boot as common_boot -from common.tasks import security -from common.tasks import network -from common.tasks import initd -from common.tasks import cleanup -from common.tasks import loopback - - -def initialize(): - pass - - -def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - host.CheckPackages(), - - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), - - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.ModifyFstab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.ConfigureDHCP(), - initd.ResolveInitScripts(), - initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), - - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) - - if manifest.bootstrapper['tarball']: - tasklist.add(bootstrap.MakeTarball()) - - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) - - -def rollback_tasks(tasklist, tasks_completed, manifest): - completed = [type(task) for task in tasks_completed] - - def counter_task(task, counter): - if task in completed and counter not in completed: - tasklist.add(counter()) - - counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) - counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) diff --git a/providers/kvm/assets/grub.d/00_header b/providers/kvm/assets/grub.d/00_header deleted file mode 100644 index 43927bc..0000000 --- a/providers/kvm/assets/grub.d/00_header +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -set -e - -# nothing to do, skip grub mkconfig for this diff --git a/providers/kvm/assets/grub.d/10_linux b/providers/kvm/assets/grub.d/10_linux deleted file mode 100644 index 0b08346..0000000 --- a/providers/kvm/assets/grub.d/10_linux +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# This file generates the old menu.lst configuration with grub2 -# It was copied from tomheadys github repo: -# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom - -prefix=/usr -exec_prefix=${prefix} -bindir=${exec_prefix}/bin -libdir=${exec_prefix}/lib -. ${libdir}/grub/grub-mkconfig_lib - -export TEXTDOMAIN=grub -export TEXTDOMAINDIR=${prefix}/share/locale - -GRUB_DEVICE=/dev/sda1 - - -cat << EOF -set default=${GRUB_DEFAULT} -set timeout=${GRUB_TIMEOUT} -insmod part_msdos -insmod ext2 -insmod gettext -set menu_color_normal=cyan/blue -set menu_color_highlight=white/blue -set root='(hd0,msdos1)' -EOF - -if ${GRUB_HIDDEN_TIMEOUT:-false}; then - printf "hiddenmenu\n" -fi - -linux_entry () -{ - os="$1" - version="$2" - args="$4" - - title="$(gettext_quoted "%s, with Linux %s")" - - cat << EOF -menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { - insmod part_msdos - insmod ext2 - set timeout=${GRUB_TIMEOUT} - set root='(hd0,msdos1)' - echo 'Loading Linux ${version}' - linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} - echo 'Loading initial ramdisk ...' - initrd ${rel_dirname}/${initrd} -} -EOF -} - -list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do - if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi - done` -prepare_boot_cache= - -while [ "x$list" != "x" ] ; do - linux=`version_find_latest $list` - basename=`basename $linux` - dirname=`dirname $linux` - rel_dirname=`make_system_path_relative_to_its_root $dirname` - version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` - alt_version=`echo $version | sed -e "s,\.old$,,g"` - linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" - - initrd= - for i in "initrd.img-${version}" "initrd-${version}.img" \ - "initrd-${version}" "initramfs-${version}.img" \ - "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ - "initrd-${alt_version}" "initramfs-${alt_version}.img"; do - if test -e "${dirname}/${i}" ; then - initrd="$i" - break - fi - done - - initramfs= - for i in "config-${version}" "config-${alt_version}"; do - if test -e "${dirname}/${i}" ; then - initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` - break - fi - done - - linux_entry "${OS}" "${version}" \ - "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" - - list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` -done diff --git a/providers/kvm/manifest-schema.json b/providers/kvm/manifest-schema.json deleted file mode 100644 index c7fbd83..0000000 --- a/providers/kvm/manifest-schema.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Kvm manifest", - "type": "object", - "properties": { - "volume": { - "type": "object", - "properties": { - "backing": { - "type": "string", - "enum": ["raw", "qcow2"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] - } - }, - "required": ["backing", "filesystem"] - } - }, - "required": ["volume"] -} diff --git a/providers/kvm/manifest.py b/providers/kvm/manifest.py deleted file mode 100644 index 0e7b141..0000000 --- a/providers/kvm/manifest.py +++ /dev/null @@ -1,16 +0,0 @@ -import base - - -class Manifest(base.Manifest): - def validate(self, data): - super(Manifest, self).validate(data) - from os import path - schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') - self.schema_validate(data, schema_path) - - def parse(self, data): - super(Manifest, self).parse(data) - self.image = data['image'] - self.virtualization = data['virtualization'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/kvm/tasks/boot.py b/providers/kvm/tasks/boot.py deleted file mode 100644 index 73a6ea0..0000000 --- a/providers/kvm/tasks/boot.py +++ /dev/null @@ -1,55 +0,0 @@ -from base import Task -from common import phases - - -class ConfigureGrub(Task): - description = 'Configuring grub for KVM' - phase = phases.system_modification - - def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - - from common.tools import log_check_call - - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - - if info.manifest.virtualization == 'virtio': - print "Using virtio" - modules_path = os.path.join(info.root, - 'etc/initramfs-tools/modules') - with open(modules_path, 'a') as modules: - modules.write("\nvirtio_pci\nvirtio_blk\n") - - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']]) - - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - - if info.manifest.virtualization == 'virtio': - from common.tools import sed_i - grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg') - sed_i(grub_cfg, 'sda', 'vda') - device_map = os.path.join(info.root, - 'boot/grub/device.map') - sed_i(device_map, 'sda', 'vda') - fstab_file = os.path.join(info.root, - 'etc/fstab') - sed_i(fstab_file, 'sda', 'vda') - diff --git a/providers/kvm/tasks/packages.py b/providers/kvm/tasks/packages.py deleted file mode 100644 index c04bce0..0000000 --- a/providers/kvm/tasks/packages.py +++ /dev/null @@ -1,48 +0,0 @@ -from base import Task -from common import phases -from common.tasks import packages -from common.tasks.host import CheckPackages - - -class HostPackages(Task): - description = 'Determining required host packages' - phase = phases.preparation - before = [CheckPackages] - after = [packages.HostPackages] - - def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') - - -class ImagePackages(Task): - description = 'Determining required image packages' - phase = phases.preparation - after = [packages.ImagePackages] - - def run(self, info): - manifest = info.manifest - include, exclude = info.img_packages - # Add some basic packages we are going to need - include.update(['parted', - 'kpartx', - # Needed for the init scripts - 'file', - # isc-dhcp-client doesn't work properly with ec2 - 'dhcpcd', - 'chkconfig', - 'openssh-client', - 'grub2' - ]) - - exclude.update(['isc-dhcp-client', - 'isc-dhcp-common', - ]) - - # In squeeze, we need a special kernel flavor for xen - kernels = {'squeeze': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, - 'wheezy': {'amd64': 'linux-image-amd64', - 'i386': 'linux-image-686', }, } - include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 6a2603c..8e31b1b 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -1,20 +1,17 @@ from manifest import Manifest from tasks import packages -from common.tasks import packages as common_packages -from common.tasks import host +from common.tasks import volume as volume_tasks from common.tasks import loopback -from common.tasks import parted +from common.tasks import partitioning from common.tasks import filesystem from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup -from common.tasks import loopback +from common.tasks import workspace def initialize(): @@ -22,60 +19,48 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - host.CheckPackages(), + from common.task_sets import base_set + from common.task_sets import volume_set + from common.task_sets import mounting_set + from common.task_sets import apt_set + from common.task_sets import locale_set + tasklist.add(*base_set) + tasklist.add(*volume_set) + tasklist.add(*mounting_set) + tasklist.add(*apt_set) + tasklist.add(*locale_set) - loopback.CreateQemuImg(), - loopback.Attach(), - parted.PartitionVolume(), - parted.MapPartitions(), - parted.FormatPartitions(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), + if manifest.volume['partitions']['type'] != 'none': + from common.task_sets import partitioning_set + tasklist.add(*partitioning_set) - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.ModifyFstab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - network.RemoveDNSInfo(), - network.ConfigureNetworkIF(), - network.ConfigureDHCP(), - initd.ResolveInitScripts(), - initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), + tasklist.add(packages.ImagePackages, - filesystem.UnmountVolume(), - parted.UnmapPartitions(), - loopback.Detach(), - filesystem.DeleteMountDir()) + loopback.Create, - if manifest.bootstrapper['tarball']: - tasklist.add(bootstrap.MakeTarball()) + boot.ConfigureGrub, + common_boot.BlackListModules, + common_boot.DisableGetTTYs, + security.EnableShadowConfig, + network.RemoveDNSInfo, + network.ConfigureNetworkIF, + network.RemoveHostname, + initd.ResolveInitScripts, + initd.InstallInitScripts, + cleanup.ClearMOTD, + cleanup.CleanTMP, - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + loopback.MoveImage) + + if manifest.bootstrapper.get('tarball', False): + tasklist.add(bootstrap.MakeTarball) + + from common.task_sets import get_fs_specific_set + tasklist.add(*get_fs_specific_set(manifest.volume['partitions'])) + + if 'boot' in manifest.volume['partitions']: + from common.task_sets import boot_partition_set + tasklist.add(*boot_partition_set) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -83,10 +68,13 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) + counter_task(loopback.Create, volume_tasks.Delete) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(parted.MapPartitions, parted.UnmapPartitions) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) - counter_task(loopback.Attach, loopback.Detach) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/virtualbox/assets/grub.d/00_header b/providers/virtualbox/assets/grub.d/00_header deleted file mode 100644 index 43927bc..0000000 --- a/providers/virtualbox/assets/grub.d/00_header +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -set -e - -# nothing to do, skip grub mkconfig for this diff --git a/providers/virtualbox/assets/grub.d/10_linux b/providers/virtualbox/assets/grub.d/10_linux deleted file mode 100644 index 0b08346..0000000 --- a/providers/virtualbox/assets/grub.d/10_linux +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh - -# This file generates the old menu.lst configuration with grub2 -# It was copied from tomheadys github repo: -# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom - -prefix=/usr -exec_prefix=${prefix} -bindir=${exec_prefix}/bin -libdir=${exec_prefix}/lib -. ${libdir}/grub/grub-mkconfig_lib - -export TEXTDOMAIN=grub -export TEXTDOMAINDIR=${prefix}/share/locale - -GRUB_DEVICE=/dev/sda1 - - -cat << EOF -set default=${GRUB_DEFAULT} -set timeout=${GRUB_TIMEOUT} -insmod part_msdos -insmod ext2 -insmod gettext -set menu_color_normal=cyan/blue -set menu_color_highlight=white/blue -set root='(hd0,msdos1)' -EOF - -if ${GRUB_HIDDEN_TIMEOUT:-false}; then - printf "hiddenmenu\n" -fi - -linux_entry () -{ - os="$1" - version="$2" - args="$4" - - title="$(gettext_quoted "%s, with Linux %s")" - - cat << EOF -menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { - insmod part_msdos - insmod ext2 - set timeout=${GRUB_TIMEOUT} - set root='(hd0,msdos1)' - echo 'Loading Linux ${version}' - linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} - echo 'Loading initial ramdisk ...' - initrd ${rel_dirname}/${initrd} -} -EOF -} - -list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do - if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi - done` -prepare_boot_cache= - -while [ "x$list" != "x" ] ; do - linux=`version_find_latest $list` - basename=`basename $linux` - dirname=`dirname $linux` - rel_dirname=`make_system_path_relative_to_its_root $dirname` - version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` - alt_version=`echo $version | sed -e "s,\.old$,,g"` - linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" - - initrd= - for i in "initrd.img-${version}" "initrd-${version}.img" \ - "initrd-${version}" "initramfs-${version}.img" \ - "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ - "initrd-${alt_version}" "initramfs-${alt_version}.img"; do - if test -e "${dirname}/${i}" ; then - initrd="$i" - break - fi - done - - initramfs= - for i in "config-${version}" "config-${alt_version}"; do - if test -e "${dirname}/${i}" ; then - initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` - break - fi - done - - linux_entry "${OS}" "${version}" \ - "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" - - list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` -done diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 52ac2ee..e9b1474 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -8,15 +8,16 @@ "properties": { "backing": { "type": "string", - "enum": ["raw", "qcow2"] + "enum": ["raw", "vdi", "qcow2"] }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["none", "mbr"] } + } } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } - }, - "required": ["volume"] + } } diff --git a/providers/virtualbox/manifest.py b/providers/virtualbox/manifest.py index 8e5bb11..8b5fbc2 100644 --- a/providers/virtualbox/manifest.py +++ b/providers/virtualbox/manifest.py @@ -12,5 +12,3 @@ class Manifest(base.Manifest): super(Manifest, self).parse(data) self.virtualization = None self.image = data['image'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index 428f70a..bddb860 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -1,34 +1,65 @@ from base import Task from common import phases +from common.tasks import apt +from common.fs.loopbackvolume import LoopbackVolume class ConfigureGrub(Task): description = 'Configuring grub' phase = phases.system_modification + after = [apt.AptUpgrade] def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - import os.path - device_map_path = os.path.join(info.root, 'boot/grub/device.map') - with open(device_map_path, 'w') as device_map: - device_map.write('(hd0) /dev/sda\n') - + import os from common.tools import log_check_call - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) - script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) - script_dst = os.path.join(info.root, 'etc/grub.d/00_header') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-initramfs', '-u']) - # Install grub in mbr - log_check_call(['/usr/sbin/grub-install', '--boot-directory=' + info.root + "/boot/", - info.bootstrap_device['path']]) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + boot_dir = os.path.join(info.root, 'boot') + grub_dir = os.path.join(boot_dir, 'grub') + + from base.fs.partitionmaps.none import NoPartitions + from base.fs.partitionmaps.gpt import GPTPartitionMap + from common.fs import remount + p_map = info.volume.partition_map + + def mk_remount_fn(fn): + def set_device_path(): + fn() + if isinstance(p_map, NoPartitions): + p_map.root.device_path = info.volume.device_path + return set_device_path + link_fn = mk_remount_fn(info.volume.link_dm_node) + unlink_fn = mk_remount_fn(info.volume.unlink_dm_node) + + # GRUB cannot deal with installing to loopback devices + # so we fake a real harddisk with dmsetup. + # Guide here: http://ebroder.net/2009/08/04/installing-grub-onto-a-disk-image/ + if isinstance(info.volume, LoopbackVolume): + remount(info.volume, link_fn) + try: + [device_path] = log_check_call(['readlink', '-f', info.volume.device_path]) + device_map_path = os.path.join(grub_dir, 'device.map') + partition_prefix = 'msdos' + if isinstance(p_map, GPTPartitionMap): + partition_prefix = 'gpt' + with open(device_map_path, 'w') as device_map: + device_map.write('(hd0) {device_path}\n'.format(device_path=device_path)) + if not isinstance(p_map, NoPartitions): + for idx, partition in enumerate(info.volume.partition_map.partitions): + [partition_path] = log_check_call(['readlink', '-f', partition.device_path]) + device_map.write('(hd0,{prefix}{idx}) {device_path}\n' + .format(device_path=partition_path, prefix=partition_prefix, idx=idx+1)) + + # Install grub + log_check_call(['/usr/sbin/chroot', info.root, + '/usr/sbin/grub-install', + # '--root-directory=' + info.root, + # '--boot-directory=' + boot_dir, + device_path]) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + except Exception as e: + if isinstance(info.volume, LoopbackVolume): + remount(info.volume, unlink_fn) + raise e + + if isinstance(info.volume, LoopbackVolume): + remount(info.volume, unlink_fn) diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 9bdd204..58b1d7f 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -1,19 +1,6 @@ from base import Task from common import phases from common.tasks import packages -from common.tasks.host import CheckPackages - - -class HostPackages(Task): - description = 'Determining required host packages' - phase = phases.preparation - before = [CheckPackages] - after = [packages.HostPackages] - - def run(self, info): - info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc', 'kpartx']) - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') class ImagePackages(Task): @@ -25,20 +12,7 @@ class ImagePackages(Task): manifest = info.manifest include, exclude = info.img_packages # Add some basic packages we are going to need - include.update(['parted', - 'kpartx', - # Needed for the init scripts - 'file', - # isc-dhcp-client doesn't work properly with ec2 - 'dhcpcd', - 'chkconfig', - 'openssh-client', - 'grub2' - ]) - - exclude.update(['isc-dhcp-client', - 'isc-dhcp-common', - ]) + include.add('grub2') # In squeeze, we need a special kernel flavor for xen kernels = {'squeeze': {'amd64': 'linux-image-amd64', diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py new file mode 100644 index 0000000..2744233 --- /dev/null +++ b/providers/virtualbox/volume.py @@ -0,0 +1,77 @@ +from common.fs.loopbackvolume import LoopbackVolume +from base.fs.exceptions import VolumeError +from common.tools import log_check_call +from common.fs import get_partitions + + +class VirtualBoxVolume(LoopbackVolume): + + extension = 'vdi' + + def _before_create(self, e): + self.image_path = e.image_path + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) + + def _check_nbd_module(self): + from base.fs.partitionmaps.none import NoPartitions + if isinstance(self.partition_map, NoPartitions): + if not self._module_loaded('nbd'): + raise VolumeError('The kernel module `nbd\' must be loaded ' + '(`modprobe nbd\') to attach .vdi images') + else: + num_partitions = len(self.partition_map.partitions) + if not self._module_loaded('nbd'): + msg = ('The kernel module `nbd\' must be loaded ' + '(`modprobe nbd max_part={num_partitions}\') to attach .vdi images' + .format(num_partitions=num_partitions)) + raise VolumeError(msg) + nbd_max_part = int(self._module_param('nbd', 'max_part')) + if nbd_max_part < num_partitions: + # Found here: http://bethesignal.org/blog/2011/01/05/how-to-mount-virtualbox-vdi-image/ + msg = ('The kernel module `nbd\' was loaded with the max_part ' + 'parameter set to {max_part}, which is below ' + 'the amount of partitions for this volume ({num_partitions}). ' + 'Reload the nbd kernel module with max_part set to at least {num_partitions} ' + '(`rmmod nbd; modprobe nbd max_part={num_partitions}\').' + .format(max_part=nbd_max_part, num_partitions=num_partitions)) + raise VolumeError(msg) + + def _before_attach(self, e): + self._check_nbd_module() + self.loop_device_path = self._find_free_nbd_device() + log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path]) + self.device_path = self.loop_device_path + + def _before_detach(self, e): + log_check_call(['/usr/bin/qemu-nbd', '--disconnect', self.loop_device_path]) + del self.loop_device_path + del self.device_path + + def _module_loaded(self, module): + import re + regexp = re.compile('^{module} +'.format(module=module)) + with open('/proc/modules') as loaded_modules: + for line in loaded_modules: + match = regexp.match(line) + if match is not None: + return True + return False + + def _module_param(self, module, param): + import os.path + param_path = os.path.join('/sys/module', module, 'parameters', param) + with open(param_path) as param: + return param.read().strip() + + # From http://lists.gnu.org/archive/html/qemu-devel/2011-11/msg02201.html + # Apparently it's not in the current qemu-nbd shipped with wheezy + def _is_nbd_used(self, device_name): + return device_name in get_partitions() + + def _find_free_nbd_device(self): + import os.path + for i in xrange(0, 15): + device_name = 'nbd' + str(i) + if not self._is_nbd_used(device_name): + return os.path.join('/dev', device_name) + raise VolumeError('Unable to find free nbd device.')