diff --git a/.gitignore b/.gitignore index a1e6d9f..ebaa1c3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ *.pyc +# Jekyll-generated files +Gemfile.lock +_site/ # When developing for ec2 `vagrant provision' is quite handy -Vagrantfile -.vagrant +/Vagrantfile +/.vagrant +/build +/dist +/bootstrap_vz.egg-info diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index fb4a18a..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,12 +0,0 @@ -# Coding standards # -* Specify the full path when invoking a command. -* Use long options whenever possible, this makes the commands invoked a lot easier to understand. -* Use tabs for indentation and spaces for alignment. -* Max line length is 110 chars. -* Multiple assignments may be aligned. -* Follow PEP8 with the exception of the following rules - * E101: Indenting with tabs and aligning with spaces - * E221: Alignment of assignments - * E241: Alignment of assignments - * E501: The line length is 110 characters not 80 - * W191: We indent with tabs not spaces diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..53c8db4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2013-2014 Anders Ingemann + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a671734 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE +include manifests/* +recursive-include bootstrapvz assets/* +recursive-include bootstrapvz *.json diff --git a/README.md b/README.md index cb64e9f..f6a2a9a 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,24 @@ bootstrap-vz =========================================== -bootstrap-vz is a fully automated bootstrapping tool for Debian. -It creates images for various virtualized platforms (at the moment: kvm, virtualbox, ec2). -The plugin architecture allows for heavy modification of standard behavior -(e.g. create a vagrant box, apply puppet manifests, run custom shell commands). +bootstrap-vz is a bootstrapping framework for Debian. +It is is specifically targeted at bootstrapping systems for virtualized environments. +bootstrap-vz runs without any user intervention and generates ready-to-boot images for +[a number of virtualization platforms](http://andsens.github.io/bootstrap-vz/providers.html). +Its aim is to provide a reproducable bootstrapping process using [manifests](http://andsens.github.io/bootstrap-vz/manifest.html) as well as supporting a high degree of customizability through plugins. -At no time is the resulting image booted, meaning there are no latent logfiles -or bash_history files. +bootstrap-vz was coded from scratch in python once the bash script architecture that was used in the +[build-debian-cloud](https://github.com/andsens/build-debian-cloud) bootstrapper reached its +limits. -The bootstrapper runs on a single json manifest file which contains all configurable -parameters. This allows you to recreate the image whenever you like so you can create -an updated version of an existing image or create the same image in multiple EC2 regions. +Documentation +------------- +The end-user documentation for bootstrap-vz is available +at [andsens.github.io/bootstrap-vz](http://andsens.github.io/bootstrap-vz). +There, you can discover [what the dependencies](http://andsens.github.io/bootstrap-vz/#dependencies) +for a specific cloud provider are, [see a list of available plugins](http://andsens.github.io/bootstrap-vz/plugins.html) +and learn [how you create a manifest](http://andsens.github.io/bootstrap-vz/manifest.html). -Dependencies ------------- -You will need to run debian wheezy with **python 2.7** and **debootstrap** installed. -Other depencies include: -* qemu-utils -* parted -* grub2 -* euca2ools -* xfsprogs (If you want to use XFS as a filesystem) -Also the following python libraries are required: -* **boto** ([version 2.14.0 or higher](https://github.com/boto/boto)) -* **jsonschema** ([version 2.0.0](https://pypi.python.org/pypi/jsonschema), only available through pip) -* **termcolor** -* **fysom** - -Bootstrapping instance store AMIs requires **euca2ools** to be installed. +Developers +---------- +The API documentation can be found at [bootstrap-vz.readthedocs.org](http://bootstrap-vz.readthedocs.org). diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py deleted file mode 100644 index 2b4bcec..0000000 --- a/base/bootstrapinfo.py +++ /dev/null @@ -1,56 +0,0 @@ - - -class BootstrapInformation(object): - def __init__(self, manifest=None, debug=False): - self.manifest = manifest - self.debug = debug - - import random - self.run_id = '{id:08x}'.format(id=random.randrange(16 ** 8)) - - import os.path - self.workspace = os.path.join(manifest.bootstrapper['workspace'], self.run_id) - - from fs import load_volume - self.volume = load_volume(self.manifest.volume, manifest.system['bootloader']) - - self.apt_mirror = self.manifest.packages.get('mirror', 'http://http.debian.net/debian') - - class DictClass(dict): - def __getattr__(self, name): - return self[name] - - def __setattr__(self, name, value): - self[name] = value - - def set_manifest_vars(obj, data): - for key, value in data.iteritems(): - if isinstance(value, dict): - obj[key] = DictClass() - set_manifest_vars(obj[key], value) - continue - if not isinstance(value, list): - obj[key] = value - - self.manifest_vars = {} - self.manifest_vars['apt_mirror'] = self.apt_mirror - set_manifest_vars(self.manifest_vars, self.manifest.data) - - from datetime import datetime - now = datetime.now() - time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', - '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', - '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] - for key in time_vars: - self.manifest_vars[key] = now.strftime(key) - - from pkg.sourceslist import SourceLists - self.source_lists = SourceLists(self.manifest_vars) - from pkg.packagelist import PackageList - self.packages = PackageList(self.manifest_vars, self.source_lists) - self.include_packages = set() - self.exclude_packages = set() - - self.host_dependencies = set() - - self.initd = {'install': {}, 'disable': []} diff --git a/base/fs/__init__.py b/base/fs/__init__.py deleted file mode 100644 index f19e189..0000000 --- a/base/fs/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ - - -def load_volume(data, bootloader): - from common.fs.loopbackvolume import LoopbackVolume - from providers.ec2.ebsvolume import EBSVolume - from common.fs.virtualdiskimage import VirtualDiskImage - from common.fs.virtualmachinedisk import VirtualMachineDisk - from partitionmaps.gpt import GPTPartitionMap - from partitionmaps.msdos import MSDOSPartitionMap - from partitionmaps.none import NoPartitions - partition_maps = {'none': NoPartitions, - 'gpt': GPTPartitionMap, - 'msdos': MSDOSPartitionMap, - } - partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader) - volume_backings = {'raw': LoopbackVolume, - 's3': LoopbackVolume, - 'vdi': VirtualDiskImage, - 'vmdk': VirtualMachineDisk, - 'ebs': EBSVolume - } - return volume_backings.get(data['backing'])(partition_map) diff --git a/base/fs/exceptions.py b/base/fs/exceptions.py deleted file mode 100644 index bc38490..0000000 --- a/base/fs/exceptions.py +++ /dev/null @@ -1,8 +0,0 @@ - - -class VolumeError(Exception): - pass - - -class PartitionError(Exception): - pass diff --git a/base/fs/partitionmaps/msdos.py b/base/fs/partitionmaps/msdos.py deleted file mode 100644 index ab71978..0000000 --- a/base/fs/partitionmaps/msdos.py +++ /dev/null @@ -1,38 +0,0 @@ -from abstract import AbstractPartitionMap -from ..partitions.msdos import MSDOSPartition -from ..partitions.msdos_swap import MSDOSSwapPartition -from common.tools import log_check_call - - -class MSDOSPartitionMap(AbstractPartitionMap): - - def __init__(self, data, bootloader): - from common.bytes import Bytes - self.partitions = [] - - def last_partition(): - return self.partitions[-1] if len(self.partitions) > 0 else None - - if 'boot' in data: - self.boot = MSDOSPartition(Bytes(data['boot']['size']), data['boot']['filesystem'], None) - self.partitions.append(self.boot) - if 'swap' in data: - self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition()) - self.partitions.append(self.swap) - self.root = MSDOSPartition(Bytes(data['root']['size']), data['root']['filesystem'], last_partition()) - self.partitions.append(self.root) - - getattr(self, 'boot', self.root).flags.append('boot') - - if bootloader == 'grub': - self.partitions[0].offset = Bytes('2MiB') - self.partitions[0].size -= self.partitions[0].offset - - super(MSDOSPartitionMap, self).__init__(bootloader) - - def _before_create(self, event): - volume = event.volume - log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, - '--', 'mklabel', 'msdos']) - for partition in self.partitions: - partition.create(volume) diff --git a/base/fs/partitionmaps/none.py b/base/fs/partitionmaps/none.py deleted file mode 100644 index 8fc7650..0000000 --- a/base/fs/partitionmaps/none.py +++ /dev/null @@ -1,15 +0,0 @@ -from ..partitions.single import SinglePartition - - -class NoPartitions(object): - - def __init__(self, data, bootloader): - from common.bytes import Bytes - self.root = SinglePartition(Bytes(data['root']['size']), data['root']['filesystem']) - self.partitions = [self.root] - - def is_blocking(self): - return self.root.fsm.current == 'mounted' - - def get_total_size(self): - return self.root.get_end() diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py deleted file mode 100644 index 5684561..0000000 --- a/base/fs/partitions/abstract.py +++ /dev/null @@ -1,86 +0,0 @@ -from abc import ABCMeta -from abc import abstractmethod -import os.path -from common.tools import log_check_call -from common.fsm_proxy import FSMProxy - - -class AbstractPartition(FSMProxy): - - __metaclass__ = ABCMeta - - events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, - {'name': 'format', 'src': 'created', 'dst': 'formatted'}, - {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, - {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, - ] - - class Mount(object): - def __init__(self, source, destination, opts): - self.source = source - self.destination = destination - self.opts = opts - - def mount(self, prefix): - mount_dir = os.path.join(prefix, self.destination) - if isinstance(self.source, AbstractPartition): - self.source.mount(destination=mount_dir) - else: - log_check_call(['/bin/mount'] + self.opts + [self.source, mount_dir]) - self.mount_dir = mount_dir - - def unmount(self): - if isinstance(self.source, AbstractPartition): - self.source.unmount() - else: - log_check_call(['/bin/umount', self.mount_dir]) - del self.mount_dir - - def __init__(self, size, filesystem): - self.size = size - self.filesystem = filesystem - self.device_path = None - self.mounts = {} - - cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} - super(AbstractPartition, self).__init__(cfg) - - def get_uuid(self): - [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) - return uuid - - @abstractmethod - def get_start(self): - pass - - def get_end(self): - return self.get_start() + self.size - - def _before_format(self, e): - mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) - log_check_call([mkfs, self.device_path]) - - def _before_mount(self, e): - log_check_call(['/bin/mount', '--types', self.filesystem, self.device_path, e.destination]) - self.mount_dir = e.destination - - def _after_mount(self, e): - for destination in sorted(self.mounts.iterkeys(), key=len): - self.mounts[destination].mount(self.mount_dir) - - def _before_unmount(self, e): - for destination in sorted(self.mounts.iterkeys(), key=len, reverse=True): - self.mounts[destination].unmount() - log_check_call(['/bin/umount', self.mount_dir]) - del self.mount_dir - - def add_mount(self, source, destination, opts=[]): - mount = self.Mount(source, destination, opts) - if self.fsm.current == 'mounted': - mount.mount(self.mount_dir) - self.mounts[destination] = mount - - def remove_mount(self, destination): - if self.fsm.current == 'mounted': - self.mounts[destination].unmount() - del self.mounts[destination] diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py deleted file mode 100644 index 216479e..0000000 --- a/base/fs/partitions/base.py +++ /dev/null @@ -1,59 +0,0 @@ -from abstract import AbstractPartition - - -class BasePartition(AbstractPartition): - - events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, - {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, - {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, - {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, - {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, - {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, - - {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, - {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, - ] - - def __init__(self, size, filesystem, previous): - self.previous = previous - from common.bytes import Bytes - self.offset = Bytes(0) - self.flags = [] - super(BasePartition, self).__init__(size, filesystem) - - def create(self, volume): - self.fsm.create(volume=volume) - - def get_index(self): - if self.previous is None: - return 1 - else: - return self.previous.get_index() + 1 - - def get_start(self): - if self.previous is None: - return self.offset - else: - return self.previous.get_end() + self.offset - - def map(self, device_path): - self.fsm.map(device_path=device_path) - - def _before_create(self, e): - from common.tools import log_check_call - create_command = ('mkpart primary {start} {end}' - .format(start=str(self.get_start()), - end=str(self.get_end()))) - log_check_call(['/sbin/parted', '--script', '--align', 'none', e.volume.device_path, - '--', create_command]) - - for flag in self.flags: - log_check_call(['/sbin/parted', '--script', e.volume.device_path, - '--', ('set {idx} {flag} on' - .format(idx=str(self.get_index()), flag=flag))]) - - def _before_map(self, e): - self.device_path = e.device_path - - def _before_unmap(self, e): - self.device_path = None diff --git a/base/fs/partitions/gpt.py b/base/fs/partitions/gpt.py deleted file mode 100644 index aed7f2f..0000000 --- a/base/fs/partitions/gpt.py +++ /dev/null @@ -1,18 +0,0 @@ -from common.tools import log_check_call -from base import BasePartition - - -class GPTPartition(BasePartition): - - def __init__(self, size, filesystem, name, previous): - self.name = name - super(GPTPartition, self).__init__(size, filesystem, previous) - - def _before_create(self, e): - super(GPTPartition, self)._before_create(e) - # partition name only works for gpt, for msdos that becomes the part-type (primary, extended, logical) - name_command = ('name {idx} {name}' - .format(idx=self.get_index(), - name=self.name)) - log_check_call(['/sbin/parted', '--script', e.volume.device_path, - '--', name_command]) diff --git a/base/fs/partitions/gpt_swap.py b/base/fs/partitions/gpt_swap.py deleted file mode 100644 index 0217770..0000000 --- a/base/fs/partitions/gpt_swap.py +++ /dev/null @@ -1,11 +0,0 @@ -from common.tools import log_check_call -from gpt import GPTPartition - - -class GPTSwapPartition(GPTPartition): - - def __init__(self, size, previous): - super(GPTSwapPartition, self).__init__(size, 'swap', 'swap', previous) - - def _before_format(self, e): - log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/msdos_swap.py b/base/fs/partitions/msdos_swap.py deleted file mode 100644 index 4b1d1dc..0000000 --- a/base/fs/partitions/msdos_swap.py +++ /dev/null @@ -1,11 +0,0 @@ -from common.tools import log_check_call -from msdos import MSDOSPartition - - -class MSDOSSwapPartition(MSDOSPartition): - - def __init__(self, size, previous): - super(MSDOSSwapPartition, self).__init__(size, 'swap', previous) - - def _before_format(self, e): - log_check_call(['/sbin/mkswap', self.device_path]) diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py deleted file mode 100644 index 828ba91..0000000 --- a/base/fs/partitions/single.py +++ /dev/null @@ -1,8 +0,0 @@ -from abstract import AbstractPartition - - -class SinglePartition(AbstractPartition): - - def get_start(self): - from common.bytes import Bytes - return Bytes(0) diff --git a/base/fs/partitions/unformatted.py b/base/fs/partitions/unformatted.py deleted file mode 100644 index bbbc357..0000000 --- a/base/fs/partitions/unformatted.py +++ /dev/null @@ -1,12 +0,0 @@ -from base import BasePartition - - -class UnformattedPartition(BasePartition): - - events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, - {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, - {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, - ] - - def __init__(self, size, previous): - super(UnformattedPartition, self).__init__(size, None, previous) diff --git a/base/log.py b/base/log.py deleted file mode 100644 index 869f878..0000000 --- a/base/log.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging - - -def get_logfile_path(manifest_path): - import os.path - from datetime import datetime - - manifest_basename = os.path.basename(manifest_path) - manifest_name, _ = os.path.splitext(manifest_basename) - timestamp = datetime.now().strftime('%Y%m%d%H%M%S') - filename = "{timestamp}_{name}.log".format(timestamp=timestamp, name=manifest_name) - return os.path.normpath(os.path.join(os.path.dirname(__file__), '../logs', filename)) - - -def setup_logger(logfile=None, debug=False): - root = logging.getLogger() - root.setLevel(logging.NOTSET) - - file_handler = logging.FileHandler(logfile) - file_handler.setFormatter(FileFormatter('[%(relativeCreated)s] %(levelname)s: %(message)s')) - file_handler.setLevel(logging.DEBUG) - root.addHandler(file_handler) - - import sys - console_handler = logging.StreamHandler(sys.stderr) - console_handler.setFormatter(ConsoleFormatter()) - if debug: - console_handler.setLevel(logging.DEBUG) - else: - console_handler.setLevel(logging.INFO) - root.addHandler(console_handler) - - -class ConsoleFormatter(logging.Formatter): - level_colors = {logging.ERROR: 'red', - logging.WARNING: 'magenta', - logging.INFO: 'blue', - } - - def format(self, record): - if(record.levelno in self.level_colors): - from termcolor import colored - record.msg = colored(record.msg, self.level_colors[record.levelno]) - return super(ConsoleFormatter, self).format(record) - - -class FileFormatter(logging.Formatter): - def format(self, record): - return super(FileFormatter, self).format(record) diff --git a/base/main.py b/base/main.py deleted file mode 100644 index 007157a..0000000 --- a/base/main.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging -log = logging.getLogger(__name__) - - -def main(): - import log - args = get_args() - logfile = log.get_logfile_path(args.manifest) - log.setup_logger(logfile=logfile, debug=args.debug) - run(args) - - -def get_args(): - from argparse import ArgumentParser - parser = ArgumentParser(description='Bootstrap Debian for the cloud.') - parser.add_argument('--debug', action='store_true', - help='Print debugging information') - parser.add_argument('--pause-on-error', action='store_true', - help='Pause on error, before rollback') - parser.add_argument('--dry-run', action='store_true', - help='Dont\'t actually run the tasks') - parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') - return parser.parse_args() - - -def run(args): - from manifest import Manifest - manifest = Manifest(args.manifest) - - from tasklist import TaskList - tasklist = TaskList() - tasklist.load('resolve_tasks', manifest) - - from bootstrapinfo import BootstrapInformation - bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug) - - try: - tasklist.run(info=bootstrap_info, dry_run=args.dry_run) - log.info('Successfully completed bootstrapping') - except (Exception, KeyboardInterrupt) as e: - log.exception(e) - if args.pause_on_error: - raw_input("Press Enter to commence rollback") - log.error('Rolling back') - - rollback_tasklist = TaskList() - - def counter_task(task, counter): - if task in tasklist.tasks_completed and counter not in tasklist.tasks_completed: - rollback_tasklist.tasks.add(counter) - rollback_tasklist.load('resolve_rollback_tasks', manifest, counter_task) - - rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run) - log.info('Successfully completed rollback') diff --git a/base/manifest.py b/base/manifest.py deleted file mode 100644 index a51f826..0000000 --- a/base/manifest.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -log = logging.getLogger(__name__) - - -class Manifest(object): - def __init__(self, path): - self.path = path - self.load() - self.validate() - self.parse() - - def load(self): - self.data = self.load_json(self.path) - provider_modname = 'providers.{provider}'.format(provider=self.data['provider']) - log.debug('Loading provider `{modname}\''.format(modname=provider_modname)) - self.modules = {'provider': __import__(provider_modname, fromlist=['providers']), - 'plugins': [], - } - if 'plugins' in self.data: - for plugin_name, plugin_data in self.data['plugins'].iteritems(): - modname = 'plugins.{plugin}'.format(plugin=plugin_name) - log.debug('Loading plugin `{modname}\''.format(modname=modname)) - plugin = __import__(modname, fromlist=['plugins']) - self.modules['plugins'].append(plugin) - - self.modules['provider'].initialize() - for module in self.modules['plugins']: - init = getattr(module, 'initialize', None) - if callable(init): - init() - - def validate(self): - from . import validate_manifest - validate_manifest(self.data, self.schema_validator, self.validation_error) - self.modules['provider'].validate_manifest(self.data, self.schema_validator, self.validation_error) - for plugin in self.modules['plugins']: - validate = getattr(plugin, 'validate_manifest', None) - if callable(validate): - validate(self.data, self.schema_validator, self.validation_error) - - def parse(self): - self.provider = self.data['provider'] - self.bootstrapper = self.data['bootstrapper'] - self.image = self.data['image'] - self.volume = self.data['volume'] - self.system = self.data['system'] - self.packages = self.data['packages'] - self.plugins = self.data['plugins'] if 'plugins' in self.data else {} - - def load_json(self, path): - import json - from minify_json import json_minify - with open(path) as stream: - return json.loads(json_minify(stream.read(), False)) - - def schema_validator(self, data, schema_path): - import jsonschema - schema = self.load_json(schema_path) - try: - jsonschema.validate(data, schema) - except jsonschema.ValidationError as e: - self.validation_error(e.message, e.path) - - def validation_error(self, message, json_path=None): - from common.exceptions import ManifestError - raise ManifestError(message, self.path, json_path) diff --git a/base/phase.py b/base/phase.py deleted file mode 100644 index 9acf825..0000000 --- a/base/phase.py +++ /dev/null @@ -1,16 +0,0 @@ - - -class Phase(object): - def __init__(self, name, description): - self.name = name - self.description = description - - def pos(self): - from common.phases import order - return next(i for i, phase in enumerate(order) if phase is self) - - def __cmp__(self, other): - return self.pos() - other.pos() - - def __str__(self): - return self.name diff --git a/base/pkg/exceptions.py b/base/pkg/exceptions.py deleted file mode 100644 index 9437f9c..0000000 --- a/base/pkg/exceptions.py +++ /dev/null @@ -1,8 +0,0 @@ - - -class PackageError(Exception): - pass - - -class SourceError(Exception): - pass diff --git a/base/pkg/packagelist.py b/base/pkg/packagelist.py deleted file mode 100644 index 762bb4a..0000000 --- a/base/pkg/packagelist.py +++ /dev/null @@ -1,57 +0,0 @@ -from exceptions import PackageError - - -class PackageList(object): - - class Remote(object): - def __init__(self, name, target): - self.name = name - self.target = target - - def __str__(self): - if self.target is None: - return self.name - else: - return '{name}/{target}'.format(name=self.name, target=self.target) - - class Local(object): - def __init__(self, path): - self.path = path - - def __str__(self): - return self.path - - def __init__(self, manifest_vars, source_lists): - self.manifest_vars = manifest_vars - self.source_lists = source_lists - self.default_target = '{system.release}'.format(**self.manifest_vars) - self.install = [] - self.remote = lambda: filter(lambda x: isinstance(x, self.Remote), self.install) - - def add(self, name, target=None): - name = name.format(**self.manifest_vars) - if target is not None: - target = target.format(**self.manifest_vars) - package = next((pkg for pkg in self.remote() if pkg.name == name), None) - if package is not None: - same_target = package.target != target - same_target = same_target or package.target is None and target == self.default_target - same_target = same_target or package.target == self.default_target and target is None - if not same_target: - msg = ('The package {name} was already added to the package list, ' - 'but with another target release ({target})').format(name=name, target=package.target) - raise PackageError(msg) - return - - check_target = target - if check_target is None: - check_target = self.default_target - if not self.source_lists.target_exists(check_target): - msg = ('The target release {target} was not found in the sources list').format(target=check_target) - raise PackageError(msg) - - self.install.append(self.Remote(name, target)) - - def add_local(self, package_path): - package_path = package_path.format(**self.manifest_vars) - self.install.append(self.Local(package_path)) diff --git a/base/task.py b/base/task.py deleted file mode 100644 index e980477..0000000 --- a/base/task.py +++ /dev/null @@ -1,17 +0,0 @@ - - -class Task(object): - phase = None - predecessors = [] - successors = [] - - class __metaclass__(type): - def __repr__(cls): - return '{module}.{task}'.format(module=cls.__module__, task=cls.__name__) - - def __str__(cls): - return repr(cls) - - @classmethod - def run(cls, info): - pass diff --git a/base/tasklist.py b/base/tasklist.py deleted file mode 100644 index 2803d7a..0000000 --- a/base/tasklist.py +++ /dev/null @@ -1,128 +0,0 @@ -from common.exceptions import TaskListError -import logging -log = logging.getLogger(__name__) - - -class TaskList(object): - - def __init__(self): - self.tasks = set() - self.tasks_completed = [] - - def load(self, function, manifest, *args): - getattr(manifest.modules['provider'], function)(self.tasks, manifest, *args) - for plugin in manifest.modules['plugins']: - fn = getattr(plugin, function, None) - if callable(fn): - fn(self.tasks, manifest, *args) - - def run(self, info={}, dry_run=False): - task_list = self.create_list() - log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(map(repr, task_list)))) - - for task in task_list: - if hasattr(task, 'description'): - log.info(task.description) - else: - log.info('Running {task}'.format(task=task)) - if not dry_run: - task.run(info) - self.tasks_completed.append(task) - - def create_list(self): - from common.phases import order - graph = {} - for task in self.tasks: - self.check_ordering(task) - successors = set() - successors.update(task.successors) - successors.update(filter(lambda succ: task in succ.predecessors, self.tasks)) - succeeding_phases = order[order.index(task.phase) + 1:] - successors.update(filter(lambda succ: succ.phase in succeeding_phases, self.tasks)) - graph[task] = filter(lambda succ: succ in self.tasks, successors) - - components = self.strongly_connected_components(graph) - cycles_found = 0 - for component in components: - if len(component) > 1: - cycles_found += 1 - log.debug('Cycle: {list}\n'.format(list=', '.join(map(repr, component)))) - if cycles_found > 0: - msg = ('{0} cycles were found in the tasklist, ' - 'consult the logfile for more information.'.format(cycles_found)) - raise TaskListError(msg) - - sorted_tasks = self.topological_sort(graph) - - return sorted_tasks - - def check_ordering(self, task): - for successor in task.successors: - if successor.phase > successor.phase: - msg = ("The task {task} is specified as running before {other}, " - "but its phase '{phase}' lies after the phase '{other_phase}'" - .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase)) - raise TaskListError(msg) - for predecessor in task.predecessors: - if task.phase < predecessor.phase: - msg = ("The task {task} is specified as running after {other}, " - "but its phase '{phase}' lies before the phase '{other_phase}'" - .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase)) - raise TaskListError(msg) - - def strongly_connected_components(self, graph): - # Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py - # Find the strongly connected components in a graph using Tarjan's algorithm. - # graph should be a dictionary mapping node names to lists of successor nodes. - - result = [] - stack = [] - low = {} - - def visit(node): - if node in low: - return - - num = len(low) - low[node] = num - stack_pos = len(stack) - stack.append(node) - - for successor in graph[node]: - visit(successor) - low[node] = min(low[node], low[successor]) - - if num == low[node]: - component = tuple(stack[stack_pos:]) - del stack[stack_pos:] - result.append(component) - for item in component: - low[item] = len(graph) - - for node in graph: - visit(node) - - return result - - def topological_sort(self, graph): - # Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py - count = {} - for node in graph: - count[node] = 0 - for node in graph: - for successor in graph[node]: - count[successor] += 1 - - ready = [node for node in graph if count[node] == 0] - - result = [] - while ready: - node = ready.pop(-1) - result.append(node) - - for successor in graph[node]: - count[successor] -= 1 - if count[successor] == 0: - ready.append(successor) - - return result diff --git a/bootstrap-vz b/bootstrap-vz index bffc990..46f8fbf 100755 --- a/bootstrap-vz +++ b/bootstrap-vz @@ -1,6 +1,5 @@ #!/usr/bin/env python - -if __name__ == '__main__' and __package__ is None: - from base import main +if __name__ == '__main__': + from bootstrapvz.base import main main() diff --git a/bootstrapvz/__init__.py b/bootstrapvz/__init__.py new file mode 100644 index 0000000..c3ba8db --- /dev/null +++ b/bootstrapvz/__init__.py @@ -0,0 +1,3 @@ + + +__version__ = '0.9' diff --git a/bootstrapvz/base/__init__.py b/bootstrapvz/base/__init__.py new file mode 100644 index 0000000..a448245 --- /dev/null +++ b/bootstrapvz/base/__init__.py @@ -0,0 +1,17 @@ +__all__ = ['Phase', 'Task', 'main'] +from phase import Phase +from task import Task +from main import main + + +def validate_manifest(data, validator, error): + """Validates the manifest using the base manifest + + Args: + data (dict): The data of the manifest + validator (function): The function that validates the manifest given the data and a path + error (function): The function tha raises an error when the validation fails + """ + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) diff --git a/bootstrapvz/base/bootstrapinfo.py b/bootstrapvz/base/bootstrapinfo.py new file mode 100644 index 0000000..a14e7ff --- /dev/null +++ b/bootstrapvz/base/bootstrapinfo.py @@ -0,0 +1,101 @@ + + +class BootstrapInformation(object): + """The BootstrapInformation class holds all information about the bootstrapping process. + The nature of the attributes of this class are rather diverse. + Tasks may set their own attributes on this class for later retrieval by another task. + Information that becomes invalid (e.g. a path to a file that has been deleted) must be removed. + """ + def __init__(self, manifest=None, debug=False): + """Instantiates a new bootstrap info object. + + Args: + manifest (Manifest): The manifest + debug (bool): Whether debugging is turned on + """ + # Set the manifest attribute. + self.manifest = manifest + self.debug = debug + + # Create a run_id. This id may be used to uniquely identify the currrent bootstrapping process + import random + self.run_id = '{id:08x}'.format(id=random.randrange(16 ** 8)) + + # Define the path to our workspace + import os.path + self.workspace = os.path.join(manifest.bootstrapper['workspace'], self.run_id) + + # Load all the volume information + from fs import load_volume + self.volume = load_volume(self.manifest.volume, manifest.system['bootloader']) + + # The default apt mirror + self.apt_mirror = self.manifest.packages.get('mirror', 'http://http.debian.net/debian') + + # Normalize the release codenames so that tasks may query for release codenames rather than + # 'stable', 'unstable' etc. This is useful when handling cases that are specific to a release. + release_codenames_path = os.path.join(os.path.dirname(__file__), 'release-codenames.json') + from bootstrapvz.common.tools import config_get + self.release_codename = config_get(release_codenames_path, [self.manifest.system['release']]) + + class DictClass(dict): + """Tiny extension of dict to allow setting and getting keys via attributes + """ + def __getattr__(self, name): + return self[name] + + def __setattr__(self, name, value): + self[name] = value + + def set_manifest_vars(obj, data): + """Runs through the manifest and creates DictClasses for every key + + Args: + obj (dict): dictionary to set the values on + data (dict): dictionary of values to set on the obj + """ + for key, value in data.iteritems(): + if isinstance(value, dict): + obj[key] = DictClass() + set_manifest_vars(obj[key], value) + continue + # Lists are not supported + if not isinstance(value, list): + obj[key] = value + + # manifest_vars is a dictionary of all the manifest values, + # with it users can cross-reference values in the manifest, so that they do not need to be written twice + self.manifest_vars = {} + self.manifest_vars['apt_mirror'] = self.apt_mirror + set_manifest_vars(self.manifest_vars, self.manifest.data) + + # Populate the manifest_vars with datetime information + # and map the datetime variables directly to the dictionary + from datetime import datetime + now = datetime.now() + time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', + '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', + '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] + for key in time_vars: + self.manifest_vars[key] = now.strftime(key) + + # Keep a list of apt sources, + # so that tasks may add to that list without having to fiddle with apt source list files. + from pkg.sourceslist import SourceLists + self.source_lists = SourceLists(self.manifest_vars) + # Keep a list of packages that should be installed, tasks can add and remove things from this list + from pkg.packagelist import PackageList + self.packages = PackageList(self.manifest_vars, self.source_lists) + + # These sets should rarely be used and specify which packages the debootstrap invocation + # should be called with. + self.include_packages = set() + self.exclude_packages = set() + + # Dictionary to specify which commands are required on the host. + # The keys are commands, while the values are either package names or urls + # that hint at how a command may be made available. + self.host_dependencies = {} + + # Lists of startup scripts that should be installed and disabled + self.initd = {'install': {}, 'disable': []} diff --git a/bootstrapvz/base/fs/__init__.py b/bootstrapvz/base/fs/__init__.py new file mode 100644 index 0000000..b575b87 --- /dev/null +++ b/bootstrapvz/base/fs/__init__.py @@ -0,0 +1,34 @@ + + +def load_volume(data, bootloader): + """Instantiates a volume that corresponds to the data in the manifest + Args: + data (dict): The 'volume' section from the manifest + bootloader (str): Name of the bootloader the system will boot with + + Returns: + Volume. The volume that represents all information pertaining to the volume we bootstrap on + """ + from bootstrapvz.common.fs.loopbackvolume import LoopbackVolume + from bootstrapvz.providers.ec2.ebsvolume import EBSVolume + from bootstrapvz.common.fs.virtualdiskimage import VirtualDiskImage + from bootstrapvz.common.fs.virtualmachinedisk import VirtualMachineDisk + # Create a mapping between valid partition maps in the manifest and their corresponding classes + from partitionmaps.gpt import GPTPartitionMap + from partitionmaps.msdos import MSDOSPartitionMap + from partitionmaps.none import NoPartitions + partition_maps = {'none': NoPartitions, + 'gpt': GPTPartitionMap, + 'msdos': MSDOSPartitionMap, + } + # Instantiate the partition map + partition_map = partition_maps.get(data['partitions']['type'])(data['partitions'], bootloader) + # Create a mapping between valid volume backings in the manifest and their corresponding classes + volume_backings = {'raw': LoopbackVolume, + 's3': LoopbackVolume, + 'vdi': VirtualDiskImage, + 'vmdk': VirtualMachineDisk, + 'ebs': EBSVolume + } + # Create the volume with the partition map as an argument + return volume_backings.get(data['backing'])(partition_map) diff --git a/bootstrapvz/base/fs/exceptions.py b/bootstrapvz/base/fs/exceptions.py new file mode 100644 index 0000000..fad7868 --- /dev/null +++ b/bootstrapvz/base/fs/exceptions.py @@ -0,0 +1,12 @@ + + +class VolumeError(Exception): + """Raised when an error occurs while interacting with the volume + """ + pass + + +class PartitionError(Exception): + """Raised when an error occurs while interacting with the partitions on the volume + """ + pass diff --git a/__init__.py b/bootstrapvz/base/fs/partitionmaps/__init__.py similarity index 100% rename from __init__.py rename to bootstrapvz/base/fs/partitionmaps/__init__.py diff --git a/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py similarity index 56% rename from base/fs/partitionmaps/abstract.py rename to bootstrapvz/base/fs/partitionmaps/abstract.py index 831d048..0d7824d 100644 --- a/base/fs/partitionmaps/abstract.py +++ b/bootstrapvz/base/fs/partitionmaps/abstract.py @@ -1,30 +1,55 @@ from abc import ABCMeta from abc import abstractmethod -from common.tools import log_check_call -from common.fsm_proxy import FSMProxy +from bootstrapvz.common.tools import log_check_call +from bootstrapvz.common.fsm_proxy import FSMProxy from ..exceptions import PartitionError class AbstractPartitionMap(FSMProxy): + """Abstract representation of a partiton map + This class is a finite state machine and represents the state of the real partition map + """ __metaclass__ = ABCMeta + # States the partition map can be in events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, ] def __init__(self, bootloader): + """ + Args: + bootloader (str): Name of the bootloader we will use for bootstrapping + """ + # Create the configuration for the state machine cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} super(AbstractPartitionMap, self).__init__(cfg) def is_blocking(self): + """Returns whether the partition map is blocking volume detach operations + + Returns: + bool. + """ return self.fsm.current == 'mapped' def get_total_size(self): + """Returns the total size the partitions occupy + + Returns: + Bytes. The size of all the partitions + """ + # We just need the endpoint of the last partition return self.partitions[-1].get_end() def create(self, volume): + """Creates the partition map + + Args: + volume (Volume): The volume to create the partition map on + """ self.fsm.create(volume=volume) @abstractmethod @@ -32,19 +57,30 @@ class AbstractPartitionMap(FSMProxy): pass def map(self, volume): + """Maps the partition map to device nodes + + Args: + volume (Volume): The volume the partition map resides on + """ self.fsm.map(volume=volume) def _before_map(self, event): + """ + Raises: + PartitionError + """ volume = event.volume try: - mappings = log_check_call(['/sbin/kpartx', '-l', volume.device_path]) + # Ask kpartx how the partitions will be mapped before actually attaching them. + mappings = log_check_call(['kpartx', '-l', volume.device_path]) import re regexp = re.compile('^(?P.+[^\d](?P\d+)) : ' '(?P\d) (?P\d+) ' '{device_path} (?P\d+)$' .format(device_path=volume.device_path)) - log_check_call(['/sbin/kpartx', '-a', volume.device_path]) + log_check_call(['kpartx', '-a', volume.device_path]) import os.path + # Run through the kpartx output and map the paths to the partitions for mapping in mappings: match = regexp.match(mapping) if match is None: @@ -53,26 +89,40 @@ class AbstractPartitionMap(FSMProxy): p_idx = int(match.group('p_idx')) - 1 self.partitions[p_idx].map(partition_path) + # Check if any partition was not mapped for idx, partition in enumerate(self.partitions): if partition.fsm.current not in ['mapped', 'formatted']: raise PartitionError('kpartx did not map partition #{idx}'.format(idx=idx + 1)) except PartitionError as e: + # Revert any mapping and reraise the error for partition in self.partitions: if not partition.fsm.can('unmap'): partition.unmap() - log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + log_check_call(['kpartx', '-d', volume.device_path]) raise e def unmap(self, volume): + """Unmaps the partition + + Args: + volume (Volume): The volume to unmap the partition map from + """ self.fsm.unmap(volume=volume) def _before_unmap(self, event): + """ + Raises: + PartitionError + """ volume = event.volume + # Run through all partitions before unmapping and make sure they can all be unmapped for partition in self.partitions: if partition.fsm.cannot('unmap'): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) - log_check_call(['/sbin/kpartx', '-d', volume.device_path]) + # Actually unmap the partitions + log_check_call(['kpartx', '-d', volume.device_path]) + # Call unmap on all partitions for partition in self.partitions: partition.unmap() diff --git a/base/fs/partitionmaps/gpt.py b/bootstrapvz/base/fs/partitionmaps/gpt.py similarity index 51% rename from base/fs/partitionmaps/gpt.py rename to bootstrapvz/base/fs/partitionmaps/gpt.py index 2502288..9edfcad 100644 --- a/base/fs/partitionmaps/gpt.py +++ b/bootstrapvz/base/fs/partitionmaps/gpt.py @@ -1,38 +1,57 @@ from abstract import AbstractPartitionMap from ..partitions.gpt import GPTPartition from ..partitions.gpt_swap import GPTSwapPartition -from common.tools import log_check_call +from bootstrapvz.common.tools import log_check_call class GPTPartitionMap(AbstractPartitionMap): + """Represents a GPT partition map + """ def __init__(self, data, bootloader): - from common.bytes import Bytes + """ + Args: + data (dict): volume.partitions part of the manifest + bootloader (str): Name of the bootloader we will use for bootstrapping + """ + from bootstrapvz.common.bytes import Bytes + # List of partitions self.partitions = [] + # Returns the last partition unless there is none def last_partition(): return self.partitions[-1] if len(self.partitions) > 0 else None + # GPT offset gpt_offset = Bytes('17KiB') + # If we are using the grub bootloader we need to create an unformatted partition + # at the beginning of the map. Its size is 1007kb, which we will steal from the + # next partition. if bootloader == 'grub': from ..partitions.unformatted import UnformattedPartition self.grub_boot = UnformattedPartition(Bytes('1007KiB'), last_partition()) self.grub_boot.offset = gpt_offset + # Mark the partition as a bios_grub partition self.grub_boot.flags.append('bios_grub') self.partitions.append(self.grub_boot) + # The boot and swap partitions are optional if 'boot' in data: - self.boot = GPTPartition(Bytes(data['boot']['size']), data['boot']['filesystem'], + self.boot = GPTPartition(Bytes(data['boot']['size']), + data['boot']['filesystem'], data['boot'].get('format_command', None), 'boot', last_partition()) self.partitions.append(self.boot) if 'swap' in data: self.swap = GPTSwapPartition(Bytes(data['swap']['size']), last_partition()) self.partitions.append(self.swap) - self.root = GPTPartition(Bytes(data['root']['size']), data['root']['filesystem'], + self.root = GPTPartition(Bytes(data['root']['size']), + data['root']['filesystem'], data['root'].get('format_command', None), 'root', last_partition()) self.partitions.append(self.root) + # Depending on whether we have a grub boot partition + # we will need to set the offset accordingly. if hasattr(self, 'grub_boot'): self.partitions[1].size -= gpt_offset self.partitions[1].size -= self.grub_boot.size @@ -43,8 +62,13 @@ class GPTPartitionMap(AbstractPartitionMap): super(GPTPartitionMap, self).__init__(bootloader) def _before_create(self, event): + """Creates the partition map + """ volume = event.volume - log_check_call(['/sbin/parted', '--script', '--align', 'none', volume.device_path, + # Disk alignment still plays a role in virtualized environment, + # but I honestly have no clue as to what best practice is here, so we choose 'none' + log_check_call(['parted', '--script', '--align', 'none', volume.device_path, '--', 'mklabel', 'gpt']) + # Create the partitions for partition in self.partitions: partition.create(volume) diff --git a/bootstrapvz/base/fs/partitionmaps/msdos.py b/bootstrapvz/base/fs/partitionmaps/msdos.py new file mode 100644 index 0000000..d21ec64 --- /dev/null +++ b/bootstrapvz/base/fs/partitionmaps/msdos.py @@ -0,0 +1,59 @@ +from abstract import AbstractPartitionMap +from ..partitions.msdos import MSDOSPartition +from ..partitions.msdos_swap import MSDOSSwapPartition +from bootstrapvz.common.tools import log_check_call + + +class MSDOSPartitionMap(AbstractPartitionMap): + """Represents a MS-DOS partition map + Sometimes also called MBR (but that confuses the hell out of me, so ms-dos it is) + """ + + def __init__(self, data, bootloader): + """ + Args: + data (dict): volume.partitions part of the manifest + bootloader (str): Name of the bootloader we will use for bootstrapping + """ + from bootstrapvz.common.bytes import Bytes + # List of partitions + self.partitions = [] + + # Returns the last partition unless there is none + def last_partition(): + return self.partitions[-1] if len(self.partitions) > 0 else None + + # The boot and swap partitions are optional + if 'boot' in data: + self.boot = MSDOSPartition(Bytes(data['boot']['size']), + data['boot']['filesystem'], data['boot'].get('format_command', None), + last_partition()) + self.partitions.append(self.boot) + if 'swap' in data: + self.swap = MSDOSSwapPartition(Bytes(data['swap']['size']), last_partition()) + self.partitions.append(self.swap) + self.root = MSDOSPartition(Bytes(data['root']['size']), + data['root']['filesystem'], data['root'].get('format_command', None), + last_partition()) + self.partitions.append(self.root) + + # Mark boot as the boot partition, or root, if boot does not exist + getattr(self, 'boot', self.root).flags.append('boot') + + # If we are using the grub bootloader, we will need to create a 2 MB offset at the beginning + # of the partitionmap and steal it from the first partition + if bootloader == 'grub': + self.partitions[0].offset = Bytes('2MiB') + self.partitions[0].size -= self.partitions[0].offset + + super(MSDOSPartitionMap, self).__init__(bootloader) + + def _before_create(self, event): + volume = event.volume + # Disk alignment still plays a role in virtualized environment, + # but I honestly have no clue as to what best practice is here, so we choose 'none' + log_check_call(['parted', '--script', '--align', 'none', volume.device_path, + '--', 'mklabel', 'msdos']) + # Create the partitions + for partition in self.partitions: + partition.create(volume) diff --git a/bootstrapvz/base/fs/partitionmaps/none.py b/bootstrapvz/base/fs/partitionmaps/none.py new file mode 100644 index 0000000..734626f --- /dev/null +++ b/bootstrapvz/base/fs/partitionmaps/none.py @@ -0,0 +1,36 @@ +from ..partitions.single import SinglePartition + + +class NoPartitions(object): + """Represents a virtual 'NoPartitions' partitionmap. + This virtual partition map exists because it is easier for tasks to + simply always deal with partition maps and then let the base abstract that away. + """ + + def __init__(self, data, bootloader): + """ + Args: + data (dict): volume.partitions part of the manifest + bootloader (str): Name of the bootloader we will use for bootstrapping + """ + from bootstrapvz.common.bytes import Bytes + # In the NoPartitions partitions map we only have a single 'partition' + self.root = SinglePartition(Bytes(data['root']['size']), + data['root']['filesystem'], data['root'].get('format_command', None)) + self.partitions = [self.root] + + def is_blocking(self): + """Returns whether the partition map is blocking volume detach operations + + Returns: + bool. + """ + return self.root.fsm.current == 'mounted' + + def get_total_size(self): + """Returns the total size the partitions occupy + + Returns: + Bytes. The size of all the partitions + """ + return self.root.get_end() diff --git a/base/fs/partitionmaps/__init__.py b/bootstrapvz/base/fs/partitions/__init__.py similarity index 100% rename from base/fs/partitionmaps/__init__.py rename to bootstrapvz/base/fs/partitions/__init__.py diff --git a/bootstrapvz/base/fs/partitions/abstract.py b/bootstrapvz/base/fs/partitions/abstract.py new file mode 100644 index 0000000..0384916 --- /dev/null +++ b/bootstrapvz/base/fs/partitions/abstract.py @@ -0,0 +1,166 @@ +from abc import ABCMeta +from abc import abstractmethod +import os.path +from bootstrapvz.common.tools import log_check_call +from bootstrapvz.common.fsm_proxy import FSMProxy + + +class AbstractPartition(FSMProxy): + """Abstract representation of a partiton + This class is a finite state machine and represents the state of the real partition + """ + + __metaclass__ = ABCMeta + + # Our states + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'created'}, + {'name': 'format', 'src': 'created', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + ] + + class Mount(object): + """Represents a mount into the partition + """ + def __init__(self, source, destination, opts): + """ + Args: + source (str,AbstractPartition): The path from where we mount or a partition + destination (str): The path of the mountpoint + opts (list): List of options to pass to the mount command + """ + self.source = source + self.destination = destination + self.opts = opts + + def mount(self, prefix): + """Performs the mount operation or forwards it to another partition + + Args: + prefix (str): Path prefix of the mountpoint + """ + mount_dir = os.path.join(prefix, self.destination) + # If the source is another partition, we tell that partition to mount itself + if isinstance(self.source, AbstractPartition): + self.source.mount(destination=mount_dir) + else: + log_check_call(['mount'] + self.opts + [self.source, mount_dir]) + self.mount_dir = mount_dir + + def unmount(self): + """Performs the unmount operation or asks the partition to unmount itself + """ + # If its a partition, it can unmount itself + if isinstance(self.source, AbstractPartition): + self.source.unmount() + else: + log_check_call(['umount', self.mount_dir]) + del self.mount_dir + + def __init__(self, size, filesystem, format_command): + """ + Args: + size (Bytes): Size of the partition + filesystem (str): Filesystem the partition should be formatted with + format_command (list): Optional format command, valid variables are fs, device_path and size + """ + self.size = size + self.filesystem = filesystem + self.format_command = format_command + # Path to the partition + self.device_path = None + # Dictionary with mount points as keys and Mount objects as values + self.mounts = {} + + # Create the configuration for our state machine + cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': {}} + super(AbstractPartition, self).__init__(cfg) + + def get_uuid(self): + """Gets the UUID of the partition + + Returns: + str. The UUID of the partition + """ + [uuid] = log_check_call(['blkid', '-s', 'UUID', '-o', 'value', self.device_path]) + return uuid + + @abstractmethod + def get_start(self): + pass + + def get_end(self): + """Gets the end of the partition + + Returns: + Bytes. The end of the partition + """ + return self.get_start() + self.size + + def _before_format(self, e): + """Formats the partition + """ + # If there is no explicit format_command define we simply call mkfs.fstype + if self.format_command is None: + format_command = ['mkfs.{fs}', '{device_path}'] + else: + format_command = self.format_command + variables = {'fs': self.filesystem, + 'device_path': self.device_path, + 'size': self.size, + } + command = map(lambda part: part.format(**variables), format_command) + # Format the partition + log_check_call(command) + + def _before_mount(self, e): + """Mount the partition + """ + log_check_call(['mount', '--types', self.filesystem, self.device_path, e.destination]) + self.mount_dir = e.destination + + def _after_mount(self, e): + """Mount any mounts associated with this partition + """ + # Make sure we mount in ascending order of mountpoint path length + # This ensures that we don't mount /dev/pts before we mount /dev + for destination in sorted(self.mounts.iterkeys(), key=len): + self.mounts[destination].mount(self.mount_dir) + + def _before_unmount(self, e): + """Unmount any mounts associated with this partition + """ + # Unmount the mounts in descending order of mounpoint path length + # You cannot unmount /dev before you have unmounted /dev/pts + for destination in sorted(self.mounts.iterkeys(), key=len, reverse=True): + self.mounts[destination].unmount() + log_check_call(['umount', self.mount_dir]) + del self.mount_dir + + def add_mount(self, source, destination, opts=[]): + """Associate a mount with this partition + Automatically mounts it + + Args: + source (str,AbstractPartition): The source of the mount + destination (str): The path to the mountpoint + opts (list): Any options that should be passed to the mount command + """ + # Create a new mount object, mount it if the partition is mounted and put it in the mounts dict + mount = self.Mount(source, destination, opts) + if self.fsm.current == 'mounted': + mount.mount(self.mount_dir) + self.mounts[destination] = mount + + def remove_mount(self, destination): + """Remove a mount from this partition + Automatically unmounts it + + Args: + destination (str): The mountpoint path of the mount that should be removed + """ + # Unmount the mount if the partition is mounted and delete it from the mounts dict + # If the mount is already unmounted and the source is a partition, this will raise an exception + if self.fsm.current == 'mounted': + self.mounts[destination].unmount() + del self.mounts[destination] diff --git a/bootstrapvz/base/fs/partitions/base.py b/bootstrapvz/base/fs/partitions/base.py new file mode 100644 index 0000000..d17881b --- /dev/null +++ b/bootstrapvz/base/fs/partitions/base.py @@ -0,0 +1,105 @@ +from abstract import AbstractPartition + + +class BasePartition(AbstractPartition): + """Represents a partition that is actually a partition (and not a virtual one like 'Single') + """ + + # Override the states of the abstract partition + # A real partition can be mapped and unmapped + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'format', 'src': 'mapped', 'dst': 'formatted'}, + {'name': 'mount', 'src': 'formatted', 'dst': 'mounted'}, + {'name': 'unmount', 'src': 'mounted', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'formatted', 'dst': 'unmapped_fmt'}, + + {'name': 'map', 'src': 'unmapped_fmt', 'dst': 'formatted'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, filesystem, format_command, previous): + """ + Args: + size (Bytes): Size of the partition + filesystem (str): Filesystem the partition should be formatted with + format_command (list): Optional format command, valid variables are fs, device_path and size + previous (BasePartition): The partition that preceeds this one + """ + # By saving the previous partition we have + # a linked list that partitions can go backwards in to find the first partition. + self.previous = previous + from bootstrapvz.common.bytes import Bytes + # Initialize the offset to 0 bytes, may be changed later + self.offset = Bytes(0) + # List of flags that parted should put on the partition + self.flags = [] + super(BasePartition, self).__init__(size, filesystem, format_command) + + def create(self, volume): + """Creates the partition + + Args: + volume (Volume): The volume to create the partition on + """ + self.fsm.create(volume=volume) + + def get_index(self): + """Gets the index of this partition in the partition map + + Returns: + int. The index of the partition in the partition map + """ + if self.previous is None: + # Partitions are 1 indexed + return 1 + else: + # Recursive call to the previous partition, walking up the chain... + return self.previous.get_index() + 1 + + def get_start(self): + """Gets the starting byte of this partition + + Returns: + Bytes. The starting byte of this partition + """ + if self.previous is None: + # If there is no previous partition, this partition begins at the offset + return self.offset + else: + # Get the end of the previous partition and add the offset of this partition + return self.previous.get_end() + self.offset + + def map(self, device_path): + """Maps the partition to a device_path + + Args: + device_path (str): The device patht his partition should be mapped to + """ + self.fsm.map(device_path=device_path) + + def _before_create(self, e): + """Creates the partition + """ + from bootstrapvz.common.tools import log_check_call + # The create command is failry simple, start and end are just Bytes objects coerced into strings + create_command = ('mkpart primary {start} {end}' + .format(start=str(self.get_start()), + end=str(self.get_end()))) + # Create the partition + log_check_call(['parted', '--script', '--align', 'none', e.volume.device_path, + '--', create_command]) + + # Set any flags on the partition + for flag in self.flags: + log_check_call(['parted', '--script', e.volume.device_path, + '--', ('set {idx} {flag} on' + .format(idx=str(self.get_index()), flag=flag))]) + + def _before_map(self, e): + # Set the device path + self.device_path = e.device_path + + def _before_unmap(self, e): + # When unmapped, the device_path ifnromation becomes invalid, so we delete it + self.device_path = None diff --git a/bootstrapvz/base/fs/partitions/gpt.py b/bootstrapvz/base/fs/partitions/gpt.py new file mode 100644 index 0000000..79dd30c --- /dev/null +++ b/bootstrapvz/base/fs/partitions/gpt.py @@ -0,0 +1,29 @@ +from bootstrapvz.common.tools import log_check_call +from base import BasePartition + + +class GPTPartition(BasePartition): + """Represents a GPT partition + """ + + def __init__(self, size, filesystem, format_command, name, previous): + """ + Args: + size (Bytes): Size of the partition + filesystem (str): Filesystem the partition should be formatted with + format_command (list): Optional format command, valid variables are fs, device_path and size + name (str): The name of the partition + previous (BasePartition): The partition that preceeds this one + """ + self.name = name + super(GPTPartition, self).__init__(size, filesystem, format_command, previous) + + def _before_create(self, e): + # Create the partition and then set the name of the partition afterwards + super(GPTPartition, self)._before_create(e) + # partition name only works for gpt, for msdos that becomes the part-type (primary, extended, logical) + name_command = ('name {idx} {name}' + .format(idx=self.get_index(), + name=self.name)) + log_check_call(['parted', '--script', e.volume.device_path, + '--', name_command]) diff --git a/bootstrapvz/base/fs/partitions/gpt_swap.py b/bootstrapvz/base/fs/partitions/gpt_swap.py new file mode 100644 index 0000000..6dc5865 --- /dev/null +++ b/bootstrapvz/base/fs/partitions/gpt_swap.py @@ -0,0 +1,18 @@ +from bootstrapvz.common.tools import log_check_call +from gpt import GPTPartition + + +class GPTSwapPartition(GPTPartition): + """Represents a GPT swap partition + """ + + def __init__(self, size, previous): + """ + Args: + size (Bytes): Size of the partition + previous (BasePartition): The partition that preceeds this one + """ + super(GPTSwapPartition, self).__init__(size, 'swap', None, 'swap', previous) + + def _before_format(self, e): + log_check_call(['mkswap', self.device_path]) diff --git a/base/fs/partitions/msdos.py b/bootstrapvz/base/fs/partitions/msdos.py similarity index 65% rename from base/fs/partitions/msdos.py rename to bootstrapvz/base/fs/partitions/msdos.py index e0f7f62..cb7d96d 100644 --- a/base/fs/partitions/msdos.py +++ b/bootstrapvz/base/fs/partitions/msdos.py @@ -2,4 +2,6 @@ from base import BasePartition class MSDOSPartition(BasePartition): + """Represents an MS-DOS partition + """ pass diff --git a/bootstrapvz/base/fs/partitions/msdos_swap.py b/bootstrapvz/base/fs/partitions/msdos_swap.py new file mode 100644 index 0000000..12085e0 --- /dev/null +++ b/bootstrapvz/base/fs/partitions/msdos_swap.py @@ -0,0 +1,18 @@ +from bootstrapvz.common.tools import log_check_call +from msdos import MSDOSPartition + + +class MSDOSSwapPartition(MSDOSPartition): + """Represents a MS-DOS swap partition + """ + + def __init__(self, size, previous): + """ + Args: + size (Bytes): Size of the partition + previous (BasePartition): The partition that preceeds this one + """ + super(MSDOSSwapPartition, self).__init__(size, 'swap', None, previous) + + def _before_format(self, e): + log_check_call(['mkswap', self.device_path]) diff --git a/bootstrapvz/base/fs/partitions/single.py b/bootstrapvz/base/fs/partitions/single.py new file mode 100644 index 0000000..297041e --- /dev/null +++ b/bootstrapvz/base/fs/partitions/single.py @@ -0,0 +1,16 @@ +from abstract import AbstractPartition + + +class SinglePartition(AbstractPartition): + """Represents a single virtual partition on an unpartitioned volume + """ + + def get_start(self): + """Gets the starting byte of this partition + + Returns: + Bytes. The starting byte of this partition + """ + from bootstrapvz.common.bytes import Bytes + # On an unpartitioned volume there is no offset and no previous partition + return Bytes(0) diff --git a/bootstrapvz/base/fs/partitions/unformatted.py b/bootstrapvz/base/fs/partitions/unformatted.py new file mode 100644 index 0000000..271f35f --- /dev/null +++ b/bootstrapvz/base/fs/partitions/unformatted.py @@ -0,0 +1,21 @@ +from base import BasePartition + + +class UnformattedPartition(BasePartition): + """Represents an unformatted partition + It cannot be mounted + """ + + # The states for our state machine. It can only be mapped, not mounted. + events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'unmapped'}, + {'name': 'map', 'src': 'unmapped', 'dst': 'mapped'}, + {'name': 'unmap', 'src': 'mapped', 'dst': 'unmapped'}, + ] + + def __init__(self, size, previous): + """ + Args: + size (Bytes): Size of the partition + previous (BasePartition): The partition that preceeds this one + """ + super(UnformattedPartition, self).__init__(size, None, None, previous) diff --git a/base/fs/volume.py b/bootstrapvz/base/fs/volume.py similarity index 51% rename from base/fs/volume.py rename to bootstrapvz/base/fs/volume.py index 631c886..50bf213 100644 --- a/base/fs/volume.py +++ b/bootstrapvz/base/fs/volume.py @@ -1,14 +1,18 @@ from abc import ABCMeta -from common.fsm_proxy import FSMProxy -from common.tools import log_check_call -from exceptions import VolumeError +from bootstrapvz.common.fsm_proxy import FSMProxy +from bootstrapvz.common.tools import log_check_call +from .exceptions import VolumeError from partitionmaps.none import NoPartitions class Volume(FSMProxy): + """Represents an abstract volume. + This class is a finite state machine and represents the state of the real volume. + """ __metaclass__ = ABCMeta + # States this volume can be in events = [{'name': 'create', 'src': 'nonexistent', 'dst': 'detached'}, {'name': 'attach', 'src': 'detached', 'dst': 'attached'}, {'name': 'link_dm_node', 'src': 'attached', 'dst': 'linked'}, @@ -18,33 +22,76 @@ class Volume(FSMProxy): ] def __init__(self, partition_map): + """ + Args: + partition_map (PartitionMap): The partition map for the volume + """ + # Path to the volume self.device_path = None self.real_device_path = None + # The partition map self.partition_map = partition_map + # The size of the volume as reported by the partition map self.size = self.partition_map.get_total_size() + # Before detaching, check that nothing would block the detachment callbacks = {'onbeforedetach': self._check_blocking} if isinstance(self.partition_map, NoPartitions): + # When the volume has no partitions, the virtual root partition path is equal to that of the volume + # Update that path whenever the path to the volume changes def set_dev_path(e): self.partition_map.root.device_path = self.device_path callbacks['onafterattach'] = set_dev_path callbacks['onlink_dm_node'] = set_dev_path callbacks['onunlink_dm_node'] = set_dev_path + # Create the configuration for our finite state machine cfg = {'initial': 'nonexistent', 'events': self.events, 'callbacks': callbacks} super(Volume, self).__init__(cfg) def _after_create(self, e): + """ + Args: + e (_e_obj): Event object containing arguments to create() + """ if isinstance(self.partition_map, NoPartitions): + # When the volume has no partitions, the virtual root partition + # is essentially created when the volume is created, forward that creation event. self.partition_map.root.create() def _check_blocking(self, e): + """Checks whether the volume is blocked + + Args: + e (_e_obj): Event object containing arguments to create() + + Raises: + VolumeError + """ + # Only the partition map can block the volume if self.partition_map.is_blocking(): raise VolumeError('The partitionmap prevents the detach procedure') def _before_link_dm_node(self, e): + """Links the volume using the device mapper + This allows us to create a 'window' into the volume that acts like a volum in itself. + Mainly it is used to fool grub into thinking that it is working with a real volume, + rather than a loopback device or a network block device. + + Args: + e (_e_obj): Event object containing arguments to create() + Arguments are: + logical_start_sector (int): The sector the volume should start at in the new volume + start_sector (int): The offset at which the volume should begin to be mapped in the new volume + sectors (int): The number of sectors that should be mapped + Read more at: http://manpages.debian.org/cgi-bin/man.cgi?query=dmsetup&apropos=0&sektion=0&manpath=Debian+7.0+wheezy&format=html&locale=en + + Raises: + VolumeError + """ import os.path - from common.fs import get_partitions + from bootstrapvz.common.fs import get_partitions + # Fetch information from /proc/partitions proc_partitions = get_partitions() device_name = os.path.basename(self.device_path) device_partition = proc_partitions[device_name] @@ -55,8 +102,10 @@ class Volume(FSMProxy): # The offset at which the volume should begin to be mapped in the new volume start_sector = getattr(e, 'start_sector', 0) + # The number of sectors that should be mapped sectors = getattr(e, 'sectors', int(self.size / 512) - start_sector) + # This is the table we send to dmsetup, so that it may create a decie mapping for us. table = ('{log_start_sec} {sectors} linear {major}:{minor} {start_sec}' .format(log_start_sec=logical_start_sector, sectors=sectors, @@ -65,6 +114,7 @@ class Volume(FSMProxy): start_sec=start_sector)) import string import os.path + # Figure out the device letter and path for letter in string.ascii_lowercase: dev_name = 'vd' + letter dev_path = os.path.join('/dev/mapper', dev_name) @@ -76,12 +126,21 @@ class Volume(FSMProxy): if not hasattr(self, 'dm_node_name'): raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - log_check_call(['/sbin/dmsetup', 'create', self.dm_node_name], table) + # Create the device mapping + log_check_call(['dmsetup', 'create', self.dm_node_name], table) + # Update the device_path but remember the old one for when we unlink the volume again self.unlinked_device_path = self.device_path self.device_path = self.dm_node_path def _before_unlink_dm_node(self, e): - log_check_call(['/sbin/dmsetup', 'remove', self.dm_node_name]) + """Unlinks the device mapping + + Args: + e (_e_obj): Event object containing arguments to create() + """ + log_check_call(['dmsetup', 'remove', self.dm_node_name]) + # Delete the no longer valid information del self.dm_node_name del self.dm_node_path + # Reset the device_path self.device_path = self.unlinked_device_path diff --git a/bootstrapvz/base/log.py b/bootstrapvz/base/log.py new file mode 100644 index 0000000..15551c8 --- /dev/null +++ b/bootstrapvz/base/log.py @@ -0,0 +1,95 @@ +"""This module holds functions and classes responsible for formatting the log output +both to a file and to the console. +.. module:: log +""" +import logging + + +def create_log_dir(): + """Creates the log directory + + Returns: + str. The path to the logdirectory + """ + log_dir_path = '/var/log/bootstrap-vz' + import os + if not os.path.exists(log_dir_path): + os.makedirs(log_dir_path) + return log_dir_path + + +def get_log_filename(manifest_path): + """Returns the path to a logfile given a manifest + The logfile name is constructed from the current timestamp and the basename of the manifest + + Args: + manifest_path (str): The path to the manifest + + Returns: + str. The path to the logfile + """ + import os.path + from datetime import datetime + + manifest_basename = os.path.basename(manifest_path) + manifest_name, _ = os.path.splitext(manifest_basename) + timestamp = datetime.now().strftime('%Y%m%d%H%M%S') + filename = "{timestamp}_{name}.log".format(timestamp=timestamp, name=manifest_name) + return filename + + +def setup_logger(logfile=None, debug=False): + """Sets up the python logger to log to both a file and the console + + Args: + logfile (str): Path to a logfile + debug (bool): Whether to log debug output to the console + """ + root = logging.getLogger() + # Make sure all logging statements are processed by our handlers, they decide the log level + root.setLevel(logging.NOTSET) + + # Create a file log handler + file_handler = logging.FileHandler(logfile) + # Absolute timestamps are rather useless when bootstrapping, it's much more interesting + # to see how long things take, so we log in a relative format instead + file_handler.setFormatter(FileFormatter('[%(relativeCreated)s] %(levelname)s: %(message)s')) + # The file log handler always logs everything + file_handler.setLevel(logging.DEBUG) + root.addHandler(file_handler) + + # Create a console log handler + import sys + console_handler = logging.StreamHandler(sys.stderr) + # We want to colorize the output to the console, so we add a formatter + console_handler.setFormatter(ConsoleFormatter()) + # Set the log level depending on the debug argument + if debug: + console_handler.setLevel(logging.DEBUG) + else: + console_handler.setLevel(logging.INFO) + root.addHandler(console_handler) + + +class ConsoleFormatter(logging.Formatter): + """Formats log statements for the console + """ + level_colors = {logging.ERROR: 'red', + logging.WARNING: 'magenta', + logging.INFO: 'blue', + } + + def format(self, record): + if(record.levelno in self.level_colors): + # Colorize the message if we have a color for it (DEBUG has no color) + from termcolor import colored + record.msg = colored(record.msg, self.level_colors[record.levelno]) + return super(ConsoleFormatter, self).format(record) + + +class FileFormatter(logging.Formatter): + """Formats log statements for output to file + Currently this is just a stub + """ + def format(self, record): + return super(FileFormatter, self).format(record) diff --git a/bootstrapvz/base/main.py b/bootstrapvz/base/main.py new file mode 100644 index 0000000..936abe4 --- /dev/null +++ b/bootstrapvz/base/main.py @@ -0,0 +1,100 @@ +"""Main module containing all the setup necessary for running the bootstrapping process +.. module:: main +""" + +import logging +log = logging.getLogger(__name__) + + +def main(): + """Main function for invoking the bootstrap process + + Raises: + Exception + """ + # Get the commandline arguments + import os + args = get_args() + # Require root privileges, except when doing a dry-run where they aren't needed + if os.geteuid() != 0 and not args.dry_run: + raise Exception('This program requires root privileges.') + # Setup logging + import log + log_dir = log.create_log_dir() + log_filename = log.get_log_filename(args.manifest) + logfile = os.path.join(log_dir, log_filename) + log.setup_logger(logfile=logfile, debug=args.debug) + # Everything has been set up, begin the bootstrapping process + run(args) + + +def get_args(): + """Creates an argument parser and returns the arguments it has parsed + """ + from argparse import ArgumentParser + parser = ArgumentParser(description='Bootstrap Debian for the cloud.') + parser.add_argument('--debug', action='store_true', + help='Print debugging information') + parser.add_argument('--pause-on-error', action='store_true', + help='Pause on error, before rollback') + parser.add_argument('--dry-run', action='store_true', + help='Dont\'t actually run the tasks') + parser.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') + return parser.parse_args() + + +def run(args): + """Runs the bootstrapping process + + Args: + args (dict): Dictionary of arguments from the commandline + """ + # Load the manifest + from manifest import Manifest + manifest = Manifest(args.manifest) + + # Get the tasklist + from tasklist import TaskList + tasklist = TaskList() + # 'resolve_tasks' is the name of the function to call on the provider and plugins + tasklist.load('resolve_tasks', manifest) + + # Create the bootstrap information object that'll be used throughout the bootstrapping process + from bootstrapinfo import BootstrapInformation + bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug) + + try: + # Run all the tasks the tasklist has gathered + tasklist.run(info=bootstrap_info, dry_run=args.dry_run) + # We're done! :-) + log.info('Successfully completed bootstrapping') + except (Exception, KeyboardInterrupt) as e: + # When an error occurs, log it and begin rollback + log.exception(e) + if args.pause_on_error: + # The --pause-on-error is useful when the user wants to inspect the volume before rollback + raw_input('Press Enter to commence rollback') + log.error('Rolling back') + + # Create a new tasklist to gather the necessary tasks for rollback + rollback_tasklist = TaskList() + + # Create a useful little function for the provider and plugins to use, + # when figuring out what tasks should be added to the rollback list. + def counter_task(task, counter): + """counter_task() adds the second argument to the rollback tasklist + if the first argument is present in the list of completed tasks + + Args: + task (Task): The task to look for in the completed tasks list + counter (Task): The task to add to the rollback tasklist + """ + if task in tasklist.tasks_completed and counter not in tasklist.tasks_completed: + rollback_tasklist.tasks.add(counter) + # Ask the provider and plugins for tasks they'd like to add to the rollback tasklist + # Any additional arguments beyond the first two are passed directly to the provider and plugins + rollback_tasklist.load('resolve_rollback_tasks', manifest, counter_task) + + # Run the rollback tasklist + rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run) + log.info('Successfully completed rollback') diff --git a/base/manifest-schema.json b/bootstrapvz/base/manifest-schema.json similarity index 92% rename from base/manifest-schema.json rename to bootstrapvz/base/manifest-schema.json index d74b36a..e4f4eb6 100644 --- a/base/manifest-schema.json +++ b/bootstrapvz/base/manifest-schema.json @@ -71,6 +71,9 @@ ] }, "minItems": 1 + }, + "install_standard": { + "type": "boolean" } }, "additionalProperties": false @@ -99,7 +102,7 @@ "additionalProperties": false } }, - "required": ["provider", "bootstrapper", "image", "volume", "system"], + "required": ["provider", "bootstrapper", "system", "volume"], "definitions": { "path": { "type": "string", @@ -141,10 +144,14 @@ "type": "object", "properties": { "size": { "$ref": "#/definitions/bytes" }, - "filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] } + "filesystem": { "enum": ["ext2", "ext3", "ext4", "xfs"] }, + "format_command": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } }, "required": ["size", "filesystem"] } - }, - "required": ["provider", "bootstrapper", "system", "packages", "volume"] + } } diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py new file mode 100644 index 0000000..58f7994 --- /dev/null +++ b/bootstrapvz/base/manifest.py @@ -0,0 +1,125 @@ +"""The Manifest module contains the manifest that providers and plugins use +to determine which tasks should be added to the tasklist, what arguments various +invocations should have etc.. +.. module:: manifest +""" +from bootstrapvz.common.tools import load_json +import logging +log = logging.getLogger(__name__) + + +class Manifest(object): + """This class holds all the information that providers and plugins need + to perform the bootstrapping process. All actions that are taken originate from + here. The manifest shall not be modified after it has been loaded. + Currently, immutability is not enforced and it would require a fair amount of code + to enforce it, instead we just rely on tasks behaving properly. + """ + def __init__(self, path): + """Initializer: Given a path we load, validate and parse the manifest. + + Args: + path (str): The path to the manifest + """ + self.path = path + self.load() + self.validate() + self.parse() + + def load(self): + """Loads the manifest. + This function not only reads the manifest but also loads the specified provider and plugins. + Once they are loaded, the initialize() function is called on each of them (if it exists). + The provider must have an initialize function. + """ + # Load the manifest JSON using the loader in common.tools + # It strips comments (which are invalid in strict json) before loading the data. + self.data = load_json(self.path) + # Get the provider name from the manifest and load the corresponding module + provider_modname = 'bootstrapvz.providers.{provider}'.format(provider=self.data['provider']) + log.debug('Loading provider `{modname}\''.format(modname=provider_modname)) + # Create a modules dict that contains the loaded provider and plugins + self.modules = {'provider': __import__(provider_modname, fromlist=['providers']), + 'plugins': [], + } + # Run through all the plugins mentioned in the manifest and load them + if 'plugins' in self.data: + for plugin_name, plugin_data in self.data['plugins'].iteritems(): + modname = 'bootstrapvz.plugins.{plugin}'.format(plugin=plugin_name) + log.debug('Loading plugin `{modname}\''.format(modname=modname)) + plugin = __import__(modname, fromlist=['plugins']) + self.modules['plugins'].append(plugin) + + # Run the initialize function on the provider and plugins + self.modules['provider'].initialize() + for module in self.modules['plugins']: + # Plugins are not required to have an initialize function + init = getattr(module, 'initialize', None) + if callable(init): + init() + + def validate(self): + """Validates the manifest using the base, provider and plugin validation functions. + Plugins are not required to have a validate_manifest function + """ + from . import validate_manifest + # Validate the manifest with the base validation function in __init__ + validate_manifest(self.data, self.schema_validator, self.validation_error) + # Run the provider validation + self.modules['provider'].validate_manifest(self.data, self.schema_validator, self.validation_error) + # Run the validation function for any plugin that has it + for plugin in self.modules['plugins']: + validate = getattr(plugin, 'validate_manifest', None) + if callable(validate): + validate(self.data, self.schema_validator, self.validation_error) + + def parse(self): + """Parses the manifest. + Well... "parsing" is a big word. + The function really just sets up some convenient attributes so that tasks + don't have to access information with info.manifest.data['section'] + but can do it with info.manifest.section. + """ + self.provider = self.data['provider'] + self.bootstrapper = self.data['bootstrapper'] + self.image = self.data['image'] + self.volume = self.data['volume'] + self.system = self.data['system'] + # The packages and plugins section is not required + self.packages = self.data['packages'] if 'packages' in self.data else {} + self.plugins = self.data['plugins'] if 'plugins' in self.data else {} + + def load_json(self, path): + """Loads JSON. Unused and will be removed. + Use common.tools.load_json instead + """ + import json + from minify_json import json_minify + with open(path) as stream: + return json.loads(json_minify(stream.read(), False)) + + def schema_validator(self, data, schema_path): + """This convenience function is passed around to all the validation functions + so that they may run a json-schema validation by giving it the data and a path to the schema. + + Args: + data (dict): Data to validate (normally the manifest data) + schema_path (str): Path to the json-schema to use for validation + """ + import jsonschema + schema = load_json(schema_path) + try: + jsonschema.validate(data, schema) + except jsonschema.ValidationError as e: + self.validation_error(e.message, e.path) + + def validation_error(self, message, json_path=None): + """This function is passed to all validation functions so that they may + raise a validation error because a custom validation of the manifest failed. + + Args: + message (str): Message to user about the error + json_path (list): A path to the location in the manifest where the error occurred + """ + from bootstrapvz.common.exceptions import ManifestError + raise ManifestError(message, self.path, json_path) diff --git a/bootstrapvz/base/phase.py b/bootstrapvz/base/phase.py new file mode 100644 index 0000000..3a5e320 --- /dev/null +++ b/bootstrapvz/base/phase.py @@ -0,0 +1,35 @@ + + +class Phase(object): + """The Phase class represents a phase a task may be in. + It has no function other than to act as an anchor in the task graph. + All phases are instantiated in common.phases + """ + + def __init__(self, name, description): + # The name of the phase + self.name = name + # The description of the phase (currently not used anywhere) + self.description = description + + def pos(self): + """Gets the position of the phase + + Returns: + int. The positional index of the phase in relation to the other phases + """ + from bootstrapvz.common.phases import order + return next(i for i, phase in enumerate(order) if phase is self) + + def __cmp__(self, other): + """Compares the phase order in relation to the other phases + """ + return self.pos() - other.pos() + + def __str__(self): + """String representation of the phase, the name suffices + + Returns: + string. + """ + return self.name diff --git a/base/pkg/__init__.py b/bootstrapvz/base/pkg/__init__.py similarity index 100% rename from base/pkg/__init__.py rename to bootstrapvz/base/pkg/__init__.py diff --git a/bootstrapvz/base/pkg/exceptions.py b/bootstrapvz/base/pkg/exceptions.py new file mode 100644 index 0000000..dc7534b --- /dev/null +++ b/bootstrapvz/base/pkg/exceptions.py @@ -0,0 +1,12 @@ + + +class PackageError(Exception): + """Raised when an error occurrs while handling the packageslist + """ + pass + + +class SourceError(Exception): + """Raised when an error occurs while handling the sourceslist + """ + pass diff --git a/bootstrapvz/base/pkg/packagelist.py b/bootstrapvz/base/pkg/packagelist.py new file mode 100644 index 0000000..23e596d --- /dev/null +++ b/bootstrapvz/base/pkg/packagelist.py @@ -0,0 +1,115 @@ +from exceptions import PackageError + + +class PackageList(object): + """Represents a list of packages + """ + + class Remote(object): + """A remote package with an optional target + """ + def __init__(self, name, target): + """ + Args: + name (str): The name of the package + target (str): The name of the target release + """ + self.name = name + self.target = target + + def __str__(self): + """Converts the package into somehting that apt-get install can parse + Returns: + string. + """ + if self.target is None: + return self.name + else: + return '{name}/{target}'.format(name=self.name, target=self.target) + + class Local(object): + """A local package + """ + def __init__(self, path): + """ + Args: + path (str): The path to the local package + """ + self.path = path + + def __str__(self): + """ + Returns: + string. The path to the local package + """ + return self.path + + def __init__(self, manifest_vars, source_lists): + """ + Args: + manifest_vars (dict): The manifest variables + source_lists (SourceLists): The sourcelists for apt + """ + self.manifest_vars = manifest_vars + self.source_lists = source_lists + # The default_target is the release we are bootstrapping + self.default_target = '{system.release}'.format(**self.manifest_vars) + # The list of packages that should be installed, this is not a set. + # We want to preserve the order in which the packages were added so that local + # packages may be installed in the correct order. + self.install = [] + # A function that filters the install list and only returns remote packages + self.remote = lambda: filter(lambda x: isinstance(x, self.Remote), self.install) + + def add(self, name, target=None): + """Adds a package to the install list + + Args: + name (str): The name of the package to install, may contain manifest vars references + target (str): The name of the target release for the package, may contain manifest vars references + + Raises: + PackageError + """ + name = name.format(**self.manifest_vars) + if target is not None: + target = target.format(**self.manifest_vars) + # Check if the package has already been added. + # If so, make sure it's the same target and raise a PackageError otherwise + package = next((pkg for pkg in self.remote() if pkg.name == name), None) + if package is not None: + # It's the same target if the target names match or one of the targets is None + # and the other is the default target. + same_target = package.target == target + same_target = same_target or package.target is None and target == self.default_target + same_target = same_target or package.target == self.default_target and target is None + if not same_target: + msg = ('The package {name} was already added to the package list, ' + 'but with target release `{target}\' instead of `{add_target}\'' + .format(name=name, target=package.target, add_target=target)) + raise PackageError(msg) + # The package has already been added, skip the checks below + return + + # Check if the target exists in the sources list, raise a PackageError if not + check_target = target + if check_target is None: + check_target = self.default_target + if not self.source_lists.target_exists(check_target): + msg = ('The target release {target} was not found in the sources list').format(target=check_target) + raise PackageError(msg) + + # Note that we maintain the target value even if it is none. + # This allows us to preserve the semantics of the default target when calling apt-get install + # Why? Try installing nfs-client/wheezy, you can't. It's a virtual package for which you cannot define + # a target release. Only `apt-get install nfs-client` works. + self.install.append(self.Remote(name, target)) + + def add_local(self, package_path): + """Adds a local package to the installation list + + Args: + package_path (str): Path to the local package, may contain manifest vars references + """ + package_path = package_path.format(**self.manifest_vars) + self.install.append(self.Local(package_path)) diff --git a/base/pkg/sourceslist.py b/bootstrapvz/base/pkg/sourceslist.py similarity index 59% rename from base/pkg/sourceslist.py rename to bootstrapvz/base/pkg/sourceslist.py index 7dc6486..0a50243 100644 --- a/base/pkg/sourceslist.py +++ b/bootstrapvz/base/pkg/sourceslist.py @@ -1,12 +1,27 @@ class SourceLists(object): + """Represents a list of sources lists for apt + """ def __init__(self, manifest_vars): + """ + Args: + manifest_vars (dict): The manifest variables + """ + # A dictionary with the name of the file in sources.list.d as the key + # That values are lists of Source objects self.sources = {} + # Save the manifest variables, we need the later on self.manifest_vars = manifest_vars def add(self, name, line): + """Adds a source to the apt sources list + + Args: + name (str): Name of the file in sources.list.d, may contain manifest vars references + line (str): The line for the source file, may contain manifest vars references + """ name = name.format(**self.manifest_vars) line = line.format(**self.manifest_vars) if name not in self.sources: @@ -14,7 +29,16 @@ class SourceLists(object): self.sources[name].append(Source(line)) def target_exists(self, target): + """Checks whether the target exists in the sources list + + Args: + target (str): Name of the target to check for, may contain manifest vars references + + Returns: + bool. Whether the target exists + """ target = target.format(**self.manifest_vars) + # Run through all the sources and return True if the target exists for lines in self.sources.itervalues(): if target in (source.distribution for source in lines): return True @@ -22,8 +46,20 @@ class SourceLists(object): class Source(object): + """Represents a single source line + """ def __init__(self, line): + """ + Args: + line (str): A apt source line + + Raises: + SourceError + """ + # Parse the source line and populate the class attributes with it + # The format is taken from `man sources.list` + # or: http://manpages.debian.org/cgi-bin/man.cgi?sektion=5&query=sources.list&apropos=0&manpath=sid&locale=en import re regexp = re.compile('^(?Pdeb|deb-src)\s+' '(\[\s*(?P.+\S)?\s*\]\s+)?' @@ -45,6 +81,12 @@ class Source(object): self.components = re.sub(' +', ' ', match['components']).split(' ') def __str__(self): + """Convert the object into a source line + This is pretty much the reverse of what we're doing in the initialization function. + + Returns: + string. + """ options = '' if len(self.options) > 0: options = ' [{options}]'.format(options=' '.join(self.options)) diff --git a/bootstrapvz/base/release-codenames.json b/bootstrapvz/base/release-codenames.json new file mode 100644 index 0000000..cac8692 --- /dev/null +++ b/bootstrapvz/base/release-codenames.json @@ -0,0 +1,22 @@ +{ // This is a mapping of Debian release names to their respective codenames + "unstable": "sid", + "testing": "jessie", + "stable": "wheezy", + "oldstable": "squeeze", + + "jessie": "jessie", + "wheezy": "wheezy", + "squeeze": "squeeze", + + // The following release names are not supported, but included of completeness sake + "lenny": "lenny", + "etch": "etch", + "sarge": "sarge", + "woody": "woody", + "potato": "potato", + "slink": "slink", + "hamm": "hamm", + "bo": "bo", + "rex": "rex", + "buzz": "buzz" +} diff --git a/bootstrapvz/base/task.py b/bootstrapvz/base/task.py new file mode 100644 index 0000000..ea92698 --- /dev/null +++ b/bootstrapvz/base/task.py @@ -0,0 +1,38 @@ + + +class Task(object): + """The task class represents a task that can be run. + It is merely a wrapper for the run function and should never be instantiated. + """ + # The phase this task is located in. + phase = None + # List of tasks that should run before this task is run + predecessors = [] + # List of tasks that should run after this task has run + successors = [] + + class __metaclass__(type): + """Metaclass to control how the class is coerced into a string + """ + def __repr__(cls): + """ + Returns: + string. + """ + return '{module}.{task}'.format(module=cls.__module__, task=cls.__name__) + + def __str__(cls): + """ + Returns: + string. + """ + return repr(cls) + + @classmethod + def run(cls, info): + """The run function, all work is done inside this function + + :param info: The bootstrap info object. + :type info: BootstrapInformation + """ + pass diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py new file mode 100644 index 0000000..1016246 --- /dev/null +++ b/bootstrapvz/base/tasklist.py @@ -0,0 +1,252 @@ +"""The tasklist module contains the TaskList class. +.. module:: tasklist +""" + +from bootstrapvz.common.exceptions import TaskListError +import logging +log = logging.getLogger(__name__) + + +class TaskList(object): + """The tasklist class aggregates all tasks that should be run + and orders them according to their dependencies. + """ + + def __init__(self): + self.tasks = set() + self.tasks_completed = [] + + def load(self, function, manifest, *args): + """Calls 'function' on the provider and all plugins that have been loaded by the manifest. + Any additional arguments are passed directly to 'function'. + The function that is called shall accept the taskset as its first argument and the manifest + as its second argument. + + Args: + function (str): Name of the function to call + manifest (Manifest): The manifest + \*args: Additional arguments that should be passed to the function that is called + """ + # Call 'function' on the provider + getattr(manifest.modules['provider'], function)(self.tasks, manifest, *args) + for plugin in manifest.modules['plugins']: + # Plugins har not required to have whatever function we call + fn = getattr(plugin, function, None) + if callable(fn): + fn(self.tasks, manifest, *args) + + def run(self, info={}, dry_run=False): + """Converts the taskgraph into a list and runs all tasks in that list + + Args: + info (dict): The bootstrap information object + dry_run (bool): Whether to actually run the tasks or simply step through them + """ + # Create a list for us to run + task_list = self.create_list() + # Output the tasklist + log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(map(repr, task_list)))) + + for task in task_list: + # Tasks are not required to have a description + if hasattr(task, 'description'): + log.info(task.description) + else: + # If there is no description, simply coerce the task into a string and print its name + log.info('Running {task}'.format(task=task)) + if not dry_run: + # Run the task + task.run(info) + # Remember which tasks have been run for later use (e.g. when rolling back, because of an error) + self.tasks_completed.append(task) + + def create_list(self): + """Creates a list of all the tasks that should be run. + """ + from bootstrapvz.common.phases import order + # Get a hold of all tasks + tasks = self.get_all_tasks() + # Make sure the taskset is a subset of all the tasks we have gathered + self.tasks.issubset(tasks) + # Create a graph over all tasks by creating a map of each tasks successors + graph = {} + for task in tasks: + # Do a sanity check first + self.check_ordering(task) + successors = set() + # Add all successors mentioned in the task + successors.update(task.successors) + # Add all tasks that mention this task as a predecessor + successors.update(filter(lambda succ: task in succ.predecessors, tasks)) + # Create a list of phases that succeed the phase of this task + succeeding_phases = order[order.index(task.phase) + 1:] + # Add all tasks that occur in above mentioned succeeding phases + successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks)) + # Map the successors to the task + graph[task] = successors + + # Use the strongly connected components algorithm to check for cycles in our task graph + components = self.strongly_connected_components(graph) + cycles_found = 0 + for component in components: + # Node of 1 is also a strongly connected component but hardly a cycle, so we filter them out + if len(component) > 1: + cycles_found += 1 + log.debug('Cycle: {list}\n'.format(list=', '.join(map(repr, component)))) + if cycles_found > 0: + msg = ('{0} cycles were found in the tasklist, ' + 'consult the logfile for more information.'.format(cycles_found)) + raise TaskListError(msg) + + # Run a topological sort on the graph, returning an ordered list + sorted_tasks = self.topological_sort(graph) + + # Filter out any tasks not in the tasklist + # We want to maintain ordering, so we don't use set intersection + sorted_tasks = filter(lambda task: task in self.tasks, sorted_tasks) + return sorted_tasks + + def get_all_tasks(self): + """Gets a list of all task classes in the package + + Returns: + list. A list of all tasks in the package + """ + # Get a generator that returns all classes in the package + import os.path + pkg_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) + classes = self.get_all_classes(pkg_path, 'bootstrapvz.') + + # lambda function to check whether a class is a task (excluding the superclass Task) + def is_task(obj): + from task import Task + return issubclass(obj, Task) and obj is not Task + return filter(is_task, classes) # Only return classes that are tasks + + def get_all_classes(self, path=None, prefix=''): + """ Given a path to a package, this function retrieves all the classes in it + + Args: + path (str): Path to the package + prefix (str): Name of the package followed by a dot + + Returns: + generator. A generator that yields classes + + Raises: + Exception + """ + import pkgutil + import importlib + import inspect + + def walk_error(module): + raise Exception('Unable to inspect module `{module}\''.format(module=module)) + walker = pkgutil.walk_packages([path], prefix, walk_error) + for _, module_name, _ in walker: + module = importlib.import_module(module_name) + classes = inspect.getmembers(module, inspect.isclass) + for class_name, obj in classes: + # We only want classes that are defined in the module, and not imported ones + if obj.__module__ == module_name: + yield obj + + def check_ordering(self, task): + """Checks the ordering of a task in relation to other tasks and their phases + This function checks for a subset of what the strongly connected components algorithm does, + but can deliver a more precise error message, namely that there is a conflict between + what a task has specified as its predecessors or successors and in which phase it is placed. + + Args: + task (Task): The task to check the ordering for + + Raises: + TaskListError + """ + for successor in task.successors: + # Run through all successors and check whether the phase of the task + # comes before the phase of a successor + if successor.phase > successor.phase: + msg = ("The task {task} is specified as running before {other}, " + "but its phase '{phase}' lies after the phase '{other_phase}'" + .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase)) + raise TaskListError(msg) + for predecessor in task.predecessors: + # Run through all predecessors and check whether the phase of the task + # comes after the phase of a predecessor + if task.phase < predecessor.phase: + msg = ("The task {task} is specified as running after {other}, " + "but its phase '{phase}' lies before the phase '{other_phase}'" + .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase)) + raise TaskListError(msg) + + def strongly_connected_components(self, graph): + """Find the strongly connected components in a graph using Tarjan's algorithm. + Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py + + Args: + graph (dict): mapping of tasks to lists of successor tasks + + Returns: + list. List of tuples that are strongly connected comoponents + """ + + result = [] + stack = [] + low = {} + + def visit(node): + if node in low: + return + + num = len(low) + low[node] = num + stack_pos = len(stack) + stack.append(node) + + for successor in graph[node]: + visit(successor) + low[node] = min(low[node], low[successor]) + + if num == low[node]: + component = tuple(stack[stack_pos:]) + del stack[stack_pos:] + result.append(component) + for item in component: + low[item] = len(graph) + + for node in graph: + visit(node) + + return result + + def topological_sort(self, graph): + """Runs a topological sort on a graph + Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py + + Args: + graph (dict): mapping of tasks to lists of successor tasks + + Returns: + list. A list of all tasks in the graph sorted according to ther dependencies + """ + count = {} + for node in graph: + count[node] = 0 + for node in graph: + for successor in graph[node]: + count[successor] += 1 + + ready = [node for node in graph if count[node] == 0] + + result = [] + while ready: + node = ready.pop(-1) + result.append(node) + + for successor in graph[node]: + count[successor] -= 1 + if count[successor] == 0: + ready.append(successor) + + return result diff --git a/common/__init__.py b/bootstrapvz/common/__init__.py similarity index 100% rename from common/__init__.py rename to bootstrapvz/common/__init__.py diff --git a/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root similarity index 100% rename from common/assets/init.d/expand-root rename to bootstrapvz/common/assets/init.d/expand-root diff --git a/common/assets/init.d/generate-ssh-hostkeys b/bootstrapvz/common/assets/init.d/generate-ssh-hostkeys similarity index 100% rename from common/assets/init.d/generate-ssh-hostkeys rename to bootstrapvz/common/assets/init.d/generate-ssh-hostkeys diff --git a/common/assets/init.d/squeeze/generate-ssh-hostkeys b/bootstrapvz/common/assets/init.d/squeeze/generate-ssh-hostkeys similarity index 100% rename from common/assets/init.d/squeeze/generate-ssh-hostkeys rename to bootstrapvz/common/assets/init.d/squeeze/generate-ssh-hostkeys diff --git a/common/bytes.py b/bootstrapvz/common/bytes.py similarity index 96% rename from common/bytes.py rename to bootstrapvz/common/bytes.py index e6f2b88..0a2aab2 100644 --- a/common/bytes.py +++ b/bootstrapvz/common/bytes.py @@ -4,9 +4,9 @@ class Bytes(object): units = {'B': 1, 'KiB': 1024, - 'MiB': 1024*1024, - 'GiB': 1024*1024*1024, - 'TiB': 1024*1024*1024*1024, + 'MiB': 1024 * 1024, + 'GiB': 1024 * 1024 * 1024, + 'TiB': 1024 * 1024 * 1024 * 1024, } def __init__(self, qty): diff --git a/common/exceptions.py b/bootstrapvz/common/exceptions.py similarity index 100% rename from common/exceptions.py rename to bootstrapvz/common/exceptions.py diff --git a/common/fs/__init__.py b/bootstrapvz/common/fs/__init__.py similarity index 92% rename from common/fs/__init__.py rename to bootstrapvz/common/fs/__init__.py index 93ba9ae..694846e 100644 --- a/common/fs/__init__.py +++ b/bootstrapvz/common/fs/__init__.py @@ -17,7 +17,7 @@ def get_partitions(): def remount(volume, fn): - from base.fs.partitionmaps.none import NoPartitions + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions p_map = volume.partition_map root_dir = p_map.root.mount_dir diff --git a/common/fs/loopbackvolume.py b/bootstrapvz/common/fs/loopbackvolume.py similarity index 59% rename from common/fs/loopbackvolume.py rename to bootstrapvz/common/fs/loopbackvolume.py index 9f42728..c0481b6 100644 --- a/common/fs/loopbackvolume.py +++ b/bootstrapvz/common/fs/loopbackvolume.py @@ -1,5 +1,5 @@ -from base.fs.volume import Volume -from common.tools import log_check_call +from bootstrapvz.base.fs.volume import Volume +from ..tools import log_check_call class LoopbackVolume(Volume): @@ -12,14 +12,14 @@ class LoopbackVolume(Volume): def _before_create(self, e): self.image_path = e.image_path vol_size = str(self.size.get_qty_in('MiB')) + 'M' - log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', self.image_path, vol_size]) + log_check_call(['qemu-img', 'create', '-f', 'raw', self.image_path, vol_size]) def _before_attach(self, e): - [self.loop_device_path] = log_check_call(['/sbin/losetup', '--show', '--find', self.image_path]) + [self.loop_device_path] = log_check_call(['losetup', '--show', '--find', self.image_path]) self.device_path = self.loop_device_path def _before_detach(self, e): - log_check_call(['/sbin/losetup', '--detach', self.loop_device_path]) + log_check_call(['losetup', '--detach', self.loop_device_path]) del self.loop_device_path del self.device_path diff --git a/common/fs/qemuvolume.py b/bootstrapvz/common/fs/qemuvolume.py similarity index 84% rename from common/fs/qemuvolume.py rename to bootstrapvz/common/fs/qemuvolume.py index 3383a0c..c079532 100644 --- a/common/fs/qemuvolume.py +++ b/bootstrapvz/common/fs/qemuvolume.py @@ -1,7 +1,7 @@ -from common.fs.loopbackvolume import LoopbackVolume -from base.fs.exceptions import VolumeError -from common.tools import log_check_call -from common.fs import get_partitions +from loopbackvolume import LoopbackVolume +from bootstrapvz.base.fs.exceptions import VolumeError +from ..tools import log_check_call +from . import get_partitions class QEMUVolume(LoopbackVolume): @@ -9,10 +9,10 @@ class QEMUVolume(LoopbackVolume): def _before_create(self, e): self.image_path = e.image_path vol_size = str(self.size.get_qty_in('MiB')) + 'M' - log_check_call(['/usr/bin/qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size]) + log_check_call(['qemu-img', 'create', '-f', self.qemu_format, self.image_path, vol_size]) def _check_nbd_module(self): - from base.fs.partitionmaps.none import NoPartitions + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions if isinstance(self.partition_map, NoPartitions): if not self._module_loaded('nbd'): msg = ('The kernel module `nbd\' must be loaded ' @@ -40,11 +40,11 @@ class QEMUVolume(LoopbackVolume): def _before_attach(self, e): self._check_nbd_module() self.loop_device_path = self._find_free_nbd_device() - log_check_call(['/usr/bin/qemu-nbd', '--connect', self.loop_device_path, self.image_path]) + log_check_call(['qemu-nbd', '--connect', self.loop_device_path, self.image_path]) self.device_path = self.loop_device_path def _before_detach(self, e): - log_check_call(['/usr/bin/qemu-nbd', '--disconnect', self.loop_device_path]) + log_check_call(['qemu-nbd', '--disconnect', self.loop_device_path]) del self.loop_device_path del self.device_path diff --git a/common/fs/virtualdiskimage.py b/bootstrapvz/common/fs/virtualdiskimage.py similarity index 84% rename from common/fs/virtualdiskimage.py rename to bootstrapvz/common/fs/virtualdiskimage.py index 8d177d6..b36c8fc 100644 --- a/common/fs/virtualdiskimage.py +++ b/bootstrapvz/common/fs/virtualdiskimage.py @@ -1,4 +1,4 @@ -from common.fs.qemuvolume import QEMUVolume +from qemuvolume import QEMUVolume class VirtualDiskImage(QEMUVolume): diff --git a/common/fs/virtualmachinedisk.py b/bootstrapvz/common/fs/virtualmachinedisk.py similarity index 94% rename from common/fs/virtualmachinedisk.py rename to bootstrapvz/common/fs/virtualmachinedisk.py index 6126aec..499c84a 100644 --- a/common/fs/virtualmachinedisk.py +++ b/bootstrapvz/common/fs/virtualmachinedisk.py @@ -1,4 +1,4 @@ -from common.fs.qemuvolume import QEMUVolume +from qemuvolume import QEMUVolume class VirtualMachineDisk(QEMUVolume): diff --git a/common/fsm_proxy.py b/bootstrapvz/common/fsm_proxy.py similarity index 100% rename from common/fsm_proxy.py rename to bootstrapvz/common/fsm_proxy.py diff --git a/base/minify_json.py b/bootstrapvz/common/minify_json.py similarity index 100% rename from base/minify_json.py rename to bootstrapvz/common/minify_json.py diff --git a/common/phases.py b/bootstrapvz/common/phases.py similarity index 96% rename from common/phases.py rename to bootstrapvz/common/phases.py index d6cf8a1..e83feab 100644 --- a/common/phases.py +++ b/bootstrapvz/common/phases.py @@ -1,4 +1,4 @@ -from base import Phase +from bootstrapvz.base.phase import Phase preparation = Phase('Preparation', 'Initializing connections, fetching data etc.') volume_creation = Phase('Volume creation', 'Creating the volume to bootstrap onto') diff --git a/common/task_sets.py b/bootstrapvz/common/task_sets.py similarity index 74% rename from common/task_sets.py rename to bootstrapvz/common/task_sets.py index d0afec4..dbf09ef 100644 --- a/common/task_sets.py +++ b/bootstrapvz/common/task_sets.py @@ -1,30 +1,32 @@ -from common.tasks import workspace -from common.tasks import packages -from common.tasks import host -from common.tasks import boot -from common.tasks import bootstrap -from common.tasks import volume -from common.tasks import filesystem -from common.tasks import partitioning -from common.tasks import cleanup -from common.tasks import apt -from common.tasks import security -from common.tasks import locale +from tasks import workspace +from tasks import packages +from tasks import host +from tasks import boot +from tasks import bootstrap +from tasks import volume +from tasks import filesystem +from tasks import partitioning +from tasks import cleanup +from tasks import apt +from tasks import security +from tasks import locale base_set = [workspace.CreateWorkspace, - host.HostDependencies, - host.CheckHostDependencies, + bootstrap.AddRequiredCommands, + host.CheckExternalCommands, bootstrap.Bootstrap, workspace.DeleteWorkspace, ] volume_set = [volume.Attach, volume.Detach, + filesystem.AddRequiredCommands, filesystem.Format, filesystem.FStab, ] -partitioning_set = [partitioning.PartitionVolume, +partitioning_set = [partitioning.AddRequiredCommands, + partitioning.PartitionVolume, partitioning.MapPartitions, partitioning.UnmapPartitions, ] @@ -63,6 +65,8 @@ def get_apt_set(manifest): base.append(apt.InstallTrustedKeys) if 'install' in manifest.packages: base.append(packages.AddManifestPackages) + if manifest.packages.get('install_standard', False): + base.append(packages.AddTaskselStandardPackages) return base @@ -72,7 +76,7 @@ locale_set = [locale.LocaleBootstrapPackage, ] -bootloader_set = {'grub': [boot.AddGrubPackage, boot.InstallGrub], +bootloader_set = {'grub': [boot.AddGrubPackage, boot.ConfigureGrub, boot.InstallGrub], 'extlinux': [boot.AddExtlinuxPackage, boot.InstallExtLinux], } diff --git a/common/tasks/__init__.py b/bootstrapvz/common/tasks/__init__.py similarity index 100% rename from common/tasks/__init__.py rename to bootstrapvz/common/tasks/__init__.py diff --git a/common/tasks/apt.py b/bootstrapvz/common/tasks/apt.py similarity index 78% rename from common/tasks/apt.py rename to bootstrapvz/common/tasks/apt.py index f440ecb..a6f424b 100644 --- a/common/tasks/apt.py +++ b/bootstrapvz/common/tasks/apt.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tools import log_check_call +from bootstrapvz.base import Task +from .. import phases +from ..tools import log_check_call import locale import os @@ -26,13 +26,18 @@ class AddDefaultSources(Task): sections = 'main' if 'sections' in info.manifest.system: sections = ' '.join(info.manifest.system['sections']) +<<<<<<< HEAD:common/tasks/apt.py info.source_lists.add('main', 'deb {apt_mirror} {system.release} '+sections) info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} '+sections) info.source_lists.add('main', 'deb http://security.debian.org/ {system.release}/updates '+sections) info.source_lists.add('main', 'deb-src http://security.debian.org/ {system.release}/updates '+sections) +======= + info.source_lists.add('main', 'deb {apt_mirror} {system.release} ' + sections) + info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} ' + sections) +>>>>>>> upstream/master:bootstrapvz/common/tasks/apt.py if info.manifest.system['release'] not in {'testing', 'unstable'}: - info.source_lists.add('main', 'deb {apt_mirror} {system.release}-updates '+sections) - info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates '+sections) + info.source_lists.add('main', 'deb {apt_mirror} {system.release}-updates ' + sections) + info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates ' + sections) class InstallTrustedKeys(Task): @@ -89,8 +94,8 @@ class AptUpdate(Task): @classmethod def run(cls, info): - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'update']) + log_check_call(['chroot', info.root, + 'apt-get', 'update']) class AptUpgrade(Task): @@ -102,15 +107,15 @@ class AptUpgrade(Task): def run(cls, info): from subprocess import CalledProcessError try: - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'install', - '--fix-broken', - '--no-install-recommends', - '--assume-yes']) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'upgrade', - '--no-install-recommends', - '--assume-yes']) + log_check_call(['chroot', info.root, + 'apt-get', 'install', + '--fix-broken', + '--no-install-recommends', + '--assume-yes']) + log_check_call(['chroot', info.root, + 'apt-get', 'upgrade', + '--no-install-recommends', + '--assume-yes']) except CalledProcessError as e: if e.returncode == 100: import logging @@ -127,9 +132,9 @@ class PurgeUnusedPackages(Task): @classmethod def run(cls, info): - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'autoremove', - '--purge']) + log_check_call(['chroot', info.root, + 'apt-get', 'autoremove', + '--purge']) class AptClean(Task): @@ -138,8 +143,8 @@ class AptClean(Task): @classmethod def run(cls, info): - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'clean']) + log_check_call(['chroot', info.root, + 'apt-get', 'clean']) lists = os.path.join(info.root, 'var/lib/apt/lists') for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]: diff --git a/common/tasks/boot.py b/bootstrapvz/common/tasks/boot.py similarity index 76% rename from common/tasks/boot.py rename to bootstrapvz/common/tasks/boot.py index d66d97a..05dab75 100644 --- a/common/tasks/boot.py +++ b/bootstrapvz/common/tasks/boot.py @@ -1,8 +1,8 @@ -from base import Task -from common import phases -from common.tasks import apt -from common.tasks import filesystem -from base.fs import partitionmaps +from bootstrapvz.base import Task +from .. import phases +import apt +import filesystem +from bootstrapvz.base.fs import partitionmaps import os.path @@ -24,7 +24,7 @@ class DisableGetTTYs(Task): @classmethod def run(cls, info): - from common.tools import sed_i + from ..tools import sed_i inittab_path = os.path.join(info.root, 'etc/inittab') tty1 = '1:2345:respawn:/sbin/getty 38400 tty1' sed_i(inittab_path, '^' + tty1, '#' + tty1) @@ -44,6 +44,20 @@ class AddGrubPackage(Task): info.packages.add('grub-pc') +class ConfigureGrub(Task): + description = 'Configuring grub' + phase = phases.system_modification + predecessors = [filesystem.FStab] + + @classmethod + def run(cls, info): + from bootstrapvz.common.tools import sed_i + grub_def = os.path.join(info.root, 'etc/default/grub') + sed_i(grub_def, '^#GRUB_TERMINAL=console', 'GRUB_TERMINAL=console') + sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT="quiet"', + 'GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0"') + + class InstallGrub(Task): description = 'Installing grub' phase = phases.system_modification @@ -51,13 +65,13 @@ class InstallGrub(Task): @classmethod def run(cls, info): - from common.fs.loopbackvolume import LoopbackVolume - from common.tools import log_check_call + from ..fs.loopbackvolume import LoopbackVolume + from ..tools import log_check_call boot_dir = os.path.join(info.root, 'boot') grub_dir = os.path.join(boot_dir, 'grub') - from common.fs import remount + from ..fs import remount p_map = info.volume.partition_map def link_fn(): @@ -91,9 +105,9 @@ class InstallGrub(Task): idx=idx + 1)) # Install grub - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/grub-install', device_path]) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + log_check_call(['chroot', info.root, + 'grub-install', device_path]) + log_check_call(['chroot', info.root, 'update-grub']) except Exception as e: if isinstance(info.volume, LoopbackVolume): remount(info.volume, unlink_fn) @@ -122,17 +136,17 @@ class InstallExtLinux(Task): @classmethod def run(cls, info): - from common.tools import log_check_call + from ..tools import log_check_call if isinstance(info.volume.partition_map, partitionmaps.gpt.GPTPartitionMap): bootloader = '/usr/lib/syslinux/gptmbr.bin' else: bootloader = '/usr/lib/extlinux/mbr.bin' - log_check_call(['/usr/sbin/chroot', info.root, - '/bin/dd', 'bs=440', 'count=1', + log_check_call(['chroot', info.root, + 'dd', 'bs=440', 'count=1', 'if=' + bootloader, 'of=' + info.volume.device_path]) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/extlinux', + log_check_call(['chroot', info.root, + 'extlinux', '--install', '/boot/extlinux']) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/extlinux-update']) + log_check_call(['chroot', info.root, + 'extlinux-update']) diff --git a/common/tasks/bootstrap.py b/bootstrapvz/common/tasks/bootstrap.py similarity index 80% rename from common/tasks/bootstrap.py rename to bootstrapvz/common/tasks/bootstrap.py index 8cb8d31..2274c60 100644 --- a/common/tasks/bootstrap.py +++ b/bootstrapvz/common/tasks/bootstrap.py @@ -1,12 +1,23 @@ -from base import Task -from common import phases -from common.exceptions import TaskError +from bootstrapvz.base import Task +from .. import phases +from ..exceptions import TaskError +import host import logging log = logging.getLogger(__name__) +class AddRequiredCommands(Task): + description = 'Adding commands required bootstrapping Debian' + phase = phases.preparation + successors = [host.CheckExternalCommands] + + @classmethod + def run(cls, info): + info.host_dependencies['debootstrap'] = 'debootstrap' + + def get_bootstrap_args(info): - executable = ['/usr/sbin/debootstrap'] + executable = ['debootstrap'] options = ['--arch=' + info.manifest.system['architecture']] if len(info.include_packages) > 0: options.append('--include=' + ','.join(info.include_packages)) @@ -34,7 +45,7 @@ class MakeTarball(Task): if os.path.isfile(info.tarball): log.debug('Found matching tarball, skipping download') else: - from common.tools import log_call + from ..tools import log_call status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments) if status != 1: msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status) @@ -52,5 +63,5 @@ class Bootstrap(Task): if hasattr(info, 'tarball'): options.extend(['--unpack-tarball=' + info.tarball]) - from common.tools import log_check_call + from ..tools import log_check_call log_check_call(executable + options + arguments) diff --git a/common/tasks/cleanup.py b/bootstrapvz/common/tasks/cleanup.py similarity index 87% rename from common/tasks/cleanup.py rename to bootstrapvz/common/tasks/cleanup.py index c6cc3c1..33957da 100644 --- a/common/tasks/cleanup.py +++ b/bootstrapvz/common/tasks/cleanup.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases import os import shutil @@ -28,8 +28,8 @@ class ShredHostkeys(Task): private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys] public = [path + '.pub' for path in private] - from common.tools import log_check_call - log_check_call(['/usr/bin/shred', '--remove'] + private + public) + from ..tools import log_check_call + log_check_call(['shred', '--remove'] + private + public) class CleanTMP(Task): diff --git a/common/tasks/development.py b/bootstrapvz/common/tasks/development.py similarity index 67% rename from common/tasks/development.py rename to bootstrapvz/common/tasks/development.py index b71504d..2fe8e18 100644 --- a/common/tasks/development.py +++ b/bootstrapvz/common/tasks/development.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases class TriggerRollback(Task): @@ -9,5 +9,5 @@ class TriggerRollback(Task): @classmethod def run(cls, info): - from common.exceptions import TaskError + from ..exceptions import TaskError raise TaskError('Trigger rollback') diff --git a/common/tasks/filesystem.py b/bootstrapvz/common/tasks/filesystem.py similarity index 85% rename from common/tasks/filesystem.py rename to bootstrapvz/common/tasks/filesystem.py index 4f79af5..71845f6 100644 --- a/common/tasks/filesystem.py +++ b/bootstrapvz/common/tasks/filesystem.py @@ -1,18 +1,30 @@ -from base import Task -from common import phases -from common.tools import log_check_call -from bootstrap import Bootstrap -from common.tasks import apt +from bootstrapvz.base import Task +from .. import phases +from ..tools import log_check_call +import apt +import bootstrap +import host import volume +class AddRequiredCommands(Task): + description = 'Adding commands required for formatting the partitions' + phase = phases.preparation + successors = [host.CheckExternalCommands] + + @classmethod + def run(cls, info): + if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): + info.host_dependencies['mkfs.xfs'] = 'xfsprogs' + + class Format(Task): description = 'Formatting the volume' phase = phases.volume_preparation @classmethod def run(cls, info): - from base.fs.partitions.unformatted import UnformattedPartition + from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition for partition in info.volume.partition_map.partitions: if not isinstance(partition, UnformattedPartition): partition.format() @@ -25,13 +37,13 @@ class TuneVolumeFS(Task): @classmethod def run(cls, info): - from base.fs.partitions.unformatted import UnformattedPartition + from bootstrapvz.base.fs.partitions.unformatted import UnformattedPartition import re # Disable the time based filesystem check for partition in info.volume.partition_map.partitions: if not isinstance(partition, UnformattedPartition): if re.match('^ext[2-4]$', partition.filesystem) is not None: - log_check_call(['/sbin/tune2fs', '-i', '0', partition.device_path]) + log_check_call(['tune2fs', '-i', '0', partition.device_path]) class AddXFSProgs(Task): @@ -90,7 +102,7 @@ class MountBoot(Task): class MountSpecials(Task): description = 'Mounting special block devices' phase = phases.os_installation - predecessors = [Bootstrap] + predecessors = [bootstrap.Bootstrap] @classmethod def run(cls, info): diff --git a/bootstrapvz/common/tasks/host.py b/bootstrapvz/common/tasks/host.py new file mode 100644 index 0000000..ad58208 --- /dev/null +++ b/bootstrapvz/common/tasks/host.py @@ -0,0 +1,31 @@ +from bootstrapvz.base import Task +from .. import phases +from ..exceptions import TaskError + + +class CheckExternalCommands(Task): + description = 'Checking availability of external commands' + phase = phases.preparation + + @classmethod + def run(cls, info): + from ..tools import log_check_call + from subprocess import CalledProcessError + import re + missing_packages = [] + for command, package in info.host_dependencies.items(): + try: + log_check_call(['type ' + command], shell=True) + except CalledProcessError: + if re.match('^https?:\/\/', package): + msg = ('The command `{command}\' is not available, ' + 'you can download the software at `{package}\'.' + .format(command=command, package=package)) + else: + msg = ('The command `{command}\' is not available, ' + 'it is located in the package `{package}\'.' + .format(command=command, package=package)) + missing_packages.append(msg) + if len(missing_packages) > 0: + msg = '\n'.join(missing_packages) + raise TaskError(msg) diff --git a/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py similarity index 84% rename from common/tasks/initd.py rename to bootstrapvz/common/tasks/initd.py index 99842f6..97adb89 100644 --- a/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -1,7 +1,7 @@ -from base import Task -from common import phases -from common.exceptions import TaskError -from common.tools import log_check_call +from bootstrapvz.base import Task +from .. import phases +from ..exceptions import TaskError +from ..tools import log_check_call from . import assets import os.path @@ -21,10 +21,10 @@ class InstallInitScripts(Task): dst = os.path.join(info.root, 'etc/init.d', name) copy(src, dst) os.chmod(dst, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '--default', name]) + log_check_call(['chroot', info.root, 'insserv', '--default', name]) for name in info.initd['disable']: - log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '--remove', name]) + log_check_call(['chroot', info.root, 'insserv', '--remove', name]) class AddExpandRoot(Task): @@ -49,8 +49,8 @@ class AddSSHKeyGeneration(Task): install = info.initd['install'] from subprocess import CalledProcessError try: - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg-query', '-W', 'openssh-server']) + log_check_call(['chroot', info.root, + 'dpkg-query', '-W', 'openssh-server']) if info.manifest.system['release'] == 'squeeze': install['generate-ssh-hostkeys'] = os.path.join(init_scripts_dir, 'squeeze/generate-ssh-hostkeys') else: @@ -83,10 +83,10 @@ class AdjustExpandRootScript(Task): if 'expand-root' not in info.initd['install']: raise TaskError('The expand-root script was not installed') - from base.fs.partitionmaps.none import NoPartitions + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions if not isinstance(info.volume.partition_map, NoPartitions): import os.path - from common.tools import sed_i + from ..tools import sed_i script = os.path.join(info.root, 'etc/init.d.expand-root') root_idx = info.volume.partition_map.root.get_index() device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx) diff --git a/common/tasks/locale.py b/bootstrapvz/common/tasks/locale.py similarity index 82% rename from common/tasks/locale.py rename to bootstrapvz/common/tasks/locale.py index 7eb15e4..989b2b6 100644 --- a/common/tasks/locale.py +++ b/bootstrapvz/common/tasks/locale.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases import os.path @@ -20,20 +20,20 @@ class GenerateLocale(Task): @classmethod def run(cls, info): - from common.tools import sed_i - from common.tools import log_check_call + from ..tools import sed_i + from ..tools import log_check_call locale_gen = os.path.join(info.root, 'etc/locale.gen') locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'], charmap=info.manifest.system['charmap']) search = '# ' + locale_str sed_i(locale_gen, search, locale_str) - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/locale-gen']) + log_check_call(['chroot', info.root, 'locale-gen']) lang = '{locale}.{charmap}'.format(locale=info.manifest.system['locale'], charmap=info.manifest.system['charmap']) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/update-locale', 'LANG=' + lang]) + log_check_call(['chroot', info.root, + 'update-locale', 'LANG=' + lang]) class SetTimezone(Task): diff --git a/common/tasks/loopback.py b/bootstrapvz/common/tasks/loopback.py similarity index 60% rename from common/tasks/loopback.py rename to bootstrapvz/common/tasks/loopback.py index 9420429..f251a97 100644 --- a/common/tasks/loopback.py +++ b/bootstrapvz/common/tasks/loopback.py @@ -1,8 +1,25 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases +import host import volume +class AddRequiredCommands(Task): + description = 'Adding commands required for creating loopback volumes' + phase = phases.preparation + successors = [host.CheckExternalCommands] + + @classmethod + def run(cls, info): + from ..fs.loopbackvolume import LoopbackVolume + if isinstance(info.volume, LoopbackVolume): + info.host_dependencies['qemu-img'] = 'qemu-utils' + info.host_dependencies['losetup'] = 'mount' + from ..fs.qemuvolume import QEMUVolume + if isinstance(info.volume, QEMUVolume): + info.host_dependencies['losetup'] = 'mount' + + class Create(Task): description = 'Creating a loopback volume' phase = phases.volume_creation diff --git a/bootstrapvz/common/tasks/network-configuration.json b/bootstrapvz/common/tasks/network-configuration.json new file mode 100644 index 0000000..4e5dce7 --- /dev/null +++ b/bootstrapvz/common/tasks/network-configuration.json @@ -0,0 +1,12 @@ +// This is a mapping of Debian release codenames to NIC configurations +// Every item in an array is a line +{ +"squeeze": ["auto lo", + "iface lo inet loopback", + "auto eth0", + "iface eth0 inet dhcp"], +"wheezy": ["auto eth0", + "iface eth0 inet dhcp"], +"jessie": ["auto eth0", + "iface eth0 inet dhcp"] +} diff --git a/common/tasks/network.py b/bootstrapvz/common/tasks/network.py similarity index 59% rename from common/tasks/network.py rename to bootstrapvz/common/tasks/network.py index 8418a84..c175b2d 100644 --- a/common/tasks/network.py +++ b/bootstrapvz/common/tasks/network.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -import os.path +from bootstrapvz.base import Task +from .. import phases +import os class RemoveDNSInfo(Task): @@ -9,10 +9,8 @@ class RemoveDNSInfo(Task): @classmethod def run(cls, info): - from os import remove - import os.path if os.path.isfile(os.path.join(info.root, 'etc/resolv.conf')): - remove(os.path.join(info.root, 'etc/resolv.conf')) + os.remove(os.path.join(info.root, 'etc/resolv.conf')) class RemoveHostname(Task): @@ -21,10 +19,8 @@ class RemoveHostname(Task): @classmethod def run(cls, info): - from os import remove - import os.path if os.path.isfile(os.path.join(info.root, 'etc/hostname')): - remove(os.path.join(info.root, 'etc/hostname')) + os.remove(os.path.join(info.root, 'etc/hostname')) class ConfigureNetworkIF(Task): @@ -33,10 +29,10 @@ class ConfigureNetworkIF(Task): @classmethod def run(cls, info): + network_config_path = os.path.join(os.path.dirname(__file__), 'network-configuration.json') + from ..tools import config_get + if_config = config_get(network_config_path, [info.release_codename]) + interfaces_path = os.path.join(info.root, 'etc/network/interfaces') - if_config = [] - with open('common/tasks/network-configuration.json') as stream: - import json - if_config = json.loads(stream.read()) with open(interfaces_path, 'a') as interfaces: - interfaces.write('\n'.join(if_config.get(info.manifest.system['release'])) + '\n') + interfaces.write('\n'.join(if_config) + '\n') diff --git a/common/tasks/packages.py b/bootstrapvz/common/tasks/packages.py similarity index 79% rename from common/tasks/packages.py rename to bootstrapvz/common/tasks/packages.py index bf171e7..602a2a0 100644 --- a/common/tasks/packages.py +++ b/bootstrapvz/common/tasks/packages.py @@ -1,6 +1,7 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from .. import phases +import apt +from ..tools import log_check_call class AddManifestPackages(Task): @@ -40,15 +41,15 @@ class InstallPackages(Task): @classmethod def install_remote(cls, info, remote_packages): import os - from common.tools import log_check_call + from ..tools import log_check_call from subprocess import CalledProcessError try: env = os.environ.copy() env['DEBIAN_FRONTEND'] = 'noninteractive' - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/apt-get', 'install', - '--no-install-recommends', - '--assume-yes'] + log_check_call(['chroot', info.root, + 'apt-get', 'install', + '--no-install-recommends', + '--assume-yes'] + map(str, remote_packages), env=env) except CalledProcessError as e: @@ -74,7 +75,6 @@ class InstallPackages(Task): @classmethod def install_local(cls, info, local_packages): from shutil import copy - from common.tools import log_check_call import os absolute_package_paths = [] @@ -90,10 +90,23 @@ class InstallPackages(Task): env = os.environ.copy() env['DEBIAN_FRONTEND'] = 'noninteractive' - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg', '--install'] + log_check_call(['chroot', info.root, + 'dpkg', '--install'] + chrooted_package_paths, env=env) for path in absolute_package_paths: os.remove(path) + + +class AddTaskselStandardPackages(Task): + description = 'Adding standard packages from tasksel' + phase = phases.package_installation + predecessors = [apt.AptUpdate] + successors = [InstallPackages] + + @classmethod + def run(cls, info): + tasksel_packages = log_check_call(['chroot', info.root, 'tasksel', '--task-packages', 'standard']) + for pkg in tasksel_packages: + info.packages.add(pkg) diff --git a/common/tasks/partitioning.py b/bootstrapvz/common/tasks/partitioning.py similarity index 59% rename from common/tasks/partitioning.py rename to bootstrapvz/common/tasks/partitioning.py index a9d982c..db75263 100644 --- a/common/tasks/partitioning.py +++ b/bootstrapvz/common/tasks/partitioning.py @@ -1,9 +1,23 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases import filesystem +import host import volume +class AddRequiredCommands(Task): + description = 'Adding commands required for partitioning the volume' + phase = phases.preparation + successors = [host.CheckExternalCommands] + + @classmethod + def run(cls, info): + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions + if not isinstance(info.volume.partition_map, NoPartitions): + info.host_dependencies['parted'] = 'parted' + info.host_dependencies['kpartx'] = 'kpartx' + + class PartitionVolume(Task): description = 'Partitioning the volume' phase = phases.volume_preparation diff --git a/common/tasks/security.py b/bootstrapvz/common/tasks/security.py similarity index 80% rename from common/tasks/security.py rename to bootstrapvz/common/tasks/security.py index ef109ee..37bd575 100644 --- a/common/tasks/security.py +++ b/bootstrapvz/common/tasks/security.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases import os.path @@ -9,8 +9,8 @@ class EnableShadowConfig(Task): @classmethod def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on']) + from ..tools import log_check_call + log_check_call(['chroot', info.root, 'shadowconfig', 'on']) class DisableSSHPasswordAuthentication(Task): @@ -19,7 +19,7 @@ class DisableSSHPasswordAuthentication(Task): @classmethod def run(cls, info): - from common.tools import sed_i + from ..tools import sed_i sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') diff --git a/common/tasks/volume.py b/bootstrapvz/common/tasks/volume.py similarity index 85% rename from common/tasks/volume.py rename to bootstrapvz/common/tasks/volume.py index 797886b..337d6b7 100644 --- a/common/tasks/volume.py +++ b/bootstrapvz/common/tasks/volume.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import workspace +from bootstrapvz.base import Task +from .. import phases +import workspace class Attach(Task): diff --git a/common/tasks/workspace.py b/bootstrapvz/common/tasks/workspace.py similarity index 85% rename from common/tasks/workspace.py rename to bootstrapvz/common/tasks/workspace.py index b840a26..bd1ddac 100644 --- a/common/tasks/workspace.py +++ b/bootstrapvz/common/tasks/workspace.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from .. import phases class CreateWorkspace(Task): diff --git a/common/tools.py b/bootstrapvz/common/tools.py similarity index 61% rename from common/tools.py rename to bootstrapvz/common/tools.py index 2833b94..8721144 100644 --- a/common/tools.py +++ b/bootstrapvz/common/tools.py @@ -1,14 +1,12 @@ - - -def log_check_call(command, stdin=None, env=None): - status, stdout, stderr = log_call(command, stdin, env) +def log_check_call(command, stdin=None, env=None, shell=False): + status, stdout, stderr = log_call(command, stdin, env, shell) if status != 0: from subprocess import CalledProcessError raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr)) return stdout -def log_call(command, stdin=None, env=None): +def log_call(command, stdin=None, env=None, shell=False): import subprocess import select @@ -22,6 +20,7 @@ def log_call(command, stdin=None, env=None): popen_args = {'args': command, 'env': env, + 'shell': shell, 'stdin': subprocess.PIPE, 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, } @@ -56,3 +55,35 @@ def sed_i(file_path, pattern, subst): import re for line in fileinput.input(files=file_path, inplace=True): print re.sub(pattern, subst, line), + + +def load_json(path): + import json + from minify_json import json_minify + with open(path) as stream: + return json.loads(json_minify(stream.read(), False)) + + +def config_get(path, config_path): + config = load_json(path) + for key in config_path: + config = config.get(key) + return config + +def copy_tree(from_path,to_path): + from shutil import copy + import os + for abs_prefix, dirs, files in os.walk(from_path): + prefix = os.path.normpath(os.path.relpath(abs_prefix, from_path)) + for path in dirs: + full_path = os.path.join(to_path, prefix, path) + if os.path.exists(full_path): + if os.path.isdir(full_path): + continue + else: + os.remove(full_path) + os.mkdir(full_path) + for path in files: + copy(os.path.join(abs_prefix, path), + os.path.join(to_path, prefix, path)) + diff --git a/base/fs/partitions/__init__.py b/bootstrapvz/plugins/__init__.py similarity index 100% rename from base/fs/partitions/__init__.py rename to bootstrapvz/plugins/__init__.py diff --git a/plugins/admin_user/__init__.py b/bootstrapvz/plugins/admin_user/__init__.py similarity index 90% rename from plugins/admin_user/__init__.py rename to bootstrapvz/plugins/admin_user/__init__.py index 2a3c56b..67f3488 100644 --- a/plugins/admin_user/__init__.py +++ b/bootstrapvz/plugins/admin_user/__init__.py @@ -8,7 +8,7 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): import tasks - from providers.ec2.tasks import initd + from bootstrapvz.providers.ec2.tasks import initd if initd.AddEC2InitScripts in taskset: taskset.add(tasks.AdminUserCredentials) diff --git a/plugins/admin_user/manifest-schema.json b/bootstrapvz/plugins/admin_user/manifest-schema.json similarity index 100% rename from plugins/admin_user/manifest-schema.json rename to bootstrapvz/plugins/admin_user/manifest-schema.json diff --git a/plugins/admin_user/tasks.py b/bootstrapvz/plugins/admin_user/tasks.py similarity index 79% rename from plugins/admin_user/tasks.py rename to bootstrapvz/plugins/admin_user/tasks.py index eb572e8..5266f27 100644 --- a/plugins/admin_user/tasks.py +++ b/bootstrapvz/plugins/admin_user/tasks.py @@ -1,7 +1,7 @@ -from base import Task -from common import phases -from common.tasks.initd import InstallInitScripts -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks.initd import InstallInitScripts +from bootstrapvz.common.tasks import apt import os @@ -21,9 +21,9 @@ class CreateAdminUser(Task): @classmethod def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/useradd', + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, + 'useradd', '--create-home', '--shell', '/bin/bash', info.manifest.plugins['admin_user']['username']]) @@ -50,7 +50,7 @@ class AdminUserCredentials(Task): @classmethod def run(cls, info): - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i getcreds_path = os.path.join(info.root, 'etc/init.d/ec2-get-credentials') username = info.manifest.plugins['admin_user']['username'] sed_i(getcreds_path, 'username=\'root\'', 'username=\'{username}\''.format(username=username)) @@ -63,11 +63,11 @@ class DisableRootLogin(Task): @classmethod def run(cls, info): from subprocess import CalledProcessError - from common.tools import log_check_call + from bootstrapvz.common.tools import log_check_call try: - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/dpkg-query', '-W', 'openssh-server']) - from common.tools import sed_i + log_check_call(['chroot', info.root, + 'dpkg-query', '-W', 'openssh-server']) + from bootstrapvz.common.tools import sed_i sshdconfig_path = os.path.join(info.root, 'etc/ssh/sshd_config') sed_i(sshdconfig_path, 'PermitRootLogin yes', 'PermitRootLogin no') except CalledProcessError: diff --git a/bootstrapvz/plugins/apt_proxy/__init__.py b/bootstrapvz/plugins/apt_proxy/__init__.py new file mode 100644 index 0000000..a5b086c --- /dev/null +++ b/bootstrapvz/plugins/apt_proxy/__init__.py @@ -0,0 +1,11 @@ +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) + + +def resolve_tasks(taskset, manifest): + import tasks + taskset.add(tasks.SetAptProxy) + if not manifest.plugins['apt_proxy'].get('persistent', False): + taskset.add(tasks.RemoveAptProxy) diff --git a/bootstrapvz/plugins/apt_proxy/manifest-schema.json b/bootstrapvz/plugins/apt_proxy/manifest-schema.json new file mode 100644 index 0000000..ae407eb --- /dev/null +++ b/bootstrapvz/plugins/apt_proxy/manifest-schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "APT proxy plugin manifest", + "type": "object", + "properties": { + "plugins": { + "type": "object", + "properties": { + "apt_proxy": { + "type": "object", + "properties": { + "address": { + "type": "string" + }, + "persistent": { + "type": "boolean" + }, + "port": { + "type": "integer" + } + }, + "required": ["address", "port"] + } + }, + "required": ["apt_proxy"] + } + }, + "required": ["plugins"] +} diff --git a/bootstrapvz/plugins/apt_proxy/tasks.py b/bootstrapvz/plugins/apt_proxy/tasks.py new file mode 100644 index 0000000..9d8dbc3 --- /dev/null +++ b/bootstrapvz/plugins/apt_proxy/tasks.py @@ -0,0 +1,28 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +import os + + +class SetAptProxy(Task): + description = 'Setting proxy for APT' + phase = phases.package_installation + successors = [apt.AptUpdate] + + @classmethod + def run(cls, info): + proxy_path = os.path.join(info.root, 'etc/apt/apt.conf.d/02proxy') + proxy_address = info.manifest.plugins['apt_proxy']['address'] + proxy_port = info.manifest.plugins['apt_proxy']['port'] + with open(proxy_path, 'w') as proxy_file: + proxy_file.write('Acquire::http {{ Proxy "http://{address}:{port}"; }};\n' + .format(address=proxy_address, port=proxy_port)) + + +class RemoveAptProxy(Task): + description = 'Removing APT proxy configuration file' + phase = phases.system_cleaning + + @classmethod + def run(cls, info): + os.remove(os.path.join(info.root, 'etc/apt/apt.conf.d/02proxy')) diff --git a/plugins/build_metadata/__init__.py b/bootstrapvz/plugins/build_metadata/__init__.py similarity index 100% rename from plugins/build_metadata/__init__.py rename to bootstrapvz/plugins/build_metadata/__init__.py diff --git a/plugins/build_metadata/manifest-schema.json b/bootstrapvz/plugins/build_metadata/manifest-schema.json similarity index 100% rename from plugins/build_metadata/manifest-schema.json rename to bootstrapvz/plugins/build_metadata/manifest-schema.json diff --git a/plugins/build_metadata/tasks.py b/bootstrapvz/plugins/build_metadata/tasks.py similarity index 88% rename from plugins/build_metadata/tasks.py rename to bootstrapvz/plugins/build_metadata/tasks.py index f47ad4f..d459721 100644 --- a/plugins/build_metadata/tasks.py +++ b/bootstrapvz/plugins/build_metadata/tasks.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases class WriteMetadata(Task): diff --git a/bootstrapvz/plugins/chef/__init__.py b/bootstrapvz/plugins/chef/__init__.py new file mode 100644 index 0000000..7ba2396 --- /dev/null +++ b/bootstrapvz/plugins/chef/__init__.py @@ -0,0 +1,14 @@ +import tasks + + +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) + + +def resolve_tasks(taskset, manifest): + taskset.add(tasks.AddPackages) + if 'assets' in manifest.plugins['chef']: + taskset.add(tasks.CheckAssetsPath) + taskset.add(tasks.CopyChefAssets) diff --git a/plugins/minimize_size/manifest-schema.json b/bootstrapvz/plugins/chef/manifest-schema.json similarity index 63% rename from plugins/minimize_size/manifest-schema.json rename to bootstrapvz/plugins/chef/manifest-schema.json index 0d7942c..9bc9d47 100644 --- a/plugins/minimize_size/manifest-schema.json +++ b/bootstrapvz/plugins/chef/manifest-schema.json @@ -1,17 +1,18 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Minimize size plugin manifest", + "title": "Puppet plugin manifest", "type": "object", "properties": { "plugins": { "type": "object", "properties": { - "minimize_size": { + "chef": { "type": "object", "properties": { - "shrink": { "type": "boolean" }, - "zerofree": { "$ref": "#/definitions/absolute_path" } - } + "assets": { "$ref": "#/definitions/absolute_path" } + }, + "minProperties": 1, + "additionalProperties": false } } } diff --git a/bootstrapvz/plugins/chef/tasks.py b/bootstrapvz/plugins/chef/tasks.py new file mode 100644 index 0000000..2f103bf --- /dev/null +++ b/bootstrapvz/plugins/chef/tasks.py @@ -0,0 +1,40 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +import os + + +class CheckAssetsPath(Task): + description = 'Checking whether the assets path exist' + phase = phases.preparation + + @classmethod + def run(cls, info): + from bootstrapvz.common.exceptions import TaskError + assets = info.manifest.plugins['chef']['assets'] + if not os.path.exists(assets): + msg = 'The assets directory {assets} does not exist.'.format(assets=assets) + raise TaskError(msg) + if not os.path.isdir(assets): + msg = 'The assets path {assets} does not point to a directory.'.format(assets=assets) + raise TaskError(msg) + + +class AddPackages(Task): + description = 'Add chef package' + phase = phases.preparation + predecessors = [apt.AddDefaultSources] + + @classmethod + def run(cls, info): + info.packages.add('chef') + + +class CopyChefAssets(Task): + description = 'Copying chef assets' + phase = phases.system_modification + + @classmethod + def run(cls, info): + from bootstrapvz.common.tools import copy_tree + copy_tree(info.manifest.plugins['chef']['assets'], os.path.join(info.root, 'etc/chef')) diff --git a/plugins/cloud_init/__init__.py b/bootstrapvz/plugins/cloud_init/__init__.py similarity index 88% rename from plugins/cloud_init/__init__.py rename to bootstrapvz/plugins/cloud_init/__init__.py index 81eee83..e1df533 100644 --- a/plugins/cloud_init/__init__.py +++ b/bootstrapvz/plugins/cloud_init/__init__.py @@ -8,8 +8,8 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): import tasks - import providers.ec2.tasks.initd as initd_ec2 - from common.tasks import initd + import bootstrapvz.providers.ec2.tasks.initd as initd_ec2 + from bootstrapvz.common.tasks import initd if manifest.system['release'] in ['wheezy', 'stable']: taskset.add(tasks.AddBackports) diff --git a/plugins/cloud_init/manifest-schema.json b/bootstrapvz/plugins/cloud_init/manifest-schema.json similarity index 100% rename from plugins/cloud_init/manifest-schema.json rename to bootstrapvz/plugins/cloud_init/manifest-schema.json diff --git a/plugins/cloud_init/tasks.py b/bootstrapvz/plugins/cloud_init/tasks.py similarity index 90% rename from plugins/cloud_init/tasks.py rename to bootstrapvz/plugins/cloud_init/tasks.py index 75700bd..51a9f2f 100644 --- a/plugins/cloud_init/tasks.py +++ b/bootstrapvz/plugins/cloud_init/tasks.py @@ -1,7 +1,7 @@ -from base import Task -from common import phases -from common.tools import log_check_call -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tools import log_check_call +from bootstrapvz.common.tasks import apt import os.path @@ -40,7 +40,7 @@ class SetUsername(Task): @classmethod def run(cls, info): - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i cloud_cfg = os.path.join(info.root, 'etc/cloud/cloud.cfg') username = info.manifest.plugins['cloud_init']['username'] search = '^ name: debian$' @@ -68,7 +68,7 @@ class SetMetadataSource(Task): logging.getLogger(__name__).warn(msg) return sources = "cloud-init cloud-init/datasources multiselect " + sources - log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/debconf-set-selections'], sources) + log_check_call(['chroot', info.root, 'debconf-set-selections'], sources) class DisableModules(Task): diff --git a/plugins/image_commands/README.md b/bootstrapvz/plugins/image_commands/README.md similarity index 100% rename from plugins/image_commands/README.md rename to bootstrapvz/plugins/image_commands/README.md diff --git a/plugins/image_commands/__init__.py b/bootstrapvz/plugins/image_commands/__init__.py similarity index 100% rename from plugins/image_commands/__init__.py rename to bootstrapvz/plugins/image_commands/__init__.py diff --git a/plugins/image_commands/manifest-schema.json b/bootstrapvz/plugins/image_commands/manifest-schema.json similarity index 100% rename from plugins/image_commands/manifest-schema.json rename to bootstrapvz/plugins/image_commands/manifest-schema.json diff --git a/plugins/image_commands/tasks.py b/bootstrapvz/plugins/image_commands/tasks.py similarity index 73% rename from plugins/image_commands/tasks.py rename to bootstrapvz/plugins/image_commands/tasks.py index 1c8e3e4..70ec370 100644 --- a/plugins/image_commands/tasks.py +++ b/bootstrapvz/plugins/image_commands/tasks.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases class ImageExecuteCommand(Task): @@ -8,7 +8,7 @@ class ImageExecuteCommand(Task): @classmethod def run(cls, info): - from common.tools import log_check_call + from bootstrapvz.common.tools import log_check_call for raw_command in info.manifest.plugins['image_commands']['commands']: command = map(lambda part: part.format(root=info.root, **info.manifest_vars), raw_command) log_check_call(command) diff --git a/plugins/minimize_size/__init__.py b/bootstrapvz/plugins/minimize_size/__init__.py similarity index 70% rename from plugins/minimize_size/__init__.py rename to bootstrapvz/plugins/minimize_size/__init__.py index d49b551..f5ab69c 100644 --- a/plugins/minimize_size/__init__.py +++ b/bootstrapvz/plugins/minimize_size/__init__.py @@ -5,9 +5,6 @@ def validate_manifest(data, validator, error): import os.path schema_path = os.path.join(os.path.dirname(__file__), 'manifest-schema.json') validator(data, schema_path) - if 'zerofree' in data['plugins']['minimize_size']: - zerofree_schema_path = os.path.join(os.path.dirname(__file__), 'manifest-schema-zerofree.json') - validator(data, zerofree_schema_path) if data['plugins']['minimize_size'].get('shrink', False) and data['volume']['backing'] != 'vmdk': error('Can only shrink vmdk images', ['plugins', 'minimize_size', 'shrink']) @@ -16,11 +13,11 @@ def resolve_tasks(taskset, manifest): taskset.update([tasks.AddFolderMounts, tasks.RemoveFolderMounts, ]) - if 'zerofree' in manifest.plugins['minimize_size']: - taskset.add(tasks.CheckZerofreePath) + if manifest.plugins['minimize_size'].get('zerofree', False): + taskset.add(tasks.AddRequiredCommands) taskset.add(tasks.Zerofree) if manifest.plugins['minimize_size'].get('shrink', False): - taskset.add(tasks.CheckVMWareDMCommand) + taskset.add(tasks.AddRequiredCommands) taskset.add(tasks.ShrinkVolume) diff --git a/plugins/minimize_size/manifest-schema-zerofree.json b/bootstrapvz/plugins/minimize_size/manifest-schema.json similarity index 67% rename from plugins/minimize_size/manifest-schema-zerofree.json rename to bootstrapvz/plugins/minimize_size/manifest-schema.json index bd6715d..0181fa2 100644 --- a/plugins/minimize_size/manifest-schema-zerofree.json +++ b/bootstrapvz/plugins/minimize_size/manifest-schema.json @@ -3,13 +3,14 @@ "title": "Minimize size plugin manifest", "type": "object", "properties": { - "volume": { + "plugins": { "type": "object", "properties": { - "partitions": { + "minimize_size": { "type": "object", "properties": { - "type": { "enum": ["none"] } + "shrink": { "type": "boolean" }, + "zerofree": { "type": "boolean" } } } } diff --git a/bootstrapvz/plugins/minimize_size/tasks.py b/bootstrapvz/plugins/minimize_size/tasks.py new file mode 100644 index 0000000..a9b4fc2 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/tasks.py @@ -0,0 +1,84 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +from bootstrapvz.common.tasks import bootstrap +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import host +from bootstrapvz.common.tasks import partitioning +from bootstrapvz.common.tasks import volume +import os + +folders = ['tmp', 'var/lib/apt/lists'] + + +class AddFolderMounts(Task): + description = 'Mounting folders for writing temporary and cache data' + phase = phases.os_installation + predecessors = [bootstrap.Bootstrap] + + @classmethod + def run(cls, info): + info.minimize_size_folder = os.path.join(info.workspace, 'minimize_size') + os.mkdir(info.minimize_size_folder) + for folder in folders: + temp_path = os.path.join(info.minimize_size_folder, folder.replace('/', '_')) + os.mkdir(temp_path) + + full_path = os.path.join(info.root, folder) + info.volume.partition_map.root.add_mount(temp_path, full_path, ['--bind']) + + +class RemoveFolderMounts(Task): + description = 'Removing folder mounts for temporary and cache data' + phase = phases.system_cleaning + successors = [apt.AptClean] + + @classmethod + def run(cls, info): + import shutil + for folder in folders: + temp_path = os.path.join(info.minimize_size_folder, folder.replace('/', '_')) + full_path = os.path.join(info.root, folder) + + info.volume.partition_map.root.remove_mount(full_path) + shutil.rmtree(temp_path) + + os.rmdir(info.minimize_size_folder) + del info.minimize_size_folder + + +class AddRequiredCommands(Task): + description = 'Adding commands required for reducing volume size' + phase = phases.preparation + successors = [host.CheckExternalCommands] + + @classmethod + def run(cls, info): + if info.manifest.plugins['minimize_size'].get('zerofree', False): + info.host_dependencies['zerofree'] = 'zerofree' + if info.manifest.plugins['minimize_size'].get('shrink', False): + link = 'https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0' + info.host_dependencies['vmware-vdiskmanager'] = link + + +class Zerofree(Task): + description = 'Zeroing unused blocks on the root partition' + phase = phases.volume_unmounting + predecessors = [filesystem.UnmountRoot] + successors = [partitioning.UnmapPartitions, volume.Detach] + + @classmethod + def run(cls, info): + from bootstrapvz.common.tools import log_check_call + log_check_call(['zerofree', info.volume.partition_map.root.device_path]) + + +class ShrinkVolume(Task): + description = 'Shrinking the volume' + phase = phases.volume_unmounting + predecessors = [volume.Detach] + + @classmethod + def run(cls, info): + from bootstrapvz.common.tools import log_check_call + log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path]) diff --git a/base/__init__.py b/bootstrapvz/plugins/ntp/__init__.py similarity index 52% rename from base/__init__.py rename to bootstrapvz/plugins/ntp/__init__.py index 4e11028..9a68d0f 100644 --- a/base/__init__.py +++ b/bootstrapvz/plugins/ntp/__init__.py @@ -1,10 +1,11 @@ -__all__ = ['Phase', 'Task', 'main'] -from phase import Phase -from task import Task -from main import main - - def validate_manifest(data, validator, error): import os.path schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) validator(data, schema_path) + + +def resolve_tasks(taskset, manifest): + import tasks + taskset.add(tasks.AddNtpPackage) + if manifest.plugins['ntp'].get('servers', False): + taskset.add(tasks.SetNtpServers) diff --git a/bootstrapvz/plugins/ntp/manifest-schema.json b/bootstrapvz/plugins/ntp/manifest-schema.json new file mode 100644 index 0000000..d15be21 --- /dev/null +++ b/bootstrapvz/plugins/ntp/manifest-schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "NTP plugin manifest", + "type": "object", + "properties": { + "plugins": { + "type": "object", + "properties": { + "ntp": { + "type": "object", + "properties": { + "servers": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1 + } + } + } + }, + "required": ["ntp"] + } + }, + "required": ["plugins"] +} diff --git a/bootstrapvz/plugins/ntp/tasks.py b/bootstrapvz/plugins/ntp/tasks.py new file mode 100644 index 0000000..12e69e5 --- /dev/null +++ b/bootstrapvz/plugins/ntp/tasks.py @@ -0,0 +1,34 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import packages + + +class AddNtpPackage(Task): + description = 'Adding NTP Package' + phase = phases.package_installation + successors = [packages.InstallPackages] + + @classmethod + def run(cls, info): + info.packages.add('ntp') + + +class SetNtpServers(Task): + description = 'Setting NTP servers' + phase = phases.system_modification + + @classmethod + def run(cls, info): + import fileinput + import os + import re + ntp_path = os.path.join(info.root, 'etc/ntp.conf') + servers = list(info.manifest.plugins['ntp']['servers']) + debian_ntp_server = re.compile('.*[0-9]\.debian\.pool\.ntp\.org.*') + for line in fileinput.input(files=ntp_path, inplace=True): + # Will write all the specified servers on the first match, then supress all other default servers + if re.match(debian_ntp_server, line): + while servers: + print 'server {server_address} iburst'.format(server_address=servers.pop(0)) + else: + print line, diff --git a/plugins/opennebula/README.md b/bootstrapvz/plugins/opennebula/README.md similarity index 78% rename from plugins/opennebula/README.md rename to bootstrapvz/plugins/opennebula/README.md index 1d96eaa..49ab0c9 100644 --- a/plugins/opennebula/README.md +++ b/bootstrapvz/plugins/opennebula/README.md @@ -11,6 +11,3 @@ It set ups the network and ssh keys. TO do so you should configure your virtual ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] FILES path_to_my_ssh_public_key.pub -Provider will install all *.pub* files in the root authorized_keys file. - -In case of an EC2 start, if the USER_EC2_DATA element is a script it will be executed. diff --git a/bootstrapvz/plugins/opennebula/__init__.py b/bootstrapvz/plugins/opennebula/__init__.py new file mode 100644 index 0000000..4164c31 --- /dev/null +++ b/bootstrapvz/plugins/opennebula/__init__.py @@ -0,0 +1,8 @@ +import tasks + + +def resolve_tasks(taskset, manifest): + if manifest.system['release'] in ['wheezy', 'stable']: + taskset.add(tasks.AddBackports) + taskset.update([tasks.AddONEContextPackage]) + diff --git a/bootstrapvz/plugins/opennebula/tasks.py b/bootstrapvz/plugins/opennebula/tasks.py new file mode 100644 index 0000000..202c5a2 --- /dev/null +++ b/bootstrapvz/plugins/opennebula/tasks.py @@ -0,0 +1,31 @@ +from bootstrapvz.base import Task +from bootstrapvz.common.tasks import apt +from bootstrapvz.common import phases +import os + + +class AddBackports(Task): + description = 'Adding backports to the apt sources' + phase = phases.preparation + + @classmethod + def run(cls, info): + if info.source_lists.target_exists('{system.release}-backports'): + import logging + msg = ('{system.release}-backports target already exists').format(**info.manifest_vars) + logging.getLogger(__name__).info(msg) + else: + info.source_lists.add('backports', 'deb {apt_mirror} {system.release}-backports main') + info.source_lists.add('backports', 'deb-src {apt_mirror} {system.release}-backports main') + +class AddONEContextPackage(Task): + description = 'Adding the OpenNebula context package' + phase = phases.preparation + predecessors = [apt.AddDefaultSources, AddBackports] + + @classmethod + def run(cls, info): + target = None + if info.manifest.system['release'] in ['wheezy', 'stable']: + target = '{system.release}-backports' + info.packages.add('opennebula-context', target) diff --git a/plugins/prebootstrapped/__init__.py b/bootstrapvz/plugins/prebootstrapped/__init__.py similarity index 77% rename from plugins/prebootstrapped/__init__.py rename to bootstrapvz/plugins/prebootstrapped/__init__.py index 1c7b024..c0bafe7 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/bootstrapvz/plugins/prebootstrapped/__init__.py @@ -2,15 +2,15 @@ from tasks import Snapshot from tasks import CopyImage from tasks import CreateFromSnapshot from tasks import CreateFromImage -from providers.ec2.tasks import ebs -from providers.virtualbox.tasks import guest_additions -from common.tasks import loopback -from common.tasks import volume -from common.tasks import locale -from common.tasks import apt -from common.tasks import bootstrap -from common.tasks import filesystem -from common.tasks import partitioning +from bootstrapvz.providers.ec2.tasks import ebs +from bootstrapvz.providers.virtualbox.tasks import guest_additions +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import locale +from bootstrapvz.common.tasks import apt +from bootstrapvz.common.tasks import bootstrap +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import partitioning def validate_manifest(data, validator, error): diff --git a/plugins/prebootstrapped/manifest-schema.json b/bootstrapvz/plugins/prebootstrapped/manifest-schema.json similarity index 100% rename from plugins/prebootstrapped/manifest-schema.json rename to bootstrapvz/plugins/prebootstrapped/manifest-schema.json diff --git a/plugins/prebootstrapped/tasks.py b/bootstrapvz/plugins/prebootstrapped/tasks.py similarity index 87% rename from plugins/prebootstrapped/tasks.py rename to bootstrapvz/plugins/prebootstrapped/tasks.py index 047cf31..018f88e 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/bootstrapvz/plugins/prebootstrapped/tasks.py @@ -1,10 +1,10 @@ -from base import Task -from common import phases -from common.tasks import volume -from common.tasks import packages -from providers.virtualbox.tasks import guest_additions -from providers.ec2.tasks import ebs -from common.fs import remount +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import packages +from bootstrapvz.providers.virtualbox.tasks import guest_additions +from bootstrapvz.providers.ec2.tasks import ebs +from bootstrapvz.common.fs import remount from shutil import copyfile import os.path import time @@ -81,7 +81,7 @@ def set_fs_states(volume): p_map = volume.partition_map partitions_state = 'attached' - from base.fs.partitionmaps.none import NoPartitions + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions if isinstance(p_map, NoPartitions): partitions_state = 'formatted' else: diff --git a/plugins/puppet/__init__.py b/bootstrapvz/plugins/puppet/__init__.py similarity index 85% rename from plugins/puppet/__init__.py rename to bootstrapvz/plugins/puppet/__init__.py index 089a0ca..458b1cd 100644 --- a/plugins/puppet/__init__.py +++ b/bootstrapvz/plugins/puppet/__init__.py @@ -8,9 +8,10 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - taskset.add(tasks.CheckPaths) taskset.add(tasks.AddPackages) if 'assets' in manifest.plugins['puppet']: + taskset.add(tasks.CheckAssetsPath) taskset.add(tasks.CopyPuppetAssets) if 'manifest' in manifest.plugins['puppet']: + taskset.add(tasks.CheckManifestPath) taskset.add(tasks.ApplyPuppetManifest) diff --git a/plugins/puppet/manifest-schema.json b/bootstrapvz/plugins/puppet/manifest-schema.json similarity index 100% rename from plugins/puppet/manifest-schema.json rename to bootstrapvz/plugins/puppet/manifest-schema.json diff --git a/plugins/puppet/tasks.py b/bootstrapvz/plugins/puppet/tasks.py similarity index 65% rename from plugins/puppet/tasks.py rename to bootstrapvz/plugins/puppet/tasks.py index 8b0559c..dca4cd7 100644 --- a/plugins/puppet/tasks.py +++ b/bootstrapvz/plugins/puppet/tasks.py @@ -1,17 +1,17 @@ -from base import Task -from common import phases -from common.tasks import apt -from common.tasks import network +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +from bootstrapvz.common.tasks import network import os -class CheckPaths(Task): - description = 'Checking whether manifest and assets paths exist' +class CheckAssetsPath(Task): + description = 'Checking whether the assets path exist' phase = phases.preparation @classmethod def run(cls, info): - from common.exceptions import TaskError + from bootstrapvz.common.exceptions import TaskError assets = info.manifest.plugins['puppet']['assets'] if not os.path.exists(assets): msg = 'The assets directory {assets} does not exist.'.format(assets=assets) @@ -20,6 +20,14 @@ class CheckPaths(Task): msg = 'The assets path {assets} does not point to a directory.'.format(assets=assets) raise TaskError(msg) + +class CheckManifestPath(Task): + description = 'Checking whether the manifest path exist' + phase = phases.preparation + + @classmethod + def run(cls, info): + from bootstrapvz.common.exceptions import TaskError manifest = info.manifest.plugins['puppet']['manifest'] if not os.path.exists(manifest): msg = 'The manifest file {manifest} does not exist.'.format(manifest=manifest) @@ -45,22 +53,8 @@ class CopyPuppetAssets(Task): @classmethod def run(cls, info): - from shutil import copy - puppet_path = os.path.join(info.root, 'etc/puppet') - puppet_assets = info.manifest.plugins['puppet']['assets'] - for abs_prefix, dirs, files in os.walk(puppet_assets): - prefix = os.path.normpath(os.path.relpath(abs_prefix, puppet_assets)) - for path in dirs: - full_path = os.path.join(puppet_path, prefix, path) - if os.path.exists(full_path): - if os.path.isdir(full_path): - continue - else: - os.remove(full_path) - os.mkdir(full_path) - for path in files: - copy(os.path.join(abs_prefix, path), - os.path.join(puppet_path, prefix, path)) + from bootstrapvz.common.tools import copy_tree + copy_tree(info.manifest.plugins['puppet']['assets'], os.path.join(info.root, 'etc/puppet')) class ApplyPuppetManifest(Task): @@ -83,11 +77,21 @@ class ApplyPuppetManifest(Task): copy(pp_manifest, manifest_dst) manifest_path = os.path.join('/', manifest_rel_dst) - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/puppet', 'apply', manifest_path]) + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, + 'puppet', 'apply', manifest_path]) os.remove(manifest_dst) - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i hosts_path = os.path.join(info.root, 'etc/hosts') sed_i(hosts_path, '127.0.0.1\s*{hostname}\n?'.format(hostname=hostname), '') + + +class EnableAgent(Task): + description = 'Enabling the puppet agent' + phase = phases.system_modification + + @classmethod + def run(cls, info): + puppet_defaults = os.path.join(info.root, 'etc/defaults/puppet') + sed_i(puppet_defaults, 'START=no', 'START=yes') diff --git a/plugins/root_password/__init__.py b/bootstrapvz/plugins/root_password/__init__.py similarity index 81% rename from plugins/root_password/__init__.py rename to bootstrapvz/plugins/root_password/__init__.py index 947b1b5..e93aa90 100644 --- a/plugins/root_password/__init__.py +++ b/bootstrapvz/plugins/root_password/__init__.py @@ -7,7 +7,7 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - from common.tasks.security import DisableSSHPasswordAuthentication + from bootstrapvz.common.tasks.security import DisableSSHPasswordAuthentication from tasks import SetRootPassword taskset.discard(DisableSSHPasswordAuthentication) taskset.add(SetRootPassword) diff --git a/plugins/root_password/manifest-schema.json b/bootstrapvz/plugins/root_password/manifest-schema.json similarity index 100% rename from plugins/root_password/manifest-schema.json rename to bootstrapvz/plugins/root_password/manifest-schema.json diff --git a/plugins/root_password/tasks.py b/bootstrapvz/plugins/root_password/tasks.py similarity index 55% rename from plugins/root_password/tasks.py rename to bootstrapvz/plugins/root_password/tasks.py index 85776cc..359a0d4 100644 --- a/plugins/root_password/tasks.py +++ b/bootstrapvz/plugins/root_password/tasks.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases class SetRootPassword(Task): @@ -8,6 +8,6 @@ class SetRootPassword(Task): @classmethod def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/chpasswd'], + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, 'chpasswd'], 'root:' + info.manifest.plugins['root_password']['password']) diff --git a/plugins/unattended_upgrades/__init__.py b/bootstrapvz/plugins/unattended_upgrades/__init__.py similarity index 100% rename from plugins/unattended_upgrades/__init__.py rename to bootstrapvz/plugins/unattended_upgrades/__init__.py diff --git a/plugins/unattended_upgrades/manifest-schema.json b/bootstrapvz/plugins/unattended_upgrades/manifest-schema.json similarity index 100% rename from plugins/unattended_upgrades/manifest-schema.json rename to bootstrapvz/plugins/unattended_upgrades/manifest-schema.json diff --git a/plugins/unattended_upgrades/tasks.py b/bootstrapvz/plugins/unattended_upgrades/tasks.py similarity index 94% rename from plugins/unattended_upgrades/tasks.py rename to bootstrapvz/plugins/unattended_upgrades/tasks.py index 2bb485a..1299588 100644 --- a/plugins/unattended_upgrades/tasks.py +++ b/bootstrapvz/plugins/unattended_upgrades/tasks.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt class AddUnattendedUpgradesPackage(Task): diff --git a/plugins/vagrant/__init__.py b/bootstrapvz/plugins/vagrant/__init__.py similarity index 83% rename from plugins/vagrant/__init__.py rename to bootstrapvz/plugins/vagrant/__init__.py index b0af5d3..f657f50 100644 --- a/plugins/vagrant/__init__.py +++ b/bootstrapvz/plugins/vagrant/__init__.py @@ -8,14 +8,14 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - from common.tasks import security - from common.tasks import loopback - from common.tasks import network + from bootstrapvz.common.tasks import security + from bootstrapvz.common.tasks import loopback + from bootstrapvz.common.tasks import network taskset.discard(security.DisableSSHPasswordAuthentication) taskset.discard(loopback.MoveImage) taskset.discard(network.RemoveHostname) - from common.tasks import volume + from bootstrapvz.common.tasks import volume taskset.update([tasks.CheckBoxPath, tasks.CreateVagrantBoxDir, tasks.AddPackages, diff --git a/bootstrapvz/plugins/vagrant/assets/Vagrantfile b/bootstrapvz/plugins/vagrant/assets/Vagrantfile new file mode 100644 index 0000000..b3164ca --- /dev/null +++ b/bootstrapvz/plugins/vagrant/assets/Vagrantfile @@ -0,0 +1,11 @@ +Vagrant::Config.run do |config| + # This Vagrantfile is auto-generated by `vagrant package` to contain + # the MAC address of the box. Custom configuration should be placed in + # the actual `Vagrantfile` in this box. + config.vm.base_mac = "[MAC_ADDRESS]" +end + +# Load include vagrant file if it exists after the auto-generated +# so it can override any of the settings +include_vagrantfile = File.expand_path("../include/_Vagrantfile", __FILE__) +load include_vagrantfile if File.exist?(include_vagrantfile) diff --git a/plugins/vagrant/assets/authorized_keys b/bootstrapvz/plugins/vagrant/assets/authorized_keys similarity index 100% rename from plugins/vagrant/assets/authorized_keys rename to bootstrapvz/plugins/vagrant/assets/authorized_keys diff --git a/plugins/vagrant/assets/box.ovf b/bootstrapvz/plugins/vagrant/assets/box.ovf similarity index 100% rename from plugins/vagrant/assets/box.ovf rename to bootstrapvz/plugins/vagrant/assets/box.ovf diff --git a/plugins/vagrant/assets/metadata.json b/bootstrapvz/plugins/vagrant/assets/metadata.json similarity index 100% rename from plugins/vagrant/assets/metadata.json rename to bootstrapvz/plugins/vagrant/assets/metadata.json diff --git a/plugins/vagrant/manifest-schema.json b/bootstrapvz/plugins/vagrant/manifest-schema.json similarity index 100% rename from plugins/vagrant/manifest-schema.json rename to bootstrapvz/plugins/vagrant/manifest-schema.json diff --git a/plugins/vagrant/tasks.py b/bootstrapvz/plugins/vagrant/tasks.py similarity index 91% rename from plugins/vagrant/tasks.py rename to bootstrapvz/plugins/vagrant/tasks.py index 4118e08..94e7805 100644 --- a/plugins/vagrant/tasks.py +++ b/bootstrapvz/plugins/vagrant/tasks.py @@ -1,7 +1,7 @@ -from base import Task -from common import phases -from common.tasks import workspace -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import workspace +from bootstrapvz.common.tasks import apt import os import shutil @@ -18,7 +18,7 @@ class CheckBoxPath(Task): box_name = '{name}.box'.format(name=box_basename) box_path = os.path.join(info.manifest.bootstrapper['workspace'], box_name) if os.path.exists(box_path): - from common.exceptions import TaskError + from bootstrapvz.common.exceptions import TaskError msg = 'The vagrant box `{name}\' already exists at `{path}\''.format(name=box_name, path=box_path) raise TaskError(msg) info.vagrant = {'box_name': box_name, @@ -60,7 +60,7 @@ class SetHostname(Task): hostname_file.write(hostname) hosts_path = os.path.join(info.root, 'etc/hosts') - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i sed_i(hosts_path, '^127.0.0.1\tlocalhost$', '127.0.0.1\tlocalhost\n127.0.0.1\t' + hostname) @@ -70,9 +70,9 @@ class CreateVagrantUser(Task): @classmethod def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/useradd', + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, + 'useradd', '--create-home', '--shell', '/bin/bash', 'vagrant']) @@ -114,9 +114,9 @@ class AddInsecurePublicKey(Task): os.chmod(authorized_keys_path, stat.S_IRUSR | stat.S_IWUSR) # We can't do this directly with python, since getpwnam gets its info from the host - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, - '/bin/chown', 'vagrant:vagrant', + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, + 'chown', 'vagrant:vagrant', '/home/vagrant/.ssh', '/home/vagrant/.ssh/authorized_keys']) @@ -126,8 +126,8 @@ class SetRootPassword(Task): @classmethod def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/chpasswd'], 'root:vagrant') + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, 'chpasswd'], 'root:vagrant') class PackageBox(Task): @@ -142,14 +142,14 @@ class PackageBox(Task): import random mac_address = '080027{mac:06X}'.format(mac=random.randrange(16 ** 6)) - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i sed_i(vagrantfile, '\\[MAC_ADDRESS\\]', mac_address) metadata_source = os.path.join(assets, 'metadata.json') metadata = os.path.join(info.vagrant['folder'], 'metadata.json') shutil.copy(metadata_source, metadata) - from common.tools import log_check_call + from bootstrapvz.common.tools import log_check_call disk_name = 'box-disk1.{ext}'.format(ext=info.volume.extension) disk_link = os.path.join(info.vagrant['folder'], disk_name) log_check_call(['ln', '-s', info.volume.image_path, disk_link]) diff --git a/plugins/__init__.py b/bootstrapvz/providers/__init__.py similarity index 100% rename from plugins/__init__.py rename to bootstrapvz/providers/__init__.py diff --git a/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py similarity index 77% rename from providers/ec2/__init__.py rename to bootstrapvz/providers/ec2/__init__.py index 5038e8d..9272d2c 100644 --- a/providers/ec2/__init__.py +++ b/bootstrapvz/providers/ec2/__init__.py @@ -7,17 +7,17 @@ import tasks.filesystem import tasks.boot import tasks.network import tasks.initd -from common.tasks import volume -from common.tasks import filesystem -from common.tasks import boot -from common.tasks import network -from common.tasks import initd -from common.tasks import partitioning -from common.tasks import loopback -from common.tasks import bootstrap -from common.tasks import security -from common.tasks import cleanup -from common.tasks import workspace +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import boot +from bootstrapvz.common.tasks import network +from bootstrapvz.common.tasks import initd +from bootstrapvz.common.tasks import partitioning +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tasks import bootstrap +from bootstrapvz.common.tasks import security +from bootstrapvz.common.tasks import cleanup +from bootstrapvz.common.tasks import workspace def initialize(): @@ -30,7 +30,7 @@ def validate_manifest(data, validator, error): import os.path validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) - from common.bytes import Bytes + from bootstrapvz.common.bytes import Bytes if data['volume']['backing'] == 'ebs': volume_size = Bytes(0) for key, partition in data['volume']['partitions'].iteritems(): @@ -51,17 +51,17 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - import common.task_sets - taskset.update(common.task_sets.base_set) - taskset.update(common.task_sets.mounting_set) - taskset.update(common.task_sets.get_apt_set(manifest)) - taskset.update(common.task_sets.locale_set) - taskset.update(common.task_sets.ssh_set) + from bootstrapvz.common import task_sets + taskset.update(task_sets.base_set) + taskset.update(task_sets.mounting_set) + taskset.update(task_sets.get_apt_set(manifest)) + taskset.update(task_sets.locale_set) + taskset.update(task_sets.ssh_set) if manifest.volume['partitions']['type'] != 'none': - taskset.update(common.task_sets.partitioning_set) + taskset.update(task_sets.partitioning_set) - taskset.update([tasks.host.HostDependencies, + taskset.update([tasks.host.AddExternalCommands, tasks.packages.DefaultPackages, tasks.connection.GetCredentials, tasks.host.GetInfo, @@ -91,13 +91,14 @@ def resolve_tasks(taskset, manifest): taskset.add(boot.AddGrubPackage) taskset.add(tasks.boot.ConfigurePVGrub) else: - taskset.update(common.task_sets.bootloader_set.get(manifest.system['bootloader'])) + taskset.update(task_sets.bootloader_set.get(manifest.system['bootloader'])) backing_specific_tasks = {'ebs': [tasks.ebs.Create, tasks.ebs.Attach, filesystem.FStab, tasks.ebs.Snapshot], - 's3': [loopback.Create, + 's3': [loopback.AddRequiredCommands, + loopback.Create, volume.Attach, tasks.filesystem.S3FStab, tasks.ami.BundleImage, @@ -112,10 +113,10 @@ def resolve_tasks(taskset, manifest): if manifest.bootstrapper.get('tarball', False): taskset.add(bootstrap.MakeTarball) - taskset.update(common.task_sets.get_fs_specific_set(manifest.volume['partitions'])) + taskset.update(task_sets.get_fs_specific_set(manifest.volume['partitions'])) if 'boot' in manifest.volume['partitions']: - taskset.update(common.task_sets.boot_partition_set) + taskset.update(task_sets.boot_partition_set) def resolve_rollback_tasks(taskset, manifest, counter_task): diff --git a/providers/ec2/assets/certs/cert-ec2.pem b/bootstrapvz/providers/ec2/assets/certs/cert-ec2.pem similarity index 100% rename from providers/ec2/assets/certs/cert-ec2.pem rename to bootstrapvz/providers/ec2/assets/certs/cert-ec2.pem diff --git a/providers/ec2/assets/grub.d/40_custom b/bootstrapvz/providers/ec2/assets/grub.d/40_custom similarity index 100% rename from providers/ec2/assets/grub.d/40_custom rename to bootstrapvz/providers/ec2/assets/grub.d/40_custom diff --git a/providers/ec2/assets/init.d/ec2-get-credentials b/bootstrapvz/providers/ec2/assets/init.d/ec2-get-credentials similarity index 100% rename from providers/ec2/assets/init.d/ec2-get-credentials rename to bootstrapvz/providers/ec2/assets/init.d/ec2-get-credentials diff --git a/providers/ec2/assets/init.d/ec2-run-user-data b/bootstrapvz/providers/ec2/assets/init.d/ec2-run-user-data similarity index 100% rename from providers/ec2/assets/init.d/ec2-run-user-data rename to bootstrapvz/providers/ec2/assets/init.d/ec2-run-user-data diff --git a/providers/ec2/ebsvolume.py b/bootstrapvz/providers/ec2/ebsvolume.py similarity index 93% rename from providers/ec2/ebsvolume.py rename to bootstrapvz/providers/ec2/ebsvolume.py index 8cbcdbf..0738c60 100644 --- a/providers/ec2/ebsvolume.py +++ b/bootstrapvz/providers/ec2/ebsvolume.py @@ -1,5 +1,5 @@ -from base.fs.volume import Volume -from base.fs.exceptions import VolumeError +from bootstrapvz.base.fs.volume import Volume +from bootstrapvz.base.fs.exceptions import VolumeError import time diff --git a/providers/ec2/manifest-schema-s3.json b/bootstrapvz/providers/ec2/manifest-schema-s3.json similarity index 100% rename from providers/ec2/manifest-schema-s3.json rename to bootstrapvz/providers/ec2/manifest-schema-s3.json diff --git a/providers/ec2/manifest-schema.json b/bootstrapvz/providers/ec2/manifest-schema.json similarity index 100% rename from providers/ec2/manifest-schema.json rename to bootstrapvz/providers/ec2/manifest-schema.json diff --git a/providers/ec2/tasks/__init__.py b/bootstrapvz/providers/ec2/tasks/__init__.py similarity index 100% rename from providers/ec2/tasks/__init__.py rename to bootstrapvz/providers/ec2/tasks/__init__.py diff --git a/bootstrapvz/providers/ec2/tasks/ami-akis.json b/bootstrapvz/providers/ec2/tasks/ami-akis.json new file mode 100644 index 0000000..79e1b66 --- /dev/null +++ b/bootstrapvz/providers/ec2/tasks/ami-akis.json @@ -0,0 +1,34 @@ +// This is a mapping of EC2 regions to processor architectures to Amazon Kernel Images +// Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs +{ +"ap-northeast-1": // Asia Pacific (Tokyo) Region + {"i386": "aki-136bf512", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-176bf516"}, // pv-grub-hd0_1.04-x86_64.gz +"ap-southeast-1": // Asia Pacific (Singapore) Region + {"i386": "aki-ae3973fc", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-503e7402"}, // pv-grub-hd0_1.04-x86_64.gz +"ap-southeast-2": // Asia Pacific (Sydney) Region + {"i386": "aki-cd62fff7", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-c362fff9"}, // pv-grub-hd0_1.04-x86_64.gz +"eu-west-1": // EU (Ireland) Region + {"i386": "aki-68a3451f", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-52a34525"}, // pv-grub-hd0_1.04-x86_64.gz +"sa-east-1": // South America (Sao Paulo) Region + {"i386": "aki-5b53f446", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-5553f448"}, // pv-grub-hd0_1.04-x86_64.gz +"us-east-1": // US East (Northern Virginia) Region + {"i386": "aki-8f9dcae6", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-919dcaf8"}, // pv-grub-hd0_1.04-x86_64.gz +"us-gov-west-1": // AWS GovCloud (US) + {"i386": "aki-1fe98d3c", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-1de98d3e"}, // pv-grub-hd0_1.04-x86_64.gz +"us-west-1": // US West (Northern California) Region + {"i386": "aki-8e0531cb", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-880531cd"}, // pv-grub-hd0_1.04-x86_64.gz +"us-west-2": // US West (Oregon) Region + {"i386": "aki-f08f11c0", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-fc8f11cc"}, // pv-grub-hd0_1.04-x86_64.gz +"cn-north-1":// China North (Beijing) Region + {"i386": "aki-908f1da9", // pv-grub-hd0_1.04-i386.gz + "amd64": "aki-9e8f1da7"} // pv-grub-hd0_1.04-x86_64.gz +} diff --git a/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py similarity index 56% rename from providers/ec2/tasks/ami.py rename to bootstrapvz/providers/ec2/tasks/ami.py index 9e0ff0b..d023df5 100644 --- a/providers/ec2/tasks/ami.py +++ b/bootstrapvz/providers/ec2/tasks/ami.py @@ -1,9 +1,9 @@ -from base import Task -from common import phases -from common.exceptions import TaskError -from common.tools import log_check_call +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.exceptions import TaskError +from bootstrapvz.common.tools import log_check_call from ebs import Snapshot -from common.tasks import workspace +from bootstrapvz.common.tasks import workspace from connection import Connect from . import assets import os.path @@ -38,8 +38,10 @@ class BundleImage(Task): def run(cls, info): bundle_name = 'bundle-{id}'.format(id=info.run_id) info.bundle_path = os.path.join(info.workspace, bundle_name) - log_check_call(['/usr/bin/euca-bundle-image', + arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + log_check_call(['euca-bundle-image', '--image', info.volume.image_path, + '--arch', arch, '--user', info.credentials['user-id'], '--privatekey', info.credentials['private-key'], '--cert', info.credentials['certificate'], @@ -63,7 +65,7 @@ class UploadImage(Task): else: s3_url = 'https://s3-{region}.amazonaws.com/'.format(region=info.host['region']) info.manifest.manifest_location = info.manifest.image['bucket'] + '/' + info.ami_name + '.manifest.xml' - log_check_call(['/usr/bin/euca-upload-bundle', + log_check_call(['euca-upload-bundle', '--bucket', info.manifest.image['bucket'], '--manifest', manifest_file, '--access-key', info.credentials['access-key'], @@ -90,48 +92,6 @@ class RegisterAMI(Task): phase = phases.image_registration predecessors = [Snapshot, UploadImage] - # Source: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs - kernel_mapping = {'ap-northeast-1': # Asia Pacific (Tokyo) Region - {'i386': 'aki-136bf512', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-176bf516' # pv-grub-hd0_1.04-x86_64.gz - }, - 'ap-southeast-1': # Asia Pacific (Singapore) Region - {'i386': 'aki-ae3973fc', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-503e7402' # pv-grub-hd0_1.04-x86_64.gz - }, - 'ap-southeast-2': # Asia Pacific (Sydney) Region - {'i386': 'aki-cd62fff7', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-c362fff9' # pv-grub-hd0_1.04-x86_64.gz - }, - 'eu-west-1': # EU (Ireland) Region - {'i386': 'aki-68a3451f', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-52a34525' # pv-grub-hd0_1.04-x86_64.gz - }, - 'sa-east-1': # South America (Sao Paulo) Region - {'i386': 'aki-5b53f446', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-5553f448' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-east-1': # US East (Northern Virginia) Region - {'i386': 'aki-8f9dcae6', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-919dcaf8' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-gov-west-1': # AWS GovCloud (US) - {'i386': 'aki-1fe98d3c', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-1de98d3e' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-west-1': # US West (Northern California) Region - {'i386': 'aki-8e0531cb', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-880531cd' # pv-grub-hd0_1.04-x86_64.gz - }, - 'us-west-2': # US West (Oregon) Region - {'i386': 'aki-f08f11c0', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-fc8f11cc' # pv-grub-hd0_1.04-x86_64.gz - }, - 'cn-north-1': # China North (Beijing) Region - {'i386': 'aki-908f1da9', # pv-grub-hd0_1.04-i386.gz - 'amd64': 'aki-9e8f1da7' # pv-grub-hd0_1.04-x86_64.gz - } - } @classmethod def run(cls, info): registration_params = {'name': info.ami_name, @@ -140,17 +100,11 @@ class RegisterAMI(Task): 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) if info.manifest.volume['backing'] == 's3': - grub_boot_device = 'hd0' registration_params['image_location'] = info.manifest.manifest_location else: root_dev_name = {'pvm': '/dev/sda', 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization']) registration_params['root_device_name'] = root_dev_name - from base.fs.partitionmaps.none import NoPartitions - if isinstance(info.volume.partition_map, NoPartitions): - grub_boot_device = 'hd0' - else: - grub_boot_device = 'hd00' from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping @@ -163,8 +117,9 @@ class RegisterAMI(Task): registration_params['virtualization_type'] = 'hvm' else: registration_params['virtualization_type'] = 'paravirtual' - registration_params['kernel_id'] = (cls.kernel_mapping - .get(info.host['region']) - .get(info.manifest.system['architecture'])) + akis_path = os.path.join(os.path.dirname(__file__), 'ami-akis.json') + from bootstrapvz.common.tools import config_get + registration_params['kernel_id'] = config_get(akis_path, [info.host['region'], + info.manifest.system['architecture']]) info.image = info.connection.register_image(**registration_params) diff --git a/providers/ec2/tasks/boot.py b/bootstrapvz/providers/ec2/tasks/boot.py similarity index 76% rename from providers/ec2/tasks/boot.py rename to bootstrapvz/providers/ec2/tasks/boot.py index b921324..664bd7d 100644 --- a/providers/ec2/tasks/boot.py +++ b/bootstrapvz/providers/ec2/tasks/boot.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases from . import assets import os @@ -26,9 +26,9 @@ class ConfigurePVGrub(Task): copy(script_src, script_dst) os.chmod(script_dst, rwxr_xr_x) - from base.fs.partitionmaps.none import NoPartitions + from bootstrapvz.base.fs.partitionmaps.none import NoPartitions if not isinstance(info.volume.partition_map, NoPartitions): - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i root_idx = info.volume.partition_map.root.get_index() grub_device = 'GRUB_DEVICE=/dev/xvda{idx}'.format(idx=root_idx) sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', grub_device) @@ -36,17 +36,17 @@ class ConfigurePVGrub(Task): sed_i(script_dst, '^\troot \(hd0\)$', grub_root) if info.manifest.volume['backing'] == 's3': - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i sed_i(script_dst, '^GRUB_DEVICE=/dev/xvda$', 'GRUB_DEVICE=/dev/xvda1') - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' 'GRUB_HIDDEN_TIMEOUT=true') sed_i(grub_def, '^#GRUB_TERMINAL=console', 'GRUB_TERMINAL=console') sed_i(grub_def, '^GRUB_CMDLINE_LINUX_DEFAULT="quiet"', 'GRUB_CMDLINE_LINUX_DEFAULT="console=hvc0"') - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - log_check_call(['/usr/sbin/chroot', info.root, - '/bin/ln', '--symbolic', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) + from bootstrapvz.common.tools import log_check_call + log_check_call(['chroot', info.root, 'update-grub']) + log_check_call(['chroot', info.root, + 'ln', '--symbolic', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) diff --git a/providers/ec2/tasks/connection.py b/bootstrapvz/providers/ec2/tasks/connection.py similarity index 95% rename from providers/ec2/tasks/connection.py rename to bootstrapvz/providers/ec2/tasks/connection.py index 73d8f8d..a776a3e 100644 --- a/providers/ec2/tasks/connection.py +++ b/bootstrapvz/providers/ec2/tasks/connection.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases import host diff --git a/providers/ec2/tasks/ebs.py b/bootstrapvz/providers/ec2/tasks/ebs.py similarity index 89% rename from providers/ec2/tasks/ebs.py rename to bootstrapvz/providers/ec2/tasks/ebs.py index b5f3fe8..e11c8cb 100644 --- a/providers/ec2/tasks/ebs.py +++ b/bootstrapvz/providers/ec2/tasks/ebs.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases class Create(Task): diff --git a/providers/ec2/tasks/filesystem.py b/bootstrapvz/providers/ec2/tasks/filesystem.py similarity index 92% rename from providers/ec2/tasks/filesystem.py rename to bootstrapvz/providers/ec2/tasks/filesystem.py index 9a77049..3e3dce4 100644 --- a/providers/ec2/tasks/filesystem.py +++ b/bootstrapvz/providers/ec2/tasks/filesystem.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases class S3FStab(Task): diff --git a/providers/ec2/tasks/host.py b/bootstrapvz/providers/ec2/tasks/host.py similarity index 54% rename from providers/ec2/tasks/host.py rename to bootstrapvz/providers/ec2/tasks/host.py index 661f01f..7701ef5 100644 --- a/providers/ec2/tasks/host.py +++ b/bootstrapvz/providers/ec2/tasks/host.py @@ -1,17 +1,18 @@ -from base import Task -from common import phases -from common.tasks import host +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import host -class HostDependencies(Task): - description = 'Adding required host packages for EC2 bootstrapping' +class AddExternalCommands(Task): + description = 'Determining required external commands for EC2 bootstrapping' phase = phases.preparation - successors = [host.CheckHostDependencies] + successors = [host.CheckExternalCommands] @classmethod def run(cls, info): if info.manifest.volume['backing'] == 's3': - info.host_dependencies.add('euca2ools') + info.host_dependencies['euca-bundle-image'] = 'euca2ools' + info.host_dependencies['euca-upload-bundle'] = 'euca2ools' class GetInfo(Task): diff --git a/providers/ec2/tasks/initd.py b/bootstrapvz/providers/ec2/tasks/initd.py similarity index 81% rename from providers/ec2/tasks/initd.py rename to bootstrapvz/providers/ec2/tasks/initd.py index 88b0121..82a9902 100644 --- a/providers/ec2/tasks/initd.py +++ b/bootstrapvz/providers/ec2/tasks/initd.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import initd +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import initd from . import assets import os.path diff --git a/providers/ec2/tasks/network.py b/bootstrapvz/providers/ec2/tasks/network.py similarity index 68% rename from providers/ec2/tasks/network.py rename to bootstrapvz/providers/ec2/tasks/network.py index cbaf4ac..8d3f695 100644 --- a/providers/ec2/tasks/network.py +++ b/bootstrapvz/providers/ec2/tasks/network.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt import os.path @@ -12,7 +12,7 @@ class EnableDHCPCDDNS(Task): def run(cls, info): # The dhcp client that ships with debian sets the DNS servers per default. # For dhcpcd we need to configure it to do that. - from common.tools import sed_i + from bootstrapvz.common.tools import sed_i dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') @@ -39,18 +39,18 @@ class InstallEnhancedNetworking(Task): import urllib urllib.urlretrieve(drivers_url, archive) - from common.tools import log_check_call - log_check_call('/bin/tar', '--ungzip', - '--extract', - '--file', archive, - '--directory', os.path.join(info.root, 'tmp')) + from bootstrapvz.common.tools import log_check_call + log_check_call('tar', '--ungzip', + '--extract', + '--file', archive, + '--directory', os.path.join(info.root, 'tmp')) src_dir = os.path.join('/tmp', os.path.basename(drivers_url), 'src') - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/make', '--directory', src_dir]) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/bin/make', 'install', - '--directory', src_dir]) + log_check_call(['chroot', info.root, + 'make', '--directory', src_dir]) + log_check_call(['chroot', info.root, + 'make', 'install', + '--directory', src_dir]) ixgbevf_conf_path = os.path.join(info.root, 'etc/modprobe.d/ixgbevf.conf') with open(ixgbevf_conf_path, 'w') as ixgbevf_conf: diff --git a/bootstrapvz/providers/ec2/tasks/packages-kernels.json b/bootstrapvz/providers/ec2/tasks/packages-kernels.json new file mode 100644 index 0000000..1e42622 --- /dev/null +++ b/bootstrapvz/providers/ec2/tasks/packages-kernels.json @@ -0,0 +1,12 @@ +// This is a mapping of Debian release codenames to processor architectures to kernel packages +{ +"squeeze": // In squeeze, we need a special kernel flavor for xen + {"i386": "linux-image-xen-686", + "amd64": "linux-image-xen-amd64"}, +"wheezy": + {"i386": "linux-image-686", + "amd64": "linux-image-amd64"}, +"jessie": + {"i386": "linux-image-686", + "amd64": "linux-image-amd64"} +} diff --git a/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py similarity index 53% rename from providers/ec2/tasks/packages.py rename to bootstrapvz/providers/ec2/tasks/packages.py index 242457a..62107cd 100644 --- a/providers/ec2/tasks/packages.py +++ b/bootstrapvz/providers/ec2/tasks/packages.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt class DefaultPackages(Task): @@ -17,10 +17,9 @@ class DefaultPackages(Task): info.exclude_packages.add('isc-dhcp-client') info.exclude_packages.add('isc-dhcp-common') - # In squeeze, we need a special kernel flavor for xen - kernels = {} - with open('providers/ec2/tasks/packages-kernels.json') as stream: - import json - kernels = json.loads(stream.read()) - kernel_package = kernels.get(info.manifest.system['release']).get(info.manifest.system['architecture']) + import os.path + kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.json') + from bootstrapvz.common.tools import config_get + kernel_package = config_get(kernel_packages_path, [info.release_codename, + info.manifest.system['architecture']]) info.packages.add(kernel_package) diff --git a/providers/kvm/README.md b/bootstrapvz/providers/kvm/README.md similarity index 100% rename from providers/kvm/README.md rename to bootstrapvz/providers/kvm/README.md diff --git a/providers/kvm/__init__.py b/bootstrapvz/providers/kvm/__init__.py similarity index 63% rename from providers/kvm/__init__.py rename to bootstrapvz/providers/kvm/__init__.py index a0fe7b3..313660b 100644 --- a/providers/kvm/__init__.py +++ b/bootstrapvz/providers/kvm/__init__.py @@ -1,14 +1,14 @@ import tasks.packages -from common.tasks import volume -from common.tasks import loopback -from common.tasks import partitioning -from common.tasks import filesystem -from common.tasks import bootstrap -from common.tasks import security -from common.tasks import network -from common.tasks import initd -from common.tasks import cleanup -from common.tasks import workspace +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tasks import partitioning +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import bootstrap +from bootstrapvz.common.tasks import security +from bootstrapvz.common.tasks import network +from bootstrapvz.common.tasks import initd +from bootstrapvz.common.tasks import cleanup +from bootstrapvz.common.tasks import workspace def initialize(): @@ -25,17 +25,17 @@ def validate_manifest(data, validator, error): def resolve_tasks(tasklist, manifest): - import common.task_sets - tasklist.update(common.task_sets.base_set) - tasklist.update(common.task_sets.volume_set) - tasklist.update(common.task_sets.mounting_set) - tasklist.update(common.task_sets.get_apt_set(manifest)) - tasklist.update(common.task_sets.locale_set) + from bootstrapvz.common import task_sets + tasklist.update(task_sets.base_set) + tasklist.update(task_sets.volume_set) + tasklist.update(task_sets.mounting_set) + tasklist.update(task_sets.get_apt_set(manifest)) + tasklist.update(task_sets.locale_set) - tasklist.update(common.task_sets.bootloader_set.get(manifest.system['bootloader'])) + tasklist.update(task_sets.bootloader_set.get(manifest.system['bootloader'])) if manifest.volume['partitions']['type'] != 'none': - tasklist.update(common.task_sets.partitioning_set) + tasklist.update(task_sets.partitioning_set) tasklist.update([tasks.packages.DefaultPackages, @@ -60,10 +60,10 @@ def resolve_tasks(tasklist, manifest): from tasks import virtio tasklist.update([virtio.VirtIO]) - tasklist.update(common.task_sets.get_fs_specific_set(manifest.volume['partitions'])) + tasklist.update(task_sets.get_fs_specific_set(manifest.volume['partitions'])) if 'boot' in manifest.volume['partitions']: - tasklist.update(common.task_sets.boot_partition_set) + tasklist.update(task_sets.boot_partition_set) def resolve_rollback_tasks(tasklist, manifest, counter_task): diff --git a/providers/kvm/manifest-schema.json b/bootstrapvz/providers/kvm/manifest-schema.json similarity index 75% rename from providers/kvm/manifest-schema.json rename to bootstrapvz/providers/kvm/manifest-schema.json index 12937c8..3586e4e 100644 --- a/providers/kvm/manifest-schema.json +++ b/bootstrapvz/providers/kvm/manifest-schema.json @@ -3,21 +3,22 @@ "title": "KVM manifest", "type": "object", "properties": { - "bootstrapper": { + "system": { "type": "object", "properties": { "virtio": { "type": "array", "items": { - "type": "string" + "type": "string", + "enum": ["virtio", + "virtio_pci", + "virtio_balloon", + "virtio_blk", + "virtio_net", + "virtio_ring"] }, "minItems": 1 - } - } - }, - "system": { - "type": "object", - "properties": { + }, "bootloader": { "type": "string", "enum": ["grub", "extlinux"] diff --git a/providers/__init__.py b/bootstrapvz/providers/kvm/tasks/__init__.py similarity index 100% rename from providers/__init__.py rename to bootstrapvz/providers/kvm/tasks/__init__.py diff --git a/providers/kvm/tasks/packages.py b/bootstrapvz/providers/kvm/tasks/packages.py similarity index 77% rename from providers/kvm/tasks/packages.py rename to bootstrapvz/providers/kvm/tasks/packages.py index 52d2639..97eec67 100644 --- a/providers/kvm/tasks/packages.py +++ b/bootstrapvz/providers/kvm/tasks/packages.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt class DefaultPackages(Task): diff --git a/providers/kvm/tasks/virtio.py b/bootstrapvz/providers/kvm/tasks/virtio.py similarity index 62% rename from providers/kvm/tasks/virtio.py rename to bootstrapvz/providers/kvm/tasks/virtio.py index a3d2370..cb3ae68 100644 --- a/providers/kvm/tasks/virtio.py +++ b/bootstrapvz/providers/kvm/tasks/virtio.py @@ -1,5 +1,5 @@ -from base import Task -from common import phases +from bootstrapvz.base import Task +from bootstrapvz.common import phases import os @@ -12,5 +12,5 @@ class VirtIO(Task): modules = os.path.join(info.root, '/etc/initramfs-tools/modules') with open(modules, "a") as modules_file: modules_file.write("\n") - for module in info.manifest.bootstrapper.get('virtio', []): - modules_file.write(module+"\n") + for module in info.manifest.system.get('virtio', []): + modules_file.write(module + "\n") diff --git a/providers/virtualbox/__init__.py b/bootstrapvz/providers/virtualbox/__init__.py similarity index 65% rename from providers/virtualbox/__init__.py rename to bootstrapvz/providers/virtualbox/__init__.py index 8719b4f..f9fa99b 100644 --- a/providers/virtualbox/__init__.py +++ b/bootstrapvz/providers/virtualbox/__init__.py @@ -1,14 +1,14 @@ import tasks.packages -from common.tasks import volume -from common.tasks import loopback -from common.tasks import partitioning -from common.tasks import filesystem -from common.tasks import bootstrap -from common.tasks import security -from common.tasks import network -from common.tasks import initd -from common.tasks import cleanup -from common.tasks import workspace +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tasks import partitioning +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import bootstrap +from bootstrapvz.common.tasks import security +from bootstrapvz.common.tasks import network +from bootstrapvz.common.tasks import initd +from bootstrapvz.common.tasks import cleanup +from bootstrapvz.common.tasks import workspace def initialize(): @@ -25,17 +25,17 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - import common.task_sets - taskset.update(common.task_sets.base_set) - taskset.update(common.task_sets.volume_set) - taskset.update(common.task_sets.mounting_set) - taskset.update(common.task_sets.get_apt_set(manifest)) - taskset.update(common.task_sets.locale_set) + from bootstrapvz.common import task_sets + taskset.update(task_sets.base_set) + taskset.update(task_sets.volume_set) + taskset.update(task_sets.mounting_set) + taskset.update(task_sets.get_apt_set(manifest)) + taskset.update(task_sets.locale_set) - taskset.update(common.task_sets.bootloader_set.get(manifest.system['bootloader'])) + taskset.update(task_sets.bootloader_set.get(manifest.system['bootloader'])) if manifest.volume['partitions']['type'] != 'none': - taskset.update(common.task_sets.partitioning_set) + taskset.update(task_sets.partitioning_set) taskset.update([tasks.packages.DefaultPackages, @@ -63,10 +63,10 @@ def resolve_tasks(taskset, manifest): if manifest.bootstrapper.get('tarball', False): taskset.add(bootstrap.MakeTarball) - taskset.update(common.task_sets.get_fs_specific_set(manifest.volume['partitions'])) + taskset.update(task_sets.get_fs_specific_set(manifest.volume['partitions'])) if 'boot' in manifest.volume['partitions']: - taskset.update(common.task_sets.boot_partition_set) + taskset.update(task_sets.boot_partition_set) def resolve_rollback_tasks(taskset, manifest, counter_task): diff --git a/providers/virtualbox/manifest-schema.json b/bootstrapvz/providers/virtualbox/manifest-schema.json similarity index 100% rename from providers/virtualbox/manifest-schema.json rename to bootstrapvz/providers/virtualbox/manifest-schema.json diff --git a/providers/kvm/tasks/__init__.py b/bootstrapvz/providers/virtualbox/tasks/__init__.py similarity index 100% rename from providers/kvm/tasks/__init__.py rename to bootstrapvz/providers/virtualbox/tasks/__init__.py diff --git a/providers/virtualbox/tasks/guest_additions.py b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py similarity index 67% rename from providers/virtualbox/tasks/guest_additions.py rename to bootstrapvz/providers/virtualbox/tasks/guest_additions.py index 37dd537..2ae7c53 100644 --- a/providers/virtualbox/tasks/guest_additions.py +++ b/bootstrapvz/providers/virtualbox/tasks/guest_additions.py @@ -1,7 +1,7 @@ -from base import Task -from common import phases -from common.tasks.packages import InstallPackages -from common.exceptions import TaskError +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks.packages import InstallPackages +from bootstrapvz.common.exceptions import TaskError class CheckGuestAdditionsPath(Task): @@ -28,9 +28,9 @@ class AddGuestAdditionsPackages(Task): info.packages.add('build-essential') info.packages.add('dkms') - from common.tools import log_check_call - [kernel_version] = log_check_call(['/usr/sbin/chroot', info.root, - '/bin/uname', '-r']) + from bootstrapvz.common.tools import log_check_call + [kernel_version] = log_check_call(['chroot', info.root, + 'uname', '-r']) kernel_headers_pkg = 'linux-headers-{version}'.format(version=kernel_version) info.packages.add(kernel_headers_pkg) @@ -51,14 +51,10 @@ class InstallGuestAdditions(Task): root.add_mount(guest_additions_path, mount_path, ['-o', 'loop']) install_script = os.path.join('/', mount_dir, 'VBoxLinuxAdditions.run') - from common.tools import log_call - status, out, err = log_call(['/usr/sbin/chroot', info.root, - install_script, '--nox11']) - # Install will exit with $?=1 because X11 isn't installed - if status != 1: - msg = ('VBoxLinuxAdditions.run exited with status {status}, ' - 'it should exit with status 1').format(status=status) - raise TaskError(msg) - + # Don't check the return code of the scripts here, because 1 not necessarily means they have failed + from bootstrapvz.common.tools import log_call + log_call(['chroot', info.root, install_script, '--nox11']) + # VBoxService process could be running, as it is not affected by DisableDaemonAutostart + log_call(['chroot', info.root, 'service', 'vboxadd-service', 'stop']) root.remove_mount(mount_path) os.rmdir(mount_path) diff --git a/providers/virtualbox/tasks/packages.py b/bootstrapvz/providers/virtualbox/tasks/packages.py similarity index 75% rename from providers/virtualbox/tasks/packages.py rename to bootstrapvz/providers/virtualbox/tasks/packages.py index 316d5dd..8235c32 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/bootstrapvz/providers/virtualbox/tasks/packages.py @@ -1,6 +1,6 @@ -from base import Task -from common import phases -from common.tasks import apt +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt class DefaultPackages(Task): diff --git a/common/tasks/host.py b/common/tasks/host.py deleted file mode 100644 index 7bf79dd..0000000 --- a/common/tasks/host.py +++ /dev/null @@ -1,49 +0,0 @@ -from base import Task -from common import phases -from common.exceptions import TaskError - - -class HostDependencies(Task): - description = 'Determining required host dependencies' - phase = phases.preparation - - @classmethod - def run(cls, info): - info.host_dependencies.add('debootstrap') - - from common.fs.loopbackvolume import LoopbackVolume - if isinstance(info.volume, LoopbackVolume): - info.host_dependencies.add('qemu-utils') - - if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions): - info.host_dependencies.add('xfsprogs') - - from base.fs.partitionmaps.none import NoPartitions - if not isinstance(info.volume.partition_map, NoPartitions): - info.host_dependencies.update(['parted', 'kpartx']) - - -class CheckHostDependencies(Task): - description = 'Checking installed host packages' - phase = phases.preparation - predecessors = [HostDependencies] - - @classmethod - def run(cls, info): - from common.tools import log_check_call - from subprocess import CalledProcessError - missing_packages = [] - for package in info.host_dependencies: - try: - import os.path - if os.path.isfile('/usr/bin/dpkg-query'): - log_check_call(['/usr/bin/dpkg-query', '-s', package]) - except CalledProcessError: - missing_packages.append(package) - if len(missing_packages) > 0: - pkgs = '\', `'.join(missing_packages) - if len(missing_packages) > 1: - msg = "The packages `{packages}\' are not installed".format(packages=pkgs) - else: - msg = "The package `{packages}\' is not installed".format(packages=pkgs) - raise TaskError(msg) diff --git a/common/tasks/network-configuration.json b/common/tasks/network-configuration.json deleted file mode 100644 index 5a3c533..0000000 --- a/common/tasks/network-configuration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "squeeze": [ - "auto lo", - "iface lo inet loopback", - "auto eth0", - "iface eth0 inet dhcp" ], - "wheezy": [ - "auto eth0", - "iface eth0 inet dhcp" ], - "jessie": [ - "auto eth0", - "iface eth0 inet dhcp" ], - "testing": [ - "auto eth0", - "iface eth0 inet dhcp" ], - "unstable": [ - "auto eth0", - "iface eth0 inet dhcp" ] -} diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..e35d885 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..66d46ad --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bootstrap-vz.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bootstrap-vz.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/bootstrap-vz" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bootstrap-vz" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/base/fs.rst b/docs/base/fs.rst new file mode 100644 index 0000000..11918c4 --- /dev/null +++ b/docs/base/fs.rst @@ -0,0 +1,95 @@ + +Filesystem handling +=================== + + +Volume +------ +.. automodule:: bootstrapvz.base.fs.volume + :members: + :private-members: + +Partitionmaps +------------- + +Abstract Partitionmap +''''''''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitionmaps.abstract + :members: + :private-members: + +GPT Partitionmap +'''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitionmaps.gpt + :members: + :private-members: + +MS-DOS Partitionmap +''''''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitionmaps.msdos + :members: + :private-members: + +No Partitionmap +''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitionmaps.none + :members: + :private-members: + + +Partitions +---------- + +Abstract partition +'''''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitions.abstract + :members: + :private-members: + +Base partition +'''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitions.base + :members: + :private-members: + +GPT partition +''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitions.gpt + :members: + :private-members: + +GPT swap partition +.................. +.. automodule:: bootstrapvz.base.fs.partitions.gpt_swap + :members: + :private-members: + +MS-DOS partition +'''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitions.msdos + :members: + :private-members: + +MS-DOS swap partition +..................... +.. automodule:: bootstrapvz.base.fs.partitions.msdos_swap + :members: + :private-members: + +Single +'''''' +.. automodule:: bootstrapvz.base.fs.partitions.single + :members: + :private-members: + +Unformatted partition +''''''''''''''''''''' +.. automodule:: bootstrapvz.base.fs.partitions.unformatted + :members: + :private-members: + +Exceptions +---------- +.. automodule:: bootstrapvz.base.fs.exceptions + :members: + :private-members: diff --git a/docs/base/index.rst b/docs/base/index.rst new file mode 100644 index 0000000..d80e702 --- /dev/null +++ b/docs/base/index.rst @@ -0,0 +1,52 @@ + +Base functionality +================== + +The base module represents concepts of the bootstrapping process that tasks can interact with +and handles the gather, sorting and running of tasks. + +.. toctree:: + :maxdepth: 2 + + fs + pkg + +Bootstrap information +--------------------- +.. automodule:: bootstrapvz.base.bootstrapinfo + :members: + :private-members: + +Manifest +-------- +.. automodule:: bootstrapvz.base.manifest + :members: + :private-members: + + +Tasklist +-------- +.. automodule:: bootstrapvz.base.tasklist + :members: + :private-members: + + +Logging +-------- +.. automodule:: bootstrapvz.base.log + :members: + :private-members: + + +Task +-------- +.. automodule:: bootstrapvz.base.task + :members: + :private-members: + + +Phase +-------- +.. automodule:: bootstrapvz.base.phase + :members: + :private-members: diff --git a/docs/base/pkg.rst b/docs/base/pkg.rst new file mode 100644 index 0000000..8159ad1 --- /dev/null +++ b/docs/base/pkg.rst @@ -0,0 +1,22 @@ + +Package handling +================ + + +Package list +------------ +.. automodule:: bootstrapvz.base.pkg.packagelist + :members: + :private-members: + +Sources list +------------ +.. automodule:: bootstrapvz.base.pkg.sourceslist + :members: + :private-members: + +Exceptions +---------- +.. automodule:: bootstrapvz.base.pkg.exceptions + :members: + :private-members: diff --git a/docs/common/fs.rst b/docs/common/fs.rst new file mode 100644 index 0000000..fdbfc51 --- /dev/null +++ b/docs/common/fs.rst @@ -0,0 +1,3 @@ + +Common volume representations +============================= diff --git a/docs/common/index.rst b/docs/common/index.rst new file mode 100644 index 0000000..ae65394 --- /dev/null +++ b/docs/common/index.rst @@ -0,0 +1,13 @@ + +Common +====== + +The common module contains features that are common to multiple providers and plugins. +It holds both a large set of shared tasks and also various tools that are used by both +the base module and tasks. + +.. toctree:: + :maxdepth: 2 + + fs + tasks/index diff --git a/docs/common/tasks/index.rst b/docs/common/tasks/index.rst new file mode 100644 index 0000000..a4b5afb --- /dev/null +++ b/docs/common/tasks/index.rst @@ -0,0 +1,6 @@ + +Shared tasks +============ + +.. toctree:: + :maxdepth: 2 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..62b62f4 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- +# +# bootstrap-vz documentation build configuration file, created by +# sphinx-quickstart on Sun Mar 23 16:17:28 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath(os.pardir)) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.coverage', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'bootstrap-vz' +copyright = u'2014, Anders Ingemann' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +from bootstrapvz import __version__ +# The short X.Y version. +version = '.'.join(__version__.split('.')[:2]) +# The full version, including alpha/beta/rc tags. +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +show_authors = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'bootstrap-vzdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'bootstrap-vz.tex', u'bootstrap-vz Documentation', + u'Anders Ingemann', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'bootstrap-vz', u'bootstrap-vz Documentation', + [u'Anders Ingemann'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'bootstrap-vz', u'bootstrap-vz Documentation', + u'Anders Ingemann', 'bootstrap-vz', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..c9db652 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,25 @@ +.. bootstrap-vz documentation master file, created by + sphinx-quickstart on Sun Mar 23 16:17:28 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to bootstrap-vz's documentation! +======================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + base/index + common/index + plugins/index + providers/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst new file mode 100644 index 0000000..690ccbc --- /dev/null +++ b/docs/plugins/index.rst @@ -0,0 +1,3 @@ + +Plugins +======= diff --git a/docs/providers/index.rst b/docs/providers/index.rst new file mode 100644 index 0000000..ef70c42 --- /dev/null +++ b/docs/providers/index.rst @@ -0,0 +1,3 @@ + +Providers +========= diff --git a/logs/.gitignore b/logs/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/logs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json index 689ad4a..01452f9 100644 --- a/manifests/kvm-virtio.manifest.json +++ b/manifests/kvm-virtio.manifest.json @@ -2,20 +2,20 @@ "provider": "kvm", "bootstrapper": { "workspace": "/target", - "mirror": "http://ftp.fr.debian.org/debian/", - "virtio" : [ "virtio_pci", "virtio_blk" ] + "mirror": "http://ftp.fr.debian.org/debian/" }, "image": { "name": "debian-{system.release}-{system.architecture}-{%y}{%m}{%d}", "description": "Debian {system.release} {system.architecture}" }, "system": { - "release": "wheezy", - "architecture": "amd64", - "bootloader": "grub", - "timezone": "UTC", - "locale": "en_US", - "charmap": "UTF-8" + "release": "wheezy", + "architecture": "amd64", + "bootloader": "grub", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8", + "virtio_modules": [ "virtio_pci", "virtio_blk" ] }, "packages": {}, "volume": { diff --git a/manifests/virtualbox-vagrant.manifest.json b/manifests/virtualbox-vagrant.manifest.json index 9842757..7359103 100644 --- a/manifests/virtualbox-vagrant.manifest.json +++ b/manifests/virtualbox-vagrant.manifest.json @@ -33,6 +33,8 @@ } }, "plugins": { - "vagrant": {} + "vagrant": { + "hostname": "localhost" + } } } diff --git a/plugins/minimize_size/tasks.py b/plugins/minimize_size/tasks.py deleted file mode 100644 index 37798ae..0000000 --- a/plugins/minimize_size/tasks.py +++ /dev/null @@ -1,101 +0,0 @@ -from base import Task -from common import phases -from common.tasks import apt -from common.tasks import bootstrap -from common.tasks import filesystem -from common.tasks import partitioning -from common.tasks import volume -import os - -folders = ['tmp', 'var/lib/apt/lists'] - - -class AddFolderMounts(Task): - description = 'Mounting folders for writing temporary and cache data' - phase = phases.os_installation - predecessors = [bootstrap.Bootstrap] - - @classmethod - def run(cls, info): - info.minimize_size_folder = os.path.join(info.workspace, 'minimize_size') - os.mkdir(info.minimize_size_folder) - for folder in folders: - temp_path = os.path.join(info.minimize_size_folder, folder.replace('/', '_')) - os.mkdir(temp_path) - - full_path = os.path.join(info.root, folder) - info.volume.partition_map.root.add_mount(temp_path, full_path, ['--bind']) - - -class RemoveFolderMounts(Task): - description = 'Removing folder mounts for temporary and cache data' - phase = phases.system_cleaning - successors = [apt.AptClean] - - @classmethod - def run(cls, info): - import shutil - for folder in folders: - temp_path = os.path.join(info.minimize_size_folder, folder.replace('/', '_')) - full_path = os.path.join(info.root, folder) - - info.volume.partition_map.root.remove_mount(full_path) - shutil.rmtree(temp_path) - - os.rmdir(info.minimize_size_folder) - del info.minimize_size_folder - - -class CheckZerofreePath(Task): - description = 'Checking path to zerofree tool' - phase = phases.preparation - - @classmethod - def run(cls, info): - from common.exceptions import TaskError - import os - zerofree = info.manifest.plugins['minimize_size']['zerofree'] - if not os.path.isfile(zerofree): - raise TaskError('The path `{path}\' does not exist or is not a file'.format(path=zerofree)) - if not os.access(zerofree, os.X_OK): - raise TaskError('The path `{path}\' is not executable'.format(path=zerofree)) - - -# Get zerofree here: http://intgat.tigress.co.uk/rmy/uml/index.html -class Zerofree(Task): - description = 'Zeroing unused blocks on the volume' - phase = phases.volume_unmounting - predecessors = [filesystem.UnmountRoot, partitioning.UnmapPartitions] - successors = [volume.Detach] - - @classmethod - def run(cls, info): - from common.tools import log_check_call - zerofree = info.manifest.plugins['minimize_size']['zerofree'] - log_check_call([zerofree, info.volume.device_path]) - - -class CheckVMWareDMCommand(Task): - description = 'Checking path to vmware-vdiskmanager tool' - phase = phases.preparation - - @classmethod - def run(cls, info): - from common.exceptions import TaskError - import os - vdiskmngr = '/usr/bin/vmware-vdiskmanager' - if not os.path.isfile(vdiskmngr): - raise TaskError('Unable to find vmware-vdiskmanager at `{path}\''.format(path=vdiskmngr)) - if not os.access(vdiskmngr, os.X_OK): - raise TaskError('vmware-vdiskmanager at `{path}\' is not executable'.format(path=vdiskmngr)) - - -class ShrinkVolume(Task): - description = 'Shrinking the volume' - phase = phases.volume_unmounting - predecessors = [volume.Detach] - - @classmethod - def run(cls, info): - from common.tools import log_check_call - log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path]) diff --git a/plugins/opennebula/__init__.py b/plugins/opennebula/__init__.py deleted file mode 100644 index 3bb0a7f..0000000 --- a/plugins/opennebula/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -import tasks - - -def resolve_tasks(taskset, manifest): - taskset.add(tasks.AddONEContextPackage) - taskset.add(tasks.OpenNebulaContext) diff --git a/plugins/opennebula/assets/one-context_3.8.1.deb b/plugins/opennebula/assets/one-context_3.8.1.deb deleted file mode 100644 index 2e81188..0000000 Binary files a/plugins/opennebula/assets/one-context_3.8.1.deb and /dev/null differ diff --git a/plugins/opennebula/assets/one-ec2.sh b/plugins/opennebula/assets/one-ec2.sh deleted file mode 100755 index e52c72d..0000000 --- a/plugins/opennebula/assets/one-ec2.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [ -n "$EC2_USER_DATA" ]; then - # Check if EC2 user data is a script, if yes, execute - if [[ $EC2_USER_DATA =~ ^#! ]]; then - echo "EC2 data is an executable script, so execute it now" - TMPFILE=$(mktemp /tmp/output.XXXXXXXXXX) - chmod 755 $TMPFILE - $TMPFILE - cat $TMPFILE - else - print "Not an executable" - fi -fi diff --git a/plugins/opennebula/assets/one-pubkey.sh b/plugins/opennebula/assets/one-pubkey.sh deleted file mode 100755 index 7d7a209..0000000 --- a/plugins/opennebula/assets/one-pubkey.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -echo "Reconfigure host ssh keys" -dpkg-reconfigure openssh-server - -if [ ! -e /root/.ssh ]; then - mkdir /root/.ssh - touch /root/.ssh/authorized_keys - chmod 600 /root/.ssh/authorized_keys -fi - -echo "Copy public ssh keys to authorized_keys" -for f in /mnt/*.pub -do - cat $f >> /root/.ssh/authorized_keys - -done diff --git a/plugins/opennebula/tasks.py b/plugins/opennebula/tasks.py deleted file mode 100644 index ecb5bb3..0000000 --- a/plugins/opennebula/tasks.py +++ /dev/null @@ -1,42 +0,0 @@ -from base import Task -from common import phases -import os - -assets = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets')) - - -class AddONEContextPackage(Task): - description = 'Adding the OpenNebula context package' - phase = phases.preparation - - @classmethod - def run(cls, info): - package = os.path.join(assets, 'one-context_3.8.1.deb') - info.packages.add_local(package) - - -class OpenNebulaContext(Task): - description = 'Setup OpenNebula init context' - phase = phases.system_modification - - @classmethod - def run(cls, info): - # Fix start - from common.tools import sed_i - vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext') - sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5') - - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', - '90', '2', '3', '4', '5', 'stop', '90', '0', '6']) - - from shutil import copy - # Load all pubkeys in root authorized_keys - script_src = os.path.join(assets, 'one-pubkey.sh') - script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') - copy(script_src, script_dst) - - # If USER_EC2_DATA is a script, execute it - script_src = os.path.join(assets, 'one-ec2.sh') - script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh') - copy(script_src, script_dst) diff --git a/providers/ec2/tasks/packages-kernels.json b/providers/ec2/tasks/packages-kernels.json deleted file mode 100644 index 3c9e39f..0000000 --- a/providers/ec2/tasks/packages-kernels.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "squeeze": { - "amd64": "linux-image-xen-amd64", - "i386" : "linux-image-xen-686" }, - "wheezy": { - "amd64": "linux-image-amd64", - "i386" : "linux-image-686" }, - "jessie": { - "amd64": "linux-image-amd64", - "i386" : "linux-image-686" }, - "testing": { - "amd64": "linux-image-amd64", - "i386" : "linux-image-686" }, - "unstable": { - "amd64": "linux-image-amd64", - "i386" : "linux-image-686" } -} diff --git a/providers/virtualbox/tasks/__init__.py b/providers/virtualbox/tasks/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9bc5358 --- /dev/null +++ b/setup.py @@ -0,0 +1,34 @@ +from setuptools import setup +from setuptools import find_packages +import os.path + + +def find_version(path): + import re + version_file = open(path).read() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + +setup(name='bootstrap-vz', + version=find_version(os.path.join(os.path.dirname(__file__), 'bootstrapvz/__init__.py')), + packages=find_packages(), + include_package_data=True, + entry_points={'console_scripts': ['bootstrap-vz = bootstrapvz.base:main']}, + install_requires=['termcolor >= 1.1.0', + 'fysom >= 1.0.15', + 'jsonschema >= 2.3.0', + ], + license='Apache License, Version 2.0', + description='Bootstrap Debian images for virtualized environments', + long_description='''bootstrap-vz is a bootstrapping framework for Debian. +It is is specifically targeted at bootstrapping systems for virtualized environments. +bootstrap-vz runs without any user intervention and generates ready-to-boot images for +a number of virtualization platforms. +Its aim is to provide a reproducable bootstrapping process using manifests +as well as supporting a high degree of customizability through plugins.''', + author='Anders Ingemann', + author_email='anders@ingemann.de', + url='http://www.python.org/sigs/distutils-sig/', + )