diff --git a/base/__init__.py b/base/__init__.py index b23f0f8..4e11028 100644 --- a/base/__init__.py +++ b/base/__init__.py @@ -1,5 +1,10 @@ -__all__ = ['Manifest', 'Phase', 'Task', 'main'] -from manifest import Manifest +__all__ = ['Phase', 'Task', 'main'] from phase import Phase from task import Task from main import main + + +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) diff --git a/base/main.py b/base/main.py index 9c84ed1..7085b31 100644 --- a/base/main.py +++ b/base/main.py @@ -24,13 +24,15 @@ def get_args(): def run(args): - from manifest import load_manifest - (provider, manifest) = load_manifest(args.manifest) + from manifest import Manifest + manifest = Manifest(args.manifest) + provider = manifest.modules['provider'] + plugins = manifest.modules['plugins'] from tasklist import TaskList tasklist = TaskList() provider.resolve_tasks(tasklist, manifest) - for plugin in manifest.loaded_plugins: + for plugin in plugins: plugin.resolve_tasks(tasklist, manifest) from bootstrapinfo import BootstrapInformation @@ -46,7 +48,7 @@ def run(args): log.error('Rolling back') rollback_tasklist = TaskList() provider.resolve_rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) - for plugin in manifest.loaded_plugins: + for plugin in plugins: resolve_rollback_tasks = getattr(plugin, 'resolve_rollback_tasks', None) if callable(resolve_rollback_tasks): resolve_rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 64fa62c..f88bd2f 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -15,6 +15,13 @@ }, "required": ["workspace"] }, + "image": { + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "required": ["name"] + }, "system": { "type": "object", "properties": { @@ -92,7 +99,7 @@ "additionalProperties": false } }, - "required": ["provider", "bootstrapper", "volume", "system"], + "required": ["provider", "bootstrapper", "image", "volume", "system"], "definitions": { "path": { "type": "string", diff --git a/base/manifest.py b/base/manifest.py index a54b935..e756626 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -2,68 +2,65 @@ import logging log = logging.getLogger(__name__) -def load_manifest(path): - data = load_json(path) - - provider_name = data['provider'] - provider = __import__('providers.{module}'.format(module=provider_name), fromlist=['providers']) - init = getattr(provider, 'initialize', None) - if callable(init): - init() - log.debug('Loaded provider `%s\'', provider_name) - manifest = provider.Manifest(path) - - manifest.validate(data) - manifest.load_plugins(data) - manifest.parse(data) - return (provider, manifest) - - -def load_json(path): - import json - from minify_json import json_minify - with open(path) as stream: - return json.loads(json_minify(stream.read(), False)) - - class Manifest(object): def __init__(self, path): self.path = path + self.load() + self.validate() + self.parse() - def validate(self, data): - from os import path - schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') - self.schema_validate(data, schema_path) + def load(self): + self.data = self.load_json(self.path) + provider_modname = 'providers.{provider}'.format(provider=self.data['provider']) + log.debug('Loading provider `{modname}\''.format(modname=provider_modname)) + self.modules = {'provider': __import__(provider_modname, fromlist=['providers']), + 'plugins': [], + } + if 'plugins' in self.data: + for plugin_name, plugin_data in self.data['plugins'].iteritems(): + modname = 'plugins.{plugin}'.format(plugin=plugin_name) + log.debug('Loading plugin `{modname}\''.format(modname=modname)) + plugin = __import__(modname, fromlist=['plugins']) + self.modules['plugins'].append(plugin) - def schema_validate(self, data, schema_path): + self.modules['provider'].initialize() + for module in self.modules['plugins']: + init = getattr(module, 'initialize', None) + if callable(init): + init() + + def validate(self): + from . import validate_manifest + validate_manifest(self.data, self.schema_validator, self.validation_error) + self.modules['provider'].validate_manifest(self.data, self.schema_validator, self.validation_error) + for plugin in self.modules['plugins']: + validate = getattr(plugin, 'validate_manifest', None) + if callable(validate): + validate(self.data, self.schema_validator, self.validation_error) + + def parse(self): + self.provider = self.data['provider'] + self.bootstrapper = self.data['bootstrapper'] + self.image = self.data['image'] + self.volume = self.data['volume'] + self.system = self.data['system'] + self.packages = self.data['packages'] + self.plugins = self.data['plugins'] if 'plugins' in self.data else {} + + def load_json(self, path): + import json + from minify_json import json_minify + with open(path) as stream: + return json.loads(json_minify(stream.read(), False)) + + def schema_validator(self, data, schema_path): import jsonschema - schema = load_json(schema_path) + schema = self.load_json(schema_path) try: jsonschema.validate(data, schema) except jsonschema.ValidationError as e: + self.validation_error(e.message, e.path) + + def validation_error(self, message, json_path=None): from common.exceptions import ManifestError - raise ManifestError(e.message, self, e.path) - - def parse(self, data): - self.data = data - self.provider = data['provider'] - self.bootstrapper = data['bootstrapper'] - self.volume = data['volume'] - self.system = data['system'] - self.packages = data['packages'] - self.plugins = data['plugins'] if 'plugins' in data else {} - - def load_plugins(self, data): - self.loaded_plugins = [] - if 'plugins' in data: - for plugin_name, plugin_data in data['plugins'].iteritems(): - modname = 'plugins.{plugin_name}'.format(plugin_name=plugin_name) - plugin = __import__(modname, fromlist=['plugins']) - init = getattr(plugin, 'initialize', None) - if callable(init): - init() - log.debug('Loaded plugin `%s\'', plugin_name) - self.loaded_plugins.append(plugin) - validate = getattr(plugin, 'validate_manifest', None) - if callable(validate): - validate(data, self.schema_validate) + raise ManifestError(message, self.path, json_path) diff --git a/common/exceptions.py b/common/exceptions.py index aed6e64..91689ae 100644 --- a/common/exceptions.py +++ b/common/exceptions.py @@ -1,16 +1,16 @@ class ManifestError(Exception): - def __init__(self, message, manifest, json_path=None): + def __init__(self, message, manifest_path, json_path=None): self.message = message - self.manifest = manifest + self.manifest_path = manifest_path self.json_path = json_path def __str__(self): if self.json_path is not None: path = '.'.join(self.json_path) - return "{2}\n\tFile: {0}\n\tJSON path: {1}".format(self.manifest.path, path, self.message) - return "{0}: {1}".format(self.manifest.path, self.message) + return "{2}\n\tFile: {0}\n\tJSON path: {1}".format(self.manifest_path, path, self.message) + return "{0}: {1}".format(self.manifest_path, self.message) class TaskListError(Exception): diff --git a/plugins/admin_user/__init__.py b/plugins/admin_user/__init__.py index 2ab4592..65db43c 100644 --- a/plugins/admin_user/__init__.py +++ b/plugins/admin_user/__init__.py @@ -1,9 +1,9 @@ -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/cloud_init/__init__.py b/plugins/cloud_init/__init__.py index 35cfa05..ab46126 100644 --- a/plugins/cloud_init/__init__.py +++ b/plugins/cloud_init/__init__.py @@ -1,9 +1,9 @@ -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/image_commands/__init__.py b/plugins/image_commands/__init__.py index f7cfa69..37e3b56 100644 --- a/plugins/image_commands/__init__.py +++ b/plugins/image_commands/__init__.py @@ -1,9 +1,9 @@ -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/prebootstrapped/__init__.py b/plugins/prebootstrapped/__init__.py index bba4010..72e0d0a 100644 --- a/plugins/prebootstrapped/__init__.py +++ b/plugins/prebootstrapped/__init__.py @@ -12,10 +12,10 @@ from common.tasks import filesystem from common.tasks import partitioning -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/root_password/__init__.py b/plugins/root_password/__init__.py index 73ccf38..f333c55 100644 --- a/plugins/root_password/__init__.py +++ b/plugins/root_password/__init__.py @@ -1,9 +1,9 @@ -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/unattended_upgrades/__init__.py b/plugins/unattended_upgrades/__init__.py index 87a6107..8331751 100644 --- a/plugins/unattended_upgrades/__init__.py +++ b/plugins/unattended_upgrades/__init__.py @@ -1,9 +1,9 @@ -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/plugins/vagrant/__init__.py b/plugins/vagrant/__init__.py index 8ba5320..cd503a9 100644 --- a/plugins/vagrant/__init__.py +++ b/plugins/vagrant/__init__.py @@ -1,10 +1,10 @@ import tasks -def validate_manifest(data, schema_validate): - from os import path - schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) - schema_validate(data, schema_path) +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) def resolve_tasks(tasklist, manifest): diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 9c1dc14..a48a2c6 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -1,4 +1,3 @@ -from manifest import Manifest import tasks.packages import tasks.connection import tasks.host @@ -27,6 +26,28 @@ def initialize(): logging.getLogger('boto').setLevel(logging.INFO) +def validate_manifest(data, validator, error): + import os.path + validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + + if data['volume']['backing'] == 'ebs': + volume_size = 1 if data['volume']['partitions']['type'] == 'mbr' else 0 + for key, partition in data['volume']['partitions'].iteritems(): + if key != 'type': + volume_size += partition['size'] + if volume_size % 1024 != 0: + msg = ('The volume size must be a multiple of 1024 when using EBS backing ' + '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)') + error(msg, ['volume', 'partitions']) + else: + validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json')) + + if data['virtualization'] == 'pvm' and data['system']['bootloader'] != 'pvgrub': + error('Paravirtualized AMIs only support pvgrub as a bootloader', ['system', 'bootloader']) + if data['virtualization'] == 'hvm' and data['system']['bootloader'] != 'extlinux': + error('HVM AMIs only support extlinux as a bootloader', ['system', 'bootloader']) + + def resolve_tasks(tasklist, manifest): from common.task_sets import base_set from common.task_sets import mounting_set diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index 4e94fe4..30666a5 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -7,9 +7,6 @@ "image": { "type": "object", "properties": { - "name": { - "type": "string" - }, "description": { "type": "string" } diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py deleted file mode 100644 index c1264e1..0000000 --- a/providers/ec2/manifest.py +++ /dev/null @@ -1,40 +0,0 @@ -import base -from common.exceptions import ManifestError - - -class Manifest(base.Manifest): - def validate(self, data): - super(Manifest, self).validate(data) - from os import path - self.schema_validate(data, path.join(path.dirname(__file__), 'manifest-schema.json')) - if data['volume']['backing'] == 'ebs': - volume_size = self._calculate_volume_size(data['volume']['partitions']) - if volume_size % 1024 != 0: - msg = ('The volume size must be a multiple of 1024 when using EBS backing ' - '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)') - raise ManifestError(msg, self) - else: - self.schema_validate(data, path.join(path.dirname(__file__), 'manifest-schema-s3.json')) - - if data['virtualization'] == 'pvm' and data['system']['bootloader'] != 'pvgrub': - raise ManifestError('Paravirtualized AMIs only support pvgrub as a bootloader', self) - if data['virtualization'] == 'hvm' and data['system']['bootloader'] != 'extlinux': - raise ManifestError('HVM AMIs only support extlinux as a bootloader', self) - - def parse(self, data): - super(Manifest, self).parse(data) - self.credentials = data['credentials'] - self.virtualization = data['virtualization'] - self.image = data['image'] - - def _calculate_volume_size(self, partitions): - if partitions['type'] == 'mbr': - size = 1 - else: - size = 0 - if 'boot' in partitions: - size += partitions['boot']['size'] - size += partitions['root']['size'] - if 'swap' in partitions: - size += partitions['swap']['size'] - return size diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 6b52d9c..9d79a33 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -150,7 +150,7 @@ class RegisterAMI(Task): grub_boot_device = 'hd0' else: root_dev_name = {'pvm': '/dev/sda', - 'hvm': '/dev/xvda'}.get(info.manifest.virtualization) + 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization']) registration_params['root_device_name'] = root_dev_name from base.fs.partitionmaps.none import NoPartitions if isinstance(info.volume.partition_map, NoPartitions): @@ -165,7 +165,7 @@ class RegisterAMI(Task): registration_params['block_device_map'] = BlockDeviceMapping() registration_params['block_device_map'][root_dev_name] = block_device - if info.manifest.virtualization == 'hvm': + if info.manifest.data['virtualization'] == 'hvm': registration_params['virtualization_type'] = 'hvm' else: registration_params['virtualization_type'] = 'paravirtual' diff --git a/providers/ec2/tasks/connection.py b/providers/ec2/tasks/connection.py index 0d937bc..6ba6158 100644 --- a/providers/ec2/tasks/connection.py +++ b/providers/ec2/tasks/connection.py @@ -16,9 +16,9 @@ class GetCredentials(Task): def get_credentials(self, manifest, keys): from os import getenv creds = {} - if all(key in manifest.credentials for key in keys): + if all(key in manifest.data['credentials'] for key in keys): for key in keys: - creds[key] = manifest.credentials[key] + creds[key] = manifest.data['credentials'][key] return creds def env_key(key): diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 3e296f9..5b8bd02 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -1,4 +1,3 @@ -from manifest import Manifest import tasks.packages from common.tasks import volume from common.tasks import loopback @@ -16,6 +15,12 @@ def initialize(): pass +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) + + def resolve_tasks(tasklist, manifest): from common.task_sets import base_set from common.task_sets import volume_set diff --git a/providers/virtualbox/manifest.py b/providers/virtualbox/manifest.py deleted file mode 100644 index 8b5fbc2..0000000 --- a/providers/virtualbox/manifest.py +++ /dev/null @@ -1,14 +0,0 @@ -import base - - -class Manifest(base.Manifest): - def validate(self, data): - super(Manifest, self).validate(data) - from os import path - schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') - self.schema_validate(data, schema_path) - - def parse(self, data): - super(Manifest, self).parse(data) - self.virtualization = None - self.image = data['image']