Remove provider specific manifests

Manifest and module loading has been refactored
Provider modules now must implement validate_manifest like plugins do
Simplified loading of manifests

base/__init__.py

@@ -1,5 +1,10 @@
-__all__ = ['Manifest', 'Phase', 'Task', 'main']
+__all__ = ['Phase', 'Task', 'main']
-from manifest import Manifest
 from phase import Phase
 from task import Task
 from main import main
+
+
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	validator(data, schema_path)

base/main.py

@@ -24,13 +24,15 @@ def get_args():
 
 
 def run(args):
-	from manifest import load_manifest
+	from manifest import Manifest
-	(provider, manifest) = load_manifest(args.manifest)
+	manifest = Manifest(args.manifest)
+	provider = manifest.modules['provider']
+	plugins = manifest.modules['plugins']
 
 	from tasklist import TaskList
 	tasklist = TaskList()
 	provider.resolve_tasks(tasklist, manifest)
-	for plugin in manifest.loaded_plugins:
+	for plugin in plugins:
 		plugin.resolve_tasks(tasklist, manifest)
 
 	from bootstrapinfo import BootstrapInformation
@@ -46,7 +48,7 @@ def run(args):
 		log.error('Rolling back')
 		rollback_tasklist = TaskList()
 		provider.resolve_rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest)
-		for plugin in manifest.loaded_plugins:
+		for plugin in plugins:
 			resolve_rollback_tasks = getattr(plugin, 'resolve_rollback_tasks', None)
 			if callable(resolve_rollback_tasks):
 				resolve_rollback_tasks(rollback_tasklist, tasklist.tasks_completed, manifest)

base/manifest-schema.json

@@ -15,6 +15,13 @@
 			},
 			"required": ["workspace"]
 		},
+		"image": {
+			"type": "object",
+			"properties": {
+				"name": { "type": "string" }
+			},
+			"required": ["name"]
+		},
 		"system": {
 			"type": "object",
 			"properties": {
@@ -92,7 +99,7 @@
 			"additionalProperties": false
 		}
 	},
-	"required": ["provider", "bootstrapper", "volume", "system"],
+	"required": ["provider", "bootstrapper", "image", "volume", "system"],
 	"definitions": {
 		"path": {
 			"type":    "string",

base/manifest.py

@@ -2,68 +2,65 @@ import logging
 log = logging.getLogger(__name__)
 
 
-def load_manifest(path):
-	data = load_json(path)
-
-	provider_name = data['provider']
-	provider = __import__('providers.{module}'.format(module=provider_name), fromlist=['providers'])
-	init = getattr(provider, 'initialize', None)
-	if callable(init):
-		init()
-	log.debug('Loaded provider `%s\'', provider_name)
-	manifest = provider.Manifest(path)
-
-	manifest.validate(data)
-	manifest.load_plugins(data)
-	manifest.parse(data)
-	return (provider, manifest)
-
-
-def load_json(path):
-	import json
-	from minify_json import json_minify
-	with open(path) as stream:
-		return json.loads(json_minify(stream.read(), False))
-
-
 class Manifest(object):
 	def __init__(self, path):
 		self.path = path
+		self.load()
+		self.validate()
+		self.parse()
 
-	def validate(self, data):
+	def load(self):
-		from os import path
+		self.data = self.load_json(self.path)
-		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
+		provider_modname = 'providers.{provider}'.format(provider=self.data['provider'])
-		self.schema_validate(data, schema_path)
+		log.debug('Loading provider `{modname}\''.format(modname=provider_modname))
+		self.modules = {'provider': __import__(provider_modname, fromlist=['providers']),
+		                'plugins': [],
+		                }
+		if 'plugins' in self.data:
+			for plugin_name, plugin_data in self.data['plugins'].iteritems():
+				modname = 'plugins.{plugin}'.format(plugin=plugin_name)
+				log.debug('Loading plugin `{modname}\''.format(modname=modname))
+				plugin = __import__(modname, fromlist=['plugins'])
+				self.modules['plugins'].append(plugin)
 
-	def schema_validate(self, data, schema_path):
+		self.modules['provider'].initialize()
+		for module in self.modules['plugins']:
+			init = getattr(module, 'initialize', None)
+			if callable(init):
+				init()
+
+	def validate(self):
+		from . import validate_manifest
+		validate_manifest(self.data, self.schema_validator, self.validation_error)
+		self.modules['provider'].validate_manifest(self.data, self.schema_validator, self.validation_error)
+		for plugin in self.modules['plugins']:
+			validate = getattr(plugin, 'validate_manifest', None)
+			if callable(validate):
+				validate(self.data, self.schema_validator, self.validation_error)
+
+	def parse(self):
+		self.provider     = self.data['provider']
+		self.bootstrapper = self.data['bootstrapper']
+		self.image        = self.data['image']
+		self.volume       = self.data['volume']
+		self.system       = self.data['system']
+		self.packages     = self.data['packages']
+		self.plugins      = self.data['plugins'] if 'plugins' in self.data else {}
+
+	def load_json(self, path):
+		import json
+		from minify_json import json_minify
+		with open(path) as stream:
+			return json.loads(json_minify(stream.read(), False))
+
+	def schema_validator(self, data, schema_path):
 		import jsonschema
-		schema = load_json(schema_path)
+		schema = self.load_json(schema_path)
 		try:
 			jsonschema.validate(data, schema)
 		except jsonschema.ValidationError as e:
+			self.validation_error(e.message, e.path)
+
+	def validation_error(self, message, json_path=None):
 			from common.exceptions import ManifestError
-			raise ManifestError(e.message, self, e.path)
+			raise ManifestError(message, self.path, json_path)
-
-	def parse(self, data):
-		self.data = data
-		self.provider = data['provider']
-		self.bootstrapper = data['bootstrapper']
-		self.volume = data['volume']
-		self.system = data['system']
-		self.packages = data['packages']
-		self.plugins = data['plugins'] if 'plugins' in data else {}
-
-	def load_plugins(self, data):
-		self.loaded_plugins = []
-		if 'plugins' in data:
-			for plugin_name, plugin_data in data['plugins'].iteritems():
-				modname = 'plugins.{plugin_name}'.format(plugin_name=plugin_name)
-				plugin = __import__(modname, fromlist=['plugins'])
-				init = getattr(plugin, 'initialize', None)
-				if callable(init):
-					init()
-				log.debug('Loaded plugin `%s\'', plugin_name)
-				self.loaded_plugins.append(plugin)
-				validate = getattr(plugin, 'validate_manifest', None)
-				if callable(validate):
-					validate(data, self.schema_validate)

common/exceptions.py

@@ -1,16 +1,16 @@
 
 
 class ManifestError(Exception):
-	def __init__(self, message, manifest, json_path=None):
+	def __init__(self, message, manifest_path, json_path=None):
 		self.message = message
-		self.manifest = manifest
+		self.manifest_path = manifest_path
 		self.json_path = json_path
 
 	def __str__(self):
 		if self.json_path is not None:
 			path = '.'.join(self.json_path)
-			return "{2}\n\tFile: {0}\n\tJSON path: {1}".format(self.manifest.path, path, self.message)
+			return "{2}\n\tFile: {0}\n\tJSON path: {1}".format(self.manifest_path, path, self.message)
-		return "{0}: {1}".format(self.manifest.path, self.message)
+		return "{0}: {1}".format(self.manifest_path, self.message)
 
 
 class TaskListError(Exception):

plugins/admin_user/__init__.py

@@ -1,9 +1,9 @@
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/cloud_init/__init__.py

@@ -1,9 +1,9 @@
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/image_commands/__init__.py

@@ -1,9 +1,9 @@
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/prebootstrapped/__init__.py

@@ -12,10 +12,10 @@ from common.tasks import filesystem
 from common.tasks import partitioning
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/root_password/__init__.py

@@ -1,9 +1,9 @@
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/unattended_upgrades/__init__.py

@@ -1,9 +1,9 @@
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

plugins/vagrant/__init__.py

@@ -1,10 +1,10 @@
 import tasks
 
 
-def validate_manifest(data, schema_validate):
+def validate_manifest(data, validator, error):
-	from os import path
+	import os.path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
+	validator(data, schema_path)
 
 
 def resolve_tasks(tasklist, manifest):

providers/ec2/__init__.py

@@ -1,4 +1,3 @@
-from manifest import Manifest
 import tasks.packages
 import tasks.connection
 import tasks.host
@@ -27,6 +26,28 @@ def initialize():
 	logging.getLogger('boto').setLevel(logging.INFO)
 
 
+def validate_manifest(data, validator, error):
+	import os.path
+	validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+
+	if data['volume']['backing'] == 'ebs':
+		volume_size = 1 if data['volume']['partitions']['type'] == 'mbr' else 0
+		for key, partition in data['volume']['partitions'].iteritems():
+			if key != 'type':
+				volume_size += partition['size']
+		if volume_size % 1024 != 0:
+			msg = ('The volume size must be a multiple of 1024 when using EBS backing '
+			       '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)')
+			error(msg, ['volume', 'partitions'])
+	else:
+		validator(data, os.path.join(os.path.dirname(__file__), 'manifest-schema-s3.json'))
+
+	if data['virtualization'] == 'pvm' and data['system']['bootloader'] != 'pvgrub':
+			error('Paravirtualized AMIs only support pvgrub as a bootloader', ['system', 'bootloader'])
+	if data['virtualization'] == 'hvm' and data['system']['bootloader'] != 'extlinux':
+			error('HVM AMIs only support extlinux as a bootloader', ['system', 'bootloader'])
+
+
 def resolve_tasks(tasklist, manifest):
 	from common.task_sets import base_set
 	from common.task_sets import mounting_set

providers/ec2/manifest-schema.json

@@ -7,9 +7,6 @@
 		"image": {
 			"type": "object",
 			"properties": {
-				"name": {
-					"type": "string"
-				},
 				"description": {
 					"type": "string"
 				}

providers/ec2/manifest.py

@@ -1,40 +0,0 @@
-import base
-from common.exceptions import ManifestError
-
-
-class Manifest(base.Manifest):
-	def validate(self, data):
-		super(Manifest, self).validate(data)
-		from os import path
-		self.schema_validate(data, path.join(path.dirname(__file__), 'manifest-schema.json'))
-		if data['volume']['backing'] == 'ebs':
-			volume_size = self._calculate_volume_size(data['volume']['partitions'])
-			if volume_size % 1024 != 0:
-				msg = ('The volume size must be a multiple of 1024 when using EBS backing '
-				       '(MBR partitioned volumes are 1MB larger than specified, for the post-mbr gap)')
-				raise ManifestError(msg, self)
-		else:
-			self.schema_validate(data, path.join(path.dirname(__file__), 'manifest-schema-s3.json'))
-
-		if data['virtualization'] == 'pvm' and data['system']['bootloader'] != 'pvgrub':
-			raise ManifestError('Paravirtualized AMIs only support pvgrub as a bootloader', self)
-		if data['virtualization'] == 'hvm' and data['system']['bootloader'] != 'extlinux':
-			raise ManifestError('HVM AMIs only support extlinux as a bootloader', self)
-
-	def parse(self, data):
-		super(Manifest, self).parse(data)
-		self.credentials    = data['credentials']
-		self.virtualization = data['virtualization']
-		self.image          = data['image']
-
-	def _calculate_volume_size(self, partitions):
-		if partitions['type'] == 'mbr':
-			size = 1
-		else:
-			size = 0
-		if 'boot' in partitions:
-			size += partitions['boot']['size']
-		size += partitions['root']['size']
-		if 'swap' in partitions:
-			size += partitions['swap']['size']
-		return size

providers/ec2/tasks/ami.py

@@ -150,7 +150,7 @@ class RegisterAMI(Task):
 			grub_boot_device = 'hd0'
 		else:
 			root_dev_name = {'pvm': '/dev/sda',
-			                 'hvm': '/dev/xvda'}.get(info.manifest.virtualization)
+			                 'hvm': '/dev/xvda'}.get(info.manifest.data['virtualization'])
 			registration_params['root_device_name'] = root_dev_name
 			from base.fs.partitionmaps.none import NoPartitions
 			if isinstance(info.volume.partition_map, NoPartitions):
@@ -165,7 +165,7 @@ class RegisterAMI(Task):
 			registration_params['block_device_map'] = BlockDeviceMapping()
 			registration_params['block_device_map'][root_dev_name] = block_device
 
-		if info.manifest.virtualization == 'hvm':
+		if info.manifest.data['virtualization'] == 'hvm':
 			registration_params['virtualization_type'] = 'hvm'
 		else:
 			registration_params['virtualization_type'] = 'paravirtual'

providers/ec2/tasks/connection.py

@@ -16,9 +16,9 @@ class GetCredentials(Task):
 	def get_credentials(self, manifest, keys):
 		from os import getenv
 		creds = {}
-		if all(key in manifest.credentials for key in keys):
+		if all(key in manifest.data['credentials'] for key in keys):
 			for key in keys:
-				creds[key] = manifest.credentials[key]
+				creds[key] = manifest.data['credentials'][key]
 			return creds
 
 		def env_key(key):

providers/virtualbox/__init__.py

@@ -1,4 +1,3 @@
-from manifest import Manifest
 import tasks.packages
 from common.tasks import volume
 from common.tasks import loopback
@@ -16,6 +15,12 @@ def initialize():
 	pass
 
 
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	validator(data, schema_path)
+
+
 def resolve_tasks(tasklist, manifest):
 	from common.task_sets import base_set
 	from common.task_sets import volume_set

providers/virtualbox/manifest.py

@@ -1,14 +0,0 @@
-import base
-
-
-class Manifest(base.Manifest):
-	def validate(self, data):
-		super(Manifest, self).validate(data)
-		from os import path
-		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
-		self.schema_validate(data, schema_path)
-
-	def parse(self, data):
-		super(Manifest, self).parse(data)
-		self.virtualization = None
-		self.image          = data['image']