Manifest validation, simplified main run fn

base/main.py

@@ -44,13 +44,7 @@ def setup_logger(args):
 
 def run(args):
 	from manifest import load_manifest
-	from manifest import get_provider
+	(provider, manifest) = load_manifest(args.manifest)
-	data     = load_manifest(args.manifest)
-	provider = get_provider(data)
-	manifest = provider.Manifest(args.manifest, data)
-
-	manifest.validate()
-	manifest.load_plugins()
 
 	from tasklist import TaskList
 	tasklist = TaskList()

base/manifest.py

@@ -3,19 +3,33 @@ log = logging.getLogger(__name__)
 
 
 def load_manifest(path):
-	import json
+	from json import load
-	return json.load(open(path))
+	data = load(open(path))
 
-
-def get_provider(data):
 	provider = __import__('providers.{module}'.format(module=data['provider']), fromlist=['providers'])
-	return provider
+	manifest = provider.Manifest(path)
 
+	manifest.validate(data)
+	manifest.parse(data)
+	manifest.load_plugins()
+	return (provider, manifest)
 
 class Manifest(object):
-	def __init__(self, path, data):
+	def __init__(self, path):
 		self.path = path
-		self.parse(data)
+
+	def validate(self, data, schema_path=None):
+		if schema_path is not None:
+			from json import load
+			from json_schema_validator.validator import Validator
+			from json_schema_validator.schema import Schema
+			from json_schema_validator.errors import ValidationError
+			schema = Schema(load(open(schema_path)))
+			try:
+				Validator.validate(schema, data)
+			except ValidationError as e:
+				from common.exceptions import ManifestError
+				raise ManifestError(e.message, self)
 
 	def parse(self, data):
 		self.provider = data['provider']
@@ -23,9 +37,6 @@ class Manifest(object):
 		self.system   = data['system']
 		self.plugins  = data['plugins']
 
-	def validate(self):
-		pass
-
 	def load_plugins(self):
 		self.loaded_plugins = []
 		for modname in self.plugins.keys():

common/__init__.py

@@ -0,0 +1 @@
+from exceptions import *

common/exceptions.py

@@ -0,0 +1,9 @@
+__all__ = ['ManifestError']
+
+
+class ManifestError(Exception):
+	def __init__(self, message, manifest):
+		self.message = message
+		self.manifest = manifest
+	def __str__(self):
+		return "Error in `manifest' {0}: {1}".format(self.manifest.path, self.message)

manifests/ec2-ebs-pvm.manifest.json

@@ -22,7 +22,7 @@
 	"volume": {
 		"backing"   : "ebs",
 		"filesystem": "ext4",
-		"size"      : "1G"   
+		"size"      : 1
 	},
 	"plugins": {
 		"admin_user": {

providers/ec2/__init__.py

@@ -5,10 +5,12 @@ def modify_tasklist(tasklist, manifest):
 	from tasks import packages
 	from tasks import ec2
 	from tasks import host
+	from tasks import ebs
 	tasklist.extend([packages.HostPackages(),
 	                 packages.ImagePackages(),
+	                 host.CheckPackages(),
 	                 ec2.GetCredentials(),
 	                 host.GetInfo(),
 	                 ec2.Connect(),
-	                 host.InstallPackages()
+	                 ebs.CreateVolume(),
                   ])

providers/ec2/manifest-schema.json

@@ -0,0 +1,20 @@
+{
+	"title": "EC2 manifest",
+	"type": "object",
+	"properties": {
+		"bootstrapdir": {
+			"type": "string"
+		},
+		"volume": {
+			"type": "object",
+			"properties": {
+				"backing": {
+					"type": "string",
+					"enum": ["ebs", "s3"]
+				}
+			},
+			"required": ["backing"]
+		}
+	},
+	"required": ["bootstrapdir", "volume"]
+}

providers/ec2/manifest.py

@@ -2,10 +2,13 @@ import base
 
 
 class Manifest(base.Manifest):
+	def validate(self, data):
+		from os import path
+		schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+		super(Manifest, self).validate(data, schema_path)
+
 	def parse(self, data):
 		super(Manifest, self).parse(data)
-		self.credentials    = data["credentials"]
+		self.credentials    = data['credentials']
-		self.virtualization = data["virtualization"]
+		self.virtualization = data['virtualization']
-
+		self.volume         = data['volume']
-	def validate(self):
-		super(Manifest, self).validate()

providers/ec2/tasks/ebs.py

@@ -0,0 +1,21 @@
+from base import Task
+
+
+class CreateVolume(Task):
+	description = 'Creating an EBS volume for bootstrapping'
+
+	def run(self, info):
+		# info.conn.create_volume(50, "us-west-2")
+# volume_id=`euca-create-volume --size $volume_size --zone "$availability_zone" | awk '{print $2}'`
+# [ -z "$volume_id" ] && die "Unable to create volume."
+# log "The EBS volume id is $volume_id"
+
+# 		for package in info.host_packages:
+# 			try:
+# 				with open(devnull, 'w') as dev_null:
+# 					subprocess.check_call(['/usr/bin/dpkg', '-s', package], stdout=dev_null, stderr=dev_null)
+# 			except subprocess.CalledProcessError:
+# 				msg = "The package ``{0}\'\' is not installed".format(package)
+# 				raise RuntimeError(msg)
+		pass
+

providers/ec2/tasks/ec2.py

@@ -6,9 +6,9 @@ class GetCredentials(Task):
 
 	def run(self, info):
 		super(GetCredentials, self).run(info)
-		info.ec2_credentials = self.get_ec2_credentials(info.manifest)
+		info.credentials = self.get_credentials(info.manifest)
 
-	def get_ec2_credentials(self, manifest):
+	def get_credentials(self, manifest):
 		from os import getenv
 		# manifest overrides environment
 		if(manifest.credentials['access-key'] and manifest.credentials['secret-key']):
@@ -31,8 +31,7 @@ class Connect(Task):
 	
 	def run(self, info):
 		super(Connect, self).run(info)
-		# import boto.ec2
+		from boto.ec2 import connect_to_region
-		# info.ec2_connection = boto.ec2.connect_to_region(info.host['region'],
+		info.connection = connect_to_region(info.host['region'],
-		#                                                  aws_access_key_id=info.ec2_credentials['access_key'],
+		                                    aws_access_key_id=info.credentials['access_key'],
-		#                                                  aws_secret_access_key=info.ec2_credentials['secret_key'])
+		                                    aws_secret_access_key=info.credentials['secret_key'])
-		# return 'ec2_connection', ec2_connection

providers/ec2/tasks/host.py

@@ -1,22 +1,29 @@
 from base import Task
 
 
+class CheckPackages(Task):
+	description = 'Checking installed host packages'
+
+	def run(self, info):
+		import subprocess
+		from os import devnull
+		for package in info.host_packages:
+			try:
+				with open(devnull, 'w') as dev_null:
+					subprocess.check_call(['/usr/bin/dpkg', '-s', package], stdout=dev_null, stderr=dev_null)
+			except subprocess.CalledProcessError:
+				msg = "The package ``{0}\'\' is not installed".format(package)
+				raise RuntimeError(msg)
+
+
 class GetInfo(Task):
-	description = 'Retrieving host information'
+	description = 'Retrieving instance metadata'
 
 	def run(self, info):
 		super(GetInfo, self).run(info)
-		# import urllib2
+		import urllib2
-		# import json
+		import json
-		# response = urllib2.urlopen('http://169.254.169.254/latest/dynamic/instance-identity/document')
+		metadata_url = 'http://169.254.169.254/latest/dynamic/instance-identity/document'
-		# info.host = json.load(response.read())
+		response = urllib2.urlopen(url=metadata_url, timeout=5)
-		# return info
+		info.host = json.load(response)
-
+		return info
-
-class InstallPackages(Task):
-	description = 'Installing host packages'
-
-	def run(self, info):
-		# Check if packages are installed with
-		# /usr/bin/dpkg -s ${name} | grep -q 'Status: install'
-		pass