From 5689b77011bcaea987bfc317a4f27d29a38d5574 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 23 Jun 2013 12:00:17 +0200 Subject: [PATCH] Manifest validation, simplified main run fn --- base/main.py | 8 +------ base/manifest.py | 31 ++++++++++++++++-------- common/__init__.py | 1 + common/exceptions.py | 9 +++++++ manifests/ec2-ebs-pvm.manifest.json | 2 +- providers/ec2/__init__.py | 4 +++- providers/ec2/manifest-schema.json | 20 ++++++++++++++++ providers/ec2/manifest.py | 13 ++++++---- providers/ec2/tasks/ebs.py | 21 ++++++++++++++++ providers/ec2/tasks/ec2.py | 13 +++++----- providers/ec2/tasks/host.py | 37 +++++++++++++++++------------ 11 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 common/__init__.py create mode 100644 common/exceptions.py create mode 100644 providers/ec2/manifest-schema.json create mode 100644 providers/ec2/tasks/ebs.py diff --git a/base/main.py b/base/main.py index eb63bd8..5e0d6a9 100644 --- a/base/main.py +++ b/base/main.py @@ -44,13 +44,7 @@ def setup_logger(args): def run(args): from manifest import load_manifest - from manifest import get_provider - data = load_manifest(args.manifest) - provider = get_provider(data) - manifest = provider.Manifest(args.manifest, data) - - manifest.validate() - manifest.load_plugins() + (provider, manifest) = load_manifest(args.manifest) from tasklist import TaskList tasklist = TaskList() diff --git a/base/manifest.py b/base/manifest.py index 047d421..015e3e2 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -3,19 +3,33 @@ log = logging.getLogger(__name__) def load_manifest(path): - import json - return json.load(open(path)) + from json import load + data = load(open(path)) - -def get_provider(data): provider = __import__('providers.{module}'.format(module=data['provider']), fromlist=['providers']) - return provider + manifest = provider.Manifest(path) + manifest.validate(data) + manifest.parse(data) + manifest.load_plugins() + return (provider, manifest) class Manifest(object): - def __init__(self, path, data): + def __init__(self, path): self.path = path - self.parse(data) + + def validate(self, data, schema_path=None): + if schema_path is not None: + from json import load + from json_schema_validator.validator import Validator + from json_schema_validator.schema import Schema + from json_schema_validator.errors import ValidationError + schema = Schema(load(open(schema_path))) + try: + Validator.validate(schema, data) + except ValidationError as e: + from common.exceptions import ManifestError + raise ManifestError(e.message, self) def parse(self, data): self.provider = data['provider'] @@ -23,9 +37,6 @@ class Manifest(object): self.system = data['system'] self.plugins = data['plugins'] - def validate(self): - pass - def load_plugins(self): self.loaded_plugins = [] for modname in self.plugins.keys(): diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..f4fa88c --- /dev/null +++ b/common/__init__.py @@ -0,0 +1 @@ +from exceptions import * diff --git a/common/exceptions.py b/common/exceptions.py new file mode 100644 index 0000000..be467e9 --- /dev/null +++ b/common/exceptions.py @@ -0,0 +1,9 @@ +__all__ = ['ManifestError'] + + +class ManifestError(Exception): + def __init__(self, message, manifest): + self.message = message + self.manifest = manifest + def __str__(self): + return "Error in `manifest' {0}: {1}".format(self.manifest.path, self.message) diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index 1048b02..2596ddd 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -22,7 +22,7 @@ "volume": { "backing" : "ebs", "filesystem": "ext4", - "size" : "1G" + "size" : 1 }, "plugins": { "admin_user": { diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 0e9b2ea..9bcd5ce 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -5,10 +5,12 @@ def modify_tasklist(tasklist, manifest): from tasks import packages from tasks import ec2 from tasks import host + from tasks import ebs tasklist.extend([packages.HostPackages(), packages.ImagePackages(), + host.CheckPackages(), ec2.GetCredentials(), host.GetInfo(), ec2.Connect(), - host.InstallPackages() + ebs.CreateVolume(), ]) diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json new file mode 100644 index 0000000..d5ba725 --- /dev/null +++ b/providers/ec2/manifest-schema.json @@ -0,0 +1,20 @@ +{ + "title": "EC2 manifest", + "type": "object", + "properties": { + "bootstrapdir": { + "type": "string" + }, + "volume": { + "type": "object", + "properties": { + "backing": { + "type": "string", + "enum": ["ebs", "s3"] + } + }, + "required": ["backing"] + } + }, + "required": ["bootstrapdir", "volume"] +} diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index fe04252..e3b25ff 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -2,10 +2,13 @@ import base class Manifest(base.Manifest): + def validate(self, data): + from os import path + schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) + super(Manifest, self).validate(data, schema_path) + def parse(self, data): super(Manifest, self).parse(data) - self.credentials = data["credentials"] - self.virtualization = data["virtualization"] - - def validate(self): - super(Manifest, self).validate() + self.credentials = data['credentials'] + self.virtualization = data['virtualization'] + self.volume = data['volume'] diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py new file mode 100644 index 0000000..bf9efb4 --- /dev/null +++ b/providers/ec2/tasks/ebs.py @@ -0,0 +1,21 @@ +from base import Task + + +class CreateVolume(Task): + description = 'Creating an EBS volume for bootstrapping' + + def run(self, info): + # info.conn.create_volume(50, "us-west-2") +# volume_id=`euca-create-volume --size $volume_size --zone "$availability_zone" | awk '{print $2}'` +# [ -z "$volume_id" ] && die "Unable to create volume." +# log "The EBS volume id is $volume_id" + +# for package in info.host_packages: +# try: +# with open(devnull, 'w') as dev_null: +# subprocess.check_call(['/usr/bin/dpkg', '-s', package], stdout=dev_null, stderr=dev_null) +# except subprocess.CalledProcessError: +# msg = "The package ``{0}\'\' is not installed".format(package) +# raise RuntimeError(msg) + pass + diff --git a/providers/ec2/tasks/ec2.py b/providers/ec2/tasks/ec2.py index 787e61e..c70b44f 100644 --- a/providers/ec2/tasks/ec2.py +++ b/providers/ec2/tasks/ec2.py @@ -6,9 +6,9 @@ class GetCredentials(Task): def run(self, info): super(GetCredentials, self).run(info) - info.ec2_credentials = self.get_ec2_credentials(info.manifest) + info.credentials = self.get_credentials(info.manifest) - def get_ec2_credentials(self, manifest): + def get_credentials(self, manifest): from os import getenv # manifest overrides environment if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): @@ -31,8 +31,7 @@ class Connect(Task): def run(self, info): super(Connect, self).run(info) - # import boto.ec2 - # info.ec2_connection = boto.ec2.connect_to_region(info.host['region'], - # aws_access_key_id=info.ec2_credentials['access_key'], - # aws_secret_access_key=info.ec2_credentials['secret_key']) - # return 'ec2_connection', ec2_connection + from boto.ec2 import connect_to_region + info.connection = connect_to_region(info.host['region'], + aws_access_key_id=info.credentials['access_key'], + aws_secret_access_key=info.credentials['secret_key']) diff --git a/providers/ec2/tasks/host.py b/providers/ec2/tasks/host.py index c9b6498..3121d93 100644 --- a/providers/ec2/tasks/host.py +++ b/providers/ec2/tasks/host.py @@ -1,22 +1,29 @@ from base import Task +class CheckPackages(Task): + description = 'Checking installed host packages' + + def run(self, info): + import subprocess + from os import devnull + for package in info.host_packages: + try: + with open(devnull, 'w') as dev_null: + subprocess.check_call(['/usr/bin/dpkg', '-s', package], stdout=dev_null, stderr=dev_null) + except subprocess.CalledProcessError: + msg = "The package ``{0}\'\' is not installed".format(package) + raise RuntimeError(msg) + + class GetInfo(Task): - description = 'Retrieving host information' + description = 'Retrieving instance metadata' def run(self, info): super(GetInfo, self).run(info) - # import urllib2 - # import json - # response = urllib2.urlopen('http://169.254.169.254/latest/dynamic/instance-identity/document') - # info.host = json.load(response.read()) - # return info - - -class InstallPackages(Task): - description = 'Installing host packages' - - def run(self, info): - # Check if packages are installed with - # /usr/bin/dpkg -s ${name} | grep -q 'Status: install' - pass + import urllib2 + import json + metadata_url = 'http://169.254.169.254/latest/dynamic/instance-identity/document' + response = urllib2.urlopen(url=metadata_url, timeout=5) + info.host = json.load(response) + return info