commit 4dbdb387cdb5fe6504884c5ebb0a345589029d57 Author: Anders Ingemann Date: Thu May 2 19:13:35 2013 +0200 Basic structure up and running diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..68bccdc --- /dev/null +++ b/common/__init__.py @@ -0,0 +1,4 @@ +from bootstrapinfo import BootstrapInformation +from manifest import Manifest +from task import Task +from tasklist import TaskList diff --git a/common/bootstrapinfo.py b/common/bootstrapinfo.py new file mode 100644 index 0000000..445161d --- /dev/null +++ b/common/bootstrapinfo.py @@ -0,0 +1,6 @@ + + +class BootstrapInformation(object): + def __init__(self, manifest=None, args=None): + self.manifest = manifest + self.args = args diff --git a/common/manifest.py b/common/manifest.py new file mode 100644 index 0000000..d78c333 --- /dev/null +++ b/common/manifest.py @@ -0,0 +1,14 @@ +import json + + +class Manifest(object): + def __init__(self, path): + self.path = path + self.parse(json.load(open(self.path))) + + def parse(self, data): + self.volume = data['volume'] + self.system = data['system'] + + def validate(self): + pass diff --git a/common/task.py b/common/task.py new file mode 100644 index 0000000..cc795f3 --- /dev/null +++ b/common/task.py @@ -0,0 +1,10 @@ + + +class Task(object): + name = None + + def __init__(self): + pass + + def run(self, info): + print 'Running ' + self.__module__ + "." + self.__class__.__name__ diff --git a/common/tasklist.py b/common/tasklist.py new file mode 100644 index 0000000..5f34fe4 --- /dev/null +++ b/common/tasklist.py @@ -0,0 +1,15 @@ + + +class TaskList(list): + def run(self, info): + for task in self: + task.run(info) + + def before(self, task): + pass + + def replace(self, task): + pass + + def after(self, task): + pass diff --git a/debian-build-cloud b/debian-build-cloud new file mode 100755 index 0000000..fcb420b --- /dev/null +++ b/debian-build-cloud @@ -0,0 +1,18 @@ +#!/usr/bin/env python + + +def main(): + from argparse import ArgumentParser + import ec2 + parser = ArgumentParser(description='Bootstrap Debian for the cloud.') + parser.add_argument('--debug', action='store_true', + help='Print debugging information') + + subparsers = parser.add_subparsers(title='providers', description='supported providers', dest='provider') + ec2.init_subparser(subparsers) + + args = parser.parse_args() + args.func(args) + +if __name__ == '__main__' and __package__ is None: + main() diff --git a/ec2/__init__.py b/ec2/__init__.py new file mode 100644 index 0000000..344904b --- /dev/null +++ b/ec2/__init__.py @@ -0,0 +1 @@ +from main import init_subparser diff --git a/ec2/ec2.py b/ec2/ec2.py new file mode 100644 index 0000000..e7ce082 --- /dev/null +++ b/ec2/ec2.py @@ -0,0 +1,40 @@ +from common import Task + + +class GetCredentials(Task): + def run(self, info): + super(GetCredentials, self).run(info) + info.ec2_credentials = self.get_ec2_credentials(info.args, info.manifest) + return info + + def get_ec2_credentials(self, args, manifest): + from os import getenv + # args override manifest override environment + if(args.access_key and args.secret_key): + return {'access_key': args.access_key, + 'secret_key': args.secret_key} + if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): + return {'access_key': manifest.credentials['access-key'], + 'secret_key': manifest.credentials['secret-key']} + if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')): + return {'access_key': getenv('EC2_ACCESS_KEY'), + 'secret_key': getenv('EC2_SECRET_KEY')} + + if(bool(args.access_key) != bool(args.secret_key)): + raise RuntimeError('Both the access key and secret key must be specified as arguments.') + if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])): + raise RuntimeError('Both the access key and secret key must be specified in the manifest.') + if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))): + raise RuntimeError('Both the access key and secret key must be specified as environment variables.') + + raise RuntimeError('No ec2 credentials found.') + + +class Connect(Task): + def run(self, info): + super(Connect, self).run(info) + # import boto.ec2 + # info.ec2_connection = boto.ec2.connect_to_region(info.host['region'], + # aws_access_key_id=info.ec2_credentials['access_key'], + # aws_secret_access_key=info.ec2_credentials['secret_key']) + return info diff --git a/ec2/host.py b/ec2/host.py new file mode 100644 index 0000000..fec31c5 --- /dev/null +++ b/ec2/host.py @@ -0,0 +1,16 @@ +from common import Task + + +class GetInfo(Task): + def run(self, info): + super(GetInfo, self).run(info) + import urllib2 + import json + # response = urllib2.urlopen('http://169.254.169.254/latest/dynamic/instance-identity/document') + # info.host = json.load(response.read()) + return info + + +class InstallPackages(Task): + def run(self, info): + pass diff --git a/ec2/main.py b/ec2/main.py new file mode 100644 index 0000000..6509043 --- /dev/null +++ b/ec2/main.py @@ -0,0 +1,37 @@ + + +def init_subparser(subparsers): + cmd = subparsers.add_parser('ec2', help='Bootstrap Debian for EC2') + cmd.add_argument('--access-key', help='AWS Access Key', metavar='ID') + cmd.add_argument('--secret-key', help='AWS Secret Key', metavar='KEY') + cmd.add_argument('manifest', help='Manifest file to use for bootstrapping', metavar='MANIFEST') + cmd.set_defaults(func=run) + + +def run(args): + from manifest import Manifest + from common import BootstrapInformation + manifest = Manifest(args.manifest) + manifest.validate() + + task_list = get_tasklist(manifest) + + info = BootstrapInformation(manifest=manifest, args=args) + task_list.run(info) + + +def get_tasklist(manifest): + from common import TaskList + import packages + import ec2 + import host + task_list = TaskList() + task_list.extend([packages.HostPackages(), + packages.ImagePackages(), + ec2.GetCredentials(), + host.GetInfo(), + ec2.Connect(), + host.InstallPackages() + ]) + + return task_list diff --git a/ec2/manifest.py b/ec2/manifest.py new file mode 100644 index 0000000..bccbba7 --- /dev/null +++ b/ec2/manifest.py @@ -0,0 +1,11 @@ +import common + + +class Manifest(common.Manifest): + def parse(self, data): + super(Manifest, self).parse(data) + self.credentials = data["credentials"] + self.virtualization = data["virtualization"] + + def validate(self): + super(Manifest, self).validate() diff --git a/ec2/packages.py b/ec2/packages.py new file mode 100644 index 0000000..803a7ca --- /dev/null +++ b/ec2/packages.py @@ -0,0 +1,55 @@ +from common import Task + + +class HostPackages(Task): + def run(self, info): + super(HostPackages, self).run(info) + info.host_pkg = self.get_host_packages(info.manifest) + return info + + def get_host_packages(self, manifest): + packages = set(['debootstrap', + # To make sure a volume is not busy before unmounting we need lsof + 'lsof', + ]) + if manifest.volume['filesystem'] == 'xfs': + packages.add('xfsprogs') + + return packages + + +class ImagePackages(Task): + def run(self, info): + super(ImagePackages, self).run(info) + info.image_pkg_include, info.image_pkg_exclude = self.get_image_packages(info.manifest) + return info + + def get_image_packages(self, manifest): + # Add some basic packages we are going to need + include = set(['udev', + 'openssh-server', + # We could bootstrap without locales, but things just suck without them, error messages etc. + 'locales', + # Needed for the init scripts + 'file', + # isc-dhcp-client doesn't work properly with ec2 + 'dhcpcd', + ]) + + if manifest.virtualization == 'pvm': + include.add('grub-pc') + + exclude = set(['isc-dhcp-client', + 'isc-dhcp-common', + ]) + + # In squeeze, we need a special kernel flavor for xen + kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64', + 'i386': 'linux-image-xen-686', }, + 'wheezy': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, } + include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + + include = include.union(manifest.system['packages']) + + return include, exclude diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json new file mode 100644 index 0000000..266a2eb --- /dev/null +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -0,0 +1,36 @@ +{ + "provider" : "ec2", + "virtualization": "pvm", + "credentials" : { + "access-key": null, + "secret-key": null + }, + + "bootstrapdir" : "/target", + "image": { + "name" : "debian-{release}-{architecture}-{virt}-{year}{month}{day}", + "description": "Debian {release} {architecture} AMI ({virt})" + }, + "system": { + "architecture": "amd64", + "release" : "wheezy", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8", + "packages" : [] + }, + "volume": { + "backing" : "ebs", + "filesystem": "ext4", + "size" : "1G" + }, + "plugins": { + "admin-user": { + "enabled": true + }, + "build-metadata": { + "enabled": false, + "path": "/root/build-metadata-{ami_name}" + } + } +}