From b92f70e54867098ba1931b272809e4cb0e9b59ad Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Thu, 27 Jun 2013 23:26:29 +0200 Subject: [PATCH] bootstrap task implemented --- base/manifest-schema.json | 33 ++++++++++++++++--- base/manifest.py | 6 +++- common/tools.py | 21 ++++++++++++ manifests/ec2-ebs-pvm.manifest.json | 10 +++--- providers/ec2/__init__.py | 4 +++ providers/ec2/tasks/bootstrap.py | 50 +++++++++++++++++++++++++++++ providers/ec2/tasks/filesystem.py | 3 +- 7 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 common/tools.py create mode 100644 providers/ec2/tasks/bootstrap.py diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 68725ec..bb0f006 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -6,8 +6,31 @@ "provider": { "type": "string" }, - "bootstrapdir": { - "type": "string" + "bootstrapper": { + "type": "object", + "properties": { + "mount_dir": { "type": "string" }, + "tarball": { "type": "boolean" }, + "tarball_dir": { "type": "string" } + }, + "required": ["mount_dir"] + }, + "system": { + "type": "object", + "properties": { + "release": { + "type": "string", + "enum": ["wheezy"] + }, + "architecture": { + "type": "string", + "enum": ["i386", "amd64"] + }, + "timezone": { "type": "string" }, + "locale": { "type": "string" }, + "charmap": { "type": "string" } + }, + "required": ["release", "architecture", "timezone", "locale", "charmap"] }, "volume": { "type": "object", @@ -32,7 +55,7 @@ } }, "additionalProperties": false - }, - "require": ["provider", "bootstrapdir"] - } + } + }, + "required": ["provider", "bootstrapper", "volume", "system"] } diff --git a/base/manifest.py b/base/manifest.py index aa77c90..89188dc 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -41,7 +41,11 @@ class Manifest(object): def parse(self, data): self.provider = data['provider'] - self.bootstrapdir = data['bootstrapdir'] + self.bootstrapper = data['bootstrapper'] + if 'tarball' not in self.bootstrapper: + self.bootstrapper['tarball'] = False + if 'tarball_dir' not in self.bootstrapper: + self.bootstrapper['tarball_dir'] = '/tmp' self.volume = data['volume'] self.system = data['system'] self.plugins = data['plugins'] diff --git a/common/tools.py b/common/tools.py new file mode 100644 index 0000000..8cac5aa --- /dev/null +++ b/common/tools.py @@ -0,0 +1,21 @@ + + +def log_command(command, logger): + import subprocess + import select + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while True: + reads = [process.stdout.fileno(), process.stderr.fileno()] + ret = select.select(reads, [], []) + for fd in ret[0]: + if fd == process.stdout.fileno(): + line = process.stdout.readline() + if line != '': + logger.debug(line) + if fd == process.stderr.fileno(): + line = process.stderr.readline() + if line != '': + logger.error(line) + if process.poll() is not None: + break + return process.returncode diff --git a/manifests/ec2-ebs-pvm.manifest.json b/manifests/ec2-ebs-pvm.manifest.json index 9bc54f9..63431df 100644 --- a/manifests/ec2-ebs-pvm.manifest.json +++ b/manifests/ec2-ebs-pvm.manifest.json @@ -6,18 +6,20 @@ "secret-key": null }, - "bootstrapdir": "/target", + "bootstrapper": { + "mount_dir": "/target", + "tarball": true + }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{year}{month}{day}", "description": "Debian {release} {architecture} AMI ({virtualization})" }, "system": { - "architecture": "amd64", "release" : "wheezy", + "architecture": "amd64", "timezone" : "UTC", "locale" : "en_US", - "charmap" : "UTF-8", - "packages" : [] + "charmap" : "UTF-8" }, "volume": { "backing" : "ebs", diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index d50cf9c..f998eaf 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -4,6 +4,7 @@ from tasks import connection from tasks import host from tasks import ebs from tasks import filesystem +from tasks import bootstrap def tasks(tasklist, manifest): @@ -18,6 +19,9 @@ def tasks(tasklist, manifest): if re.search('ext.', manifest.volume['filesystem'].lower()): tasklist.add(filesystem.TuneVolumeFS()) tasklist.add(filesystem.CreateMountDir(), filesystem.MountVolume()) + if manifest.bootstrapper['tarball']: + tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.Bootstrap()) from common.tasks import TriggerRollback tasklist.add(TriggerRollback()) diff --git a/providers/ec2/tasks/bootstrap.py b/providers/ec2/tasks/bootstrap.py new file mode 100644 index 0000000..2293841 --- /dev/null +++ b/providers/ec2/tasks/bootstrap.py @@ -0,0 +1,50 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from common.tools import log_command +import logging +log = logging.getLogger(__name__) + + +def get_bootstrap_args(info): + executable = ['/usr/sbin/debootstrap'] + options = ['--arch=' + info.manifest.system['architecture']] + include, exclude = info.img_packages + if len(include) > 0: + options.append('--include=' + ','.join(include)) + if len(exclude) > 0: + options.append('--exclude=' + ','.join(exclude)) + arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian'] + return executable, options, arguments + + +class MakeTarball(Task): + description = 'Creating bootstrap tarball' + phase = phases.os_installation + + def run(self, info): + from hashlib import sha1 + import os.path + executable, options, arguments = get_bootstrap_args(info) + tarball_id = sha1(repr(frozenset(options + arguments))).hexdigest()[0:8] + tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) + info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + + command = executable + options + ['--make-tarball=' + info.tarball] + arguments + if log_command(command, log) != 0: + raise TaskError('Unable to create bootstrap tarball') + + +class Bootstrap(Task): + description = 'Installing Debian' + phase = phases.os_installation + after = [MakeTarball] + + def run(self, info): + executable, options, arguments = get_bootstrap_args(info) + if hasattr(info, 'tarball'): + options.extend(['--unpack-tarball=' + info.tarball]) + command = executable + options + arguments + command = executable + options + ['--make-tarball=' + info.tarball] + arguments + if log_command(command, log) != 0: + raise TaskError('Unable to bootstrap') diff --git a/providers/ec2/tasks/filesystem.py b/providers/ec2/tasks/filesystem.py index b0e79ee..5158bdb 100644 --- a/providers/ec2/tasks/filesystem.py +++ b/providers/ec2/tasks/filesystem.py @@ -45,7 +45,8 @@ class CreateMountDir(Task): def run(self, info): import os - info.root = '{bs_dir}/{vol_id}'.format(bs_dir=info.manifest.bootstrapdir, vol_id=info.volume.id) + mount_dir = info.manifest.bootstrapper['mount_dir'] + info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id) # Works recursively, fails if last part exists, which is exaclty what we want. os.makedirs(info.root)