diff --git a/manifests/ec2-s3-pvm.manifest.json b/manifests/ec2-s3-pvm.manifest.json new file mode 100644 index 0000000..8b70f67 --- /dev/null +++ b/manifests/ec2-s3-pvm.manifest.json @@ -0,0 +1,29 @@ +{ + "provider": "ec2", + "virtualization": "pvm", + "credentials": { + "access-key": null, + "secret-key": null + }, + + "bootstrapper": { + "mount_dir": "/target" + }, + "image": { + "name": "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + "bucket": "" + }, + "system": { + "release": "wheezy", + "architecture": "amd64", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "volume": { + "backing": "s3", + "filesystem": "ext4", + "size": 1024 + } +} diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index e4abbe7..dd09d3c 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -5,6 +5,7 @@ from tasks import connection from tasks import host from tasks import ami from tasks import ebs +from tasks import loopback from tasks import filesystem from tasks import bootstrap from tasks import locale @@ -28,20 +29,13 @@ def tasks(tasklist, manifest): connection.GetCredentials(), host.GetInfo(), ami.AMIName(), - connection.Connect()) - if manifest.volume['backing'].lower() == 'ebs': - tasklist.add(ebs.Create(), - ebs.Attach()) - tasklist.add(filesystem.FormatVolume()) - if manifest.volume['filesystem'].lower() == 'xfs': - tasklist.add(filesystem.AddXFSProgs()) - if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - tasklist.add(filesystem.TuneVolumeFS()) - tasklist.add(filesystem.CreateMountDir(), - filesystem.MountVolume()) - if manifest.bootstrapper['tarball']: - tasklist.add(bootstrap.MakeTarball()) - tasklist.add(bootstrap.Bootstrap(), + connection.Connect(), + + filesystem.FormatVolume(), + filesystem.CreateMountDir(), + filesystem.MountVolume(), + + bootstrap.Bootstrap(), filesystem.MountSpecials(), locale.GenerateLocale(), locale.SetTimezone(), @@ -69,11 +63,27 @@ def tasks(tasklist, manifest): filesystem.UnmountSpecials(), filesystem.UnmountVolume(), filesystem.DeleteMountDir()) - if manifest.volume['backing'].lower() == 'ebs': - tasklist.add(ebs.Detach(), - ebs.Snapshot(), - ebs.Delete()) - tasklist.add(ami.RegisterAMI()) + + if manifest.bootstrapper['tarball']: + tasklist.add(bootstrap.MakeTarball()) + + backing_specific_tasks = {'ebs': [ebs.Create(), + ebs.Attach(), + ebs.Detach(), + ebs.Snapshot(), + ami.RegisterAMI(), + ebs.Delete()], + 's3': [loopback.Create(), + loopback.Attach(), + loopback.Detach(), + loopback.Delete()]} + tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + + filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], + 'ext2': [filesystem.TuneVolumeFS()], + 'ext3': [filesystem.TuneVolumeFS()], + 'ext4': [filesystem.TuneVolumeFS()]} + tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) def rollback_tasks(tasklist, tasks_completed, manifest): diff --git a/providers/ec2/manifest-schema-s3.json b/providers/ec2/manifest-schema-s3.json new file mode 100644 index 0000000..d807841 --- /dev/null +++ b/providers/ec2/manifest-schema-s3.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "EC2 manifest for instance store AMIs", + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "bucket": { + "type": "string" + } + }, + "required": ["bucket"] + } + }, + "required": ["image"] +} diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index 01813a7..1d35897 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -8,9 +8,13 @@ class Manifest(base.Manifest): from os import path schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') self.schema_validate(data, schema_path) - if data['volume']['backing'] == 'ebs' and data['volume']['size'] % 1024 != 0: - msg = 'The volume size must be a multiple of 1024 when using EBS backing' - raise ManifestError(msg, self) + if data['volume']['backing'] == 'ebs': + if data['volume']['size'] % 1024 != 0: + msg = 'The volume size must be a multiple of 1024 when using EBS backing' + raise ManifestError(msg, self) + else: + schema_path = path.join(path.dirname(__file__), 'manifest-schema-s3.json') + self.schema_validate(data, schema_path) def parse(self, data): super(Manifest, self).parse(data) @@ -19,3 +23,7 @@ class Manifest(base.Manifest): self.image = data['image'] if data['volume']['backing'] == 'ebs': self.ebs_volume_size = data['volume']['size'] / 1024 + if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3': + self.volume['loopback_dir'] = '/tmp' + if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3': + self.image['bundle_dir'] = '/tmp' diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 600b68e..00eac3c 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -35,6 +35,31 @@ class AMIName(Task): info.ami_description = ami_description +class BundleImage(Task): + description = 'Bundling the image' + phase = phases.image_registration + + def run(self, info): + import os.path + bundle_name = 'bundle-{id:x}'.format(id=info.run_id) + info.bundle_dir = os.path.join(info.manifest.image['bundle_dir'], bundle_name) + # from euca2ools.commands.bundle.bundleimage import BundleImage + # bundler = BundleImage() + # bundler. + # euca-upload-bundle -b "${S3_BUCKET}" -m "${bundledir}/${ami_name}.manifest.xml" + pass + + +class RemoveBundle(Task): + description = 'Removing the bundle files' + phase = phases.cleaning + + def run(self, info): + from shutil import rmtree + rmtree(info.bundle_dir) + del info.bundle_dir + + class RegisterAMI(Task): description = 'Registering the image as an AMI' phase = phases.image_registration diff --git a/providers/ec2/tasks/loopback.py b/providers/ec2/tasks/loopback.py new file mode 100644 index 0000000..e1f15d1 --- /dev/null +++ b/providers/ec2/tasks/loopback.py @@ -0,0 +1,46 @@ +from base import Task +from common import phases +from common.tools import log_check_call + + +class Create(Task): + description = 'Creating a loopback volume' + phase = phases.volume_creation + + def run(self, info): + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + import os.path + info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + log_check_call(['/bin/dd', + 'if=/dev/zero', 'of='+info.loopback_file, + 'bs=1M', 'seek='+info.manifest.volume['size'], 'count=0']) + + +class Attach(Task): + description = 'Attaching the loopback volume' + phase = phases.volume_creation + after = [Create] + + def run(self, info): + info.bootstrap_device = {} + info.bootstrap_device['path'] = log_check_call(['/sbin/losetup', '--find']) + log_check_call(['/sbin/losetup', info.bootstrap_device['path'], info.loopback_file]) + + +class Detach(Task): + description = 'Detaching the loopback volume' + phase = phases.volume_unmounting + + def run(self, info): + log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['path']]) + del info.bootstrap_device + + +class Delete(Task): + description = 'Deleting the loopback volume' + phase = phases.cleaning + + def run(self, info): + from os import remove + remove(info.bootstrap_device['path']) + del info.loopback_file