Began work on S3 backed instances. Refactored adding tasks.

Got stuck on the bundle image call, it uses argparse.
Somehow we need to fake the arguments.

manifests/ec2-s3-pvm.manifest.json

@@ -0,0 +1,29 @@
+{
+	"provider":       "ec2",
+	"virtualization": "pvm",
+	"credentials":    {
+		"access-key": null,
+		"secret-key": null
+	},
+
+	"bootstrapper": {
+		"mount_dir": "/target"
+	},
+	"image": {
+		"name":        "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} AMI ({virtualization})"
+		"bucket":      ""
+	},
+	"system": {
+		"release":      "wheezy",
+		"architecture": "amd64",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"volume": {
+		"backing":      "s3",
+		"filesystem":   "ext4",
+		"size":         1024
+	}
+}

providers/ec2/__init__.py

@@ -5,6 +5,7 @@ from tasks import connection
 from tasks import host
 from tasks import ami
 from tasks import ebs
+from tasks import loopback
 from tasks import filesystem
 from tasks import bootstrap
 from tasks import locale
@@ -28,20 +29,13 @@ def tasks(tasklist, manifest):
 	             connection.GetCredentials(),
 	             host.GetInfo(),
 	             ami.AMIName(),
-	             connection.Connect())
-	if manifest.volume['backing'].lower() == 'ebs':
-		tasklist.add(ebs.Create(),
-		             ebs.Attach())
-	tasklist.add(filesystem.FormatVolume())
-	if manifest.volume['filesystem'].lower() == 'xfs':
-		tasklist.add(filesystem.AddXFSProgs())
-	if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
-		tasklist.add(filesystem.TuneVolumeFS())
-	tasklist.add(filesystem.CreateMountDir(),
-	             filesystem.MountVolume())
-	if manifest.bootstrapper['tarball']:
-		tasklist.add(bootstrap.MakeTarball())
-	tasklist.add(bootstrap.Bootstrap(),
+	             connection.Connect(),
+
+	             filesystem.FormatVolume(),
+	             filesystem.CreateMountDir(),
+	             filesystem.MountVolume(),
+
+	             bootstrap.Bootstrap(),
 	             filesystem.MountSpecials(),
 	             locale.GenerateLocale(),
 	             locale.SetTimezone(),
@@ -69,11 +63,27 @@ def tasks(tasklist, manifest):
 	             filesystem.UnmountSpecials(),
 	             filesystem.UnmountVolume(),
 	             filesystem.DeleteMountDir())
-	if manifest.volume['backing'].lower() == 'ebs':
-		tasklist.add(ebs.Detach(),
-		             ebs.Snapshot(),
-		             ebs.Delete())
-	tasklist.add(ami.RegisterAMI())
+
+	if manifest.bootstrapper['tarball']:
+		tasklist.add(bootstrap.MakeTarball())
+
+	backing_specific_tasks = {'ebs': [ebs.Create(),
+	                                  ebs.Attach(),
+	                                  ebs.Detach(),
+	                                  ebs.Snapshot(),
+	                                  ami.RegisterAMI(),
+	                                  ebs.Delete()],
+	                          's3': [loopback.Create(),
+	                                 loopback.Attach(),
+	                                 loopback.Detach(),
+	                                 loopback.Delete()]}
+	tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
+
+	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
+	                             'ext2': [filesystem.TuneVolumeFS()],
+	                             'ext3': [filesystem.TuneVolumeFS()],
+	                             'ext4': [filesystem.TuneVolumeFS()]}
+	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
 
 
 def rollback_tasks(tasklist, tasks_completed, manifest):

providers/ec2/manifest-schema-s3.json

@@ -0,0 +1,17 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "EC2 manifest for instance store AMIs",
+	"type": "object",
+	"properties": {
+		"image": {
+			"type": "object",
+			"properties": {
+				"bucket": {
+					"type": "string"
+				}
+			},
+			"required": ["bucket"]
+		}
+	},
+	"required": ["image"]
+}

providers/ec2/manifest.py

@@ -8,9 +8,13 @@ class Manifest(base.Manifest):
 		from os import path
 		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
 		self.schema_validate(data, schema_path)
-		if data['volume']['backing'] == 'ebs' and data['volume']['size'] % 1024 != 0:
-			msg = 'The volume size must be a multiple of 1024 when using EBS backing'
-			raise ManifestError(msg, self)
+		if data['volume']['backing'] == 'ebs':
+			if data['volume']['size'] % 1024 != 0:
+				msg = 'The volume size must be a multiple of 1024 when using EBS backing'
+				raise ManifestError(msg, self)
+		else:
+			schema_path = path.join(path.dirname(__file__), 'manifest-schema-s3.json')
+			self.schema_validate(data, schema_path)
 
 	def parse(self, data):
 		super(Manifest, self).parse(data)
@@ -19,3 +23,7 @@ class Manifest(base.Manifest):
 		self.image          = data['image']
 		if data['volume']['backing'] == 'ebs':
 			self.ebs_volume_size = data['volume']['size'] / 1024
+		if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3':
+			self.volume['loopback_dir'] = '/tmp'
+		if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3':
+			self.image['bundle_dir'] = '/tmp'

providers/ec2/tasks/ami.py

@@ -35,6 +35,31 @@ class AMIName(Task):
 		info.ami_description = ami_description
 
 
+class BundleImage(Task):
+	description = 'Bundling the image'
+	phase = phases.image_registration
+
+	def run(self, info):
+		import os.path
+		bundle_name = 'bundle-{id:x}'.format(id=info.run_id)
+		info.bundle_dir = os.path.join(info.manifest.image['bundle_dir'], bundle_name)
+		# from euca2ools.commands.bundle.bundleimage import BundleImage
+		# bundler = BundleImage()
+		# bundler.
+		# euca-upload-bundle -b "${S3_BUCKET}" -m "${bundledir}/${ami_name}.manifest.xml"
+		pass
+
+
+class RemoveBundle(Task):
+	description = 'Removing the bundle files'
+	phase = phases.cleaning
+
+	def run(self, info):
+		from shutil import rmtree
+		rmtree(info.bundle_dir)
+		del info.bundle_dir
+
+
 class RegisterAMI(Task):
 	description = 'Registering the image as an AMI'
 	phase = phases.image_registration

providers/ec2/tasks/loopback.py

@@ -0,0 +1,46 @@
+from base import Task
+from common import phases
+from common.tools import log_check_call
+
+
+class Create(Task):
+	description = 'Creating a loopback volume'
+	phase = phases.volume_creation
+
+	def run(self, info):
+		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
+		import os.path
+		info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		log_check_call(['/bin/dd',
+		                'if=/dev/zero', 'of='+info.loopback_file,
+		                'bs=1M', 'seek='+info.manifest.volume['size'], 'count=0'])
+
+
+class Attach(Task):
+	description = 'Attaching the loopback volume'
+	phase = phases.volume_creation
+	after = [Create]
+
+	def run(self, info):
+		info.bootstrap_device = {}
+		info.bootstrap_device['path'] = log_check_call(['/sbin/losetup', '--find'])
+		log_check_call(['/sbin/losetup', info.bootstrap_device['path'], info.loopback_file])
+
+
+class Detach(Task):
+	description = 'Detaching the loopback volume'
+	phase = phases.volume_unmounting
+
+	def run(self, info):
+		log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['path']])
+		del info.bootstrap_device
+
+
+class Delete(Task):
+	description = 'Deleting the loopback volume'
+	phase = phases.cleaning
+
+	def run(self, info):
+		from os import remove
+		remove(info.bootstrap_device['path'])
+		del info.loopback_file