Place everything in info.workspace

instead of spreading files all around the disk

base/bootstrapinfo.py

@@ -8,3 +8,6 @@ class BootstrapInformation(object):
 		self.debug = debug
 		import random
 		self.run_id = random.randrange(16 ** 8)
+		import os.path
+		workspace_dirname = '{id:x}'.format(id=self.run_id)
+		self.workspace = os.path.join(manifest.bootstrapper['workspace'], workspace_dirname)

base/manifest-schema.json

@@ -11,8 +11,7 @@
 			"properties": {
 				"workspace": { "type": "string" },
 				"mirror":   { "type": "string" },
-				"tarball": { "type": "boolean" },
+				"tarball": { "type": "boolean" }
-				"tarball_dir": { "type": "string" }
 			},
 			"required": ["workspace"]
 		},

base/manifest.py

@@ -49,10 +49,6 @@ class Manifest(object):
 		self.bootstrapper = data['bootstrapper']
 		if 'mirror' not in self.bootstrapper:
 			self.bootstrapper['mirror'] = 'http://http.debian.net/debian'
-		if 'tarball' not in self.bootstrapper:
-			self.bootstrapper['tarball'] = False
-		if 'tarball_dir' not in self.bootstrapper and self.bootstrapper['tarball']:
-			self.bootstrapper['tarball_dir'] = '/tmp'
 		self.volume = data['volume']
 		self.system = data['system']
 		self.plugins = data['plugins'] if 'plugins' in data else {}

common/fs/loopbackvolume.py

@@ -23,6 +23,8 @@ class LoopbackVolume(Volume):
 	                  {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'},
 	                  ]
 
+	extension = 'raw'
+
 	def __init__(self, partition_map, callbacks={}):
 		callbacks.update({'onbeforecreate': self._create,
 		                  'onbeforeattach': self._attach,

common/tasks/bootstrap.py

@@ -29,7 +29,7 @@ class MakeTarball(Task):
 		hash_args = [arg for arg in arguments if arg != info.root]
 		tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8]
 		tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id)
-		info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename)
+		info.tarball = os.path.join(info.manifest.bootstrapper['workspace'], tarball_filename)
 		if os.path.isfile(info.tarball):
 			log.debug('Found matching tarball, skipping download')
 		else:

common/tasks/filesystem.py

@@ -41,9 +41,7 @@ class CreateMountDir(Task):
 
 	def run(self, info):
 		import os
-		workspace = info.manifest.bootstrapper['workspace']
+		info.root = os.path.join(info.workspace, 'root')
-		info.root = '{workspace}/{id:x}'.format(workspace=workspace, id=info.run_id)
-		# Works recursively, fails if last part exists, which is exactly what we want.
 		os.makedirs(info.root)
 
 

common/tasks/loopback.py

@@ -9,9 +9,8 @@ class Create(Task):
 	before = [volume.Attach]
 
 	def run(self, info):
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
 		import os.path
-		image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
+		image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension))
 		info.volume.create(image_path)
 
 
@@ -21,8 +20,8 @@ class MoveImage(Task):
 
 	def run(self, info):
 		import os.path
-		image_basename = os.path.basename(info.volume.image_path)
+		filename = 'loopback-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension)
-		destination = os.path.join(info.manifest.bootstrapper['workspace'], image_basename)
+		destination = os.path.join(info.bootstrapper['workspace'], filename)
 		import shutil
 		shutil.move(info.volume.image_path, destination)
 		import logging

common/tasks/volume.py

@@ -1,5 +1,6 @@
 from base import Task
 from common import phases
+from common.tasks import workspace
 
 
 class Attach(Task):
@@ -21,6 +22,7 @@ class Detach(Task):
 class Delete(Task):
 	description = 'Deleting the volume'
 	phase = phases.cleaning
+	before = [workspace.DeleteWorkspace]
 
 	def run(self, info):
 		info.volume.delete()

common/tasks/workspace.py

@@ -0,0 +1,20 @@
+from base import Task
+from common import phases
+
+
+class CreateWorkspace(Task):
+	description = 'Creating workspace'
+	phase = phases.preparation
+
+	def run(self, info):
+		import os
+		os.makedirs(info.workspace)
+
+
+class DeleteWorkspace(Task):
+	description = 'Deleting workspace'
+	phase = phases.cleaning
+
+	def run(self, info):
+		import os
+		os.rmdir(info.workspace)

manifests/kvm-virtio.manifest.json

@@ -20,8 +20,7 @@
 	"volume": {
 		"backing"   : "raw",
 		"filesystem": "ext4",
-		"size"      : 1024,
+		"size"      : 1024
-		"loopback_dir" : "/tmp"
 	},
 	"plugins": {
 		"user_packages": {

manifests/virtualbox.manifest.json

@@ -19,7 +19,6 @@
 	},
 	"volume": {
 		"backing":         "raw",
-		"loopback_dir":    "/tmp",
 		"partitions": {
 			"boot": {
 				"size":       12,

plugins/prebootstrapped/tasks.py

@@ -51,9 +51,9 @@ class CopyImage(Task):
 		import os.path
 		from shutil import copyfile
 		loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id)
-		image_copy_path = os.path.join('/tmp', loopback_backup_name)
+		destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name)
-		copyfile(info.volume.image_path, image_copy_path)
+		copyfile(info.volume.image_path, destination)
-		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path)
+		msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination)
 		log.info(msg)
 
 
@@ -65,8 +65,7 @@ class CreateFromImage(Task):
 	def run(self, info):
 		import os.path
 		from shutil import copyfile
-		loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id)
+		info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension))
-		info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename)
 		loopback_backup_path = info.manifest.plugins['prebootstrapped']['image']
 		copyfile(loopback_backup_path, info.volume.image_path)
 

providers/ec2/__init__.py

@@ -74,7 +74,7 @@ def tasks(tasklist, manifest):
 	             filesystem.DeleteMountDir(),
 	             ami.RegisterAMI())
 
-	if manifest.bootstrapper['tarball']:
+	if manifest.bootstrapper.get('tarball', False):
 		tasklist.add(bootstrap.MakeTarball())
 
 	backing_specific_tasks = {'ebs': [ebs.Create(),

providers/ec2/manifest.py

@@ -23,7 +23,3 @@ class Manifest(base.Manifest):
 		self.image          = data['image']
 		if data['volume']['backing'] == 'ebs':
 			self.ebs_volume_size = data['volume']['size'] / 1024
-		if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3':
-			self.volume['loopback_dir'] = '/tmp'
-		if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3':
-			self.image['bundle_dir'] = '/tmp'

providers/ec2/tasks/ami.py

@@ -3,6 +3,7 @@ from common import phases
 from common.exceptions import TaskError
 from common.tools import log_check_call
 from ebs import Snapshot
+from common.tasks import workspace
 from connection import Connect
 import os.path
 
@@ -45,7 +46,7 @@ class BundleImage(Task):
 
 	def run(self, info):
 		bundle_name = 'bundle-{id:x}'.format(id=info.run_id)
-		info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name)
+		info.bundle_path = os.path.join(info.workspace, bundle_name)
 		log_check_call(['/usr/bin/euca-bundle-image',
 		                '--image', info.loopback_file,
 		                '--user', info.credentials['user-id'],
@@ -80,6 +81,7 @@ class UploadImage(Task):
 class RemoveBundle(Task):
 	description = 'Removing the bundle files'
 	phase = phases.cleaning
+	before = [workspace.DeleteWorkspace]
 
 	def run(self, info):
 		from shutil import rmtree

providers/kvm/__init__.py

@@ -68,7 +68,7 @@ def tasks(tasklist, manifest):
 	             loopback.Detach(),
 	             filesystem.DeleteMountDir())
 
-	if manifest.bootstrapper['tarball']:
+	if manifest.bootstrapper.get('tarball', False):
 		tasklist.add(bootstrap.MakeTarball())
 
 	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],

providers/kvm/manifest.py

@@ -12,5 +12,3 @@ class Manifest(base.Manifest):
 		super(Manifest, self).parse(data)
 		self.image          = data['image']
 		self.virtualization = data['virtualization']
-		if 'loopback_dir' not in self.volume:
-			self.volume['loopback_dir'] = '/tmp'

providers/virtualbox/__init__.py

@@ -15,6 +15,7 @@ from common.tasks import security
 from common.tasks import network
 from common.tasks import initd
 from common.tasks import cleanup
+from common.tasks import workspace
 
 
 def initialize():
@@ -22,7 +23,8 @@ def initialize():
 
 
 def tasks(tasklist, manifest):
-	tasklist.add(packages.HostPackages(),
+	tasklist.add(workspace.CreateWorkspace(),
+	             packages.HostPackages(),
 	             common_packages.HostPackages(),
 	             packages.ImagePackages(),
 	             common_packages.ImagePackages(),
@@ -67,9 +69,10 @@ def tasks(tasklist, manifest):
 	             partitioning.UnmapPartitions(),
 	             volume_tasks.Detach(),
 	             filesystem.DeleteMountDir(),
-	             loopback.MoveImage())
+	             loopback.MoveImage(),
+	             workspace.DeleteWorkspace())
 
-	if manifest.bootstrapper['tarball']:
+	if manifest.bootstrapper.get('tarball', False):
 		tasklist.add(bootstrap.MakeTarball())
 
 	partitions = manifest.volume['partitions']
@@ -105,3 +108,4 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
 	counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
 	counter_task(volume_tasks.Attach, volume_tasks.Detach)
+	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)

providers/virtualbox/manifest-schema.json

@@ -8,7 +8,7 @@
 			"properties": {
 				"backing": {
 					"type": "string",
-					"enum": ["raw", "qcow2"]
+					"enum": ["raw", "vdi", "qcow2"]
 				}
 				// "filesystem": {
 				// 	"type": "string",

providers/virtualbox/manifest.py

@@ -12,5 +12,3 @@ class Manifest(base.Manifest):
 		super(Manifest, self).parse(data)
 		self.virtualization = None
 		self.image          = data['image']
-		if 'loopback_dir' not in self.volume:
-			self.volume['loopback_dir'] = '/tmp'

providers/virtualbox/volume.py

@@ -4,8 +4,8 @@ from common.tools import log_check_call
 
 class VirtualBoxVolume(LoopbackVolume):
 
-	def create(self, image_path):
+	extension = 'vdi'
-		super(VirtualBoxVolume, self).create(self)
+
-		self.image_path = image_path
+	def _create(self, e):
+		self.image_path = e.image_path
 		log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M'])
-		self.created = True