From 0eb5eecfc2082cb8daec6eb7b10b0a88414fcb8b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 15 Sep 2013 16:59:56 +0200 Subject: [PATCH] Place everything in info.workspace instead of spreading files all around the disk --- base/bootstrapinfo.py | 3 +++ base/manifest-schema.json | 3 +-- base/manifest.py | 4 ---- common/fs/loopbackvolume.py | 2 ++ common/tasks/bootstrap.py | 2 +- common/tasks/filesystem.py | 4 +--- common/tasks/loopback.py | 7 +++---- common/tasks/volume.py | 2 ++ common/tasks/workspace.py | 20 ++++++++++++++++++++ manifests/kvm-virtio.manifest.json | 3 +-- manifests/virtualbox.manifest.json | 1 - plugins/prebootstrapped/tasks.py | 9 ++++----- providers/ec2/__init__.py | 2 +- providers/ec2/manifest.py | 4 ---- providers/ec2/tasks/ami.py | 4 +++- providers/kvm/__init__.py | 2 +- providers/kvm/manifest.py | 2 -- providers/virtualbox/__init__.py | 10 +++++++--- providers/virtualbox/manifest-schema.json | 2 +- providers/virtualbox/manifest.py | 2 -- providers/virtualbox/volume.py | 8 ++++---- 21 files changed, 55 insertions(+), 41 deletions(-) create mode 100644 common/tasks/workspace.py diff --git a/base/bootstrapinfo.py b/base/bootstrapinfo.py index 07c954b..ff425b9 100644 --- a/base/bootstrapinfo.py +++ b/base/bootstrapinfo.py @@ -8,3 +8,6 @@ class BootstrapInformation(object): self.debug = debug import random self.run_id = random.randrange(16 ** 8) + import os.path + workspace_dirname = '{id:x}'.format(id=self.run_id) + self.workspace = os.path.join(manifest.bootstrapper['workspace'], workspace_dirname) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index e66ef96..1d751b3 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -11,8 +11,7 @@ "properties": { "workspace": { "type": "string" }, "mirror": { "type": "string" }, - "tarball": { "type": "boolean" }, - "tarball_dir": { "type": "string" } + "tarball": { "type": "boolean" } }, "required": ["workspace"] }, diff --git a/base/manifest.py b/base/manifest.py index 7aebdae..df17372 100644 --- a/base/manifest.py +++ b/base/manifest.py @@ -49,10 +49,6 @@ class Manifest(object): self.bootstrapper = data['bootstrapper'] if 'mirror' not in self.bootstrapper: self.bootstrapper['mirror'] = 'http://http.debian.net/debian' - if 'tarball' not in self.bootstrapper: - self.bootstrapper['tarball'] = False - if 'tarball_dir' not in self.bootstrapper and self.bootstrapper['tarball']: - self.bootstrapper['tarball_dir'] = '/tmp' self.volume = data['volume'] self.system = data['system'] self.plugins = data['plugins'] if 'plugins' in data else {} diff --git a/common/fs/loopbackvolume.py b/common/fs/loopbackvolume.py index c8ffb1e..f47807b 100644 --- a/common/fs/loopbackvolume.py +++ b/common/fs/loopbackvolume.py @@ -23,6 +23,8 @@ class LoopbackVolume(Volume): {'name': 'unlink_dm_node', 'src': 'linked', 'dst': 'partitioned'}, ] + extension = 'raw' + def __init__(self, partition_map, callbacks={}): callbacks.update({'onbeforecreate': self._create, 'onbeforeattach': self._attach, diff --git a/common/tasks/bootstrap.py b/common/tasks/bootstrap.py index 68ffc5f..42e1da9 100644 --- a/common/tasks/bootstrap.py +++ b/common/tasks/bootstrap.py @@ -29,7 +29,7 @@ class MakeTarball(Task): hash_args = [arg for arg in arguments if arg != info.root] tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) - info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + info.tarball = os.path.join(info.manifest.bootstrapper['workspace'], tarball_filename) if os.path.isfile(info.tarball): log.debug('Found matching tarball, skipping download') else: diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 82333b6..a0e4075 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -41,9 +41,7 @@ class CreateMountDir(Task): def run(self, info): import os - workspace = info.manifest.bootstrapper['workspace'] - info.root = '{workspace}/{id:x}'.format(workspace=workspace, id=info.run_id) - # Works recursively, fails if last part exists, which is exactly what we want. + info.root = os.path.join(info.workspace, 'root') os.makedirs(info.root) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 81d6f10..c453d6a 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -9,9 +9,8 @@ class Create(Task): before = [volume.Attach] def run(self, info): - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) import os.path - image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) info.volume.create(image_path) @@ -21,8 +20,8 @@ class MoveImage(Task): def run(self, info): import os.path - image_basename = os.path.basename(info.volume.image_path) - destination = os.path.join(info.manifest.bootstrapper['workspace'], image_basename) + filename = 'loopback-{id:x}.{ext}'.format(id=info.run_id, ext=info.volume.extension) + destination = os.path.join(info.bootstrapper['workspace'], filename) import shutil shutil.move(info.volume.image_path, destination) import logging diff --git a/common/tasks/volume.py b/common/tasks/volume.py index aa2a557..64be8eb 100644 --- a/common/tasks/volume.py +++ b/common/tasks/volume.py @@ -1,5 +1,6 @@ from base import Task from common import phases +from common.tasks import workspace class Attach(Task): @@ -21,6 +22,7 @@ class Detach(Task): class Delete(Task): description = 'Deleting the volume' phase = phases.cleaning + before = [workspace.DeleteWorkspace] def run(self, info): info.volume.delete() diff --git a/common/tasks/workspace.py b/common/tasks/workspace.py new file mode 100644 index 0000000..836443b --- /dev/null +++ b/common/tasks/workspace.py @@ -0,0 +1,20 @@ +from base import Task +from common import phases + + +class CreateWorkspace(Task): + description = 'Creating workspace' + phase = phases.preparation + + def run(self, info): + import os + os.makedirs(info.workspace) + + +class DeleteWorkspace(Task): + description = 'Deleting workspace' + phase = phases.cleaning + + def run(self, info): + import os + os.rmdir(info.workspace) diff --git a/manifests/kvm-virtio.manifest.json b/manifests/kvm-virtio.manifest.json index d14a11d..7076d92 100644 --- a/manifests/kvm-virtio.manifest.json +++ b/manifests/kvm-virtio.manifest.json @@ -20,8 +20,7 @@ "volume": { "backing" : "raw", "filesystem": "ext4", - "size" : 1024, - "loopback_dir" : "/tmp" + "size" : 1024 }, "plugins": { "user_packages": { diff --git a/manifests/virtualbox.manifest.json b/manifests/virtualbox.manifest.json index b376127..d61e30f 100644 --- a/manifests/virtualbox.manifest.json +++ b/manifests/virtualbox.manifest.json @@ -19,7 +19,6 @@ }, "volume": { "backing": "raw", - "loopback_dir": "/tmp", "partitions": { "boot": { "size": 12, diff --git a/plugins/prebootstrapped/tasks.py b/plugins/prebootstrapped/tasks.py index 57d0610..f4eb1fa 100644 --- a/plugins/prebootstrapped/tasks.py +++ b/plugins/prebootstrapped/tasks.py @@ -51,9 +51,9 @@ class CopyImage(Task): import os.path from shutil import copyfile loopback_backup_name = 'loopback-{id:x}.img.backup'.format(id=info.run_id) - image_copy_path = os.path.join('/tmp', loopback_backup_name) - copyfile(info.volume.image_path, image_copy_path) - msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=image_copy_path) + destination = os.path.join(info.manifest.bootstrapper['workspace'], loopback_backup_name) + copyfile(info.volume.image_path, destination) + msg = 'A copy of the bootstrapped volume was created. Path: {path}'.format(path=destination) log.info(msg) @@ -65,8 +65,7 @@ class CreateFromImage(Task): def run(self, info): import os.path from shutil import copyfile - loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) - info.volume.image_path = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + info.volume.image_path = os.path.join(info.workspace, 'volume.{ext}'.format(ext=info.volume.extension)) loopback_backup_path = info.manifest.plugins['prebootstrapped']['image'] copyfile(loopback_backup_path, info.volume.image_path) diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index ee63e71..533ccf7 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -74,7 +74,7 @@ def tasks(tasklist, manifest): filesystem.DeleteMountDir(), ami.RegisterAMI()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) backing_specific_tasks = {'ebs': [ebs.Create(), diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index 1d35897..abab855 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -23,7 +23,3 @@ class Manifest(base.Manifest): self.image = data['image'] if data['volume']['backing'] == 'ebs': self.ebs_volume_size = data['volume']['size'] / 1024 - if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3': - self.volume['loopback_dir'] = '/tmp' - if 'bundle_dir' not in self.image and self.volume['backing'].lower() == 's3': - self.image['bundle_dir'] = '/tmp' diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index be82f43..81c99ed 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -3,6 +3,7 @@ from common import phases from common.exceptions import TaskError from common.tools import log_check_call from ebs import Snapshot +from common.tasks import workspace from connection import Connect import os.path @@ -45,7 +46,7 @@ class BundleImage(Task): def run(self, info): bundle_name = 'bundle-{id:x}'.format(id=info.run_id) - info.bundle_path = os.path.join(info.manifest.image['bundle_dir'], bundle_name) + info.bundle_path = os.path.join(info.workspace, bundle_name) log_check_call(['/usr/bin/euca-bundle-image', '--image', info.loopback_file, '--user', info.credentials['user-id'], @@ -80,6 +81,7 @@ class UploadImage(Task): class RemoveBundle(Task): description = 'Removing the bundle files' phase = phases.cleaning + before = [workspace.DeleteWorkspace] def run(self, info): from shutil import rmtree diff --git a/providers/kvm/__init__.py b/providers/kvm/__init__.py index 724a250..8337e7b 100644 --- a/providers/kvm/__init__.py +++ b/providers/kvm/__init__.py @@ -68,7 +68,7 @@ def tasks(tasklist, manifest): loopback.Detach(), filesystem.DeleteMountDir()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], diff --git a/providers/kvm/manifest.py b/providers/kvm/manifest.py index 0e7b141..8ae196b 100644 --- a/providers/kvm/manifest.py +++ b/providers/kvm/manifest.py @@ -12,5 +12,3 @@ class Manifest(base.Manifest): super(Manifest, self).parse(data) self.image = data['image'] self.virtualization = data['virtualization'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index b5a0fce..eff464c 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -15,6 +15,7 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup +from common.tasks import workspace def initialize(): @@ -22,7 +23,8 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), + tasklist.add(workspace.CreateWorkspace(), + packages.HostPackages(), common_packages.HostPackages(), packages.ImagePackages(), common_packages.ImagePackages(), @@ -67,9 +69,10 @@ def tasks(tasklist, manifest): partitioning.UnmapPartitions(), volume_tasks.Detach(), filesystem.DeleteMountDir(), - loopback.MoveImage()) + loopback.MoveImage(), + workspace.DeleteWorkspace()) - if manifest.bootstrapper['tarball']: + if manifest.bootstrapper.get('tarball', False): tasklist.add(bootstrap.MakeTarball()) partitions = manifest.volume['partitions'] @@ -105,3 +108,4 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) counter_task(filesystem.MountBoot, filesystem.UnmountBoot) counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 0d51f6e..5e2b58b 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -8,7 +8,7 @@ "properties": { "backing": { "type": "string", - "enum": ["raw", "qcow2"] + "enum": ["raw", "vdi", "qcow2"] } // "filesystem": { // "type": "string", diff --git a/providers/virtualbox/manifest.py b/providers/virtualbox/manifest.py index 8e5bb11..8b5fbc2 100644 --- a/providers/virtualbox/manifest.py +++ b/providers/virtualbox/manifest.py @@ -12,5 +12,3 @@ class Manifest(base.Manifest): super(Manifest, self).parse(data) self.virtualization = None self.image = data['image'] - if 'loopback_dir' not in self.volume: - self.volume['loopback_dir'] = '/tmp' diff --git a/providers/virtualbox/volume.py b/providers/virtualbox/volume.py index 3425729..1af3d66 100644 --- a/providers/virtualbox/volume.py +++ b/providers/virtualbox/volume.py @@ -4,8 +4,8 @@ from common.tools import log_check_call class VirtualBoxVolume(LoopbackVolume): - def create(self, image_path): - super(VirtualBoxVolume, self).create(self) - self.image_path = image_path + extension = 'vdi' + + def _create(self, e): + self.image_path = e.image_path log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'vdi', self.image_path, str(self.size) + 'M']) - self.created = True