From 7e32263315aa2066a6fc891a6c31b750403eda6c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 10 Aug 2013 20:03:20 +0200 Subject: [PATCH] Generalize parted feature Remove filesystem module from ONE tasks --- common/tasks/filesystem.py | 19 ++++--- common/tasks/loopback.py | 12 +++++ common/tasks/parted.py | 52 ++++++++++++++++++ manifests/one-ide.manifest.json | 4 +- manifests/one-virtio.manifest.json | 4 +- providers/one/__init__.py | 22 ++++---- providers/one/manifest.py | 2 + providers/one/tasks/filesystem.py | 85 ------------------------------ 8 files changed, 94 insertions(+), 106 deletions(-) create mode 100644 common/tasks/parted.py delete mode 100644 providers/one/tasks/filesystem.py diff --git a/common/tasks/filesystem.py b/common/tasks/filesystem.py index 5254772..37c37a8 100644 --- a/common/tasks/filesystem.py +++ b/common/tasks/filesystem.py @@ -13,6 +13,7 @@ class FormatVolume(Task): dev_path = info.bootstrap_device['path'] mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) log_check_call([mkfs, dev_path]) + info.bootstrap_device['partitions'] = {'root_path': info.bootstrap_device['path']} class TuneVolumeFS(Task): @@ -21,9 +22,8 @@ class TuneVolumeFS(Task): after = [FormatVolume] def run(self, info): - dev_path = info.bootstrap_device['path'] # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) + log_check_call(['/sbin/tune2fs', '-i', '0', info.bootstrap_device['partitions']['root_path']]) class AddXFSProgs(Task): @@ -61,7 +61,7 @@ class MountVolume(Task): log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], - info.bootstrap_device['path'], + info.bootstrap_device['partitions']['root_path'], info.root]) @@ -120,7 +120,14 @@ class ModifyFstab(Task): if info.manifest.volume['filesystem'].lower() == 'xfs': mount_opts.append('nobarrier') fstab_path = os.path.join(info.root, 'etc/fstab') + + device = '/dev/sda1' + if info.manifest.virtualization == 'virtio': + device = '/dev/vda1' + if info.manifest.virtualization == 'pvm': + device = '/dev/xvda1' with open(fstab_path, 'a') as fstab: - fstab.write(('/dev/xvda1 / {filesystem} {mount_opts} 1 1\n' - .format(filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts)))) + fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' + .format(device=device, + filesystem=info.manifest.volume['filesystem'].lower(), + mount_opts=','.join(mount_opts))) diff --git a/common/tasks/loopback.py b/common/tasks/loopback.py index 631c995..8d483ca 100644 --- a/common/tasks/loopback.py +++ b/common/tasks/loopback.py @@ -17,6 +17,18 @@ class Create(Task): 'bs=1M', 'seek='+str(info.manifest.volume['size']), 'count=0']) +class CreateQemuImg(Task): + description = 'Creating a loopback volume with qemu' + phase = phases.volume_preparation + + def run(self, info): + loopback_filename = 'loopback-{id:x}.img'.format(id=info.run_id) + import os.path + info.loopback_file = os.path.join(info.manifest.volume['loopback_dir'], loopback_filename) + log_check_call(['/usr/bin/qemu-img', 'create', '-f', 'raw', + info.loopback_file, str(info.manifest.volume['size'])+'M']) + + class Attach(Task): description = 'Attaching the loopback volume' phase = phases.volume_creation diff --git a/common/tasks/parted.py b/common/tasks/parted.py new file mode 100644 index 0000000..e1916c9 --- /dev/null +++ b/common/tasks/parted.py @@ -0,0 +1,52 @@ +from base import Task +from common import phases +from common.tools import log_check_call +from common.tasks import UnmountVolume + + +class PartitionVolume(Task): + description = 'Partitioning the volume' + phase = phases.volume_preparation + + def run(self, info): + # parted + log_check_call(['parted', '-a', 'optimal', + '-s', info.bootstrap_device['path'], + 'mklabel', 'msdos']) + log_check_call(['parted', '-a', 'optimal', + '-s', info.bootstrap_device['path'], + '--', 'mkpart', 'primary', 'ext4', '32k', '-1']) + log_check_call(['parted', + '-s', info.bootstrap_device['path'], + '--', 'set', '1', 'boot', 'on']) + + +class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + after = [Partition] + + def run(self, info): + log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) + root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper')+'p1' + info.bootstrap_device['partitions'] = {'root_path': root_partition_path} + + +class FormatPartitions(Task): + description = 'Formatting the partitions' + phase = phases.volume_preparation + after = [MapPartitions] + + def run(self, info): + log_check_call(['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), + '-m', '1', '-v', info.bootstrap_device['partitions']['root_path']]) + + +class UnmapPartitions(Task): + description = 'Removing volume partitions mapping' + phase = phases.volume_unmounting + after = [UnmountVolume] + + def run(self, info): + log_check_call(['kpartx', '-d', info.loopback_file]) + del info.bootstrap_device['partitions']['root_path'] diff --git a/manifests/one-ide.manifest.json b/manifests/one-ide.manifest.json index 0180534..34bce75 100644 --- a/manifests/one-ide.manifest.json +++ b/manifests/one-ide.manifest.json @@ -7,9 +7,7 @@ }, "bootstrapper": { - "mount_dir": "/mnt/target", - "image_file": "/tmp/one.img", - "tarball": true + "mount_dir": "/target" }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", diff --git a/manifests/one-virtio.manifest.json b/manifests/one-virtio.manifest.json index 03a9b2a..29b62cc 100644 --- a/manifests/one-virtio.manifest.json +++ b/manifests/one-virtio.manifest.json @@ -7,9 +7,7 @@ }, "bootstrapper": { - "mount_dir": "/mnt/target", - "image_file": "/tmp/one.img", - "tarball": true + "mount_dir": "/target" }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 49eec72..430e99b 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -2,8 +2,8 @@ from manifest import Manifest from tasks import packages from common.tasks import packages as common_packages from tasks import host -from tasks import filesystem -from common.tasks import filesystem as common_filesystem +from common.tasks import parted +from common.tasks import filesystem from common.tasks import bootstrap from common.tasks import locale from common.tasks import apt @@ -27,12 +27,14 @@ def tasks(tasklist, manifest): common_packages.ImagePackages(), host.CheckPackages(), - filesystem.FormatVolume(), - common_filesystem.CreateMountDir(), + parted.PartitionVolume(), + parted.MapPartitions(), + parted.FormatPartitions(), + filesystem.CreateMountDir(), filesystem.MountVolume(), bootstrap.Bootstrap(), - common_filesystem.MountSpecials(), + filesystem.MountSpecials(), locale.GenerateLocale(), context.OpenNebulaContext(), locale.SetTimezone(), @@ -59,10 +61,11 @@ def tasks(tasklist, manifest): apt.PurgeUnusedPackages(), apt.AptClean(), apt.EnableDaemonAutostart(), - common_filesystem.UnmountSpecials(), + filesystem.UnmountSpecials(), filesystem.UnmountVolume(), - common_filesystem.DeleteMountDir()) + filesystem.UnmapPartitions(), + filesystem.DeleteMountDir()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) @@ -81,6 +84,7 @@ def rollback_tasks(tasklist, tasks_completed, manifest): if task in completed and counter not in completed: tasklist.add(counter()) - counter_task(common_filesystem.CreateMountDir, common_filesystem.DeleteMountDir) + counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) + counter_task(filesystem.MapPartitions, filesystem.UnmapPartitions) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) - counter_task(common_filesystem.MountSpecials, common_filesystem.UnmountSpecials) + counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/one/manifest.py b/providers/one/manifest.py index d2e0f2c..f3f26a6 100644 --- a/providers/one/manifest.py +++ b/providers/one/manifest.py @@ -13,3 +13,5 @@ class Manifest(base.Manifest): self.credentials = data['credentials'] self.virtualization = data['virtualization'] self.image = data['image'] + if 'loopback_dir' not in self.volume and self.volume['backing'].lower() == 's3': + self.volume['loopback_dir'] = '/tmp' diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py deleted file mode 100644 index 50d6dd8..0000000 --- a/providers/one/tasks/filesystem.py +++ /dev/null @@ -1,85 +0,0 @@ -from base import Task -from common import phases -from common.exceptions import TaskError -from common.tools import log_check_call -from common.filesystem import filesystem - - -class FormatVolume(Task): - description = 'Formatting the volume' - phase = phases.volume_preparation - - def run(self, info): - mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] - log_check_call(mkmount) - - loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] - log_check_call(loopcmd) - - # parted - log_check_call(['parted', '-a', 'optimal', '-s', '/dev/loop0', "mklabel", "msdos"]) - log_check_call(['parted', '-a', 'optimal', '-s', '/dev/loop0', "--", "mkpart", "primary", "ext4", "32k", "-1"]) - log_check_call(['parted', '-s', '/dev/loop0', "--", "set", "1", "boot", "on"]) - - log_check_call(['kpartx', '-a', '-v', '/dev/loop0']) - mkfs = ['/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/mapper/loop0p1'] - log_check_call(mkfs) - - -class TuneVolumeFS(Task): - description = 'Tuning the bootstrap volume filesystem' - phase = phases.volume_preparation - after = [FormatVolume] - - def run(self, info): - dev_path = '/dev/mapper/loop0p1' - # Disable the time based filesystem check - log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) - - -class MountVolume(Task): - description = 'Mounting the bootstrap volume' - phase = phases.volume_mounting - after = [filesystem.CreateMountDir] - - def run(self, info): - with open('/proc/mounts') as mounts: - for mount in mounts: - if info.root in mount: - msg = 'Something is already mounted at {root}'.format(root=info.root) - raise TaskError(msg) - - log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/mapper/loop0p1', info.root]) - - -class UnmountVolume(Task): - description = 'Unmounting the bootstrap volume' - phase = phases.volume_unmounting - after = [filesystem.UnmountSpecials] - - def run(self, info): - log_check_call(['/bin/umount', info.root]) - log_check_call(['kpartx', '-d', info.manifest.bootstrapper['image_file']]) - - -class ModifyFstab(Task): - description = 'Adding root volume to the fstab' - phase = phases.system_modification - - def run(self, info): - import os.path - mount_opts = ['defaults'] - if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: - mount_opts.append('barrier=0') - if info.manifest.volume['filesystem'].lower() == 'xfs': - mount_opts.append('nobarrier') - fstab_path = os.path.join(info.root, 'etc/fstab') - with open(fstab_path, 'a') as fstab: - device = '/dev/sda1' - if info.manifest.virtualization == 'virtio': - device = '/dev/vda1' - - fstab.write('{device} / {filesystem} {mount_opts} 1 1\n' - .format(device=device, - filesystem=info.manifest.volume['filesystem'].lower(), - mount_opts=','.join(mount_opts)))