From c81045cc6e0f6a2c90002ab50219a15c337630f1 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 3 May 2014 12:22:51 +0200 Subject: [PATCH] Simplify adding tasks There is now a get_standard_groups function --- bootstrapvz/common/task_groups.py | 169 ++++++++++++------- bootstrapvz/providers/azure/__init__.py | 28 +-- bootstrapvz/providers/ec2/__init__.py | 63 +++---- bootstrapvz/providers/gce/__init__.py | 27 +-- bootstrapvz/providers/kvm/__init__.py | 28 +-- bootstrapvz/providers/virtualbox/__init__.py | 31 +--- 6 files changed, 130 insertions(+), 216 deletions(-) diff --git a/bootstrapvz/common/task_groups.py b/bootstrapvz/common/task_groups.py index 5c982ec..882476f 100644 --- a/bootstrapvz/common/task_groups.py +++ b/bootstrapvz/common/task_groups.py @@ -10,89 +10,130 @@ from tasks import cleanup from tasks import apt from tasks import security from tasks import locale +from tasks import network -base_set = [workspace.CreateWorkspace, - bootstrap.AddRequiredCommands, - host.CheckExternalCommands, - bootstrap.Bootstrap, - workspace.DeleteWorkspace, - ] -volume_set = [volume.Attach, - volume.Detach, - filesystem.AddRequiredCommands, - filesystem.Format, - filesystem.FStab, - ] +def get_standard_groups(manifest): + group = [] + group.extend(get_base_group(manifest)) + group.extend(volume_group) + if manifest.volume['partitions']['type'] != 'none': + group.extend(partitioning_group) + if 'boot' in manifest.volume['partitions']: + group.extend(boot_partition_group) + group.extend(mounting_group) + group.extend(get_fs_specific_group(manifest)) + group.extend(get_network_group(manifest)) + group.extend(get_apt_group(manifest)) + group.extend(locale_group) + group.extend(bootloader_group.get(manifest.system['bootloader'], [])) + group.extend(cleanup_group) + return group -partitioning_set = [partitioning.AddRequiredCommands, - partitioning.PartitionVolume, - partitioning.MapPartitions, - partitioning.UnmapPartitions, - ] -boot_partition_set = [filesystem.CreateBootMountDir, - filesystem.MountBoot, - ] +def get_base_group(manifest): + group = [workspace.CreateWorkspace, + bootstrap.AddRequiredCommands, + host.CheckExternalCommands, + bootstrap.Bootstrap, + workspace.DeleteWorkspace, + ] + if manifest.bootstrapper.get('tarball', False): + group.append(bootstrap.MakeTarball) + return group -mounting_set = [filesystem.CreateMountDir, - filesystem.MountRoot, - filesystem.MountSpecials, - filesystem.UnmountRoot, - filesystem.DeleteMountDir, + +volume_group = [volume.Attach, + volume.Detach, + filesystem.AddRequiredCommands, + filesystem.Format, + filesystem.FStab, ] -ssh_set = [security.DisableSSHPasswordAuthentication, - security.DisableSSHDNSLookup, - cleanup.ShredHostkeys, - ] +partitioning_group = [partitioning.AddRequiredCommands, + partitioning.PartitionVolume, + partitioning.MapPartitions, + partitioning.UnmapPartitions, + ] + +boot_partition_group = [filesystem.CreateBootMountDir, + filesystem.MountBoot, + ] + +mounting_group = [filesystem.CreateMountDir, + filesystem.MountRoot, + filesystem.MountSpecials, + filesystem.UnmountRoot, + filesystem.DeleteMountDir, + ] + +ssh_group = [security.DisableSSHPasswordAuthentication, + security.DisableSSHDNSLookup, + cleanup.ShredHostkeys, + ] -def get_apt_set(manifest): - base = [apt.AddDefaultSources, - apt.WriteSources, - apt.DisableDaemonAutostart, - apt.AptUpdate, - apt.AptUpgrade, - packages.InstallPackages, - apt.PurgeUnusedPackages, - apt.AptClean, - apt.EnableDaemonAutostart, - ] +def get_network_group(manifest): + group = [network.ConfigureNetworkIF, + network.RemoveDNSInfo] + if manifest.system.get('hostname', False): + group.append(network.SetHostname) + else: + group.append(network.RemoveHostname) + return group + + +def get_apt_group(manifest): + group = [apt.AddDefaultSources, + apt.WriteSources, + apt.DisableDaemonAutostart, + apt.AptUpdate, + apt.AptUpgrade, + packages.InstallPackages, + apt.PurgeUnusedPackages, + apt.AptClean, + apt.EnableDaemonAutostart, + ] if 'sources' in manifest.packages: - base.append(apt.AddManifestSources) + group.append(apt.AddManifestSources) if 'trusted-keys' in manifest.packages: - base.append(apt.InstallTrustedKeys) + group.append(apt.InstallTrustedKeys) if 'preferences' in manifest.packages: - base.append(apt.AddManifestPreferences) - base.append(apt.WritePreferences) + group.append(apt.AddManifestPreferences) + group.append(apt.WritePreferences) if 'install' in manifest.packages: - base.append(packages.AddManifestPackages) + group.append(packages.AddManifestPackages) if manifest.packages.get('install_standard', False): - base.append(packages.AddTaskselStandardPackages) - return base + group.append(packages.AddTaskselStandardPackages) + return group -locale_set = [locale.LocaleBootstrapPackage, - locale.GenerateLocale, - locale.SetTimezone, - ] +locale_group = [locale.LocaleBootstrapPackage, + locale.GenerateLocale, + locale.SetTimezone, + ] -bootloader_set = {'grub': [boot.AddGrubPackage, boot.ConfigureGrub, boot.InstallGrub], - 'extlinux': [boot.AddExtlinuxPackage, boot.InstallExtLinux], - } +bootloader_group = {'grub': [boot.AddGrubPackage, boot.ConfigureGrub, boot.InstallGrub], + 'extlinux': [boot.AddExtlinuxPackage, boot.InstallExtLinux], + } -def get_fs_specific_set(partitions): - task_set = {'ext2': [filesystem.TuneVolumeFS], - 'ext3': [filesystem.TuneVolumeFS], - 'ext4': [filesystem.TuneVolumeFS], - 'xfs': [filesystem.AddXFSProgs], - } - tasks = set() +def get_fs_specific_group(manifest): + partitions = manifest.volume['partitions'] + fs_specific_tasks = {'ext2': [filesystem.TuneVolumeFS], + 'ext3': [filesystem.TuneVolumeFS], + 'ext4': [filesystem.TuneVolumeFS], + 'xfs': [filesystem.AddXFSProgs], + } + group = set() if 'boot' in partitions: - tasks.update(task_set.get(partitions['boot']['filesystem'], [])) + group.update(fs_specific_tasks.get(partitions['boot']['filesystem'], [])) if 'root' in partitions: - tasks.update(task_set.get(partitions['root']['filesystem'], [])) - return tasks + group.update(fs_specific_tasks.get(partitions['root']['filesystem'], [])) + return list(group) + + +cleanup_group = [cleanup.ClearMOTD, + cleanup.CleanTMP, + ] diff --git a/bootstrapvz/providers/azure/__init__.py b/bootstrapvz/providers/azure/__init__.py index ff9fedf..0d7d015 100644 --- a/bootstrapvz/providers/azure/__init__.py +++ b/bootstrapvz/providers/azure/__init__.py @@ -28,44 +28,18 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): from bootstrapvz.common import task_groups - taskset.update(task_groups.base_set) - taskset.update(task_groups.volume_set) - taskset.update(task_groups.mounting_set) - taskset.update(task_groups.get_apt_set(manifest)) - taskset.update(task_groups.locale_set) - - taskset.update(task_groups.bootloader_set.get(manifest.system['bootloader'])) - - if manifest.volume['partitions']['type'] != 'none': - taskset.update(task_groups.partitioning_set) - - if manifest.system.get('hostname', False): - taskset.add(network.SetHostname) - else: - taskset.add(network.RemoveHostname) + taskset.update(task_groups.get_standard_groups(manifest)) taskset.update([tasks.packages.DefaultPackages, loopback.Create, security.EnableShadowConfig, - network.RemoveDNSInfo, - network.ConfigureNetworkIF, initd.AddSSHKeyGeneration, initd.InstallInitScripts, tasks.packages.Waagent, tasks.boot.ConfigureGrub, - cleanup.ClearMOTD, - cleanup.CleanTMP, tasks.image.ConvertToVhd, ]) - if manifest.bootstrapper.get('tarball', False): - taskset.add(bootstrap.MakeTarball) - - taskset.update(task_groups.get_fs_specific_set(manifest.volume['partitions'])) - - if 'boot' in manifest.volume['partitions']: - taskset.update(task_groups.boot_partition_set) - def resolve_rollback_tasks(taskset, manifest, counter_task): counter_task(loopback.Create, volume.Delete) diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py index 1cac583..59c47a6 100644 --- a/bootstrapvz/providers/ec2/__init__.py +++ b/bootstrapvz/providers/ec2/__init__.py @@ -52,19 +52,8 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): from bootstrapvz.common import task_groups - taskset.update(task_groups.base_set) - taskset.update(task_groups.mounting_set) - taskset.update(task_groups.get_apt_set(manifest)) - taskset.update(task_groups.locale_set) - taskset.update(task_groups.ssh_set) - - if manifest.volume['partitions']['type'] != 'none': - taskset.update(task_groups.partitioning_set) - - if manifest.system.get('hostname', False): - taskset.add(network.SetHostname) - else: - taskset.add(network.RemoveHostname) + taskset.update(task_groups.get_standard_groups(manifest)) + taskset.update(task_groups.ssh_group) taskset.update([tasks.host.AddExternalCommands, tasks.packages.DefaultPackages, @@ -75,8 +64,6 @@ def resolve_tasks(taskset, manifest): boot.BlackListModules, boot.DisableGetTTYs, security.EnableShadowConfig, - network.RemoveDNSInfo, - network.ConfigureNetworkIF, tasks.network.EnableDHCPCDDNS, initd.AddExpandRoot, initd.AddSSHKeyGeneration, @@ -84,8 +71,6 @@ def resolve_tasks(taskset, manifest): tasks.initd.AddEC2InitScripts, initd.InstallInitScripts, initd.AdjustExpandRootScript, - cleanup.ClearMOTD, - cleanup.CleanTMP, tasks.ami.RegisterAMI, ]) @@ -93,36 +78,30 @@ def resolve_tasks(taskset, manifest): if manifest.system['bootloader'] == 'pvgrub': taskset.add(boot.AddGrubPackage) taskset.add(tasks.boot.ConfigurePVGrub) - else: - taskset.update(task_groups.bootloader_set.get(manifest.system['bootloader'])) - backing_specific_tasks = {'ebs': [tasks.host.GetInstanceMetadata, - tasks.ebs.Create, - tasks.ebs.Attach, - filesystem.FStab, - tasks.ebs.Snapshot], - 's3': [loopback.AddRequiredCommands, - tasks.host.SetRegion, - loopback.Create, - volume.Attach, - tasks.filesystem.S3FStab, - tasks.ami.BundleImage, - tasks.ami.UploadImage, - tasks.ami.RemoveBundle]} - taskset.update(backing_specific_tasks.get(manifest.volume['backing'].lower())) + if manifest.volume['backing'].lower() == 'ebs': + taskset.update([tasks.host.GetInstanceMetadata, + tasks.ebs.Create, + tasks.ebs.Snapshot, + ]) + taskset.add(tasks.ebs.Attach) + taskset.discard(volume.Attach) + + if manifest.volume['backing'].lower() == 's3': + taskset.update([loopback.AddRequiredCommands, + tasks.host.SetRegion, + loopback.Create, + tasks.filesystem.S3FStab, + tasks.ami.BundleImage, + tasks.ami.UploadImage, + tasks.ami.RemoveBundle, + ]) + taskset.discard(filesystem.FStab) + taskset.update([filesystem.Format, - volume.Detach, volume.Delete, ]) - if manifest.bootstrapper.get('tarball', False): - taskset.add(bootstrap.MakeTarball) - - taskset.update(task_groups.get_fs_specific_set(manifest.volume['partitions'])) - - if 'boot' in manifest.volume['partitions']: - taskset.update(task_groups.boot_partition_set) - def resolve_rollback_tasks(taskset, manifest, counter_task): counter_task(tasks.ebs.Create, volume.Delete) diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py index cf849f8..a9c464d 100644 --- a/bootstrapvz/providers/gce/__init__.py +++ b/bootstrapvz/providers/gce/__init__.py @@ -4,14 +4,9 @@ import tasks.configuration import tasks.image import tasks.host import tasks.packages -from bootstrapvz.common.tasks import volume from bootstrapvz.common.tasks import loopback -from bootstrapvz.common.tasks import partitioning -from bootstrapvz.common.tasks import filesystem from bootstrapvz.common.tasks import security -from bootstrapvz.common.tasks import network from bootstrapvz.common.tasks import initd -from bootstrapvz.common.tasks import workspace import bootstrapvz.plugins.cloud_init.tasks @@ -26,17 +21,7 @@ def validate_manifest(data, validator, error): def resolve_tasks(tasklist, manifest): - import bootstrapvz.common.task_groups - tasklist.update(bootstrapvz.common.task_groups.base_set) - tasklist.update(bootstrapvz.common.task_groups.volume_set) - tasklist.update(bootstrapvz.common.task_groups.mounting_set) - tasklist.update(bootstrapvz.common.task_groups.get_apt_set(manifest)) - tasklist.update(bootstrapvz.common.task_groups.locale_set) - - tasklist.update(bootstrapvz.common.task_groups.bootloader_set.get(manifest.system['bootloader'])) - - if manifest.volume['partitions']['type'] != 'none': - tasklist.update(bootstrapvz.common.task_groups.partitioning_set) + tasklist.update(bootstrapvz.common.task_groups.get_standard_groups(manifest)) tasklist.update([bootstrapvz.plugins.cloud_init.tasks.AddBackports, loopback.Create, @@ -49,14 +34,9 @@ def resolve_tasks(tasklist, manifest): tasks.configuration.GatherReleaseInformation, security.EnableShadowConfig, - network.RemoveDNSInfo, - network.RemoveHostname, - network.ConfigureNetworkIF, tasks.host.DisableIPv6, - tasks.host.SetHostname, tasks.boot.ConfigureGrub, initd.AddSSHKeyGeneration, - initd.InstallInitScripts, tasks.apt.CleanGoogleRepositoriesAndKeys, loopback.MoveImage, @@ -68,11 +48,6 @@ def resolve_tasks(tasklist, manifest): if 'gce_project' in manifest.image: tasklist.add(tasks.image.RegisterImage) - tasklist.update(bootstrapvz.common.task_groups.get_fs_specific_set(manifest.volume['partitions'])) - - if 'boot' in manifest.volume['partitions']: - tasklist.update(bootstrapvz.common.task_groups.boot_partition_set) - def resolve_rollback_tasks(tasklist, manifest, counter_task): counter_task(loopback.Create, volume.Delete) diff --git a/bootstrapvz/providers/kvm/__init__.py b/bootstrapvz/providers/kvm/__init__.py index f712a87..2343ef1 100644 --- a/bootstrapvz/providers/kvm/__init__.py +++ b/bootstrapvz/providers/kvm/__init__.py @@ -26,46 +26,20 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): from bootstrapvz.common import task_groups - taskset.update(task_groups.base_set) - taskset.update(task_groups.volume_set) - taskset.update(task_groups.mounting_set) - taskset.update(task_groups.get_apt_set(manifest)) - taskset.update(task_groups.locale_set) - - taskset.update(task_groups.bootloader_set.get(manifest.system['bootloader'])) - - if manifest.volume['partitions']['type'] != 'none': - taskset.update(task_groups.partitioning_set) - - if manifest.system.get('hostname', False): - taskset.add(network.SetHostname) - else: - taskset.add(network.RemoveHostname) + taskset.update(task_groups.get_standard_groups(manifest)) taskset.update([tasks.packages.DefaultPackages, loopback.Create, security.EnableShadowConfig, - network.RemoveDNSInfo, - network.ConfigureNetworkIF, initd.AddSSHKeyGeneration, initd.InstallInitScripts, - cleanup.ClearMOTD, - cleanup.CleanTMP, loopback.MoveImage, ]) - if manifest.bootstrapper.get('tarball', False): - taskset.add(bootstrap.MakeTarball) - if manifest.bootstrapper.get('virtio', []): from tasks import virtio taskset.update([virtio.VirtIO]) - taskset.update(task_groups.get_fs_specific_set(manifest.volume['partitions'])) - - if 'boot' in manifest.volume['partitions']: - taskset.update(task_groups.boot_partition_set) - def resolve_rollback_tasks(taskset, manifest, counter_task): counter_task(loopback.Create, volume.Delete) diff --git a/bootstrapvz/providers/virtualbox/__init__.py b/bootstrapvz/providers/virtualbox/__init__.py index 1a63993..882c971 100644 --- a/bootstrapvz/providers/virtualbox/__init__.py +++ b/bootstrapvz/providers/virtualbox/__init__.py @@ -26,34 +26,13 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): from bootstrapvz.common import task_groups - taskset.update(task_groups.base_set) - taskset.update(task_groups.volume_set) - taskset.update(task_groups.mounting_set) - taskset.update(task_groups.get_apt_set(manifest)) - taskset.update(task_groups.locale_set) - - taskset.update(task_groups.bootloader_set.get(manifest.system['bootloader'])) - - if manifest.volume['partitions']['type'] != 'none': - taskset.update(task_groups.partitioning_set) - - if manifest.system.get('hostname', False): - taskset.add(network.SetHostname) - else: - taskset.add(network.RemoveHostname) + taskset.update(task_groups.get_standard_groups(manifest)) taskset.update([tasks.packages.DefaultPackages, - loopback.Create, - security.EnableShadowConfig, - network.RemoveDNSInfo, - network.ConfigureNetworkIF, initd.AddSSHKeyGeneration, initd.InstallInitScripts, - cleanup.ClearMOTD, - cleanup.CleanTMP, - loopback.MoveImage, ]) @@ -64,14 +43,6 @@ def resolve_tasks(taskset, manifest): guest_additions.InstallGuestAdditions, ]) - if manifest.bootstrapper.get('tarball', False): - taskset.add(bootstrap.MakeTarball) - - taskset.update(task_groups.get_fs_specific_set(manifest.volume['partitions'])) - - if 'boot' in manifest.volume['partitions']: - taskset.update(task_groups.boot_partition_set) - def resolve_rollback_tasks(taskset, manifest, counter_task): counter_task(loopback.Create, volume.Delete)