From e1ab4dc1aeadee6d7c61b1084f8dab473245415e Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Wed, 9 Oct 2013 00:09:34 +0200 Subject: [PATCH] EC2 provider can now bootstrap EBS volumes again Use tasksets in EC2 provider --- base/fs/__init__.py | 2 +- base/fs/partitions/abstract.py | 8 -- base/fs/partitions/base.py | 3 + base/fs/partitions/single.py | 2 - common/task_sets.py | 4 + providers/ec2/__init__.py | 143 +++++++++++----------- providers/ec2/{volume.py => ebsvolume.py} | 15 ++- providers/ec2/manifest-schema.json | 6 +- providers/ec2/manifest.py | 17 ++- providers/ec2/tasks/ami.py | 9 +- providers/ec2/tasks/ebs.py | 11 +- providers/ec2/tasks/packages.py | 7 +- providers/virtualbox/__init__.py | 3 - 13 files changed, 119 insertions(+), 111 deletions(-) rename providers/ec2/{volume.py => ebsvolume.py} (82%) diff --git a/base/fs/__init__.py b/base/fs/__init__.py index 6b4763b..5817220 100644 --- a/base/fs/__init__.py +++ b/base/fs/__init__.py @@ -2,7 +2,7 @@ def load_volume(data): from common.fs.loopbackvolume import LoopbackVolume - from providers.ec2.volume import EBSVolume + from providers.ec2.ebsvolume import EBSVolume from providers.virtualbox.volume import VirtualBoxVolume from partitionmaps.gpt import GPTPartitionMap from partitionmaps.mbr import MBRPartitionMap diff --git a/base/fs/partitions/abstract.py b/base/fs/partitions/abstract.py index 9f80618..bb2354c 100644 --- a/base/fs/partitions/abstract.py +++ b/base/fs/partitions/abstract.py @@ -1,5 +1,4 @@ from abc import ABCMeta -from abc import abstractmethod from common.tools import log_check_call from common.fsm_proxy import FSMProxy @@ -26,13 +25,6 @@ class AbstractPartition(FSMProxy): [uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path]) return uuid - def create(self, volume): - self.fsm.create(volume=volume) - - @abstractmethod - def _before_create(self, e): - pass - def _before_format(self, e): mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem) log_check_call([mkfs, self.device_path]) diff --git a/base/fs/partitions/base.py b/base/fs/partitions/base.py index 0769de5..3f5792f 100644 --- a/base/fs/partitions/base.py +++ b/base/fs/partitions/base.py @@ -18,6 +18,9 @@ class BasePartition(AbstractPartition): self.previous = previous super(BasePartition, self).__init__(size, filesystem) + def create(self, volume): + self.fsm.create(volume=volume) + def get_index(self): if self.previous is None: return 1 diff --git a/base/fs/partitions/single.py b/base/fs/partitions/single.py index dd82477..2017c91 100644 --- a/base/fs/partitions/single.py +++ b/base/fs/partitions/single.py @@ -2,6 +2,4 @@ from abstract import AbstractPartition class SinglePartition(AbstractPartition): - - def _before_create(self, e): pass diff --git a/common/task_sets.py b/common/task_sets.py index f29f0e1..07e7a31 100644 --- a/common/task_sets.py +++ b/common/task_sets.py @@ -1,6 +1,7 @@ from common.tasks import workspace from common.tasks import packages from common.tasks import host +from common.tasks import bootstrap from common.tasks import volume from common.tasks import filesystem from common.tasks import partitioning @@ -13,11 +14,14 @@ base_set = [workspace.CreateWorkspace, packages.HostPackages, packages.ImagePackages, host.CheckPackages, + bootstrap.Bootstrap, workspace.DeleteWorkspace, ] volume_set = [volume.Attach, volume.Detach, + filesystem.Format, + filesystem.FStab, ] partitioning_set = [partitioning.PartitionVolume, diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 68c47c5..9741c7a 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -1,18 +1,15 @@ from manifest import Manifest import logging from tasks import packages -from common.tasks import packages as common_packages from tasks import connection from tasks import host -from common.tasks import host as common_host from tasks import ami -from common.tasks import volume +from common.tasks import volume as volume_tasks from tasks import ebs +from common.tasks import partitioning from common.tasks import loopback from common.tasks import filesystem from common.tasks import bootstrap -from common.tasks import locale -from common.tasks import apt from tasks import boot from common.tasks import boot as common_boot from common.tasks import security @@ -21,6 +18,7 @@ from common.tasks import network as common_network from tasks import initd from common.tasks import initd as common_initd from common.tasks import cleanup +from common.tasks import workspace def initialize(): @@ -29,74 +27,66 @@ def initialize(): def tasks(tasklist, manifest): - tasklist.add(packages.HostPackages(), - common_packages.HostPackages(), - packages.ImagePackages(), - common_packages.ImagePackages(), - common_host.CheckPackages(), - connection.GetCredentials(), - host.GetInfo(), - ami.AMIName(), - connection.Connect(), + from common.task_sets import base_set + from common.task_sets import mounting_set + from common.task_sets import apt_set + from common.task_sets import locale_set + from common.task_sets import ssh_set + tasklist.add(*base_set) + tasklist.add(*mounting_set) + tasklist.add(*apt_set) + tasklist.add(*locale_set) + tasklist.add(*ssh_set) - filesystem.FormatVolume(), - filesystem.CreateMountDir(), - filesystem.MountVolume(), + if manifest.volume['partitions']['type'] != 'none': + from common.task_sets import partitioning_set + tasklist.add(*partitioning_set) - bootstrap.Bootstrap(), - filesystem.MountSpecials(), - locale.GenerateLocale(), - locale.SetTimezone(), - apt.DisableDaemonAutostart(), - apt.AptSources(), - apt.AptUpgrade(), - boot.ConfigureGrub(), - filesystem.FStab(), - common_boot.BlackListModules(), - common_boot.DisableGetTTYs(), - security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), - security.DisableSSHDNSLookup(), - common_network.RemoveDNSInfo(), - common_network.ConfigureNetworkIF(), - network.EnableDHCPCDDNS(), - common_initd.ResolveInitScripts(), - initd.AddEC2InitScripts(), - common_initd.InstallInitScripts(), - cleanup.ClearMOTD(), - cleanup.ShredHostkeys(), - cleanup.CleanTMP(), - apt.PurgeUnusedPackages(), - apt.AptClean(), - apt.EnableDaemonAutostart(), - filesystem.UnmountSpecials(), + tasklist.add(packages.HostPackages, + packages.ImagePackages, + connection.GetCredentials, + host.GetInfo, + ami.AMIName, + connection.Connect, - filesystem.UnmountVolume(), - filesystem.DeleteMountDir(), - ami.RegisterAMI()) + boot.ConfigureGrub, + common_boot.BlackListModules, + common_boot.DisableGetTTYs, + security.EnableShadowConfig, + common_network.RemoveDNSInfo, + common_network.ConfigureNetworkIF, + network.EnableDHCPCDDNS, + common_initd.ResolveInitScripts, + initd.AddEC2InitScripts, + common_initd.InstallInitScripts, + cleanup.ClearMOTD, + cleanup.CleanTMP, + + ami.RegisterAMI) + + backing_specific_tasks = {'ebs': [ebs.Create, + ebs.Attach, + ebs.Snapshot], + 's3': [loopback.Create, + volume_tasks.Attach, + ami.BundleImage, + ami.UploadImage, + ami.RemoveBundle]} + tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + tasklist.add(filesystem.Format, + filesystem.FStab, + volume_tasks.Detach, + volume_tasks.Delete) if manifest.bootstrapper.get('tarball', False): - tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.MakeTarball) - backing_specific_tasks = {'ebs': [ebs.Create(), - volume.Attach(), - volume.Detach(), - ebs.Snapshot(), - volume.Delete()], - 's3': [loopback.Create(), - volume.Attach(), - volume.Detach(), - ami.BundleImage(), - ami.UploadImage(), - volume.Delete(), - ami.RemoveBundle()]} - tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower())) + from common.task_sets import get_fs_specific_set + tasklist.add(*get_fs_specific_set(manifest.volume['partitions'])) - filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()], - 'ext2': [filesystem.TuneVolumeFS()], - 'ext3': [filesystem.TuneVolumeFS()], - 'ext4': [filesystem.TuneVolumeFS()]} - tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower())) + if 'boot' in manifest.volume['partitions']: + from common.task_sets import boot_partition_set + tasklist.add(*boot_partition_set) def rollback_tasks(tasklist, tasks_completed, manifest): @@ -104,14 +94,19 @@ def rollback_tasks(tasklist, tasks_completed, manifest): def counter_task(task, counter): if task in completed and counter not in completed: - tasklist.add(counter()) + tasklist.add(counter) - if manifest.volume['backing'].lower() == 'ebs': - counter_task(ebs.Create, volume.Delete) - counter_task(volume.Attach, volume.Detach) - if manifest.volume['backing'].lower() == 's3': - counter_task(loopback.Create, volume.Delete) - counter_task(volume.Attach, volume.Detach) + counter_task(ebs.Create, volume_tasks.Delete) + counter_task(ebs.Attach, volume_tasks.Detach) + + counter_task(loopback.Create, volume_tasks.Delete) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) - counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) + counter_task(filesystem.MountBoot, filesystem.UnmountBoot) + counter_task(volume_tasks.Attach, volume_tasks.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/providers/ec2/volume.py b/providers/ec2/ebsvolume.py similarity index 82% rename from providers/ec2/volume.py rename to providers/ec2/ebsvolume.py index 16e4e96..29e9e41 100644 --- a/providers/ec2/volume.py +++ b/providers/ec2/ebsvolume.py @@ -5,18 +5,18 @@ import time class EBSVolume(Volume): - volume = None - def create(self, conn, zone): - super(EBSVolume, self).create(self) + self.fsm.create(connection=conn, zone=zone) + + def _before_create(self, e): + conn = e.connection + zone = e.zone import math - # TODO: Warn if volume size is not a multiple of 1024 - size = int(math.ceil(self.partition_map.get_volume_size() / 1024)) + size = int(math.ceil(self.partition_map.get_total_size() / 1024)) self.volume = conn.create_volume(size, zone) while self.volume.volume_state() != 'available': time.sleep(5) self.volume.update() - self.created = True def attach(self, instance_id): self.fsm.attach(instance_id=instance_id) @@ -25,7 +25,7 @@ class EBSVolume(Volume): instance_id = e.instance_id import os.path import string - for letter in string.ascii_lowercase: + for letter in string.ascii_lowercase[5:]: dev_path = os.path.join('/dev', 'xvd' + letter) if not os.path.exists(dev_path): self.device_path = dev_path @@ -47,7 +47,6 @@ class EBSVolume(Volume): self.volume.update() def _before_delete(self, e): - super(EBSVolume, self).delete(self) self.volume.delete() def snapshot(self): diff --git a/providers/ec2/manifest-schema.json b/providers/ec2/manifest-schema.json index ffa360d..43fda5d 100644 --- a/providers/ec2/manifest-schema.json +++ b/providers/ec2/manifest-schema.json @@ -20,13 +20,9 @@ "backing": { "type": "string", "enum": ["ebs", "s3"] - }, - "filesystem": { - "type": "string", - "enum": ["ext2", "ext3", "ext4", "xfs"] } }, - "required": ["backing", "filesystem"] + "required": ["backing"] } }, "required": ["volume"] diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py index abab855..9bfea5c 100644 --- a/providers/ec2/manifest.py +++ b/providers/ec2/manifest.py @@ -9,7 +9,8 @@ class Manifest(base.Manifest): schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') self.schema_validate(data, schema_path) if data['volume']['backing'] == 'ebs': - if data['volume']['size'] % 1024 != 0: + volume_size = self._calculate_volume_size(data['volume']['partitions']) + if volume_size % 1024 != 0: msg = 'The volume size must be a multiple of 1024 when using EBS backing' raise ManifestError(msg, self) else: @@ -21,5 +22,15 @@ class Manifest(base.Manifest): self.credentials = data['credentials'] self.virtualization = data['virtualization'] self.image = data['image'] - if data['volume']['backing'] == 'ebs': - self.ebs_volume_size = data['volume']['size'] / 1024 + + def _calculate_volume_size(self, partitions): + if partitions['type'] == 'mbr': + size = 1 + else: + size = 0 + if 'boot' in partitions: + size += partitions['boot']['size'] + size += partitions['root']['size'] + if 'swap' in partitions: + size += partitions['swap']['size'] + return size diff --git a/providers/ec2/tasks/ami.py b/providers/ec2/tasks/ami.py index 81c99ed..4cb4577 100644 --- a/providers/ec2/tasks/ami.py +++ b/providers/ec2/tasks/ami.py @@ -117,11 +117,14 @@ class RegisterAMI(Task): arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - if info.manifest.volume['backing'] == 'ebs': + from providers.ec2.ebsvolume import EBSVolume + from common.fs.loopbackvolume import LoopbackVolume + + if isinstance(info.volume, EBSVolume): from boto.ec2.blockdevicemapping import BlockDeviceType from boto.ec2.blockdevicemapping import BlockDeviceMapping block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=info.manifest.ebs_volume_size) + size=info.volume.partition_map.get_total_size()/1024) block_device_map = BlockDeviceMapping() block_device_map['/dev/sda1'] = block_device @@ -129,7 +132,7 @@ class RegisterAMI(Task): architecture=arch, kernel_id=kernel_id, root_device_name='/dev/sda1', block_device_map=block_device_map) - if info.manifest.volume['backing'] == 's3': + if isinstance(info.volume, LoopbackVolume): image_location = ('{bucket}/{ami_name}.manifest.xml' .format(bucket=info.manifest.image['bucket'], ami_name=info.ami_name)) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index bb84a44..e8c363e 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -1,17 +1,24 @@ from base import Task from common import phases -from common.tasks import volume class Create(Task): description = 'Creating the EBS volume' phase = phases.volume_creation - before = [volume.Attach] def run(self, info): info.volume.create(info.connection, info.host['availabilityZone']) +class Attach(Task): + description = 'Attaching the volume' + phase = phases.volume_creation + after = [Create] + + def run(self, info): + info.volume.attach(info.host['instanceId']) + + class Snapshot(Task): description = 'Creating a snapshot of the EBS volume' phase = phases.image_registration diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py index a884b7f..46f1222 100644 --- a/providers/ec2/tasks/packages.py +++ b/providers/ec2/tasks/packages.py @@ -11,8 +11,10 @@ class HostPackages(Task): after = [packages.HostPackages] def run(self, info): - if info.manifest.volume['filesystem'] == 'xfs': - info.host_packages.add('xfsprogs') + for partition in info.volume.partition_map.partitions: + if partition.filesystem == 'xfs': + info.host_packages.add('xfsprogs') + break if info.manifest.volume['backing'] == 's3': info.host_packages.add('euca2ools') @@ -25,6 +27,7 @@ class ImagePackages(Task): def run(self, info): manifest = info.manifest include, exclude = info.img_packages + include.add('openssh-server') include.add('file') # Needed for the init scripts include.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2 if manifest.virtualization == 'pvm': diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 175ca84..70412be 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -38,11 +38,8 @@ def tasks(tasklist, manifest): packages.ImagePackages, loopback.Create, - filesystem.Format, - bootstrap.Bootstrap, boot.ConfigureGrub, - filesystem.FStab, common_boot.BlackListModules, common_boot.DisableGetTTYs, security.EnableShadowConfig,