kevinheyer/bootstrap-vz
1
0
Fork 0
mirror of https://github.com/kevingruesser/bootstrap-vz.git synced 2025-10-10 09:09:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

EC2 provider can now bootstrap EBS volumes again

Browse source 
Use tasksets in EC2 provider
This commit is contained in:
Anders Ingemann 2013-10-09 00:09:34 +02:00
parent 754e414742
commit e1ab4dc1ae
13 changed files with 119 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
base/fs/__init__.py
View file

@ -2,7 +2,7 @@
def load_volume(data):
from common.fs.loopbackvolume import LoopbackVolume
from providers.ec2.volume import EBSVolume
from providers.ec2.ebsvolume import EBSVolume
from providers.virtualbox.volume import VirtualBoxVolume
from partitionmaps.gpt import GPTPartitionMap
from partitionmaps.mbr import MBRPartitionMap

8
base/fs/partitions/abstract.py
View file

@ -1,5 +1,4 @@
from abc import ABCMeta
from abc import abstractmethod
from common.tools import log_check_call
from common.fsm_proxy import FSMProxy
@ -26,13 +25,6 @@ class AbstractPartition(FSMProxy):
[uuid] = log_check_call(['/sbin/blkid', '-s', 'UUID', '-o', 'value', self.device_path])
return uuid
def create(self, volume):
self.fsm.create(volume=volume)
@abstractmethod
def _before_create(self, e):
pass
def _before_format(self, e):
mkfs = '/sbin/mkfs.{fs}'.format(fs=self.filesystem)
log_check_call([mkfs, self.device_path])

3
base/fs/partitions/base.py
View file

@ -18,6 +18,9 @@ class BasePartition(AbstractPartition):
self.previous = previous
super(BasePartition, self).__init__(size, filesystem)
def create(self, volume):
self.fsm.create(volume=volume)
def get_index(self):
if self.previous is None:
return 1

2
base/fs/partitions/single.py
View file

@ -2,6 +2,4 @@ from abstract import AbstractPartition
class SinglePartition(AbstractPartition):
def _before_create(self, e):
pass

4
common/task_sets.py
View file

@ -1,6 +1,7 @@
from common.tasks import workspace
from common.tasks import packages
from common.tasks import host
from common.tasks import bootstrap
from common.tasks import volume
from common.tasks import filesystem
from common.tasks import partitioning
@ -13,11 +14,14 @@ base_set = [workspace.CreateWorkspace,
packages.HostPackages,
packages.ImagePackages,
host.CheckPackages,
bootstrap.Bootstrap,
workspace.DeleteWorkspace,
]
volume_set = [volume.Attach,
volume.Detach,
filesystem.Format,
filesystem.FStab,
]
partitioning_set = [partitioning.PartitionVolume,

143
providers/ec2/__init__.py
View file

@ -1,18 +1,15 @@
from manifest import Manifest
import logging
from tasks import packages
from common.tasks import packages as common_packages
from tasks import connection
from tasks import host
from common.tasks import host as common_host
from tasks import ami
from common.tasks import volume
from common.tasks import volume as volume_tasks
from tasks import ebs
from common.tasks import partitioning
from common.tasks import loopback
from common.tasks import filesystem
from common.tasks import bootstrap
from common.tasks import locale
from common.tasks import apt
from tasks import boot
from common.tasks import boot as common_boot
from common.tasks import security
@ -21,6 +18,7 @@ from common.tasks import network as common_network
from tasks import initd
from common.tasks import initd as common_initd
from common.tasks import cleanup
from common.tasks import workspace
def initialize():
@ -29,74 +27,66 @@ def initialize():
def tasks(tasklist, manifest):
tasklist.add(packages.HostPackages(),
common_packages.HostPackages(),
packages.ImagePackages(),
common_packages.ImagePackages(),
common_host.CheckPackages(),
connection.GetCredentials(),
host.GetInfo(),
ami.AMIName(),
connection.Connect(),
from common.task_sets import base_set
from common.task_sets import mounting_set
from common.task_sets import apt_set
from common.task_sets import locale_set
from common.task_sets import ssh_set
tasklist.add(*base_set)
tasklist.add(*mounting_set)
tasklist.add(*apt_set)
tasklist.add(*locale_set)
tasklist.add(*ssh_set)
filesystem.FormatVolume(),
filesystem.CreateMountDir(),
filesystem.MountVolume(),
if manifest.volume['partitions']['type'] != 'none':
from common.task_sets import partitioning_set
tasklist.add(*partitioning_set)
bootstrap.Bootstrap(),
filesystem.MountSpecials(),
locale.GenerateLocale(),
locale.SetTimezone(),
apt.DisableDaemonAutostart(),
apt.AptSources(),
apt.AptUpgrade(),
boot.ConfigureGrub(),
filesystem.FStab(),
common_boot.BlackListModules(),
common_boot.DisableGetTTYs(),
security.EnableShadowConfig(),
security.DisableSSHPasswordAuthentication(),
security.DisableSSHDNSLookup(),
common_network.RemoveDNSInfo(),
common_network.ConfigureNetworkIF(),
network.EnableDHCPCDDNS(),
common_initd.ResolveInitScripts(),
initd.AddEC2InitScripts(),
common_initd.InstallInitScripts(),
cleanup.ClearMOTD(),
cleanup.ShredHostkeys(),
cleanup.CleanTMP(),
apt.PurgeUnusedPackages(),
apt.AptClean(),
apt.EnableDaemonAutostart(),
filesystem.UnmountSpecials(),
tasklist.add(packages.HostPackages,
packages.ImagePackages,
connection.GetCredentials,
host.GetInfo,
ami.AMIName,
connection.Connect,
filesystem.UnmountVolume(),
filesystem.DeleteMountDir(),
ami.RegisterAMI())
boot.ConfigureGrub,
common_boot.BlackListModules,
common_boot.DisableGetTTYs,
security.EnableShadowConfig,
common_network.RemoveDNSInfo,
common_network.ConfigureNetworkIF,
network.EnableDHCPCDDNS,
common_initd.ResolveInitScripts,
initd.AddEC2InitScripts,
common_initd.InstallInitScripts,
cleanup.ClearMOTD,
cleanup.CleanTMP,
ami.RegisterAMI)
backing_specific_tasks = {'ebs': [ebs.Create,
ebs.Attach,
ebs.Snapshot],
's3': [loopback.Create,
volume_tasks.Attach,
ami.BundleImage,
ami.UploadImage,
ami.RemoveBundle]}
tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
tasklist.add(filesystem.Format,
filesystem.FStab,
volume_tasks.Detach,
volume_tasks.Delete)
if manifest.bootstrapper.get('tarball', False):
tasklist.add(bootstrap.MakeTarball())
tasklist.add(bootstrap.MakeTarball)
backing_specific_tasks = {'ebs': [ebs.Create(),
volume.Attach(),
volume.Detach(),
ebs.Snapshot(),
volume.Delete()],
's3': [loopback.Create(),
volume.Attach(),
volume.Detach(),
ami.BundleImage(),
ami.UploadImage(),
volume.Delete(),
ami.RemoveBundle()]}
tasklist.add(*backing_specific_tasks.get(manifest.volume['backing'].lower()))
from common.task_sets import get_fs_specific_set
tasklist.add(*get_fs_specific_set(manifest.volume['partitions']))
filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
'ext2': [filesystem.TuneVolumeFS()],
'ext3': [filesystem.TuneVolumeFS()],
'ext4': [filesystem.TuneVolumeFS()]}
tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
if 'boot' in manifest.volume['partitions']:
from common.task_sets import boot_partition_set
tasklist.add(*boot_partition_set)
def rollback_tasks(tasklist, tasks_completed, manifest):
@ -104,14 +94,19 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
def counter_task(task, counter):
if task in completed and counter not in completed:
tasklist.add(counter())
tasklist.add(counter)
if manifest.volume['backing'].lower() == 'ebs':
counter_task(ebs.Create, volume.Delete)
counter_task(volume.Attach, volume.Detach)
if manifest.volume['backing'].lower() == 's3':
counter_task(loopback.Create, volume.Delete)
counter_task(volume.Attach, volume.Detach)
counter_task(ebs.Create, volume_tasks.Delete)
counter_task(ebs.Attach, volume_tasks.Detach)
counter_task(loopback.Create, volume_tasks.Delete)
counter_task(volume_tasks.Attach, volume_tasks.Detach)
counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
counter_task(filesystem.MountBoot, filesystem.UnmountBoot)
counter_task(volume_tasks.Attach, volume_tasks.Detach)
counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)

15
providers/ec2/volume.py → providers/ec2/ebsvolume.py
View file

@ -5,18 +5,18 @@ import time
class EBSVolume(Volume):
volume = None
def create(self, conn, zone):
super(EBSVolume, self).create(self)
self.fsm.create(connection=conn, zone=zone)
def _before_create(self, e):
conn = e.connection
zone = e.zone
import math
# TODO: Warn if volume size is not a multiple of 1024
size = int(math.ceil(self.partition_map.get_volume_size() / 1024))
size = int(math.ceil(self.partition_map.get_total_size() / 1024))
self.volume = conn.create_volume(size, zone)
while self.volume.volume_state() != 'available':
time.sleep(5)
self.volume.update()
self.created = True
def attach(self, instance_id):
self.fsm.attach(instance_id=instance_id)
@ -25,7 +25,7 @@ class EBSVolume(Volume):
instance_id = e.instance_id
import os.path
import string
for letter in string.ascii_lowercase:
for letter in string.ascii_lowercase[5:]:
dev_path = os.path.join('/dev', 'xvd' + letter)
if not os.path.exists(dev_path):
self.device_path = dev_path
@ -47,7 +47,6 @@ class EBSVolume(Volume):
self.volume.update()
def _before_delete(self, e):
super(EBSVolume, self).delete(self)
self.volume.delete()
def snapshot(self):

6
providers/ec2/manifest-schema.json
View file

@ -20,13 +20,9 @@
"backing": {
"type": "string",
"enum": ["ebs", "s3"]
},
"filesystem": {
"type": "string",
"enum": ["ext2", "ext3", "ext4", "xfs"]
}
},
"required": ["backing", "filesystem"]
"required": ["backing"]
}
},
"required": ["volume"]

17
providers/ec2/manifest.py
View file

@ -9,7 +9,8 @@ class Manifest(base.Manifest):
schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
self.schema_validate(data, schema_path)
if data['volume']['backing'] == 'ebs':
if data['volume']['size'] % 1024 != 0:
volume_size = self._calculate_volume_size(data['volume']['partitions'])
if volume_size % 1024 != 0:
msg = 'The volume size must be a multiple of 1024 when using EBS backing'
raise ManifestError(msg, self)
else:
@ -21,5 +22,15 @@ class Manifest(base.Manifest):
self.credentials = data['credentials']
self.virtualization = data['virtualization']
self.image = data['image']
if data['volume']['backing'] == 'ebs':
self.ebs_volume_size = data['volume']['size'] / 1024
def _calculate_volume_size(self, partitions):
if partitions['type'] == 'mbr':
size = 1
else:
size = 0
if 'boot' in partitions:
size += partitions['boot']['size']
size += partitions['root']['size']
if 'swap' in partitions:
size += partitions['swap']['size']
return size

9
providers/ec2/tasks/ami.py
View file

@ -117,11 +117,14 @@ class RegisterAMI(Task):
arch = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
kernel_id = self.kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
if info.manifest.volume['backing'] == 'ebs':
from providers.ec2.ebsvolume import EBSVolume
from common.fs.loopbackvolume import LoopbackVolume
if isinstance(info.volume, EBSVolume):
from boto.ec2.blockdevicemapping import BlockDeviceType
from boto.ec2.blockdevicemapping import BlockDeviceMapping
block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
size=info.manifest.ebs_volume_size)
size=info.volume.partition_map.get_total_size()/1024)
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = block_device
@ -129,7 +132,7 @@ class RegisterAMI(Task):
architecture=arch, kernel_id=kernel_id,
root_device_name='/dev/sda1',
block_device_map=block_device_map)
if info.manifest.volume['backing'] == 's3':
if isinstance(info.volume, LoopbackVolume):
image_location = ('{bucket}/{ami_name}.manifest.xml'
.format(bucket=info.manifest.image['bucket'],
ami_name=info.ami_name))

11
providers/ec2/tasks/ebs.py
View file

@ -1,17 +1,24 @@
from base import Task
from common import phases
from common.tasks import volume
class Create(Task):
description = 'Creating the EBS volume'
phase = phases.volume_creation
before = [volume.Attach]
def run(self, info):
info.volume.create(info.connection, info.host['availabilityZone'])
class Attach(Task):
description = 'Attaching the volume'
phase = phases.volume_creation
after = [Create]
def run(self, info):
info.volume.attach(info.host['instanceId'])
class Snapshot(Task):
description = 'Creating a snapshot of the EBS volume'
phase = phases.image_registration

7
providers/ec2/tasks/packages.py
View file

@ -11,8 +11,10 @@ class HostPackages(Task):
after = [packages.HostPackages]
def run(self, info):
if info.manifest.volume['filesystem'] == 'xfs':
info.host_packages.add('xfsprogs')
for partition in info.volume.partition_map.partitions:
if partition.filesystem == 'xfs':
info.host_packages.add('xfsprogs')
break
if info.manifest.volume['backing'] == 's3':
info.host_packages.add('euca2ools')
@ -25,6 +27,7 @@ class ImagePackages(Task):
def run(self, info):
manifest = info.manifest
include, exclude = info.img_packages
include.add('openssh-server')
include.add('file') # Needed for the init scripts
include.add('dhcpcd') # isc-dhcp-client doesn't work properly with ec2
if manifest.virtualization == 'pvm':

3
providers/virtualbox/__init__.py
View file

@ -38,11 +38,8 @@ def tasks(tasklist, manifest):
packages.ImagePackages,
loopback.Create,
filesystem.Format,
bootstrap.Bootstrap,
boot.ConfigureGrub,
filesystem.FStab,
common_boot.BlackListModules,
common_boot.DisableGetTTYs,
security.EnableShadowConfig,