kevinheyer/bootstrap-vz
1
0
Fork 0
mirror of https://github.com/kevingruesser/bootstrap-vz.git synced 2025-08-22 09:50:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #353 from vorlock/master

Browse source 
ENA support & migrate to boto3
This commit is contained in:
Anders Ingemann 2017-01-23 09:40:04 +01:00 • committed by GitHub
commit 6866adf734
9 changed files with 136 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
bootstrapvz/plugins/ec2_launch/tasks.py
View file

@ -1,7 +1,7 @@
import logging
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.providers.ec2.tasks import ami
import logging
# TODO: Merge with the method available in wip-integration-tests branch
@ -23,19 +23,21 @@ class LaunchEC2Instance(Task):
@classmethod
def run(cls, info):
conn = info._ec2['connection']
r = conn.run_instances(info._ec2['image'],
security_group_ids=info.manifest.plugins['ec2_launch'].get('security_group_ids'),
key_name=info.manifest.plugins['ec2_launch'].get('ssh_key'),
instance_type=info.manifest.plugins['ec2_launch'].get('instance_type',
'm3.medium'))
info._ec2['instance'] = r.instances[0]
r = conn.run_instances(ImageId=info._ec2['image']['ImageId'],
MinCount=1,
MaxCount=1,
SecurityGroupIds=info.manifest.plugins['ec2_launch'].get('security_group_ids'),
KeyName=info.manifest.plugins['ec2_launch'].get('ssh_key'),
InstanceType=info.manifest.plugins['ec2_launch'].get('instance_type',
'm3.medium'))
info._ec2['instance'] = r['Instances'][0]
if 'tags' in info.manifest.plugins['ec2_launch']:
def apply_format(v):
return v.format(**info.manifest_vars)
tags = info.manifest.plugins['ec2_launch']['tags']
r = {k: apply_format(v) for k, v in tags.items()}
conn.create_tags([info._ec2['instance'].id], r)
raw_tags = info.manifest.plugins['ec2_launch']['tags']
formatted_tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()}
tags = [{'Key': k, 'Value': v} for k, v in formatted_tags.items()]
conn.create_tags(Resources=[info._ec2['instance']['InstanceId']],
Tags=tags)
class PrintPublicIPAddress(Task):
@ -45,21 +47,21 @@ class PrintPublicIPAddress(Task):
@classmethod
def run(cls, info):
ec2 = info._ec2
conn = info._ec2['connection']
logger = logging.getLogger(__name__)
filename = info.manifest.plugins['ec2_launch']['print_public_ip']
if not filename:
filename = '/dev/null'
f = open(filename, 'w')
def instance_has_ip():
ec2['instance'].update()
return ec2['instance'].ip_address
if waituntil(instance_has_ip, timeout=120, interval=5):
logger.info('******* EC2 IP ADDRESS: %s *******' % ec2['instance'].ip_address)
f.write(ec2['instance'].ip_address)
else:
try:
waiter = conn.get_waiter('instance_status_ok')
waiter.wait(InstanceIds=[info._ec2['instance']['InstanceId']],
Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
info._ec2['instance'] = conn.describe_instances(InstanceIds=[info._ec2['instance']['InstanceId']])['Reservations'][0]['Instances'][0]
logger.info('******* EC2 IP ADDRESS: %s *******' % info._ec2['instance']['PublicIpAddress'])
f.write(info._ec2['instance']['PublicIpAddress'])
except:
logger.error('Could not get IP address for the instance')
f.write('')

4
bootstrapvz/providers/ec2/__init__.py
View file

@ -68,6 +68,9 @@ def resolve_tasks(taskset, manifest):
tasks.tuning.BlackListModules,
boot.BlackListModules,
boot.DisableGetTTYs,
tasks.boot.AddXenGrubConsoleOutputDevice,
grub.WriteGrubConfig,
tasks.boot.UpdateGrubConfig,
initd.AddExpandRoot,
initd.RemoveHWClock,
initd.InstallInitScripts,
@ -133,6 +136,7 @@ def resolve_tasks(taskset, manifest):
if manifest.provider.get('enhanced_networking', None) == 'simple':
taskset.update([kernel.AddDKMSPackages,
tasks.network.InstallEnhancedNetworking,
tasks.network.InstallENANetworking,
kernel.UpdateInitramfs])
taskset.update([filesystem.Format,

50
bootstrapvz/providers/ec2/ebsvolume.py
View file

@ -1,6 +1,5 @@
from bootstrapvz.base.fs.volume import Volume
from bootstrapvz.base.fs.exceptions import VolumeError
import time
class EBSVolume(Volume):
@ -9,21 +8,25 @@ class EBSVolume(Volume):
self.fsm.create(connection=conn, zone=zone)
def _before_create(self, e):
conn = e.connection
self.conn = e.connection
zone = e.zone
size = self.size.bytes.get_qty_in('GiB')
self.volume = conn.create_volume(size, zone, volume_type='gp2')
while self.volume.volume_state() != 'available':
time.sleep(5)
self.volume.update()
self.volume = self.conn.create_volume(Size=size,
AvailabilityZone=zone,
VolumeType='gp2')
self.vol_id = self.volume['VolumeId']
waiter = self.conn.get_waiter('volume_available')
waiter.wait(VolumeIds=[self.vol_id],
Filters=[{'Name': 'status', 'Values': ['available']}])
def attach(self, instance_id):
self.fsm.attach(instance_id=instance_id)
def _before_attach(self, e):
instance_id = e.instance_id
import os.path
import string
self.instance_id = e.instance_id
for letter in string.ascii_lowercase[5:]:
dev_path = os.path.join('/dev', 'xvd' + letter)
if not os.path.exists(dev_path):
@ -34,25 +37,30 @@ class EBSVolume(Volume):
if self.device_path is None:
raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
self.volume.attach(instance_id, self.ec2_device_path)
while self.volume.attachment_state() != 'attached':
time.sleep(2)
self.volume.update()
self.conn.attach_volume(VolumeId=self.vol_id,
InstanceId=self.instance_id,
Device=self.ec2_device_path)
waiter = self.conn.get_waiter('volume_in_use')
waiter.wait(VolumeIds=[self.vol_id],
Filters=[{'Name': 'attachment.status', 'Values': ['attached']}])
def _before_detach(self, e):
self.volume.detach()
while self.volume.attachment_state() is not None:
time.sleep(2)
self.volume.update()
self.conn.detach_volume(VolumeId=self.vol_id,
InstanceId=self.instance_id,
Device=self.ec2_device_path)
waiter = self.conn.get_waiter('volume_available')
waiter.wait(VolumeIds=[self.vol_id],
Filters=[{'Name': 'status', 'Values': ['available']}])
del self.ec2_device_path
self.device_path = None
def _before_delete(self, e):
self.volume.delete()
self.conn.delete_volume(VolumeId=self.vol_id)
def snapshot(self):
snapshot = self.volume.create_snapshot()
while snapshot.status != 'completed':
time.sleep(2)
snapshot.update()
return snapshot
snapshot = self.conn.create_snapshot(VolumeId=self.vol_id)
self.snap_id = snapshot['SnapshotId']
waiter = self.conn.get_waiter('snapshot_completed')
waiter.wait(SnapshotIds=[self.snap_id],
Filters=[{'Name': 'status', 'Values': ['completed']}])
return self.snap_id

45
bootstrapvz/providers/ec2/tasks/ami.py
View file

@ -21,9 +21,9 @@ class AMIName(Task):
ami_name = info.manifest.name.format(**info.manifest_vars)
ami_description = info.manifest.provider['description'].format(**info.manifest_vars)
images = info._ec2['connection'].get_all_images(owners=['self'])
images = info._ec2['connection'].describe_images(Owners=['self'])['Images']
for image in images:
if ami_name == image.name:
if ami_name == image['Name']:
msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name)
raise TaskError(msg)
info._ec2['ami_name'] = ami_name
@ -93,41 +93,46 @@ class RegisterAMI(Task):
@classmethod
def run(cls, info):
registration_params = {'name': info._ec2['ami_name'],
'description': info._ec2['ami_description']}
registration_params['architecture'] = {'i386': 'i386',
registration_params = {'Name': info._ec2['ami_name'],
'Description': info._ec2['ami_description']}
registration_params['Architecture'] = {'i386': 'i386',
'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
if info.manifest.volume['backing'] == 's3':
registration_params['image_location'] = info._ec2['manifest_location']
registration_params['ImageLocation'] = info._ec2['manifest_location']
else:
root_dev_name = {'pvm': '/dev/sda',
'hvm': '/dev/xvda'}.get(info.manifest.provider['virtualization'])
registration_params['root_device_name'] = root_dev_name
registration_params['RootDeviceName'] = root_dev_name
from boto.ec2.blockdevicemapping import BlockDeviceType
from boto.ec2.blockdevicemapping import BlockDeviceMapping
block_device = BlockDeviceType(snapshot_id=info._ec2['snapshot'].id, delete_on_termination=True,
size=info.volume.size.bytes.get_qty_in('GiB'), volume_type='gp2')
registration_params['block_device_map'] = BlockDeviceMapping()
registration_params['block_device_map'][root_dev_name] = block_device
block_device = [{'DeviceName': root_dev_name,
'Ebs': {
'SnapshotId': info._ec2['snapshot'],
'VolumeSize': info.volume.size.bytes.get_qty_in('GiB'),
'VolumeType': 'gp2',
'DeleteOnTermination': True}}]
registration_params['BlockDeviceMappings'] = block_device
if info.manifest.provider['virtualization'] == 'hvm':
registration_params['virtualization_type'] = 'hvm'
registration_params['VirtualizationType'] = 'hvm'
else:
registration_params['virtualization_type'] = 'paravirtual'
registration_params['VirtualizationType'] = 'paravirtual'
akis_path = rel_path(__file__, 'ami-akis.yml')
from bootstrapvz.common.tools import config_get
registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'],
info.manifest.system['architecture']])
registration_params['kernel_id'] = config_get(akis_path,
[info._ec2['region'],
info.manifest.system['architecture']])
if info.manifest.provider.get('enhanced_networking', None) == 'simple':
registration_params['sriov_net_support'] = 'simple'
registration_params['SriovNetSupport'] = 'simple'
registration_params['EnaSupport'] = True
info._ec2['image'] = info._ec2['connection'].register_image(**registration_params)
# Setting up tags on the AMI
if 'tags' in info.manifest.data:
raw_tags = info.manifest.data['tags']
tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()}
info._ec2['connection'].create_tags(info._ec2['image'], tags)
formatted_tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()}
tags = [{'Key': k, 'Value': v} for k, v in formatted_tags.items()]
info._ec2['connection'].create_tags(Resources=[info._ec2['image']['ImageId']],
Tags=tags)

9
bootstrapvz/providers/ec2/tasks/connection.py
View file

@ -38,6 +38,7 @@ class GetCredentials(Task):
def env_key(key):
return ('aws-' + key).upper().replace('-', '_')
if all(getenv(env_key(key)) is not None for key in keys):
for key in keys:
creds[key] = getenv(env_key(key))
@ -45,6 +46,7 @@ class GetCredentials(Task):
def provider_key(key):
return key.replace('-', '_')
import boto.provider
provider = boto.provider.Provider('aws')
if all(getattr(provider, provider_key(key)) is not None for key in keys):
@ -64,7 +66,7 @@ class Connect(Task):
@classmethod
def run(cls, info):
from boto.ec2 import connect_to_region
import boto3
connect_args = {
'aws_access_key_id': info.credentials['access-key'],
'aws_secret_access_key': info.credentials['secret-key']
@ -73,4 +75,7 @@ class Connect(Task):
if 'security-token' in info.credentials:
connect_args['security_token'] = info.credentials['security-token']
info._ec2['connection'] = connect_to_region(info._ec2['region'], **connect_args)
info._ec2['connection'] = boto3.Session(info._ec2['region'],
info.credentials['access-key'],
info.credentials['secret-key'])
info._ec2['connection'] = boto3.client('ec2', region_name=info._ec2['region'])

8
bootstrapvz/providers/ec2/tasks/ebs.py
View file

@ -28,3 +28,11 @@ class Snapshot(Task):
@classmethod
def run(cls, info):
info._ec2['snapshot'] = info.volume.snapshot()
# Setting up tags on the snapshot
if 'tags' in info.manifest.data:
raw_tags = info.manifest.data['tags']
formatted_tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()}
tags = [{'Key': k, 'Value': v} for k, v in formatted_tags.items()]
info._ec2['connection'].create_tags(Resources=[info._ec2['snapshot']],
Tags=tags)

40
bootstrapvz/providers/ec2/tasks/network.py
View file

@ -101,7 +101,8 @@ class InstallEnhancedNetworking(Task):
log_check_call(['tar', '--ungzip',
'--extract',
'--file', archive,
'--directory', os.path.join(info.root, 'usr', 'src')])
'--directory', os.path.join(info.root, 'usr',
'src')])
with open(os.path.join(module_path, 'dkms.conf'), 'w') as dkms_conf:
dkms_conf.write("""PACKAGE_NAME="ixgbevf"
@ -118,4 +119,39 @@ AUTOINSTALL="yes"
for task in ['add', 'build', 'install']:
# Invoke DKMS task using specified kernel module (-m) and version (-v)
log_check_call(['chroot', info.root,
'dkms', task, '-m', 'ixgbevf', '-v', version, '-k', info.kernel_version])
'dkms', task, '-m', 'ixgbevf', '-v', version, '-k',
info.kernel_version])
class InstallENANetworking(Task):
description = '***** Installing ENA networking kernel driver using DKMS'
phase = phases.system_modification
successors = [kernel.UpdateInitramfs]
@classmethod
def run(cls, info):
version = '1.0.0'
drivers_url = 'https://github.com/amzn/amzn-drivers'
module_path = os.path.join(info.root, 'usr', 'src',
'amzn-drivers-%s' % (version))
from bootstrapvz.common.tools import log_check_call
log_check_call(['git', 'clone', drivers_url, module_path])
with open(os.path.join(module_path, 'dkms.conf'), 'w') as dkms_conf:
dkms_conf.write("""PACKAGE_NAME="ena"
PACKAGE_VERSION="%s"
CLEAN="make -C kernel/linux/ena clean"
MAKE="make -C kernel/linux/ena/ BUILD_KERNEL=${kernelver}"
BUILT_MODULE_NAME[0]="ena"
BUILT_MODULE_LOCATION="kernel/linux/ena"
DEST_MODULE_LOCATION[0]="/updates"
DEST_MODULE_NAME[0]="ena"
AUTOINSTALL="yes"
""" % (version))
for task in ['add', 'build', 'install']:
# Invoke DKMS task using specified kernel module (-m) and version (-v)
log_check_call(['chroot', info.root,
'dkms', task, '-m', 'amzn-drivers', '-v', version,
'-k', info.kernel_version])

2
manifests/official/ec2/ebs-stretch-amd64-hvm.yml
View file

@ -2,7 +2,7 @@
name: debian-{system.release}-{system.architecture}-{provider.virtualization}-{%Y}-{%m}-{%d}-{%H}{%M}-ebs
tags:
Name: "Stretch 9.0 alpha"
Debian: "9.0~{%Y}{%m}{%d}"
Debian: "9.0~{%Y}{%m}{%d}{%H}{%M}"
provider:
name: ec2
virtualization: hvm

1
setup.py
View file

@ -24,6 +24,7 @@ setup(name='bootstrap-vz',
'jsonschema >= 2.3.0',
'pyyaml >= 3.10',
'boto >= 2.14.0',
'boto3 >= 1.4.2',
'docopt >= 0.6.1',
'pyrfc3339 >= 1.0',
'requests >= 2.4.3',