From 89eedae5fcc43657aab828f47d50332d284ff3ad Mon Sep 17 00:00:00 2001 From: Marcin Kulisz Date: Sat, 12 Nov 2016 19:58:51 +0000 Subject: [PATCH] Porting EC2 provider to boto3 --- bootstrapvz/providers/ec2/ebsvolume.py | 50 +++++++++++-------- bootstrapvz/providers/ec2/tasks/ami.py | 44 ++++++++-------- bootstrapvz/providers/ec2/tasks/connection.py | 7 ++- bootstrapvz/providers/ec2/tasks/ebs.py | 7 +++ 4 files changed, 65 insertions(+), 43 deletions(-) diff --git a/bootstrapvz/providers/ec2/ebsvolume.py b/bootstrapvz/providers/ec2/ebsvolume.py index 049094e..a3aa8d8 100644 --- a/bootstrapvz/providers/ec2/ebsvolume.py +++ b/bootstrapvz/providers/ec2/ebsvolume.py @@ -1,6 +1,5 @@ from bootstrapvz.base.fs.volume import Volume from bootstrapvz.base.fs.exceptions import VolumeError -import time class EBSVolume(Volume): @@ -9,21 +8,25 @@ class EBSVolume(Volume): self.fsm.create(connection=conn, zone=zone) def _before_create(self, e): - conn = e.connection + self.conn = e.connection zone = e.zone size = self.size.bytes.get_qty_in('GiB') - self.volume = conn.create_volume(size, zone, volume_type='gp2') - while self.volume.volume_state() != 'available': - time.sleep(5) - self.volume.update() + self.volume = self.conn.create_volume(Size=size, + AvailabilityZone=zone, + VolumeType='gp2') + self.vol_id = self.volume['VolumeId'] + waiter = self.conn.get_waiter('volume_available') + waiter.wait(VolumeIds=[self.vol_id], + Filters=[{'Name': 'status', 'Values': ['available']}]) def attach(self, instance_id): self.fsm.attach(instance_id=instance_id) def _before_attach(self, e): - instance_id = e.instance_id import os.path import string + + self.instance_id = e.instance_id for letter in string.ascii_lowercase[5:]: dev_path = os.path.join('/dev', 'xvd' + letter) if not os.path.exists(dev_path): @@ -34,25 +37,30 @@ class EBSVolume(Volume): if self.device_path is None: raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - self.volume.attach(instance_id, self.ec2_device_path) - while self.volume.attachment_state() != 'attached': - time.sleep(2) - self.volume.update() + self.conn.attach_volume(VolumeId=self.vol_id, + InstanceId=self.instance_id, + Device=self.ec2_device_path) + waiter = self.conn.get_waiter('volume_in_use') + waiter.wait(VolumeIds=[self.vol_id], + Filters=[{'Name': 'attachment.status', 'Values': ['attached']}]) def _before_detach(self, e): - self.volume.detach() - while self.volume.attachment_state() is not None: - time.sleep(2) - self.volume.update() + self.conn.detach_volume(VolumeId=self.vol_id, + InstanceId=self.instance_id, + Device=self.ec2_device_path) + waiter = self.conn.get_waiter('volume_available') + waiter.wait(VolumeIds=[self.vol_id], + Filters=[{'Name': 'status', 'Values': ['available']}]) del self.ec2_device_path self.device_path = None def _before_delete(self, e): - self.volume.delete() + self.conn.delete_volume(VolumeId=self.vol_id) def snapshot(self): - snapshot = self.volume.create_snapshot() - while snapshot.status != 'completed': - time.sleep(2) - snapshot.update() - return snapshot + snapshot = self.conn.create_snapshot(VolumeId=self.vol_id) + self.snap_id = snapshot['SnapshotId'] + waiter = self.conn.get_waiter('snapshot_completed') + waiter.wait(SnapshotIds=[self.snap_id], + Filters=[{'Name': 'status', 'Values': ['completed']}]) + return self.snap_id diff --git a/bootstrapvz/providers/ec2/tasks/ami.py b/bootstrapvz/providers/ec2/tasks/ami.py index 48b49e6..77fe195 100644 --- a/bootstrapvz/providers/ec2/tasks/ami.py +++ b/bootstrapvz/providers/ec2/tasks/ami.py @@ -21,9 +21,9 @@ class AMIName(Task): ami_name = info.manifest.name.format(**info.manifest_vars) ami_description = info.manifest.provider['description'].format(**info.manifest_vars) - images = info._ec2['connection'].get_all_images(owners=['self']) + images = info._ec2['connection'].describe_images(Owners=['self'])['Images'] for image in images: - if ami_name == image.name: + if ami_name == image['Name']: msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name) raise TaskError(msg) info._ec2['ami_name'] = ami_name @@ -93,41 +93,45 @@ class RegisterAMI(Task): @classmethod def run(cls, info): - registration_params = {'name': info._ec2['ami_name'], - 'description': info._ec2['ami_description']} - registration_params['architecture'] = {'i386': 'i386', + registration_params = {'Name': info._ec2['ami_name'], + 'Description': info._ec2['ami_description']} + registration_params['Architecture'] = {'i386': 'i386', 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) if info.manifest.volume['backing'] == 's3': - registration_params['image_location'] = info._ec2['manifest_location'] + registration_params['ImageLocation'] = info._ec2['manifest_location'] else: root_dev_name = {'pvm': '/dev/sda', 'hvm': '/dev/xvda'}.get(info.manifest.provider['virtualization']) - registration_params['root_device_name'] = root_dev_name + registration_params['RootDeviceName'] = root_dev_name - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info._ec2['snapshot'].id, delete_on_termination=True, - size=info.volume.size.bytes.get_qty_in('GiB'), volume_type='gp2') - registration_params['block_device_map'] = BlockDeviceMapping() - registration_params['block_device_map'][root_dev_name] = block_device + block_device = [{'DeviceName': root_dev_name, + 'Ebs': { + 'SnapshotId': info._ec2['snapshot'], + 'VolumeSize': info.volume.size.bytes.get_qty_in('GiB'), + 'VolumeType': 'gp2', + 'DeleteOnTermination': True}}] + registration_params['BlockDeviceMappings'] = block_device if info.manifest.provider['virtualization'] == 'hvm': - registration_params['virtualization_type'] = 'hvm' + registration_params['VirtualizationType'] = 'hvm' else: - registration_params['virtualization_type'] = 'paravirtual' + registration_params['VirtualizationType'] = 'paravirtual' akis_path = rel_path(__file__, 'ami-akis.yml') from bootstrapvz.common.tools import config_get - registration_params['kernel_id'] = config_get(akis_path, [info._ec2['region'], - info.manifest.system['architecture']]) + registration_params['kernel_id'] = config_get(akis_path, + [info._ec2['region'], + info.manifest.system['architecture']]) if info.manifest.provider.get('enhanced_networking', None) == 'simple': - registration_params['sriov_net_support'] = 'simple' + registration_params['SriovNetSupport'] = 'simple' info._ec2['image'] = info._ec2['connection'].register_image(**registration_params) # Setting up tags on the AMI if 'tags' in info.manifest.data: raw_tags = info.manifest.data['tags'] - tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()} - info._ec2['connection'].create_tags(info._ec2['image'], tags) + formatted_tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()} + tags = [{'Key': k, 'Value': v} for k, v in formatted_tags.items()] + info._ec2['connection'].create_tags(Resources=[info._ec2['image']['ImageId']], + Tags=tags) diff --git a/bootstrapvz/providers/ec2/tasks/connection.py b/bootstrapvz/providers/ec2/tasks/connection.py index f597f72..b7302e3 100644 --- a/bootstrapvz/providers/ec2/tasks/connection.py +++ b/bootstrapvz/providers/ec2/tasks/connection.py @@ -64,7 +64,7 @@ class Connect(Task): @classmethod def run(cls, info): - from boto.ec2 import connect_to_region + import boto3 connect_args = { 'aws_access_key_id': info.credentials['access-key'], 'aws_secret_access_key': info.credentials['secret-key'] @@ -73,4 +73,7 @@ class Connect(Task): if 'security-token' in info.credentials: connect_args['security_token'] = info.credentials['security-token'] - info._ec2['connection'] = connect_to_region(info._ec2['region'], **connect_args) + info._ec2['connection'] = boto3.Session(info._ec2['region'], + info.credentials['access-key'], + info.credentials['secret-key']) + info._ec2['connection'] = boto3.client('ec2', region_name=info._ec2['region']) diff --git a/bootstrapvz/providers/ec2/tasks/ebs.py b/bootstrapvz/providers/ec2/tasks/ebs.py index f50aa6b..a5d336f 100644 --- a/bootstrapvz/providers/ec2/tasks/ebs.py +++ b/bootstrapvz/providers/ec2/tasks/ebs.py @@ -28,3 +28,10 @@ class Snapshot(Task): @classmethod def run(cls, info): info._ec2['snapshot'] = info.volume.snapshot() + +# # Setting up tags on the snapshot +# if 'tags' in info.manifest.data: +# raw_tags = info.manifest.data['tags'] +# tags = {k: v.format(**info.manifest_vars) for k, v in raw_tags.items()} +# info._ec2['connection'](Resources=[info._ec2['snapshot']], +# Tags=[tags])