From 96028f96e10343623db5d6a6b64625a692fc0574 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Mon, 24 Jun 2013 23:12:39 +0200 Subject: [PATCH] Various improvements and additions. I couldn't be bothered to untangle this, so here it goes: * Log colors depending on loglevel * Simplified Filelogger * Remove description=None from basetask * create_list creates task list from argument now * Task rollback feature: If a task fails, the tasklist calls rollback() on the completed tasks in reverse order * Added TaskException to common.exceptions as a base to extend from * Added TriggerRollback task to common.tasks for development purposes * An EBS volume for bootstrapping is now created and attached to the instance (including rollback actions) * EC2 Connect task now depends on host.GetInfo --- base/log.py | 29 +++++++-------------- base/task.py | 1 - base/tasklist.py | 37 +++++++++++++++++++------- common/exceptions.py | 4 ++- common/tasks.py | 11 ++++++++ providers/ec2/__init__.py | 3 +++ providers/ec2/tasks/connection.py | 3 ++- providers/ec2/tasks/ebs.py | 43 +++++++++++++++++++++++-------- 8 files changed, 88 insertions(+), 43 deletions(-) create mode 100644 common/tasks.py diff --git a/base/log.py b/base/log.py index 581f0ae..4341f8a 100644 --- a/base/log.py +++ b/base/log.py @@ -32,29 +32,18 @@ def setup_logger(logfile=None, debug=False): class ConsoleFormatter(logging.Formatter): + level_colors = { + logging.ERROR: 'red', + logging.WARNING: 'magenta', + logging.INFO: 'blue', + } def format(self, record): - from task import Task - if(isinstance(record.msg, Task)): - task = record.msg - if(task.description is not None): - return '\033[0;34m{description}\033[0m'.format(description=task.description) - else: - return '\033[0;34mRunning {task}\033[0m'.format(task=task) + if(record.levelno in self.level_colors): + from termcolor import colored, cprint + record.msg = colored(record.msg, self.level_colors[record.levelno]) return super(ConsoleFormatter, self).format(record) class FileFormatter(logging.Formatter): def format(self, record): - from task import Task - from datetime import datetime - if(isinstance(record.msg, Task)): - task = record.msg - if(task.description is not None): - record.msg = '{description} (running {task})'.format(task=task, description=task.description) - else: - record.msg = 'Running {task}'.format(task=task) - message = super(FileFormatter, self).format(record) - record.msg = task - else: - message = super(FileFormatter, self).format(record) - return message + return super(FileFormatter, self).format(record) diff --git a/base/task.py b/base/task.py index 56b8da2..bd6c345 100644 --- a/base/task.py +++ b/base/task.py @@ -2,7 +2,6 @@ from common.exceptions import TaskListError class Task(object): - description = None phase = None before = [] diff --git a/base/tasklist.py b/base/tasklist.py index 0eade0a..192160a 100644 --- a/base/tasklist.py +++ b/base/tasklist.py @@ -22,21 +22,41 @@ class TaskList(object): return next(task for task in self.tasks if type(task) is ref) def run(self, bootstrap_info): - task_list = self.create_list() + task_list = self.create_list(self.tasks) log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list))) - for task in task_list: - log.info(task) - task.run(bootstrap_info) - def create_list(self): + tasks_completed = [] + try: + for task in task_list: + if hasattr(task, 'description'): + log.info(task.description) + else: + log.info('Running {task}'.format(task=task)) + task.run(bootstrap_info) + tasks_completed.append(task) + except Exception, e: + log.exception(e) + log.error('Rolling back') + for task in reversed(tasks_completed): + rollback = getattr(task, 'rollback', None) + if not callable(rollback): + continue + if hasattr(task, 'rollback_description'): + log.warning(task.rollback_description) + else: + log.warning('Rolling back {task}'.format(task=task)) + task.rollback(bootstrap_info) + log.info('Successfully completed rollback') + + def create_list(self, tasks): from common.phases import order graph = {} - for task in self.tasks: + for task in tasks: graph[task] = [] graph[task].extend([self.get(succ) for succ in task.before]) - graph[task].extend([succ for succ in self.tasks if type(task) in succ.after]) + graph[task].extend([succ for succ in tasks if type(task) in succ.after]) succeeding_phases = order[order.index(task.phase)+1:] - graph[task].extend([succ for succ in self.tasks if succ.phase in succeeding_phases]) + graph[task].extend([succ for succ in tasks if succ.phase in succeeding_phases]) components = self.strongly_connected_components(graph) @@ -72,7 +92,6 @@ class TaskList(object): stack.append(node) for successor in graph[node]: - # print successor visit(successor) low[node] = min(low[node], low[successor]) diff --git a/common/exceptions.py b/common/exceptions.py index 340b59c..5e325f5 100644 --- a/common/exceptions.py +++ b/common/exceptions.py @@ -1,4 +1,3 @@ -__all__ = ['ManifestError'] class ManifestError(Exception): @@ -17,3 +16,6 @@ class TaskListError(Exception): self.message = message def __str__(self): return "Error in tasklist: {0}".format(self.message) + +class TaskException(Exception): + pass diff --git a/common/tasks.py b/common/tasks.py new file mode 100644 index 0000000..c919c08 --- /dev/null +++ b/common/tasks.py @@ -0,0 +1,11 @@ +from base import Task +import phases + + +class TriggerRollback(Task): + phase = phases.cleanup + + description = 'Triggering a rollback by throwing an exception' + def run(self, info): + from common.exceptions import TaskException + raise TaskException('Trigger rollback') diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py index 5588c26..3f7af94 100644 --- a/providers/ec2/__init__.py +++ b/providers/ec2/__init__.py @@ -10,3 +10,6 @@ def tasks(tasklist, manifest): connection.GetCredentials(), host.GetInfo(), connection.Connect()) if manifest.volume['backing'].lower() == 'ebs': tasklist.add(ebs.CreateVolume(), ebs.AttachVolume()) + + from common.tasks import TriggerRollback + tasklist.add(TriggerRollback()) diff --git a/providers/ec2/tasks/connection.py b/providers/ec2/tasks/connection.py index f17b171..818160e 100644 --- a/providers/ec2/tasks/connection.py +++ b/providers/ec2/tasks/connection.py @@ -1,5 +1,6 @@ from base import Task from common import phases +import host class GetCredentials(Task): @@ -31,7 +32,7 @@ class GetCredentials(Task): class Connect(Task): description = 'Connecting to EC2' phase = phases.preparation - after = [GetCredentials] + after = [GetCredentials, host.GetInfo] def run(self, info): super(Connect, self).run(info) diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py index 1e25855..d3f149f 100644 --- a/providers/ec2/tasks/ebs.py +++ b/providers/ec2/tasks/ebs.py @@ -2,39 +2,60 @@ from base import Task from common import phases from common.exceptions import TaskException from connection import Connect +import time + class CreateVolume(Task): - description = 'Creating an EBS volume for bootstrapping' phase = phases.volume_creation after = [Connect] + description = 'Creating an EBS volume for bootstrapping' def run(self, info): volume_size = int(info.manifest.volume['size']/1024) - info.volume = info.conn.create_volume(volume_size, info.host['availabilityZone']) + + info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone']) + while info.volume.volume_state() != 'available': + time.sleep(5) + info.volume.update() + + rollback_description = 'Deleting the EBS volume' + def rollback(self, info): + info.volume.delete() + del info.volume class AttachVolume(Task): - description = 'Attaching the EBS volume' phase = phases.volume_creation after = [CreateVolume] + description = 'Attaching the EBS volume' def run(self, info): def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" + """Generates the characters from `c1` to `c2`, inclusive.""" for c in xrange(ord(c1), ord(c2)+1): yield chr(c) import os.path - import os.stat - from stat import S_ISBLK - for letter in char_range('a', 'z'): + info.bootstrap_device = {} + for letter in char_range('f', 'z'): dev_path = os.path.join('/dev', 'xvd' + letter) - mode = os.stat(dev_path).st_mode - if S_ISBLK(mode): - info.bootstrap_device = {'path': dev_path} + if not os.path.exists(dev_path): + info.bootstrap_device['path'] = dev_path + info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) break if 'path' not in info.bootstrap_device: raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - info.conn.volume.attach(info.host['instanceId'], info.bootstrap_device['path']) + + info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) + while info.volume.attachment_state() != 'attached': + time.sleep(2) + info.volume.update() + + rollback_description = 'Detaching the EBS volume' + def rollback(self, info): + info.volume.detach() + while info.volume.attachment_state() is not None: + time.sleep(2) + info.volume.update() class VolumeError(TaskException): pass