Various improvements and additions.

I couldn't be bothered to untangle this, so here it goes:
* Log colors depending on loglevel
* Simplified Filelogger
* Remove description=None from basetask
* create_list creates task list from argument now
* Task rollback feature: If a task fails, the tasklist calls rollback() on the completed tasks in reverse order
* Added TaskException to common.exceptions as a base to extend from
* Added TriggerRollback task to common.tasks for development purposes
* An EBS volume for bootstrapping is now created and attached to the instance (including rollback actions)
* EC2 Connect task now depends on host.GetInfo

base/log.py

@@ -32,29 +32,18 @@ def setup_logger(logfile=None, debug=False):
 
 
 class ConsoleFormatter(logging.Formatter):
+	level_colors = {
+		logging.ERROR: 'red',
+		logging.WARNING: 'magenta',
+		logging.INFO: 'blue',
+	}
 	def format(self, record):
-		from task import Task
+		if(record.levelno in self.level_colors):
-		if(isinstance(record.msg, Task)):
+			from termcolor import colored, cprint
-			task = record.msg
+			record.msg = colored(record.msg, self.level_colors[record.levelno])
-			if(task.description is not None):
-				return '\033[0;34m{description}\033[0m'.format(description=task.description)
-			else:
-				return '\033[0;34mRunning {task}\033[0m'.format(task=task)
 		return super(ConsoleFormatter, self).format(record)
 
 
 class FileFormatter(logging.Formatter):
 	def format(self, record):
-		from task import Task
+		return super(FileFormatter, self).format(record)
-		from datetime import datetime
-		if(isinstance(record.msg, Task)):
-			task = record.msg
-			if(task.description is not None):
-				record.msg = '{description} (running {task})'.format(task=task, description=task.description)
-			else:
-				record.msg = 'Running {task}'.format(task=task)
-			message = super(FileFormatter, self).format(record)
-			record.msg = task
-		else:
-			message = super(FileFormatter, self).format(record)
-		return message

base/task.py

@@ -2,7 +2,6 @@ from common.exceptions import TaskListError
 
 
 class Task(object):
-	description = None
 	
 	phase = None
 	before = []

base/tasklist.py

@@ -22,21 +22,41 @@ class TaskList(object):
 		return next(task for task in self.tasks if type(task) is ref)
 
 	def run(self, bootstrap_info):
-		task_list = self.create_list()
+		task_list = self.create_list(self.tasks)
 		log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(repr(task) for task in task_list)))
-		for task in task_list:
-			log.info(task)
-			task.run(bootstrap_info)
 
-	def create_list(self):
+		tasks_completed = []
+		try:
+			for task in task_list:
+				if hasattr(task, 'description'):
+					log.info(task.description)
+				else:
+					log.info('Running {task}'.format(task=task))
+				task.run(bootstrap_info)
+				tasks_completed.append(task)
+		except Exception, e:
+			log.exception(e)
+			log.error('Rolling back')
+			for task in reversed(tasks_completed):
+				rollback = getattr(task, 'rollback', None)
+				if not callable(rollback):
+					continue
+				if hasattr(task, 'rollback_description'):
+					log.warning(task.rollback_description)
+				else:
+					log.warning('Rolling back {task}'.format(task=task))
+				task.rollback(bootstrap_info)
+			log.info('Successfully completed rollback')
+
+	def create_list(self, tasks):
 		from common.phases import order
 		graph = {}
-		for task in self.tasks:
+		for task in tasks:
 			graph[task] = []
 			graph[task].extend([self.get(succ) for succ in task.before])
-			graph[task].extend([succ for succ in self.tasks if type(task) in succ.after])
+			graph[task].extend([succ for succ in tasks if type(task) in succ.after])
 			succeeding_phases = order[order.index(task.phase)+1:]
-			graph[task].extend([succ for succ in self.tasks if succ.phase in succeeding_phases])
+			graph[task].extend([succ for succ in tasks if succ.phase in succeeding_phases])
 
 
 		components = self.strongly_connected_components(graph)
@@ -72,7 +92,6 @@ class TaskList(object):
 			stack.append(node)
 
 			for successor in graph[node]:
-				# print successor
 				visit(successor)
 				low[node] = min(low[node], low[successor])
 

common/exceptions.py

@@ -1,4 +1,3 @@
-__all__ = ['ManifestError']
 
 
 class ManifestError(Exception):
@@ -17,3 +16,6 @@ class TaskListError(Exception):
 		self.message = message
 	def __str__(self):
 		return "Error in tasklist: {0}".format(self.message)
+
+class TaskException(Exception):
+	pass

common/tasks.py

@@ -0,0 +1,11 @@
+from base import Task
+import phases
+
+
+class TriggerRollback(Task):
+	phase = phases.cleanup
+
+	description = 'Triggering a rollback by throwing an exception'
+	def run(self, info):
+		from common.exceptions import TaskException
+		raise TaskException('Trigger rollback')

providers/ec2/__init__.py

@@ -10,3 +10,6 @@ def tasks(tasklist, manifest):
 	             connection.GetCredentials(), host.GetInfo(), connection.Connect())
 	if manifest.volume['backing'].lower() == 'ebs':
 		tasklist.add(ebs.CreateVolume(), ebs.AttachVolume())
+
+	from common.tasks import TriggerRollback
+	tasklist.add(TriggerRollback())

providers/ec2/tasks/connection.py

@@ -1,5 +1,6 @@
 from base import Task
 from common import phases
+import host
 
 
 class GetCredentials(Task):
@@ -31,7 +32,7 @@ class GetCredentials(Task):
 class Connect(Task):
 	description = 'Connecting to EC2'
 	phase = phases.preparation
-	after = [GetCredentials]
+	after = [GetCredentials, host.GetInfo]
 	
 	def run(self, info):
 		super(Connect, self).run(info)

providers/ec2/tasks/ebs.py

@@ -2,21 +2,32 @@ from base import Task
 from common import phases
 from common.exceptions import TaskException
 from connection import Connect
+import time
+
 
 class CreateVolume(Task):
-	description = 'Creating an EBS volume for bootstrapping'
 	phase = phases.volume_creation
 	after = [Connect]
 
+	description = 'Creating an EBS volume for bootstrapping'
 	def run(self, info):
 		volume_size = int(info.manifest.volume['size']/1024)
-		info.volume = info.conn.create_volume(volume_size, info.host['availabilityZone'])
+
+		info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone'])
+		while info.volume.volume_state() != 'available':
+			time.sleep(5)
+			info.volume.update()
+
+	rollback_description = 'Deleting the EBS volume'
+	def rollback(self, info):
+		info.volume.delete()
+		del info.volume
 
 class AttachVolume(Task):
-	description = 'Attaching the EBS volume'
 	phase = phases.volume_creation
 	after = [CreateVolume]
 
+	description = 'Attaching the EBS volume'
 	def run(self, info):
 		def char_range(c1, c2):
 			"""Generates the characters from `c1` to `c2`, inclusive."""
@@ -24,17 +35,27 @@ class AttachVolume(Task):
 				yield chr(c)
 
 		import os.path
-		import os.stat
+		info.bootstrap_device = {}
-		from stat import S_ISBLK
+		for letter in char_range('f', 'z'):
-		for letter in char_range('a', 'z'):
 			dev_path = os.path.join('/dev', 'xvd' + letter)
-			mode = os.stat(dev_path).st_mode
+			if not os.path.exists(dev_path):
-			if S_ISBLK(mode):
+				info.bootstrap_device['path'] = dev_path
-				info.bootstrap_device = {'path': dev_path}
+				info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter)
 				break
 		if 'path' not in info.bootstrap_device:
 			raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
-		info.conn.volume.attach(info.host['instanceId'], info.bootstrap_device['path'])
+
+		info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path'])
+		while info.volume.attachment_state() != 'attached':
+			time.sleep(2)
+			info.volume.update()
+
+	rollback_description = 'Detaching the EBS volume'
+	def rollback(self, info):
+		info.volume.detach()
+		while info.volume.attachment_state() is not None:
+			time.sleep(2)
+			info.volume.update()
 
 class VolumeError(TaskException):
 	pass