bootstrap-vz/base/tasklist.py

from common.exceptions import TaskListError
import logging
log = logging.getLogger(__name__)


class TaskList(object):

	def __init__(self):
		self.tasks = set()
		self.tasks_completed = []

	def load(self, function, manifest, *args):
		getattr(manifest.modules['provider'], function)(self.tasks, manifest, *args)
		for plugin in manifest.modules['plugins']:
			fn = getattr(plugin, function, None)
			if callable(fn):
				fn(self.tasks, manifest, *args)

	def run(self, info={}, dry_run=False):
		task_list = self.create_list()
		log.debug('Tasklist:\n\t{list}'.format(list='\n\t'.join(map(repr, task_list))))

		for task in task_list:
			if hasattr(task, 'description'):
				log.info(task.description)
			else:
				log.info('Running {task}'.format(task=task))
			if not dry_run:
				task.run(info)
			self.tasks_completed.append(task)

	def create_list(self):
		from common.phases import order
		# Get a hold of all tasks
		tasks = self.get_all_tasks()
		# Make sure the taskset is a subset of all the tasks we have gathered
		self.tasks.issubset(tasks)
		# Create a graph over all tasks by creating a map of each tasks successors
		graph = {}
		for task in tasks:
			# Do a sanity check first
			self.check_ordering(task)
			successors = set()
			# Add all successors mentioned in the task
			successors.update(task.successors)
			# Add all tasks that mention this task as a predecessor
			successors.update(filter(lambda succ: task in succ.predecessors, tasks))
			# Create a list of phases that succeed the phase of this task
			succeeding_phases = order[order.index(task.phase) + 1:]
			# Add all tasks that occur in above mentioned succeeding phases
			successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks))
			# Map the successors to the task
			graph[task] = successors

		components = self.strongly_connected_components(graph)
		cycles_found = 0
		for component in components:
			if len(component) > 1:
				cycles_found += 1
				log.debug('Cycle: {list}\n'.format(list=', '.join(map(repr, component))))
		if cycles_found > 0:
			msg = ('{0} cycles were found in the tasklist, '
			       'consult the logfile for more information.'.format(cycles_found))
			raise TaskListError(msg)

		# Run a topological sort on the graph, returning an ordered list
		sorted_tasks = self.topological_sort(graph)

		# Filter out any tasks not in the tasklist
		# We want to maintain ordering, so we don't use set intersection
		sorted_tasks = filter(lambda task: task in self.tasks, sorted_tasks)
		return sorted_tasks

	def get_all_tasks(self):
		# Get a generator that returns all classes in the package
		classes = self.get_all_classes('..')

		# lambda function to check whether a class is a task (excluding the superclass Task)
		def is_task(obj):
			from task import Task
			return issubclass(obj, Task) and obj is not Task
		return filter(is_task, classes)  # Only return classes that are tasks

	# Given a path, retrieve all the classes in it
	def get_all_classes(self, path=None):
		import pkgutil
		import importlib
		import inspect

		def walk_error(module):
			raise Exception('Unable to inspect module `{module}\''.format(module=module))
		walker = pkgutil.walk_packages(path, '', walk_error)
		for _, module_name, _ in walker:
			module = importlib.import_module(module_name)
			classes = inspect.getmembers(module, inspect.isclass)
			for class_name, obj in classes:
				# We only want classes that are defined in the module, and not imported ones
				if obj.__module__ == module_name:
						yield obj

	def check_ordering(self, task):
		for successor in task.successors:
			if successor.phase > successor.phase:
				msg = ("The task {task} is specified as running before {other}, "
				       "but its phase '{phase}' lies after the phase '{other_phase}'"
				       .format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
				raise TaskListError(msg)
		for predecessor in task.predecessors:
			if task.phase < predecessor.phase:
				msg = ("The task {task} is specified as running after {other}, "
				       "but its phase '{phase}' lies before the phase '{other_phase}'"
				       .format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
				raise TaskListError(msg)

	def strongly_connected_components(self, graph):
		# Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
		# Find the strongly connected components in a graph using Tarjan's algorithm.
		# graph should be a dictionary mapping node names to lists of successor nodes.

		result = []
		stack = []
		low = {}

		def visit(node):
			if node in low:
				return

			num = len(low)
			low[node] = num
			stack_pos = len(stack)
			stack.append(node)

			for successor in graph[node]:
				visit(successor)
				low[node] = min(low[node], low[successor])

			if num == low[node]:
				component = tuple(stack[stack_pos:])
				del stack[stack_pos:]
				result.append(component)
				for item in component:
					low[item] = len(graph)

		for node in graph:
			visit(node)

		return result

	def topological_sort(self, graph):
		# Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
		count = {}
		for node in graph:
			count[node] = 0
		for node in graph:
			for successor in graph[node]:
				count[successor] += 1

		ready = [node for node in graph if count[node] == 0]

		result = []
		while ready:
			node = ready.pop(-1)
			result.append(node)

			for successor in graph[node]:
				count[successor] -= 1
				if count[successor] == 0:
					ready.append(successor)

		return result