mirror of
https://github.com/kevingruesser/bootstrap-vz.git
synced 2025-08-22 18:00:35 +00:00
f3d633780f
String concatenation can at times be easier to read that format(). One should choose whichever approach is more readable.
252 lines
8.3 KiB
Python
252 lines
8.3 KiB
Python
"""The tasklist module contains the TaskList class.
|
|
.. module:: tasklist
|
|
"""
|
|
|
|
from bootstrapvz.common.exceptions import TaskListError
|
|
import logging
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class TaskList(object):
|
|
"""The tasklist class aggregates all tasks that should be run
|
|
and orders them according to their dependencies.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.tasks = set()
|
|
self.tasks_completed = []
|
|
|
|
def load(self, function, manifest, *args):
|
|
"""Calls 'function' on the provider and all plugins that have been loaded by the manifest.
|
|
Any additional arguments are passed directly to 'function'.
|
|
The function that is called shall accept the taskset as its first argument and the manifest
|
|
as its second argument.
|
|
|
|
Args:
|
|
function (str): Name of the function to call
|
|
manifest (Manifest): The manifest
|
|
\*args: Additional arguments that should be passed to the function that is called
|
|
"""
|
|
# Call 'function' on the provider
|
|
getattr(manifest.modules['provider'], function)(self.tasks, manifest, *args)
|
|
for plugin in manifest.modules['plugins']:
|
|
# Plugins are not required to have whatever function we call
|
|
fn = getattr(plugin, function, None)
|
|
if callable(fn):
|
|
fn(self.tasks, manifest, *args)
|
|
|
|
def run(self, info, dry_run=False):
|
|
"""Converts the taskgraph into a list and runs all tasks in that list
|
|
|
|
Args:
|
|
info (dict): The bootstrap information object
|
|
dry_run (bool): Whether to actually run the tasks or simply step through them
|
|
"""
|
|
# Create a list for us to run
|
|
task_list = self.create_list()
|
|
# Output the tasklist
|
|
log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list))))
|
|
|
|
for task in task_list:
|
|
# Tasks are not required to have a description
|
|
if hasattr(task, 'description'):
|
|
log.info(task.description)
|
|
else:
|
|
# If there is no description, simply coerce the task into a string and print its name
|
|
log.info('Running ' + str(task))
|
|
if not dry_run:
|
|
# Run the task
|
|
task.run(info)
|
|
# Remember which tasks have been run for later use (e.g. when rolling back, because of an error)
|
|
self.tasks_completed.append(task)
|
|
|
|
def create_list(self):
|
|
"""Creates a list of all the tasks that should be run.
|
|
"""
|
|
from bootstrapvz.common.phases import order
|
|
# Get a hold of all tasks
|
|
tasks = self.get_all_tasks()
|
|
# Make sure the taskset is a subset of all the tasks we have gathered
|
|
self.tasks.issubset(tasks)
|
|
# Create a graph over all tasks by creating a map of each tasks successors
|
|
graph = {}
|
|
for task in tasks:
|
|
# Do a sanity check first
|
|
self.check_ordering(task)
|
|
successors = set()
|
|
# Add all successors mentioned in the task
|
|
successors.update(task.successors)
|
|
# Add all tasks that mention this task as a predecessor
|
|
successors.update(filter(lambda succ: task in succ.predecessors, tasks))
|
|
# Create a list of phases that succeed the phase of this task
|
|
succeeding_phases = order[order.index(task.phase) + 1:]
|
|
# Add all tasks that occur in above mentioned succeeding phases
|
|
successors.update(filter(lambda succ: succ.phase in succeeding_phases, tasks))
|
|
# Map the successors to the task
|
|
graph[task] = successors
|
|
|
|
# Use the strongly connected components algorithm to check for cycles in our task graph
|
|
components = self.strongly_connected_components(graph)
|
|
cycles_found = 0
|
|
for component in components:
|
|
# Node of 1 is also a strongly connected component but hardly a cycle, so we filter them out
|
|
if len(component) > 1:
|
|
cycles_found += 1
|
|
log.debug('Cycle: {list}\n' + (', '.join(map(repr, component))))
|
|
if cycles_found > 0:
|
|
msg = ('{num} cycles were found in the tasklist, '
|
|
'consult the logfile for more information.'.format(num=cycles_found))
|
|
raise TaskListError(msg)
|
|
|
|
# Run a topological sort on the graph, returning an ordered list
|
|
sorted_tasks = self.topological_sort(graph)
|
|
|
|
# Filter out any tasks not in the tasklist
|
|
# We want to maintain ordering, so we don't use set intersection
|
|
sorted_tasks = filter(lambda task: task in self.tasks, sorted_tasks)
|
|
return sorted_tasks
|
|
|
|
def get_all_tasks(self):
|
|
"""Gets a list of all task classes in the package
|
|
|
|
Returns:
|
|
list. A list of all tasks in the package
|
|
"""
|
|
# Get a generator that returns all classes in the package
|
|
import os.path
|
|
pkg_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
|
|
classes = self.get_all_classes(pkg_path, 'bootstrapvz.')
|
|
|
|
# lambda function to check whether a class is a task (excluding the superclass Task)
|
|
def is_task(obj):
|
|
from task import Task
|
|
return issubclass(obj, Task) and obj is not Task
|
|
return filter(is_task, classes) # Only return classes that are tasks
|
|
|
|
def get_all_classes(self, path=None, prefix=''):
|
|
""" Given a path to a package, this function retrieves all the classes in it
|
|
|
|
Args:
|
|
path (str): Path to the package
|
|
prefix (str): Name of the package followed by a dot
|
|
|
|
Returns:
|
|
generator. A generator that yields classes
|
|
|
|
Raises:
|
|
Exception
|
|
"""
|
|
import pkgutil
|
|
import importlib
|
|
import inspect
|
|
|
|
def walk_error(module):
|
|
raise Exception('Unable to inspect module ' + module)
|
|
walker = pkgutil.walk_packages([path], prefix, walk_error)
|
|
for _, module_name, _ in walker:
|
|
module = importlib.import_module(module_name)
|
|
classes = inspect.getmembers(module, inspect.isclass)
|
|
for class_name, obj in classes:
|
|
# We only want classes that are defined in the module, and not imported ones
|
|
if obj.__module__ == module_name:
|
|
yield obj
|
|
|
|
def check_ordering(self, task):
|
|
"""Checks the ordering of a task in relation to other tasks and their phases
|
|
This function checks for a subset of what the strongly connected components algorithm does,
|
|
but can deliver a more precise error message, namely that there is a conflict between
|
|
what a task has specified as its predecessors or successors and in which phase it is placed.
|
|
|
|
Args:
|
|
task (Task): The task to check the ordering for
|
|
|
|
Raises:
|
|
TaskListError
|
|
"""
|
|
for successor in task.successors:
|
|
# Run through all successors and check whether the phase of the task
|
|
# comes before the phase of a successor
|
|
if task.phase > successor.phase:
|
|
msg = ("The task {task} is specified as running before {other}, "
|
|
"but its phase '{phase}' lies after the phase '{other_phase}'"
|
|
.format(task=task, other=successor, phase=task.phase, other_phase=successor.phase))
|
|
raise TaskListError(msg)
|
|
for predecessor in task.predecessors:
|
|
# Run through all predecessors and check whether the phase of the task
|
|
# comes after the phase of a predecessor
|
|
if task.phase < predecessor.phase:
|
|
msg = ("The task {task} is specified as running after {other}, "
|
|
"but its phase '{phase}' lies before the phase '{other_phase}'"
|
|
.format(task=task, other=predecessor, phase=task.phase, other_phase=predecessor.phase))
|
|
raise TaskListError(msg)
|
|
|
|
def strongly_connected_components(self, graph):
|
|
"""Find the strongly connected components in a graph using Tarjan's algorithm.
|
|
Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
|
|
|
|
Args:
|
|
graph (dict): mapping of tasks to lists of successor tasks
|
|
|
|
Returns:
|
|
list. List of tuples that are strongly connected comoponents
|
|
"""
|
|
|
|
result = []
|
|
stack = []
|
|
low = {}
|
|
|
|
def visit(node):
|
|
if node in low:
|
|
return
|
|
|
|
num = len(low)
|
|
low[node] = num
|
|
stack_pos = len(stack)
|
|
stack.append(node)
|
|
|
|
for successor in graph[node]:
|
|
visit(successor)
|
|
low[node] = min(low[node], low[successor])
|
|
|
|
if num == low[node]:
|
|
component = tuple(stack[stack_pos:])
|
|
del stack[stack_pos:]
|
|
result.append(component)
|
|
for item in component:
|
|
low[item] = len(graph)
|
|
|
|
for node in graph:
|
|
visit(node)
|
|
|
|
return result
|
|
|
|
def topological_sort(self, graph):
|
|
"""Runs a topological sort on a graph
|
|
Source: http://www.logarithmic.net/pfh-files/blog/01208083168/sort.py
|
|
|
|
Args:
|
|
graph (dict): mapping of tasks to lists of successor tasks
|
|
|
|
Returns:
|
|
list. A list of all tasks in the graph sorted according to ther dependencies
|
|
"""
|
|
count = {}
|
|
for node in graph:
|
|
count[node] = 0
|
|
for node in graph:
|
|
for successor in graph[node]:
|
|
count[successor] += 1
|
|
|
|
ready = [node for node in graph if count[node] == 0]
|
|
|
|
result = []
|
|
while ready:
|
|
node = ready.pop(-1)
|
|
result.append(node)
|
|
|
|
for successor in graph[node]:
|
|
count[successor] -= 1
|
|
if count[successor] == 0:
|
|
ready.append(successor)
|
|
|
|
return result
|