From e556366c194328a616a4f0af090819dbe6584914 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 3 May 2015 13:07:26 +0200 Subject: [PATCH] Fix #98. External plugin architecture implemented --- bootstrapvz/README.rst | 18 ++--- bootstrapvz/__init__.py | 2 +- bootstrapvz/base/manifest.py | 24 ++++-- bootstrapvz/base/tasklist.py | 29 ++++--- docs/developers/index.rst | 1 + docs/developers/plugins.rst | 146 +++++++++++++++++++++++++++++++++++ docs/taskoverview.py | 5 +- 7 files changed, 198 insertions(+), 27 deletions(-) create mode 100644 docs/developers/plugins.rst diff --git a/bootstrapvz/README.rst b/bootstrapvz/README.rst index 2ca112d..ff99d38 100644 --- a/bootstrapvz/README.rst +++ b/bootstrapvz/README.rst @@ -11,17 +11,17 @@ dependency graph where directed edges dictate precedence. Each task is a simple class that defines its predecessor tasks and successor tasks via attributes. Here is an example: -:: +.. code:: python - class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - predecessors = [PartitionVolume] - successors = [filesystem.Format] + class MapPartitions(Task): + description = 'Mapping volume partitions' + phase = phases.volume_preparation + predecessors = [PartitionVolume] + successors = [filesystem.Format] - @classmethod - def run(cls, info): - info.volume.partition_map.map(info.volume) + @classmethod + def run(cls, info): + info.volume.partition_map.map(info.volume) In this case the attributes define that the task at hand should run after the ``PartitionVolume`` task — i.e. after volume has been diff --git a/bootstrapvz/__init__.py b/bootstrapvz/__init__.py index 245a299..e0f4722 100644 --- a/bootstrapvz/__init__.py +++ b/bootstrapvz/__init__.py @@ -1,3 +1,3 @@ -__version__ = '0.9.0' +__version__ = '0.9.5' diff --git a/bootstrapvz/base/manifest.py b/bootstrapvz/base/manifest.py index 0daad4f..53a87da 100644 --- a/bootstrapvz/base/manifest.py +++ b/bootstrapvz/base/manifest.py @@ -52,18 +52,32 @@ class Manifest(object): """ # Get the provider name from the manifest and load the corresponding module provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name'] - log.debug('Loading provider ' + provider_modname) + log.debug('Loading provider ' + self.data['provider']['name']) # Create a modules dict that contains the loaded provider and plugins import importlib self.modules = {'provider': importlib.import_module(provider_modname), 'plugins': [], } # Run through all the plugins mentioned in the manifest and load them + from pkg_resources import iter_entry_points if 'plugins' in self.data: - for plugin_name, plugin_data in self.data['plugins'].iteritems(): - modname = 'bootstrapvz.plugins.' + plugin_name - log.debug('Loading plugin ' + modname) - plugin = importlib.import_module(modname) + for plugin_name in self.data['plugins'].keys(): + log.debug('Loading plugin ' + plugin_name) + try: + # Internal bootstrap-vz plugins take precedence wrt. plugin name + modname = 'bootstrapvz.plugins.' + plugin_name + plugin = importlib.import_module(modname) + except ImportError: + entry_points = list(iter_entry_points('bootstrapvz.plugins', name=plugin_name)) + num_entry_points = len(entry_points) + if num_entry_points < 1: + raise + if num_entry_points > 1: + msg = ('Unable to load plugin {name}, ' + 'there are {num} entry points to choose from.' + .format(name=plugin_name, num=num_entry_points)) + raise ImportError(msg) + plugin = entry_points[0].load() self.modules['plugins'].append(plugin) def validate(self): diff --git a/bootstrapvz/base/tasklist.py b/bootstrapvz/base/tasklist.py index 9f88056..0503e26 100644 --- a/bootstrapvz/base/tasklist.py +++ b/bootstrapvz/base/tasklist.py @@ -21,8 +21,13 @@ class TaskList(object): :param dict info: The bootstrap information object :param bool dry_run: Whether to actually run the tasks or simply step through them """ + # Get a hold of every task we can find, so that we can topologically sort + # all tasks, rather than just the subset we are going to run. + from bootstrapvz.common import tasks as common_tasks + modules = [common_tasks, info.manifest.modules['provider']] + info.manifest.modules['plugins'] + all_tasks = set(get_all_tasks(modules)) # Create a list for us to run - task_list = create_list(self.tasks) + task_list = create_list(self.tasks, all_tasks) # Output the tasklist log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list)))) @@ -61,12 +66,10 @@ def load_tasks(function, manifest, *args): return tasks -def create_list(taskset): +def create_list(taskset, all_tasks): """Creates a list of all the tasks that should be run. """ from bootstrapvz.common.phases import order - # Get a hold of all tasks - all_tasks = get_all_tasks() # Make sure all_tasks is a superset of the resolved taskset if not all_tasks >= taskset: msg = ('bootstrap-vz generated a list of all available tasks. ' @@ -115,23 +118,27 @@ def create_list(taskset): return sorted_tasks -def get_all_tasks(): +def get_all_tasks(modules): """Gets a list of all task classes in the package :return: A list of all tasks in the package :rtype: list """ - # Get a generator that returns all classes in the package import os.path - pkg_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) - exclude_pkgs = ['bootstrapvz.base', 'bootstrapvz.remote'] - classes = get_all_classes(pkg_path, 'bootstrapvz.', exclude_pkgs) + # Get generators that return all classes in a module + generators = [] + for module in modules: + module_path = os.path.dirname(module.__file__) + module_prefix = module.__name__ + '.' + generators.append(get_all_classes(module_path, module_prefix)) + import itertools + classes = itertools.chain(*generators) # lambda function to check whether a class is a task (excluding the superclass Task) def is_task(obj): from task import Task return issubclass(obj, Task) and obj is not Task - return set(filter(is_task, classes)) # Only return classes that are tasks + return filter(is_task, classes) # Only return classes that are tasks def get_all_classes(path=None, prefix='', excludes=[]): @@ -150,7 +157,7 @@ def get_all_classes(path=None, prefix='', excludes=[]): def walk_error(module_name): if not any(map(lambda excl: module_name.startswith(excl), excludes)): - raise Exception('Unable to inspect module ' + module_name) + raise TaskListError('Unable to inspect module ' + module_name) walker = pkgutil.walk_packages([path], prefix, walk_error) for _, module_name, _ in walker: if any(map(lambda excl: module_name.startswith(excl), excludes)): diff --git a/docs/developers/index.rst b/docs/developers/index.rst index fa243d4..e8bd9fa 100644 --- a/docs/developers/index.rst +++ b/docs/developers/index.rst @@ -6,6 +6,7 @@ Developers :hidden: contributing + plugins documentation switches taskoverview diff --git a/docs/developers/plugins.rst b/docs/developers/plugins.rst new file mode 100644 index 0000000..119eecc --- /dev/null +++ b/docs/developers/plugins.rst @@ -0,0 +1,146 @@ +Developing plugins +================== + +Developing a plugin for bootstrap-vz is a fairly straightforward process, +since there is very little code overhead. + +The process is the same whether you create an `internal <#internal-plugins>`__ +or an `external <#external-plugins>`__ plugin (though you need to add +some code for package management when creating an external plugin) + +Start by creating an ``__init__.py`` in your plugin folder. +The only obligatory function you need to implement is ``resolve_tasks()``. +This function adds tasks to be run to the tasklist: + +.. code:: python + + def resolve_tasks(taskset, manifest): + taskset.add(tasks.DoSomething) + +The manifest variable holds the manifest the user specified, +with it you can determine settings for your plugin and e.g. +check of which release of Debian bootstrap-vz will create an image. + +A task is a class with a static ``run()`` function and some meta-information: + +.. code:: python + + class DoSomething(Task): + description = 'Doing something' + phase = phases.volume_preparation + predecessors = [PartitionVolume] + successors = [filesystem.Format] + + @classmethod + def run(cls, info): + pass + +To read more about tasks and their ordering, check out the section on +`how bootstrap-vz works `__. + + +Besides the ``resolve_tasks()`` function, there is also the ``resolve_rollback_tasks()`` +function, which comes into play when something has gone awry while bootstrapping. +It should be used to clean up anything that was created during the bootstrapping +process. If you created temporary files for example, you can add a task to the +rollback taskset that deletes those files, you might even already have it because +you run it after an image has been successfully bootstrapped: + +.. code:: python + + def resolve_rollback_tasks(taskset, manifest, completed, counter_task): + counter_task(taskset, tasks.DoSomething, tasks.UndoSomething) + +In ``resolve_rollback_tasks()`` you have access to the taskset +(this time it contains tasks that will be run during rollback), the manifest, and +the tasks that have already been run before the bootstrapping aborted (``completed``). + +The last parameter is the ``counter_task()`` function, with it you can specify that +a specific task (2nd param) has to be in the taskset (1st param) for the rollback +task (3rd param) to be added. This saves code and makes it more readable than +running through the completed tasklist and checking each completed task. + +You can also specify a ``validate_manifest()`` function. +Typically it looks like this: + +.. code:: python + + def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml')) + validator(data, schema_path) + +This code validates the manifest against a schema in your plugin folder. +The schema is a `JSON schema `__, since bootstrap-vz +supports `yaml `__, you can avoid a lot of curly braces +quotes: + +.. code:: yaml + + $schema: http://json-schema.org/draft-04/schema# + title: Example plugin manifest + type: object + properties: + plugins: + type: object + properties: + example: + type: object + properties: + message: {type: string} + required: [message] + additionalProperties: false + +In the schema above we check that the ``example`` plugin has a single property +named ``message`` with a string value (setting ``additionalProperties`` to ``false`` +makes sure that users don't misspell optional attributes). + +Internal plugins +---------------- +Internal plugins are part of the bootstrap-vz package and distributed with it. +If you have developed a plugin that you think should be part of the package +because a lot of people might use it you can send a pull request to get it +included (just remember to `read the guidelines `__ first). + +External plugins +----------------- +External plugins are packages distributed separately from bootstrap-vz. +Separate distribution makes sense when your plugin solves a narrow problem scope +specific to your use-case or when the plugin contains proprietary code that you +would not like to share. +They integrate with bootstrap-vz by exposing an entry-point through ``setup.py``: + +.. code:: python + + setup(name='example-plugin', + version=0.9.5, + packages=find_packages(), + include_package_data=True, + entry_points={'bootstrapvz.plugins': ['plugin_name = package_name.module_name']}, + install_requires=['bootstrap-vz >= 0.9.5'], + ) + +Beyond ``setup.py`` the package might need a ``MANIFEST.in`` so that assets +like ``manifest-schema.yml`` are included when the package is built: + +.. code:: + + include example/manifest-schema.yml + include example/README.rst + +To test your package from source you can run ``python setup.py develop`` +to register the package so that bootstrap-vz can find the entry-point of your +plugin. + +An example plugin is available at ``__, +you can use it as a starting point for your own plugin. + +Installing external plugins +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Some plugins may not find their way to the python package index +(especially if it's in a private repo). They can of course still be installed +using pip: + +.. code:: sh + + pip install git+ssh://git@github.com/username/repo#egg=plugin_name diff --git a/docs/taskoverview.py b/docs/taskoverview.py index 97ecbb0..fa6a29b 100755 --- a/docs/taskoverview.py +++ b/docs/taskoverview.py @@ -6,8 +6,11 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..')) def generate_graph_data(): + import bootstrapvz.common.tasks + import bootstrapvz.providers + import bootstrapvz.plugins from bootstrapvz.base.tasklist import get_all_tasks - tasks = get_all_tasks() + tasks = get_all_tasks([bootstrapvz.common.tasks, bootstrapvz.providers, bootstrapvz.plugins]) def distinct(seq): seen = set()