Fix #98. External plugin architecture implemented

bootstrapvz/README.rst

@@ -11,7 +11,7 @@ dependency graph where directed edges dictate precedence. Each task is
 a simple class that defines its predecessor tasks and successor tasks
 via attributes. Here is an example:
 
-::
+.. code:: python
 
 	class MapPartitions(Task):
 		description = 'Mapping volume partitions'

bootstrapvz/__init__.py

@@ -1,3 +1,3 @@
 
 
-__version__ = '0.9.0'
+__version__ = '0.9.5'

bootstrapvz/base/manifest.py

@@ -52,18 +52,32 @@ class Manifest(object):
 		"""
 		# Get the provider name from the manifest and load the corresponding module
 		provider_modname = 'bootstrapvz.providers.' + self.data['provider']['name']
-		log.debug('Loading provider ' + provider_modname)
+		log.debug('Loading provider ' + self.data['provider']['name'])
 		# Create a modules dict that contains the loaded provider and plugins
 		import importlib
 		self.modules = {'provider': importlib.import_module(provider_modname),
 		                'plugins': [],
 		                }
 		# Run through all the plugins mentioned in the manifest and load them
+		from pkg_resources import iter_entry_points
 		if 'plugins' in self.data:
-			for plugin_name, plugin_data in self.data['plugins'].iteritems():
+			for plugin_name in self.data['plugins'].keys():
+				log.debug('Loading plugin ' + plugin_name)
+				try:
+					# Internal bootstrap-vz plugins take precedence wrt. plugin name
 					modname = 'bootstrapvz.plugins.' + plugin_name
-				log.debug('Loading plugin ' + modname)
 					plugin = importlib.import_module(modname)
+				except ImportError:
+					entry_points = list(iter_entry_points('bootstrapvz.plugins', name=plugin_name))
+					num_entry_points = len(entry_points)
+					if num_entry_points < 1:
+						raise
+					if num_entry_points > 1:
+						msg = ('Unable to load plugin {name}, '
+						       'there are {num} entry points to choose from.'
+						       .format(name=plugin_name, num=num_entry_points))
+						raise ImportError(msg)
+					plugin = entry_points[0].load()
 				self.modules['plugins'].append(plugin)
 
 	def validate(self):

bootstrapvz/base/tasklist.py

@@ -21,8 +21,13 @@ class TaskList(object):
 		:param dict info: The bootstrap information object
 		:param bool dry_run: Whether to actually run the tasks or simply step through them
 		"""
+		# Get a hold of every task we can find, so that we can topologically sort
+		# all tasks, rather than just the subset we are going to run.
+		from bootstrapvz.common import tasks as common_tasks
+		modules = [common_tasks, info.manifest.modules['provider']] + info.manifest.modules['plugins']
+		all_tasks = set(get_all_tasks(modules))
 		# Create a list for us to run
-		task_list = create_list(self.tasks)
+		task_list = create_list(self.tasks, all_tasks)
 		# Output the tasklist
 		log.debug('Tasklist:\n\t' + ('\n\t'.join(map(repr, task_list))))
 
@@ -61,12 +66,10 @@ def load_tasks(function, manifest, *args):
 	return tasks
 
 
-def create_list(taskset):
+def create_list(taskset, all_tasks):
 	"""Creates a list of all the tasks that should be run.
 	"""
 	from bootstrapvz.common.phases import order
-	# Get a hold of all tasks
-	all_tasks = get_all_tasks()
 	# Make sure all_tasks is a superset of the resolved taskset
 	if not all_tasks >= taskset:
 		msg = ('bootstrap-vz generated a list of all available tasks. '
@@ -115,23 +118,27 @@ def create_list(taskset):
 	return sorted_tasks
 
 
-def get_all_tasks():
+def get_all_tasks(modules):
 	"""Gets a list of all task classes in the package
 
 	:return: A list of all tasks in the package
 	:rtype: list
 	"""
-	# Get a generator that returns all classes in the package
 	import os.path
-	pkg_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+	# Get generators that return all classes in a module
-	exclude_pkgs = ['bootstrapvz.base', 'bootstrapvz.remote']
+	generators = []
-	classes = get_all_classes(pkg_path, 'bootstrapvz.', exclude_pkgs)
+	for module in modules:
+		module_path = os.path.dirname(module.__file__)
+		module_prefix = module.__name__ + '.'
+		generators.append(get_all_classes(module_path, module_prefix))
+	import itertools
+	classes = itertools.chain(*generators)
 
 	# lambda function to check whether a class is a task (excluding the superclass Task)
 	def is_task(obj):
 		from task import Task
 		return issubclass(obj, Task) and obj is not Task
-	return set(filter(is_task, classes))  # Only return classes that are tasks
+	return filter(is_task, classes)  # Only return classes that are tasks
 
 
 def get_all_classes(path=None, prefix='', excludes=[]):
@@ -150,7 +157,7 @@ def get_all_classes(path=None, prefix='', excludes=[]):
 
 	def walk_error(module_name):
 		if not any(map(lambda excl: module_name.startswith(excl), excludes)):
-			raise Exception('Unable to inspect module ' + module_name)
+			raise TaskListError('Unable to inspect module ' + module_name)
 	walker = pkgutil.walk_packages([path], prefix, walk_error)
 	for _, module_name, _ in walker:
 		if any(map(lambda excl: module_name.startswith(excl), excludes)):

docs/developers/index.rst

@@ -6,6 +6,7 @@ Developers
    :hidden:
 
    contributing
+   plugins
    documentation
    switches
    taskoverview

docs/developers/plugins.rst

@@ -0,0 +1,146 @@
+Developing plugins
+==================
+
+Developing a plugin for bootstrap-vz is a fairly straightforward process,
+since there is very little code overhead.
+
+The process is the same whether you create an `internal <#internal-plugins>`__
+or an `external <#external-plugins>`__ plugin (though you need to add
+some code for package management when creating an external plugin)
+
+Start by creating an ``__init__.py`` in your plugin folder.
+The only obligatory function you need to implement is ``resolve_tasks()``.
+This function adds tasks to be run to the tasklist:
+
+.. code:: python
+
+	def resolve_tasks(taskset, manifest):
+		taskset.add(tasks.DoSomething)
+
+The manifest variable holds the manifest the user specified,
+with it you can determine settings for your plugin and e.g.
+check of which release of Debian bootstrap-vz will create an image.
+
+A task is a class with a static ``run()`` function and some meta-information:
+
+.. code:: python
+
+	class DoSomething(Task):
+		description = 'Doing something'
+		phase = phases.volume_preparation
+		predecessors = [PartitionVolume]
+		successors = [filesystem.Format]
+
+		@classmethod
+		def run(cls, info):
+			pass
+
+To read more about tasks and their ordering, check out the section on
+`how bootstrap-vz works <index.html#tasks>`__.
+
+
+Besides the ``resolve_tasks()`` function, there is also the ``resolve_rollback_tasks()``
+function, which comes into play when something has gone awry while bootstrapping.
+It should be used to clean up anything that was created during the bootstrapping
+process. If you created temporary files for example, you can add a task to the
+rollback taskset that deletes those files, you might even already have it because
+you run it after an image has been successfully bootstrapped:
+
+.. code:: python
+
+	def resolve_rollback_tasks(taskset, manifest, completed, counter_task):
+		counter_task(taskset, tasks.DoSomething, tasks.UndoSomething)
+
+In  ``resolve_rollback_tasks()`` you have access to the taskset
+(this time it contains tasks that will be run during rollback), the manifest, and
+the tasks that have already been run before the bootstrapping aborted (``completed``).
+
+The last parameter is the ``counter_task()`` function, with it you can specify that
+a specific task (2nd param) has to be in the taskset (1st param) for the rollback
+task (3rd param) to be added. This saves code and makes it more readable than
+running through the completed tasklist and checking each completed task.
+
+You can also specify a ``validate_manifest()`` function.
+Typically it looks like this:
+
+.. code:: python
+
+	def validate_manifest(data, validator, error):
+		import os.path
+		schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
+		validator(data, schema_path)
+
+This code validates the manifest against a schema in your plugin folder.
+The schema is a `JSON schema <http://json-schema.org/>`__, since bootstrap-vz
+supports `yaml <http://yaml.org/>`__, you can avoid a lot of curly braces
+quotes:
+
+.. code:: yaml
+
+  $schema: http://json-schema.org/draft-04/schema#
+  title: Example plugin manifest
+  type: object
+  properties:
+    plugins:
+      type: object
+      properties:
+        example:
+          type: object
+          properties:
+            message: {type: string}
+          required: [message]
+          additionalProperties: false
+
+In the schema above we check that the ``example`` plugin has a single property
+named ``message`` with a string value (setting ``additionalProperties`` to ``false``
+makes sure that users don't misspell optional attributes).
+
+Internal plugins
+----------------
+Internal plugins are part of the bootstrap-vz package and distributed with it.
+If you have developed a plugin that you think should be part of the package
+because a lot of people might use it you can send a pull request to get it
+included (just remember to `read the guidelines <contributing.html>`__ first).
+
+External plugins
+-----------------
+External plugins are packages distributed separately from bootstrap-vz.
+Separate distribution makes sense when your plugin solves a narrow problem scope
+specific to your use-case or when the plugin contains proprietary code that you
+would not like to share.
+They integrate with bootstrap-vz by exposing an entry-point through ``setup.py``:
+
+.. code:: python
+
+	setup(name='example-plugin',
+	      version=0.9.5,
+	      packages=find_packages(),
+	      include_package_data=True,
+	      entry_points={'bootstrapvz.plugins': ['plugin_name = package_name.module_name']},
+	      install_requires=['bootstrap-vz >= 0.9.5'],
+	      )
+
+Beyond ``setup.py`` the package might need a ``MANIFEST.in`` so that assets
+like ``manifest-schema.yml`` are included when the package is built:
+
+.. code::
+
+	include example/manifest-schema.yml
+	include example/README.rst
+
+To test your package from source you can run ``python setup.py develop``
+to register the package so that bootstrap-vz can find the entry-point of your
+plugin.
+
+An example plugin is available at `<https://github.com/andsens/bootstrap-vz-example-plugin>`__,
+you can use it as a starting point for your own plugin.
+
+Installing external plugins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Some plugins may not find their way to the python package index
+(especially if it's in a private repo). They can of course still be installed
+using pip:
+
+.. code:: sh
+
+	pip install git+ssh://git@github.com/username/repo#egg=plugin_name

docs/taskoverview.py

@@ -6,8 +6,11 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 
 
 def generate_graph_data():
+	import bootstrapvz.common.tasks
+	import bootstrapvz.providers
+	import bootstrapvz.plugins
 	from bootstrapvz.base.tasklist import get_all_tasks
-	tasks = get_all_tasks()
+	tasks = get_all_tasks([bootstrapvz.common.tasks, bootstrapvz.providers, bootstrapvz.plugins])
 
 	def distinct(seq):
 		seen = set()