Task dependencies, bogus implementation

base/__init__.py

@@ -1,4 +1,5 @@
-__all__ = ['Manifest', 'Task', 'main']
+__all__ = ['Manifest', 'Phase', 'Task', 'main']
 from manifest import Manifest
 from task import Task
+from phase import Phase
 from main import main

base/main.py

@@ -48,8 +48,9 @@ def run(args):
 
 	from tasklist import TaskList
 	tasklist = TaskList()
-	provider.modify_tasklist(tasklist, manifest)
+	provider.tasks(tasklist, manifest)
-	tasklist.plugins(manifest)
+	for plugin in manifest.loaded_plugins:
+		plugin.tasks(tasklist, manifest)
 
 	from bootstrapinfo import BootstrapInformation
 	bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug)

base/phase.py

@@ -0,0 +1,14 @@
+
+
+class Phase(object):
+	description = None
+
+	def __init__(self):
+		pass
+
+	def __cmp__(self, other):
+		from common.phases import order
+		return order.index(self) - order.index(other)
+
+	def __str__(self):
+		return '{name}'.format(name=self.__class__.__name__)

base/task.py

@@ -1,13 +1,56 @@
+from functools import total_ordering
 
 
+@total_ordering
 class Task(object):
 	description = None
+	
+	phase = None
+	before = []
+	after = []
 
 	def __init__(self):
-		pass
+		self._check_ordering()
 
 	def run(self, info):
 		pass
 
+	def _after(self, other):
+		return self.phase > other.phase or type(self) in other.before or type(other) in self.after
+
+	def _before(self, other):
+		return self.phase < other.phase or type(other) in self.before or type(self) in other.after
+
+	def __lt__(self, other):
+		return self._before(other) and not self._after(other)
+
+	def __gt__(self, other):
+		return not self._before(other) and self._after(other)
+
+	def __eq__(self, other):
+		return not self._before(other) and not self._after(other)
+
+	def __ne__(self, other):
+		return self._before(other) or self._after(other)
+
 	def __str__(self):
 		return '{module}.{task}'.format(module=self.__module__, task=self.__class__.__name__)
+
+	def _check_ordering(self):
+		for task in self.before:
+			if self.phase > task.phase:
+				msg = ("The task {self} is specified as running before {other}, "
+				       "but its phase {phase} lies after the phase {other_phase}"
+				       .format(self, other, self.phase, other.phase))
+				raise TaskOrderError(msg)
+		for task in self.after:
+			if self.phase < task.phase:
+				msg = ("The task {self} is specified as running after {other}, "
+				       "but its phase {phase} lies before the phase {other_phase}"
+				       .format(self=self, other=other, phase=self.phase, other_phase=other.phase))
+				raise TaskOrderError(msg)
+		conflict = next(iter(set(self.before) &  set(self.after)), None)
+		if conflict is not None:
+				msg = ("The task {self} is specified as running both before and after {conflict}"
+				       .format(self=self, conflict=conflict))
+				raise TaskOrderError(msg)

base/tasklist.py

@@ -2,33 +2,29 @@ import logging
 log = logging.getLogger(__name__)
 
 
-class TaskList(list):
+class TaskList(object):
 
-	def plugins(self, manifest):
+	def __init__(self):
-		for plugin in manifest.loaded_plugins:
+		self.tasks = set()
-			plugin.modify_tasklist(self, manifest)
+
+	def add(self, *args):
+		self.tasks.update(args)
+
+	def remove(self, task):
+		self.tasks.discard(self.get(task))
+
+	def replace(self, task, replacement):
+		self.remove(task)
+		self.add(replacement)
+
+	def get(self, task):
+		return next(task for task in self.tasks if type(task) is ref)
 
 	def run(self, bootstrap_info):
-		for task in self:
+		log.debug('Tasklist before:\n{list}'.format(list=',\n'.join(str(task) for task in self.tasks) ))
+		task_list = sorted(self.tasks)
+		log.debug('Tasklist:\n{list}'.format(list=',\n'.join(str(task) for task in task_list) ))
+		return
+		for task in tasks:
 			log.info(task)
 			task.run(bootstrap_info)
-
-	def before(self, ref, task):
-		log.debug('Inserting %s before %s.%s', task, ref.__module__, ref.__name__)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.insert(i, task)
-
-	def replace(self, ref, task):
-		log.debug('Replacing %s.%s with %s', ref.__module__, ref.__name__, task)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.pop(i)
-		self.insert(i, task)
-
-	def after(self, ref, task):
-		log.debug('Inserting %s after %s.%s', task, ref.__module__, ref.__name__)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.insert(i+1, task)
-
-	def append(self, task):
-		super(TaskList, self).append(task)
-		log.debug('Appending %s', task)

common/exceptions.py

@@ -6,4 +6,12 @@ class ManifestError(Exception):
 		self.message = message
 		self.manifest = manifest
 	def __str__(self):
-		return "Error in `manifest' {0}: {1}".format(self.manifest.path, self.message)
+		return "Error in manifest {0}: {1}".format(self.manifest.path, self.message)
+
+
+class TaskOrderError(Exception):
+	def __init__(self, message, task):
+		self.message = message
+		self.task = task
+	def __str__(self):
+		return "Error in task order: {1}".format(self.message)

common/phases.py

@@ -0,0 +1,44 @@
+from base import Phase
+
+
+class Preparation(Phase):
+	description = 'Initializing connections, fetching data etc.'
+
+class VolumeCreation(Phase):
+	description = 'Creating the volume to bootstrap onto'
+
+class VolumePreparation(Phase):
+	description = 'Formatting the bootstrap volume'
+
+class VolumeMounting(Phase):
+	description = 'Mounting bootstrap volume'
+
+class InstallOS(Phase):
+	description = 'Installing the operating system'
+
+class ModifySystem(Phase):
+	description = 'Installing software, modifying configuration files etc.'
+
+class CleanSystem(Phase):
+	description = 'Removing sensitive data, temporary files and other leftovers'
+
+class UnmountVolume(Phase):
+	description = 'Unmounting the bootstrap volume'
+
+class RegisterImage(Phase):
+	description = 'Uploading/Registering with the provider'
+
+class Cleanup(Phase):
+	description = 'Removing temporary files'
+
+order = [Preparation,
+         VolumeCreation,
+         VolumePreparation,
+         VolumeMounting,
+         InstallOS,
+         ModifySystem,
+         CleanSystem,
+         UnmountVolume,
+         RegisterImage,
+         Cleanup
+        ]

plugins/admin_user/__init__.py

@@ -1,6 +1,5 @@
 
 
-def modify_tasklist(tasklist, manifest):
+def tasks(tasklist, manifest):
-	from providers.ec2.tasks.packages import ImagePackages
 	from adminuser import AddSudoPackage
-	tasklist.after(ImagePackages, AddSudoPackage())
+	tasklist.add(AddSudoPackage())

plugins/admin_user/adminuser.py

@@ -1,8 +1,14 @@
 from base import Task
+from common import phases
+from providers.ec2.tasks.packages import ImagePackages
+from providers.ec2.tasks.host import CheckPackages
 
 
 class AddSudoPackage(Task):
 	description = 'Adding ``sudo\'\' to the image packages'
+	phase = phases.Preparation
+	after = [ImagePackages]
+	before = [CheckPackages]
 
 	def run(self, info):
 		super(AddSudoPackage, self).run(info)

plugins/build_metadata/__init__.py

@@ -1,5 +1,5 @@
 
 
-def modify_tasklist(tasklist, manifest):
+def tasks(tasklist, manifest):
 	from buildmetadata import PrintInfo
-	tasklist.append(PrintInfo())
+	tasklist.add(PrintInfo())

plugins/build_metadata/buildmetadata.py

@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class PrintInfo(Task):
 	description = 'Printing `info\' to the console'
+	phase = phases.InstallOS
 
 	def run(self, info):
 		super(PrintInfo, self).run(info)

providers/ec2/__init__.py

@@ -1,16 +1,12 @@
 from manifest import Manifest
 
 
-def modify_tasklist(tasklist, manifest):
+def tasks(tasklist, manifest):
 	from tasks import packages
 	from tasks import ec2
 	from tasks import host
 	from tasks import ebs
-	tasklist.extend([packages.HostPackages(),
+	tasklist.add(packages.HostPackages(), packages.ImagePackages(), host.CheckPackages(),
-	                 packages.ImagePackages(),
+	             ec2.GetCredentials(), host.GetInfo(), ec2.Connect())
-	                 host.CheckPackages(),
+	if manifest.volume['backing'] is 'ebs':
-	                 ec2.GetCredentials(),
+		tasklist.add(ebs.CreateVolume())
-	                 host.GetInfo(),
-	                 ec2.Connect(),
-	                 ebs.CreateVolume(),
-                  ])

providers/ec2/manifest.py

@@ -11,4 +11,3 @@ class Manifest(base.Manifest):
 		super(Manifest, self).parse(data)
 		self.credentials    = data['credentials']
 		self.virtualization = data['virtualization']
-		self.volume         = data['volume']

providers/ec2/tasks/connection.py

@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class GetCredentials(Task):
 	description = 'Getting AWS credentials'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(GetCredentials, self).run(info)
@@ -28,6 +30,8 @@ class GetCredentials(Task):
 
 class Connect(Task):
 	description = 'Connecting to EC2'
+	phase = phases.Preparation
+	after = [GetCredentials]
 	
 	def run(self, info):
 		super(Connect, self).run(info)

providers/ec2/tasks/ebs.py

@@ -1,8 +1,11 @@
 from base import Task
-
+from common import phases
+from connection import Connect
 
 class CreateVolume(Task):
 	description = 'Creating an EBS volume for bootstrapping'
+	phase = phases.VolumeCreation
+	after = [Connect]
 
 	def run(self, info):
 		# info.conn.create_volume(50, "us-west-2")

providers/ec2/tasks/host.py

@@ -1,8 +1,11 @@
 from base import Task
-
+from common import phases
+from providers.ec2.tasks import packages
 
 class CheckPackages(Task):
 	description = 'Checking installed host packages'
+	phase = phases.Preparation
+	after = [packages.HostPackages, packages.ImagePackages]
 
 	def run(self, info):
 		import subprocess
@@ -18,6 +21,7 @@ class CheckPackages(Task):
 
 class GetInfo(Task):
 	description = 'Retrieving instance metadata'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(GetInfo, self).run(info)

providers/ec2/tasks/packages.py

@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class HostPackages(Task):
 	description = 'Determining required host packages'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(HostPackages, self).run(info)
@@ -18,6 +20,7 @@ class HostPackages(Task):
 
 class ImagePackages(Task):
 	description = 'Determining required image packages'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(ImagePackages, self).run(info)