From 1e4af40b3c8873cf1659516aadb4906e7ab23c62 Mon Sep 17 00:00:00 2001
From: Anders Ingemann <anders@ingemann.de>
Date: Sun, 23 Jun 2013 15:26:08 +0200
Subject: [PATCH] Task dependencies, bogus implementation

---
 base/__init__.py                              |  3 +-
 base/main.py                                  |  5 +-
 base/phase.py                                 | 14 ++++++
 base/task.py                                  | 45 +++++++++++++++++-
 base/tasklist.py                              | 46 +++++++++----------
 common/exceptions.py                          | 10 +++-
 common/phases.py                              | 44 ++++++++++++++++++
 plugins/admin_user/__init__.py                |  5 +-
 plugins/admin_user/adminuser.py               |  6 +++
 plugins/build_metadata/__init__.py            |  4 +-
 plugins/build_metadata/buildmetadata.py       |  2 +
 providers/ec2/__init__.py                     | 14 ++----
 providers/ec2/manifest.py                     |  1 -
 providers/ec2/tasks/{ec2.py => connection.py} |  4 ++
 providers/ec2/tasks/ebs.py                    |  5 +-
 providers/ec2/tasks/host.py                   |  6 ++-
 providers/ec2/tasks/packages.py               |  3 ++
 17 files changed, 170 insertions(+), 47 deletions(-)
 create mode 100644 base/phase.py
 create mode 100644 common/phases.py
 rename providers/ec2/tasks/{ec2.py => connection.py} (93%)

diff --git a/base/__init__.py b/base/__init__.py
index 7d46658..1690bcc 100644
--- a/base/__init__.py
+++ b/base/__init__.py
@@ -1,4 +1,5 @@
-__all__ = ['Manifest', 'Task', 'main']
+__all__ = ['Manifest', 'Phase', 'Task', 'main']
 from manifest import Manifest
 from task import Task
+from phase import Phase
 from main import main
diff --git a/base/main.py b/base/main.py
index 5e0d6a9..45b4ce9 100644
--- a/base/main.py
+++ b/base/main.py
@@ -48,8 +48,9 @@ def run(args):
 
 	from tasklist import TaskList
 	tasklist = TaskList()
-	provider.modify_tasklist(tasklist, manifest)
-	tasklist.plugins(manifest)
+	provider.tasks(tasklist, manifest)
+	for plugin in manifest.loaded_plugins:
+		plugin.tasks(tasklist, manifest)
 
 	from bootstrapinfo import BootstrapInformation
 	bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug)
diff --git a/base/phase.py b/base/phase.py
new file mode 100644
index 0000000..8b50d1f
--- /dev/null
+++ b/base/phase.py
@@ -0,0 +1,14 @@
+
+
+class Phase(object):
+	description = None
+
+	def __init__(self):
+		pass
+
+	def __cmp__(self, other):
+		from common.phases import order
+		return order.index(self) - order.index(other)
+
+	def __str__(self):
+		return '{name}'.format(name=self.__class__.__name__)
diff --git a/base/task.py b/base/task.py
index be00e38..8592fed 100644
--- a/base/task.py
+++ b/base/task.py
@@ -1,13 +1,56 @@
+from functools import total_ordering
 
 
+@total_ordering
 class Task(object):
 	description = None
+	
+	phase = None
+	before = []
+	after = []
 
 	def __init__(self):
-		pass
+		self._check_ordering()
 
 	def run(self, info):
 		pass
 
+	def _after(self, other):
+		return self.phase > other.phase or type(self) in other.before or type(other) in self.after
+
+	def _before(self, other):
+		return self.phase < other.phase or type(other) in self.before or type(self) in other.after
+
+	def __lt__(self, other):
+		return self._before(other) and not self._after(other)
+
+	def __gt__(self, other):
+		return not self._before(other) and self._after(other)
+
+	def __eq__(self, other):
+		return not self._before(other) and not self._after(other)
+
+	def __ne__(self, other):
+		return self._before(other) or self._after(other)
+
 	def __str__(self):
 		return '{module}.{task}'.format(module=self.__module__, task=self.__class__.__name__)
+
+	def _check_ordering(self):
+		for task in self.before:
+			if self.phase > task.phase:
+				msg = ("The task {self} is specified as running before {other}, "
+				       "but its phase {phase} lies after the phase {other_phase}"
+				       .format(self, other, self.phase, other.phase))
+				raise TaskOrderError(msg)
+		for task in self.after:
+			if self.phase < task.phase:
+				msg = ("The task {self} is specified as running after {other}, "
+				       "but its phase {phase} lies before the phase {other_phase}"
+				       .format(self=self, other=other, phase=self.phase, other_phase=other.phase))
+				raise TaskOrderError(msg)
+		conflict = next(iter(set(self.before) &  set(self.after)), None)
+		if conflict is not None:
+				msg = ("The task {self} is specified as running both before and after {conflict}"
+				       .format(self=self, conflict=conflict))
+				raise TaskOrderError(msg)
diff --git a/base/tasklist.py b/base/tasklist.py
index 32fcbc0..55660b6 100644
--- a/base/tasklist.py
+++ b/base/tasklist.py
@@ -2,33 +2,29 @@ import logging
 log = logging.getLogger(__name__)
 
 
-class TaskList(list):
+class TaskList(object):
 
-	def plugins(self, manifest):
-		for plugin in manifest.loaded_plugins:
-			plugin.modify_tasklist(self, manifest)
+	def __init__(self):
+		self.tasks = set()
+
+	def add(self, *args):
+		self.tasks.update(args)
+
+	def remove(self, task):
+		self.tasks.discard(self.get(task))
+
+	def replace(self, task, replacement):
+		self.remove(task)
+		self.add(replacement)
+
+	def get(self, task):
+		return next(task for task in self.tasks if type(task) is ref)
 
 	def run(self, bootstrap_info):
-		for task in self:
+		log.debug('Tasklist before:\n{list}'.format(list=',\n'.join(str(task) for task in self.tasks) ))
+		task_list = sorted(self.tasks)
+		log.debug('Tasklist:\n{list}'.format(list=',\n'.join(str(task) for task in task_list) ))
+		return
+		for task in tasks:
 			log.info(task)
 			task.run(bootstrap_info)
-
-	def before(self, ref, task):
-		log.debug('Inserting %s before %s.%s', task, ref.__module__, ref.__name__)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.insert(i, task)
-
-	def replace(self, ref, task):
-		log.debug('Replacing %s.%s with %s', ref.__module__, ref.__name__, task)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.pop(i)
-		self.insert(i, task)
-
-	def after(self, ref, task):
-		log.debug('Inserting %s after %s.%s', task, ref.__module__, ref.__name__)
-		i = next(i for i, task in enumerate(self) if type(task) is ref)
-		self.insert(i+1, task)
-
-	def append(self, task):
-		super(TaskList, self).append(task)
-		log.debug('Appending %s', task)
diff --git a/common/exceptions.py b/common/exceptions.py
index be467e9..0245a0c 100644
--- a/common/exceptions.py
+++ b/common/exceptions.py
@@ -6,4 +6,12 @@ class ManifestError(Exception):
 		self.message = message
 		self.manifest = manifest
 	def __str__(self):
-		return "Error in `manifest' {0}: {1}".format(self.manifest.path, self.message)
+		return "Error in manifest {0}: {1}".format(self.manifest.path, self.message)
+
+
+class TaskOrderError(Exception):
+	def __init__(self, message, task):
+		self.message = message
+		self.task = task
+	def __str__(self):
+		return "Error in task order: {1}".format(self.message)
diff --git a/common/phases.py b/common/phases.py
new file mode 100644
index 0000000..7809385
--- /dev/null
+++ b/common/phases.py
@@ -0,0 +1,44 @@
+from base import Phase
+
+
+class Preparation(Phase):
+	description = 'Initializing connections, fetching data etc.'
+
+class VolumeCreation(Phase):
+	description = 'Creating the volume to bootstrap onto'
+
+class VolumePreparation(Phase):
+	description = 'Formatting the bootstrap volume'
+
+class VolumeMounting(Phase):
+	description = 'Mounting bootstrap volume'
+
+class InstallOS(Phase):
+	description = 'Installing the operating system'
+
+class ModifySystem(Phase):
+	description = 'Installing software, modifying configuration files etc.'
+
+class CleanSystem(Phase):
+	description = 'Removing sensitive data, temporary files and other leftovers'
+
+class UnmountVolume(Phase):
+	description = 'Unmounting the bootstrap volume'
+
+class RegisterImage(Phase):
+	description = 'Uploading/Registering with the provider'
+
+class Cleanup(Phase):
+	description = 'Removing temporary files'
+
+order = [Preparation,
+         VolumeCreation,
+         VolumePreparation,
+         VolumeMounting,
+         InstallOS,
+         ModifySystem,
+         CleanSystem,
+         UnmountVolume,
+         RegisterImage,
+         Cleanup
+        ]
diff --git a/plugins/admin_user/__init__.py b/plugins/admin_user/__init__.py
index b83eb05..49725d6 100644
--- a/plugins/admin_user/__init__.py
+++ b/plugins/admin_user/__init__.py
@@ -1,6 +1,5 @@
 
 
-def modify_tasklist(tasklist, manifest):
-	from providers.ec2.tasks.packages import ImagePackages
+def tasks(tasklist, manifest):
 	from adminuser import AddSudoPackage
-	tasklist.after(ImagePackages, AddSudoPackage())
+	tasklist.add(AddSudoPackage())
diff --git a/plugins/admin_user/adminuser.py b/plugins/admin_user/adminuser.py
index b8512f3..9ed33d8 100644
--- a/plugins/admin_user/adminuser.py
+++ b/plugins/admin_user/adminuser.py
@@ -1,8 +1,14 @@
 from base import Task
+from common import phases
+from providers.ec2.tasks.packages import ImagePackages
+from providers.ec2.tasks.host import CheckPackages
 
 
 class AddSudoPackage(Task):
 	description = 'Adding ``sudo\'\' to the image packages'
+	phase = phases.Preparation
+	after = [ImagePackages]
+	before = [CheckPackages]
 
 	def run(self, info):
 		super(AddSudoPackage, self).run(info)
diff --git a/plugins/build_metadata/__init__.py b/plugins/build_metadata/__init__.py
index 18bc0f1..7970eca 100644
--- a/plugins/build_metadata/__init__.py
+++ b/plugins/build_metadata/__init__.py
@@ -1,5 +1,5 @@
 
 
-def modify_tasklist(tasklist, manifest):
+def tasks(tasklist, manifest):
 	from buildmetadata import PrintInfo
-	tasklist.append(PrintInfo())
+	tasklist.add(PrintInfo())
diff --git a/plugins/build_metadata/buildmetadata.py b/plugins/build_metadata/buildmetadata.py
index 8f10600..35834bc 100644
--- a/plugins/build_metadata/buildmetadata.py
+++ b/plugins/build_metadata/buildmetadata.py
@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class PrintInfo(Task):
 	description = 'Printing `info\' to the console'
+	phase = phases.InstallOS
 
 	def run(self, info):
 		super(PrintInfo, self).run(info)
diff --git a/providers/ec2/__init__.py b/providers/ec2/__init__.py
index 9bcd5ce..ade5422 100644
--- a/providers/ec2/__init__.py
+++ b/providers/ec2/__init__.py
@@ -1,16 +1,12 @@
 from manifest import Manifest
 
 
-def modify_tasklist(tasklist, manifest):
+def tasks(tasklist, manifest):
 	from tasks import packages
 	from tasks import ec2
 	from tasks import host
 	from tasks import ebs
-	tasklist.extend([packages.HostPackages(),
-	                 packages.ImagePackages(),
-	                 host.CheckPackages(),
-	                 ec2.GetCredentials(),
-	                 host.GetInfo(),
-	                 ec2.Connect(),
-	                 ebs.CreateVolume(),
-                  ])
+	tasklist.add(packages.HostPackages(), packages.ImagePackages(), host.CheckPackages(),
+	             ec2.GetCredentials(), host.GetInfo(), ec2.Connect())
+	if manifest.volume['backing'] is 'ebs':
+		tasklist.add(ebs.CreateVolume())
diff --git a/providers/ec2/manifest.py b/providers/ec2/manifest.py
index e3b25ff..fa93d28 100644
--- a/providers/ec2/manifest.py
+++ b/providers/ec2/manifest.py
@@ -11,4 +11,3 @@ class Manifest(base.Manifest):
 		super(Manifest, self).parse(data)
 		self.credentials    = data['credentials']
 		self.virtualization = data['virtualization']
-		self.volume         = data['volume']
diff --git a/providers/ec2/tasks/ec2.py b/providers/ec2/tasks/connection.py
similarity index 93%
rename from providers/ec2/tasks/ec2.py
rename to providers/ec2/tasks/connection.py
index c70b44f..66cc1e1 100644
--- a/providers/ec2/tasks/ec2.py
+++ b/providers/ec2/tasks/connection.py
@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class GetCredentials(Task):
 	description = 'Getting AWS credentials'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(GetCredentials, self).run(info)
@@ -28,6 +30,8 @@ class GetCredentials(Task):
 
 class Connect(Task):
 	description = 'Connecting to EC2'
+	phase = phases.Preparation
+	after = [GetCredentials]
 	
 	def run(self, info):
 		super(Connect, self).run(info)
diff --git a/providers/ec2/tasks/ebs.py b/providers/ec2/tasks/ebs.py
index bf9efb4..0e2a1af 100644
--- a/providers/ec2/tasks/ebs.py
+++ b/providers/ec2/tasks/ebs.py
@@ -1,8 +1,11 @@
 from base import Task
-
+from common import phases
+from connection import Connect
 
 class CreateVolume(Task):
 	description = 'Creating an EBS volume for bootstrapping'
+	phase = phases.VolumeCreation
+	after = [Connect]
 
 	def run(self, info):
 		# info.conn.create_volume(50, "us-west-2")
diff --git a/providers/ec2/tasks/host.py b/providers/ec2/tasks/host.py
index 3121d93..515f56c 100644
--- a/providers/ec2/tasks/host.py
+++ b/providers/ec2/tasks/host.py
@@ -1,8 +1,11 @@
 from base import Task
-
+from common import phases
+from providers.ec2.tasks import packages
 
 class CheckPackages(Task):
 	description = 'Checking installed host packages'
+	phase = phases.Preparation
+	after = [packages.HostPackages, packages.ImagePackages]
 
 	def run(self, info):
 		import subprocess
@@ -18,6 +21,7 @@ class CheckPackages(Task):
 
 class GetInfo(Task):
 	description = 'Retrieving instance metadata'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(GetInfo, self).run(info)
diff --git a/providers/ec2/tasks/packages.py b/providers/ec2/tasks/packages.py
index 1621c07..7e59b75 100644
--- a/providers/ec2/tasks/packages.py
+++ b/providers/ec2/tasks/packages.py
@@ -1,8 +1,10 @@
 from base import Task
+from common import phases
 
 
 class HostPackages(Task):
 	description = 'Determining required host packages'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(HostPackages, self).run(info)
@@ -18,6 +20,7 @@ class HostPackages(Task):
 
 class ImagePackages(Task):
 	description = 'Determining required image packages'
+	phase = phases.Preparation
 
 	def run(self, info):
 		super(ImagePackages, self).run(info)