Merge remote and local package array in manifest

This allows us to specify the order of installation for packages
Fixes #127

base/manifest-schema.json

@@ -53,27 +53,20 @@
 					"additionalProperties": false,
 					"minItems": 1
 				},
-				"remote": {
+				"trusted-keys": {
+					"type": "array",
+					"items": { "$ref": "#/definitions/absolute_path" },
+					"minItems": 1
+				},
+				"install": {
 					"type": "array",
 					"items": {
 						"anyOf": [
-							{
+							{ "pattern": "^[^/]+(/[^/]+)?$" },
-								"type": "object",
+							{ "$ref": "#/definitions/absolute_path" }
-								"properties": {
-									"name": {"type": "string"},
-									"target": {"type": "string"}
-								},
-								"required": ["name"]
-							},
-							{ "type": "string" }
 						]
 					},
 					"minItems": 1
-				},
-				"local": {
-					"type": "array",
-					"items": { "$ref": "#/definitions/absolute_path" },
-					"minItems": 1
 				}
 			},
 			"additionalProperties": false

base/phase.py

@@ -7,7 +7,7 @@ class Phase(object):
 
 	def pos(self):
 		from common.phases import order
-		return (i for i, phase in enumerate(order) if phase is self).next()
+		return next(i for i, phase in enumerate(order) if phase is self)
 
 	def __cmp__(self, other):
 		return self.pos() - other.pos()

base/pkg/packagelist.py

@@ -3,29 +3,55 @@ from exceptions import PackageError
 
 class PackageList(object):
 
+	class Remote(object):
+		def __init__(self, name, target):
+			self.name = name
+			self.target = target
+
+		def __str__(self):
+			if self.target is None:
+				return self.name
+			else:
+				return '{name}/{target}'.format(name=self.name, target=self.target)
+
+	class Local(object):
+		def __init__(self, path):
+			self.path = path
+
+		def __str__(self):
+			return self.path
+
 	def __init__(self, manifest_vars, source_lists):
 		self.manifest_vars = manifest_vars
 		self.source_lists = source_lists
-		self.remote = {}
+		self.default_target = '{system.release}'.format(**self.manifest_vars)
-		self.local = set()
+		self.install = []
+		self.remote = lambda: filter(lambda x: isinstance(x, self.Remote), self.install)
 
 	def add(self, name, target=None):
-		if target is None:
-			target = '{system.release}'
 		name = name.format(**self.manifest_vars)
-		target = target.format(**self.manifest_vars)
+		if target is not None:
-		if name in self.remote:
+			target = target.format(**self.manifest_vars)
-			if self.remote[name] != target:
+		package = next((pkg for pkg in self.remote() if pkg.name == name), None)
+		if package is not None:
+			same_target = package.target != target
+			same_target = same_target or package.target is None and target == self.default_target
+			same_target = same_target or package.target == self.default_target and target is None
+			if not same_target:
 				msg = ('The package {name} was already added to the package list, '
-				       'but with another target release ({target})').format(name=name, target=self.remote[name])
+				       'but with another target release ({target})').format(name=name, target=package.target)
 				raise PackageError(msg)
 			return
 
-		if not self.source_lists.target_exists(target):
+		check_target = target
-			msg = ('The target release {target} was not found in the sources list').format(target=target)
+		if check_target is None:
+			check_target = self.default_target
+		if not self.source_lists.target_exists(check_target):
+			msg = ('The target release {target} was not found in the sources list').format(target=check_target)
 			raise PackageError(msg)
-		self.remote[name] = target
+
+		self.install.append(self.Remote(name, target))
 
 	def add_local(self, package_path):
 		package_path = package_path.format(**self.manifest_vars)
-		self.local.add(package_path)
+		self.install.append(self.Local(package_path))

common/task_sets.py

@@ -52,18 +52,17 @@ def get_apt_set(manifest):
 	        apt.DisableDaemonAutostart,
 	        apt.AptUpdate,
 	        apt.AptUpgrade,
-	        packages.InstallRemotePackages,
+	        packages.InstallPackages,
-	        packages.InstallLocalPackages,
 	        apt.PurgeUnusedPackages,
 	        apt.AptClean,
 	        apt.EnableDaemonAutostart,
 	        ]
 	if 'sources' in manifest.packages:
 		base.append(apt.AddManifestSources)
-	if 'remote' in manifest.packages:
+	if 'trusted-keys' in manifest.packages:
-		base.append(apt.AddRemoteManifestPackages)
+		base.append(apt.InstallTrustedKeys)
-	if 'local' in manifest.packages:
+	if 'install' in manifest.packages:
-		base.append(apt.AddLocalManifestPackages)
+		base.append(packages.AddManifestPackages)
 	return base
 
 

common/tasks/apt.py

@@ -11,10 +11,9 @@ class AddManifestSources(Task):
 
 	@classmethod
 	def run(cls, info):
-		if 'sources' in info.manifest.packages:
+		for name, lines in info.manifest.packages['sources'].iteritems():
-			for name, lines in info.manifest.packages['sources'].iteritems():
+			for line in lines:
-				for line in lines:
+				info.source_lists.add(name, line)
-					info.source_lists.add(name, line)
 
 
 class AddDefaultSources(Task):
@@ -24,45 +23,29 @@ class AddDefaultSources(Task):
 
 	@classmethod
 	def run(cls, info):
-		if info.source_lists.target_exists('{system.release}'):
+		info.source_lists.add('main', 'deb     {apt_mirror} {system.release} main')
-			import logging
+		info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} main')
-			msg = ('{system.release} target already exists').format(**info.manifest_vars)
+		info.source_lists.add('main', 'deb     {apt_mirror} {system.release}-updates main')
-			logging.getLogger(__name__).info(msg)
+		info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates main')
-		else:
-			info.source_lists.add('main', 'deb     {apt_mirror} {system.release} main')
-			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release} main')
-			info.source_lists.add('main', 'deb     {apt_mirror} {system.release}-updates main')
-			info.source_lists.add('main', 'deb-src {apt_mirror} {system.release}-updates main')
 
 
-class AddRemoteManifestPackages(Task):
+class InstallTrustedKeys(Task):
-	description = 'Adding remote packages from the manifest'
+	description = 'Installing trusted keys'
-	phase = phases.preparation
+	phase = phases.package_installation
-	predecessors = [AddDefaultSources]
 
 	@classmethod
 	def run(cls, info):
-		for package in info.manifest.packages['remote']:
+		from shutil import copy
-			if isinstance(package, dict):
+		for key_path in info.manifest.packages['trusted-keys']:
-				info.packages.add(package['name'], package.get('target', None))
+			key_name = os.path.basename(key_path)
-			else:
+			destination = os.path.join(info.root, 'etc/apt/trusted.gpg.d', key_name)
-				info.packages.add(package, None)
+			copy(key_path, destination)
-
-
-class AddLocalManifestPackages(Task):
-	description = 'Adding local packages from the manifest'
-	phase = phases.preparation
-	predecessors = [AddDefaultSources]
-
-	@classmethod
-	def run(cls, info):
-		for package_path in info.manifest.packages['local']:
-			info.packages.add_local(package_path)
 
 
 class WriteSources(Task):
 	description = 'Writing aptitude sources to disk'
 	phase = phases.package_installation
+	predecessors = [InstallTrustedKeys]
 
 	@classmethod
 	def run(cls, info):

common/tasks/packages.py

@@ -3,26 +3,43 @@ from common import phases
 from common.tasks import apt
 
 
-class InstallRemotePackages(Task):
+class AddManifestPackages(Task):
-	description = 'Installing remote packages'
+	description = 'Adding packages from the manifest'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources]
+
+	@classmethod
+	def run(cls, info):
+		import re
+		remote = re.compile('^(?P<name>[^/]+)(/(?P<target>[^/]+))?$')
+		for package in info.manifest.packages['install']:
+			match = remote.match(package).groupdict()
+			if match is not None:
+				info.packages.add(match['name'], match['target'])
+			else:
+				info.packages.add_local(package)
+
+
+class InstallPackages(Task):
+	description = 'Installing packages'
 	phase = phases.package_installation
 	predecessors = [apt.AptUpgrade]
 
 	@classmethod
 	def run(cls, info):
-		if len(info.packages.remote) == 0:
+		batch = []
-			return
+		actions = {info.packages.Remote: cls.install_remote,
+		           info.packages.Local: cls.install_local}
+		for i, package in enumerate(info.packages.install):
+			batch.append(package)
+			next_package = info.packages.install[i + 1] if i + 1 < len(info.packages.install) else None
+			if next_package is None or package.__class__ is not next_package.__class__:
+				actions[package.__class__](info, batch)
+				batch = []
+
+	@classmethod
+	def install_remote(cls, info, remote_packages):
 		import os
-
-		packages = []
-		for name, target in info.packages.remote.iteritems():
-			packages.append('{name}/{target}'.format(name=name, target=target))
-
-		import logging
-		msg = ('The following packages will be installed (package/target-release):'
-		       '\n{packages}\n').format(packages='\n'.join(packages))
-		logging.getLogger(__name__).debug(msg)
-
 		from common.tools import log_check_call
 		from subprocess import CalledProcessError
 		try:
@@ -32,9 +49,10 @@ class InstallRemotePackages(Task):
 			                '/usr/bin/apt-get', 'install',
 			                                    '--no-install-recommends',
 			                                    '--assume-yes']
-			               + packages,
+			               + map(str, remote_packages),
 			               env=env)
 		except CalledProcessError as e:
+			import logging
 			disk_stat = os.statvfs(info.root)
 			root_free_mb = disk_stat.f_bsize * disk_stat.f_bavail / 1024 / 1024
 			disk_stat = os.statvfs(os.path.join(info.root, 'boot'))
@@ -53,31 +71,29 @@ class InstallRemotePackages(Task):
 					logging.getLogger(__name__).warn(msg)
 			raise e
 
-
-class InstallLocalPackages(Task):
-	description = 'Installing local packages'
-	phase = phases.package_installation
-	predecessors = [apt.AptUpgrade]
-	successors = [InstallRemotePackages]
-
 	@classmethod
-	def run(cls, info):
+	def install_local(cls, info, local_packages):
-		if len(info.packages.local) == 0:
-			return
 		from shutil import copy
 		from common.tools import log_check_call
 		import os
 
-		for package_src in info.packages.local:
+		absolute_package_paths = []
+		chrooted_package_paths = []
+		for package_src in local_packages:
 			pkg_name = os.path.basename(package_src)
 			package_rel_dst = os.path.join('tmp', pkg_name)
 			package_dst = os.path.join(info.root, package_rel_dst)
 			copy(package_src, package_dst)
-
+			absolute_package_paths.append(package_dst)
 			package_path = os.path.join('/', package_rel_dst)
-			env = os.environ.copy()
+			chrooted_package_paths.append(package_path)
-			env['DEBIAN_FRONTEND'] = 'noninteractive'
+
-			log_check_call(['/usr/sbin/chroot', info.root,
+		env = os.environ.copy()
-			                '/usr/bin/dpkg', '--install', package_path],
+		env['DEBIAN_FRONTEND'] = 'noninteractive'
-			               env=env)
+		log_check_call(['/usr/sbin/chroot', info.root,
-			os.remove(package_dst)
+		                '/usr/bin/dpkg', '--install']
+		               + chrooted_package_paths,
+		               env=env)
+
+		for path in absolute_package_paths:
+			os.remove(path)

plugins/prebootstrapped/tasks.py

@@ -15,7 +15,7 @@ log = logging.getLogger(__name__)
 class Snapshot(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
 	phase = phases.package_installation
-	predecessors = [packages.InstallRemotePackages, guest_additions.InstallGuestAdditions]
+	predecessors = [packages.InstallPackages, guest_additions.InstallGuestAdditions]
 
 	@classmethod
 	def run(cls, info):
@@ -49,7 +49,7 @@ class CreateFromSnapshot(Task):
 class CopyImage(Task):
 	description = 'Creating a snapshot of the bootstrapped volume'
 	phase = phases.package_installation
-	predecessors = [packages.InstallRemotePackages, guest_additions.InstallGuestAdditions]
+	predecessors = [packages.InstallPackages, guest_additions.InstallGuestAdditions]
 
 	@classmethod
 	def run(cls, info):

providers/virtualbox/tasks/guest_additions.py

@@ -1,6 +1,6 @@
 from base import Task
 from common import phases
-from common.tasks.packages import InstallRemotePackages
+from common.tasks.packages import InstallPackages
 from common.exceptions import TaskError
 
 
@@ -20,7 +20,7 @@ class CheckGuestAdditionsPath(Task):
 class AddGuestAdditionsPackages(Task):
 	description = 'Adding packages to support Guest Additions installation'
 	phase = phases.package_installation
-	successors = [InstallRemotePackages]
+	successors = [InstallPackages]
 
 	@classmethod
 	def run(cls, info):
@@ -38,7 +38,7 @@ class AddGuestAdditionsPackages(Task):
 class InstallGuestAdditions(Task):
 	description = 'Installing the VirtualBox Guest Additions'
 	phase = phases.package_installation
-	predecessors = [InstallRemotePackages]
+	predecessors = [InstallPackages]
 
 	@classmethod
 	def run(cls, info):