Integrated package plugin with base system

New phase introduced "package installation" (fixes #114)
Apt source lines are now parsed, this allows to verify the target release of added packages.
All packages (except locales) are now installed *after* bootstrapping (fixes #123)
Added env argument to log_(check_)call
HostDependencies have been refactored a little

base/bootstrapinfo.py

@@ -3,12 +3,24 @@
 class BootstrapInformation(object):
 	def __init__(self, manifest=None, debug=False):
 		self.manifest = manifest
-		from fs import load_volume
-		self.volume = load_volume(self.manifest.volume)
 		self.debug = debug
+
 		import random
 		self.run_id = '{id:08x}'.format(id=random.randrange(16 ** 8))
+
 		import os.path
 		self.workspace = os.path.join(manifest.bootstrapper['workspace'], self.run_id)
 
+		from fs import load_volume
+		self.volume = load_volume(self.manifest.volume)
+
+		from pkg.source import SourceLists
+		self.source_lists = SourceLists(self.manifest)
+		from pkg.package import PackageList
+		self.packages = PackageList(self.source_lists, self.manifest)
+		self.include_packages = set()
+		self.exclude_packages = set()
+
+		self.host_dependencies = set()
+
 		self.initd = {'install': {}, 'disable': []}

base/manifest-schema.json

@@ -26,6 +26,46 @@
 			},
 			"required": ["release", "architecture", "timezone", "locale", "charmap"]
 		},
+		"packages": {
+			"type": "object",
+			"properties": {
+				"sources": {
+					"type": "object",
+					"patternProperties": {
+						"^\\w+$": {
+							"type": "array",
+							"items": {"type": "string"},
+							"minItems": 1
+						}
+					},
+					"additionalProperties": false,
+					"minItems": 1
+				},
+				"remote": {
+					"type": "array",
+					"items": {
+						"anyOf": [
+							{
+								"type": "object",
+								"properties": {
+									"name": {"type": "string"},
+									"target": {"type": "string"}
+								},
+								"required": ["name"]
+							},
+							{ "type": "string" }
+						]
+					},
+					"minItems": 1
+				},
+				"local": {
+					"type": "array",
+					"items": { "$ref": "#/definitions/absolute_path" },
+					"minItems": 1
+				}
+			},
+			"additionalProperties": false
+		},
 		"volume": {
 			"type": "object",
 			"properties": {
@@ -56,6 +96,10 @@
 			"type":    "string",
 			"pattern": "^[^\\0]+$"
 		},
+		"absolute_path": {
+			"type":    "string",
+			"pattern": "^/[^\\0]+$"
+		},
 		"no_partitions": {
 			"type": "object",
 			"properties": {

base/pkg/__init__.py

@@ -0,0 +1,4 @@
+
+
+def load_packages(manifest):
+	pass

base/pkg/exceptions.py

@@ -0,0 +1,8 @@
+
+
+class PackageError(Exception):
+	pass
+
+
+class SourceError(Exception):
+	pass

base/pkg/packagelist.py

@@ -0,0 +1,40 @@
+from exceptions import PackageError
+
+
+class PackageList(object):
+
+	def __init__(self, sources_list, manifest):
+		self.sources_list = sources_list
+		self.default_target = manifest.system['release']
+		self.remote = {}
+		self.local = set()
+		if 'remote' in manifest.packages:
+			manifest_vars = {'release':      manifest.system['release'],
+			                 'architecture': manifest.system['architecture']}
+			for package in manifest.packages['remote']:
+				target = None
+				if isinstance(package, dict):
+					name = package['name'].format(**manifest_vars)
+					if 'target' in package:
+						target = package['target'].format(**manifest_vars)
+				else:
+					name = package.format(**manifest_vars)
+				self.add(name, target)
+		if 'local' in manifest.packages:
+			for package_path in manifest.packages['local']:
+				self.local.add(package_path)
+
+	def add(self, name, target=None):
+		if target is None:
+			target = self.default_target
+		if name in self.remote:
+			if self.remote[name] != target:
+				msg = ('The package {name} was already added to the package list, '
+				       'but with another target release ({target})').format(name=name, target=self.remote[name])
+				raise PackageError(msg)
+			return
+
+		if not self.sources_list.target_exists(target):
+			msg = ('The target release {target} was not found in the sources list').format(target=target)
+			raise PackageError(msg)
+		self.remote[name] = target

base/pkg/sourceslist.py

@@ -0,0 +1,63 @@
+
+
+class SourceLists(object):
+
+	def __init__(self, manifest):
+		self.sources = {}
+		self.manifest_vars = {'release':      manifest.system['release'],
+		                      'architecture': manifest.system['architecture'],
+		                      'apt_mirror':   'http://http.debian.net/debian'}
+		if 'sources' in manifest.packages:
+			for name, lines in manifest.packages['sources'].iteritems():
+				for line in lines:
+					self.add_source(name, '{line}\n'.format(line=line.format(**self.manifest_vars)))
+
+	def add_source(self, name, line):
+		name = name.format(**self.manifest_vars)
+		if name not in self.sources:
+			self.sources[name] = []
+		self.sources[name].append(Source(line))
+
+	def target_exists(self, target):
+		for lines in self.sources.itervalues():
+			if target in (source.distribution for source in lines):
+				return True
+		return False
+
+
+class Source(object):
+
+	def __init__(self, line):
+		import re
+		regexp = re.compile('^(?P<type>deb|deb-src)\s+'
+		                    '(\[\s*(?P<options>.+\S)?\s*\]\s+)?'
+		                    '(?P<uri>\S+)\s+'
+		                    '(?P<distribution>\S+)'
+		                    '(\s+(?P<components>.+\S))?\s*$')
+		match = regexp.match(line).groupdict()
+		if match is None:
+			from exceptions import SourceError
+			raise SourceError('Unable to parse source line `{line}\''.format(line=line))
+		self.type = match['type']
+		self.options = []
+		if match['options'] is not None:
+			self.options = re.sub(' +', ' ', match['options']).split(' ')
+		self.uri = match['uri']
+		self.distribution = match['distribution']
+		self.components = []
+		if match['components'] is not None:
+			self.components = re.sub(' +', ' ', match['components']).split(' ')
+
+	def __str__(self):
+		options = ''
+		if len(self.options) > 0:
+			options = ' [{options}]'.format(options=' '.join(self.options))
+
+		components = ''
+		if len(self.components) > 0:
+			components = ' {components}'.format(components=' '.join(self.components))
+
+		return ('{type}{options} {uri}'
+		        ' {distribution}{components}').format(type=self.type, options=options,
+		                                              uri=self.uri, distribution=self.distribution,
+		                                              components=components)

common/phases.py

@@ -5,7 +5,8 @@ volume_creation = Phase('Volume creation', 'Creating the volume to bootstrap ont
 volume_preparation = Phase('Volume preparation', 'Formatting the bootstrap volume')
 volume_mounting = Phase('Volume mounting', 'Mounting bootstrap volume')
 os_installation = Phase('OS installation', 'Installing the operating system')
-system_modification = Phase('System modification', 'Installing software, modifying configuration files etc.')
+package_installation = Phase('Package installation', 'Installing software')
+system_modification = Phase('System modification', 'Modifying configuration files, adding resources, etc.')
 system_cleaning = Phase('System cleaning', 'Removing sensitive data, temporary files and other leftovers')
 volume_unmounting = Phase('Volume unmounting', 'Unmounting the bootstrap volume')
 image_registration = Phase('Image registration', 'Uploading/Registering with the provider')
@@ -16,6 +17,7 @@ order = [preparation,
          volume_preparation,
          volume_mounting,
          os_installation,
+         package_installation,
          system_modification,
          system_cleaning,
          volume_unmounting,

common/task_sets.py

@@ -11,9 +11,8 @@ from common.tasks import security
 from common.tasks import locale
 
 base_set = [workspace.CreateWorkspace,
-            packages.HostPackages,
+            host.HostDependencies,
-            packages.ImagePackages,
+            host.CheckHostDependencies,
-            host.CheckPackages,
             bootstrap.Bootstrap,
             workspace.DeleteWorkspace,
             ]
@@ -45,16 +44,19 @@ ssh_set = [security.DisableSSHPasswordAuthentication,
            cleanup.ShredHostkeys,
            ]
 
-apt_set = [apt.DisableDaemonAutostart,
+apt_set = [apt.WriteSources,
-           apt.AptSources,
+           apt.DisableDaemonAutostart,
            apt.AptUpdate,
            apt.AptUpgrade,
+           packages.InstallRemotePackages,
+           packages.InstallLocalPackages,
            apt.PurgeUnusedPackages,
            apt.AptClean,
            apt.EnableDaemonAutostart,
            ]
 
-locale_set = [locale.GenerateLocale,
+locale_set = [locale.LocaleBootstrapPackage,
+              locale.GenerateLocale,
               locale.SetTimezone,
               ]
 

common/tasks/apt.py

@@ -1,31 +1,28 @@
 from base import Task
 from common import phases
 from common.tools import log_check_call
-import network
 import locale
 import os
 
 
-class AptSources(Task):
+class WriteSources(Task):
-	description = 'Adding aptitude sources'
+	description = 'Writing aptitude sources to disk'
-	phase = phases.system_modification
+	phase = phases.package_installation
 
 	def run(self, info):
-		sources_path = os.path.join(info.root, 'etc/apt/sources.list')
+		for name, sources in info.source_lists.sources.iteritems():
-		with open(sources_path, 'w') as apt_sources:
+			if name == 'main':
-			apt_sources.write(('deb     {apt_mirror} {release} main\n'
+				list_path = os.path.join(info.root, 'etc/apt/sources.list')
-			                   'deb-src {apt_mirror} {release} main\n'
+			else:
-			                   .format(apt_mirror='http://http.debian.net/debian',
+				list_path = os.path.join(info.root, 'etc/apt/sources.list.d/', name + '.list')
-			                           release=info.manifest.system['release'])))
+			with open(list_path, 'w') as source_list:
-			apt_sources.write(('deb     {apt_mirror} {release}/updates main\n'
+				for source in sources:
-			                   'deb-src {apt_mirror} {release}/updates main\n'
+					source_list.write('{line}\n'.format(line=str(source)))
-			                   .format(apt_mirror='http://security.debian.org/',
-			                           release=info.manifest.system['release'])))
 
 
 class DisableDaemonAutostart(Task):
 	description = 'Disabling daemon autostart'
-	phase = phases.system_modification
+	phase = phases.package_installation
 
 	def run(self, info):
 		rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d')
@@ -41,27 +38,19 @@ class DisableDaemonAutostart(Task):
 
 class AptUpdate(Task):
 	description = 'Updating the package cache'
-	phase = phases.system_modification
+	phase = phases.package_installation
-	predecessors = [locale.GenerateLocale, AptSources]
+	predecessors = [locale.GenerateLocale, WriteSources]
-	successors = [network.RemoveDNSInfo]
 
 	def run(self, info):
 		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get',
-		                                               '--fix-broken',
-		                                               '--assume-yes',
-		                                               'install'])
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '--assume-yes', 'upgrade'])
 
 
 class AptUpgrade(Task):
 	description = 'Upgrading packages and fixing broken dependencies'
-	phase = phases.system_modification
+	phase = phases.package_installation
 	predecessors = [AptUpdate, DisableDaemonAutostart]
-	successors = [network.RemoveDNSInfo]
 
 	def run(self, info):
-		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update'])
 		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get',
 		                                               '--fix-broken',
 		                                               '--assume-yes',

common/tasks/bootstrap.py

@@ -8,11 +8,10 @@ log = logging.getLogger(__name__)
 def get_bootstrap_args(info):
 	executable = ['/usr/sbin/debootstrap']
 	options = ['--arch=' + info.manifest.system['architecture']]
-	include, exclude = info.img_packages
+	if len(info.include_packages) > 0:
-	if len(include) > 0:
+		options.append('--include=' + ','.join(info.include_packages))
-		options.append('--include=' + ','.join(include))
+	if len(info.exclude_packages) > 0:
-	if len(exclude) > 0:
+		options.append('--exclude=' + ','.join(info.exclude_packages))
-		options.append('--exclude=' + ','.join(exclude))
 	arguments = [info.manifest.system['release'], info.root, info.manifest.bootstrapper['mirror']]
 	return executable, options, arguments
 

common/tasks/filesystem.py

@@ -32,8 +32,7 @@ class AddXFSProgs(Task):
 	phase = phases.preparation
 
 	def run(self, info):
-		include, exclude = info.img_packages
+		info.packages.add('xfsprogs')
-		include.add('xfsprogs')
 
 
 class CreateMountDir(Task):

common/tasks/host.py

@@ -1,18 +1,36 @@
 from base import Task
 from common import phases
 from common.exceptions import TaskError
-import packages
 
 
-class CheckPackages(Task):
+class HostDependencies(Task):
+	description = 'Determining required host dependencies'
+	phase = phases.preparation
+
+	def run(self, info):
+		info.host_dependencies.add('debootstrap')
+
+		from common.fs.loopbackvolume import LoopbackVolume
+		if isinstance(info.volume, LoopbackVolume):
+			info.host_dependencies.add('qemu-utils')
+
+		if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions):
+			info.host_dependencies.add('xfsprogs')
+
+		from base.fs.partitionmaps.none import NoPartitions
+		if not isinstance(info.volume.partition_map, NoPartitions):
+			info.host_dependencies.update(['parted', 'kpartx'])
+
+
+class CheckHostDependencies(Task):
 	description = 'Checking installed host packages'
 	phase = phases.preparation
-	predecessors = [packages.HostPackages, packages.ImagePackages]
+	predecessors = [HostDependencies]
 
 	def run(self, info):
 		from common.tools import log_check_call
 		from subprocess import CalledProcessError
-		for package in info.host_packages:
+		for package in info.host_dependencies:
 			try:
 				log_check_call(['/usr/bin/dpkg-query', '-s', package])
 			except CalledProcessError:

common/tasks/locale.py

@@ -3,9 +3,19 @@ from common import phases
 import os.path
 
 
+class LocaleBootstrapPackage(Task):
+	description = 'Adding locale package to bootstrap installation'
+	phase = phases.preparation
+
+	def run(self, info):
+		# We could bootstrap without locales, but things just suck without them
+		# eg. error messages when running apt
+		info.include_packages.add('locales')
+
+
 class GenerateLocale(Task):
 	description = 'Generating the selected locale'
-	phase = phases.system_modification
+	phase = phases.package_installation
 
 	def run(self, info):
 		from common.tools import sed_i

common/tasks/packages.py

@@ -1,33 +1,51 @@
 from base import Task
 from common import phases
+from common.tasks import apt
 
 
-class HostPackages(Task):
+class InstallRemotePackages(Task):
-	description = 'Determining required host packages'
+	description = 'Installing remote packages'
-	phase = phases.preparation
+	phase = phases.package_installation
+	predecessors = [apt.AptUpgrade]
 
 	def run(self, info):
-		info.host_packages = set()
+		if len(info.packages.remote) == 0:
-		info.host_packages.add('debootstrap')
+			return
+		import os
+		from common.tools import log_check_call
 
-		from common.fs.loopbackvolume import LoopbackVolume
+		packages = []
-		if isinstance(info.volume, LoopbackVolume):
+		for name, target in info.packages.remote.iteritems():
-			info.host_packages.add('qemu-utils')
+			packages.append('{name}/{target}'.format(name=name, target=target))
 
-		if 'xfs' in (p.filesystem for p in info.volume.partition_map.partitions):
+		env = os.environ.copy()
-			info.host_packages.add('xfsprogs')
+		env['DEBIAN_FRONTEND'] = 'noninteractive'
-
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install',
-		from base.fs.partitionmaps.none import NoPartitions
+		                '--assume-yes'] + packages,
-		if not isinstance(info.volume.partition_map, NoPartitions):
+		               env=env)
-			info.host_packages.update(['parted', 'kpartx'])
 
 
-class ImagePackages(Task):
+class InstallLocalPackages(Task):
-	description = 'Determining required image packages'
+	description = 'Installing local packages'
-	phase = phases.preparation
+	phase = phases.package_installation
+	predecessors = [apt.AptUpgrade]
+	successors = [InstallRemotePackages]
 
 	def run(self, info):
-		info.img_packages = set(), set()
+		if len(info.packages.local) == 0:
-		include, exclude = info.img_packages
+			return
-		# We could bootstrap without locales, but things just suck without them, error messages etc.
+		from shutil import copy
-		include.add('locales')
+		from common.tools import log_check_call
+		import os
+
+		for package_src in info.packages.local:
+			pkg_name = os.path.basename(package_src)
+			package_dst = os.path.join('/tmp', pkg_name)
+			copy(package_src, os.path.join(info.root, package_dst))
+
+			env = os.environ.copy()
+			env['DEBIAN_FRONTEND'] = 'noninteractive'
+			log_check_call(['/usr/sbin/chroot', info.root,
+			                '/usr/bin/dpkg', '--install', package_dst],
+			               env=env)
+			os.remove(package_dst)

common/tools.py

@@ -1,14 +1,14 @@
 
 
-def log_check_call(command, stdin=None):
+def log_check_call(command, stdin=None, env=None):
-	status, stdout, stderr = log_call(command, stdin)
+	status, stdout, stderr = log_call(command, stdin, env)
 	if status != 0:
 		from subprocess import CalledProcessError
 		raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr))
 	return stdout
 
 
-def log_call(command, stdin=None):
+def log_call(command, stdin=None, env=None):
 	import subprocess
 	import select
 
@@ -18,13 +18,19 @@ def log_call(command, stdin=None):
 	log = logging.getLogger(__name__ + command_log)
 	log.debug('Executing: {command}'.format(command=' '.join(command)))
 
+	popen_args = {'args':   command,
+	              'env':    env,
+	              'stdin':  subprocess.PIPE,
+	              'stdout': subprocess.PIPE,
+	              'stderr': subprocess.PIPE, }
 	if stdin is not None:
-		process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		popen_args['stdin'] = subprocess.PIPE
+		process = subprocess.Popen(**popen_args)
 		process.stdin.write(stdin + "\n")
 		process.stdin.flush()
 		process.stdin.close()
 	else:
-		process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		process = subprocess.Popen(**popen_args)
 	stdout = []
 	stderr = []
 	while True:

manifests/ec2-ebs-debian-official-amd64-hvm.manifest.json

@@ -30,19 +30,19 @@
 			}
 		}
 	},
-	"plugins": {
+	"packages": {
-		"packages": {
+		"sources": {
-			"sources": {
+			"backports": [
-				"backports": [
+				"deb     {apt_mirror} {release}-backports main",
-					"deb     {apt_mirror} {release}-backports main",
+				"deb-src {apt_mirror} {release}-backports main"
-					"deb-src {apt_mirror} {release}-backports main"
-				]
-			},
-			"remote": [
-				"sudo",
-				{ "name": "cloud-init", "target": "{release}-backports" }
 			]
 		},
+		"remote": [
+			"sudo",
+			{ "name": "cloud-init", "target": "{release}-backports" }
+		]
+	},
+	"plugins": {
 		"cloud_init": {
 			"username": "admin",
 			//"metadata_sources": "Ec2",

manifests/ec2-ebs-debian-official-amd64-pvm.manifest.json

@@ -30,19 +30,19 @@
 			}
 		}
 	},
-	"plugins": {
+	"packages": {
-		"packages": {
+		"sources": {
-			"sources": {
+			"backports": [
-				"backports": [
+				"deb     {apt_mirror} {release}-backports main",
-					"deb     {apt_mirror} {release}-backports main",
+				"deb-src {apt_mirror} {release}-backports main"
-					"deb-src {apt_mirror} {release}-backports main"
-				]
-			},
-			"remote": [
-				"sudo",
-				{ "name": "cloud-init", "target": "{release}-backports" }
 			]
 		},
+		"remote": [
+			"sudo",
+			{ "name": "cloud-init", "target": "{release}-backports" }
+		]
+	},
+	"plugins": {
 		"cloud_init": {
 			"username": "admin",
 			//"metadata_sources": "Ec2",

manifests/ec2-ebs-debian-official-i386-pvm.manifest.json

@@ -30,19 +30,19 @@
 			}
 		}
 	},
-	"plugins": {
+	"packages": {
-		"packages": {
+		"sources": {
-			"sources": {
+			"backports": [
-				"backports": [
+				"deb     {apt_mirror} {release}-backports main",
-					"deb     {apt_mirror} {release}-backports main",
+				"deb-src {apt_mirror} {release}-backports main"
-					"deb-src {apt_mirror} {release}-backports main"
-				]
-			},
-			"remote": [
-				"sudo",
-				{ "name": "cloud-init", "target": "{release}-backports" }
 			]
 		},
+		"remote": [
+			"sudo",
+			{ "name": "cloud-init", "target": "{release}-backports" }
+		]
+	},
+	"plugins": {
 		"cloud_init": {
 			"username": "admin",
                         //"metadata_sources": "Ec2",

plugins/admin_user/tasks.py

@@ -1,20 +1,15 @@
 from base import Task
 from common import phases
-from common.tasks.packages import ImagePackages
-from common.tasks.host import CheckPackages
 from common.tasks.initd import InstallInitScripts
-from plugins.packages.tasks import InstallRemotePackages
 import os
 
 
 class AddSudoPackage(Task):
 	description = 'Adding ``sudo\'\' to the image packages'
 	phase = phases.preparation
-	predecessors = [ImagePackages]
-	successors = [CheckPackages]
 
 	def run(self, info):
-		info.img_packages[0].add('sudo')
+		info.packages.add('sudo')
 
 
 class CreateAdminUser(Task):
@@ -58,7 +53,6 @@ class AdminUserCredentials(Task):
 class DisableRootLogin(Task):
 	description = 'Disabling SSH login for root'
 	phase = phases.system_modification
-	predecessors = [InstallRemotePackages]
 
 	def run(self, info):
 		from subprocess import CalledProcessError

plugins/packages/__init__.py

@@ -1,17 +0,0 @@
-
-
-def validate_manifest(data, schema_validate):
-	from os import path
-	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
-	schema_validate(data, schema_path)
-
-
-def resolve_tasks(tasklist, manifest):
-	from tasks import AptSources, InstallRemotePackages, InstallLocalPackages
-	packages = manifest.plugins['packages']
-	if 'sources' in packages:
-		tasklist.add(AptSources)
-	if 'remote' in packages:
-		tasklist.add(InstallRemotePackages)
-	if 'local' in packages:
-		tasklist.add(InstallLocalPackages)

plugins/packages/manifest-schema.json

@@ -1,62 +0,0 @@
-{
-	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "Packages plugin manifest",
-	"type": "object",
-	"properties": {
-		"plugins": {
-			"type": "object",
-			"properties": {
-				"packages": {
-					"type": "object",
-					"properties": {
-						"sources": {
-							"type": "object",
-							"patternProperties": {
-								"^\\w+$": {
-									"type": "array",
-									"items": {"type": "string"},
-									"minItems": 1
-								}
-							},
-							"additionalProperties": false,
-							"minItems": 1
-						},
-						"remote": {
-							"type": "array",
-							"items": {
-								"anyOf": [
-									{
-										"type": "object",
-										"properties": {
-											"name": {"type": "string"},
-											"target": {"type": "string"}
-										},
-										"required": ["name"]
-									},
-									{ "type": "string" }
-								]
-							},
-							"minItems": 1
-						},
-						"local": {
-							"type": "array",
-							"items": { "$ref": "#/definitions/absolute_path" },
-							"minItems": 1
-						}
-					},
-					"anyOf": [
-						{"required": ["sources"]},
-						{"required": ["remote"]},
-						{"required": ["local"]}
-          ]
-				}
-			}
-		}
-	},
-	"definitions": {
-		"absolute_path": {
-			"type":    "string",
-			"pattern": "^/[^\\0]+$"
-		}
-	}
-}

plugins/packages/tasks.py

@@ -1,63 +0,0 @@
-from base import Task
-from common import phases
-from common.tasks import network
-from common.tasks import apt
-import os.path
-
-
-class AptSources(Task):
-	description = 'Adding additional aptitude sources'
-	phase = phases.system_modification
-	predecessors = [apt.AptSources]
-	successors = [apt.AptUpdate]
-
-	def run(self, info):
-		manifest_vars = {'release':      info.manifest.system['release'],
-		                 'architecture': info.manifest.system['architecture'],
-		                 'apt_mirror':   'http://http.debian.net/debian'}
-		for name, lines in info.manifest.plugins['packages']['sources'].iteritems():
-			list_path = os.path.join(info.root, 'etc/apt/sources.list.d/', name + '.list')
-			with open(list_path, 'a') as source_list:
-				for line in lines:
-					source_list.write('{line}\n'.format(line=line.format(**manifest_vars)))
-
-
-class InstallRemotePackages(Task):
-	description = 'Installing remote packages'
-	phase = phases.system_modification
-	predecessors = [apt.AptUpdate, apt.DisableDaemonAutostart]
-	successors = [network.RemoveDNSInfo]
-
-	def run(self, info):
-		manifest_vars = {'release':      info.manifest.system['release'],
-		                 'architecture': info.manifest.system['architecture']}
-		from common.tools import log_check_call
-		for package in info.manifest.plugins['packages']['remote']:
-			target_release = []
-			if isinstance(package, basestring):
-				name = package
-			else:
-				name = package['name'].format(**manifest_vars)
-				if hasattr(package, 'target'):
-					target = package['target'].format(**manifest_vars)
-					target_release = ['--targe-release', target]
-			log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'install',
-			                '--assume-yes'] + target_release + [name])
-
-
-class InstallLocalPackages(Task):
-	description = 'Installing local packages'
-	phase = phases.system_modification
-
-	def run(self, info):
-		from shutil import copy
-		from os import remove
-		from common.tools import log_check_call
-
-		for package_src in info.manifest.plugins['packages']['local']:
-			pkg_name = os.path.basename(package_src)
-			package_dst = os.path.join('/tmp', pkg_name)
-			copy(package_src, os.path.join(info.root, package_dst))
-			log_check_call(['/usr/sbin/chroot', info.root,
-			                '/usr/bin/dpkg', '--install', package_dst])
-			remove(package_dst)

plugins/unattended_upgrades/tasks.py

@@ -1,17 +1,13 @@
 from base import Task
 from common import phases
-from common.tasks.packages import ImagePackages
-from common.tasks.host import CheckPackages
 
 
 class AddUnattendedUpgradesPackage(Task):
 	description = 'Adding ``unattended-upgrades\'\' to the image packages'
 	phase = phases.preparation
-	predecessors = [ImagePackages]
-	successors = [CheckPackages]
 
 	def run(self, info):
-		info.img_packages[0].add('unattended-upgrades')
+		info.packages.add('unattended-upgrades')
 
 
 class EnablePeriodicUpgrades(Task):

providers/ec2/__init__.py

@@ -43,8 +43,8 @@ def resolve_tasks(tasklist, manifest):
 		from common.task_sets import partitioning_set
 		tasklist.add(*partitioning_set)
 
-	tasklist.add(packages.HostPackages,
+	tasklist.add(packages.HostDependencies,
-	             packages.ImagePackages,
+	             packages.DefaultPackages,
 	             connection.GetCredentials,
 	             host.GetInfo,
 	             ami.AMIName,

providers/ec2/tasks/host.py

@@ -1,5 +1,16 @@
 from base import Task
 from common import phases
+from common.tasks import host
+
+
+class HostDependencies(Task):
+	description = 'Adding more required host packages'
+	phase = phases.preparation
+	successors = [host.CheckHostDependencies]
+
+	def run(self, info):
+		if info.manifest.volume['backing'] == 's3':
+			info.host_dependencies.add('euca2ools')
 
 
 class GetInfo(Task):

providers/ec2/tasks/packages.py

@@ -1,39 +1,24 @@
 from base import Task
 from common import phases
-from common.tasks import packages
-from common.tasks.host import CheckPackages
 
 
-class HostPackages(Task):
+class DefaultPackages(Task):
-	description = 'Adding more required host packages'
+	description = 'Adding image packages required for EC2'
 	phase = phases.preparation
-	predecessors = [packages.HostPackages]
-	successors = [CheckPackages]
 
 	def run(self, info):
-		if info.manifest.volume['backing'] == 's3':
+		info.packages.add('openssh-server')
-			info.host_packages.add('euca2ools')
+		info.packages.add('file')  # Needed for the init scripts
+		info.packages.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
+		info.packages.add('grub-pc')
 
-
+		info.exclude_packages.add('isc-dhcp-client')
-class ImagePackages(Task):
+		info.exclude_packages.add('isc-dhcp-common')
-	description = 'Adding more required image packages'
-	phase = phases.preparation
-	predecessors = [packages.ImagePackages]
-
-	def run(self, info):
-		manifest = info.manifest
-		include, exclude = info.img_packages
-		include.add('openssh-server')
-		include.add('file')  # Needed for the init scripts
-		include.add('dhcpcd')  # isc-dhcp-client doesn't work properly with ec2
-		include.add('grub-pc')
-
-		exclude.add('isc-dhcp-client')
-		exclude.add('isc-dhcp-common')
 
 		# In squeeze, we need a special kernel flavor for xen
 		kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64',
 		                       'i386':  'linux-image-xen-686', },
 		           'wheezy':  {'amd64': 'linux-image-amd64',
 		                       'i386':  'linux-image-686', }, }
-		include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))
+		kernel_package = kernels.get(info.manifest.system['release']).get(info.manifest.system['architecture'])
+		info.packages.add(kernel_package)

providers/virtualbox/__init__.py

@@ -33,7 +33,7 @@ def resolve_tasks(tasklist, manifest):
 		from common.task_sets import partitioning_set
 		tasklist.add(*partitioning_set)
 
-	tasklist.add(packages.ImagePackages,
+	tasklist.add(packages.DefaultPackages,
 
 	             loopback.Create,
 

providers/virtualbox/tasks/guest_additions.py

@@ -1,7 +1,6 @@
 from base import Task
 from common import phases
-from common.tasks.packages import ImagePackages
+from common.tasks.packages import InstallRemotePackages
-from common.tasks.host import CheckPackages
 from common.tasks.filesystem import FStab
 from common.exceptions import TaskError
 
@@ -21,20 +20,18 @@ class CheckGuestAdditionsPath(Task):
 class AddGuestAdditionsPackages(Task):
 	description = 'Adding packages to support Guest Additions installation'
 	phase = phases.preparation
-	predecessors = [ImagePackages]
-	successors = [CheckPackages]
 
 	def run(self, info):
-		info.img_packages[0].update(['bzip2',
+		info.packages.add('bzip2')
-		                             'build-essential',
+		info.packages.add('build-essential')
-		                             'dkms',
+		info.packages.add('dkms')
-		                             ])
+		# info.packages.add('linux-headers-3.2.0-4-amd64')  # linux-headers-$(uname -r)
 
 
 class InstallGuestAdditions(Task):
 	description = 'Installing the VirtualBox Guest Additions'
 	phase = phases.system_modification
-	predecessors = [FStab]
+	predecessors = [FStab, InstallRemotePackages]
 
 	def run(self, info):
 		import os

providers/virtualbox/tasks/packages.py

@@ -3,20 +3,14 @@ from common import phases
 from common.tasks import packages
 
 
-class ImagePackages(Task):
+class DefaultPackages(Task):
-	description = 'Determining required image packages'
+	description = 'Adding image packages required for virtualbox'
 	phase = phases.preparation
-	predecessors = [packages.ImagePackages]
 
 	def run(self, info):
-		manifest = info.manifest
-		include, exclude = info.img_packages
 		# Add some basic packages we are going to need
-		include.add('grub2')
+		info.packages.add('grub2')
 
-		# In squeeze, we need a special kernel flavor for xen
+		kernels = {'amd64': 'linux-image-amd64',
-		kernels = {'squeeze': {'amd64': 'linux-image-amd64',
+		           'i386':  'linux-image-686', }
-		                       'i386':  'linux-image-686', },
+		info.packages.add(kernels.get(info.manifest.system['architecture']))
-		           'wheezy':  {'amd64': 'linux-image-amd64',
-		                       'i386':  'linux-image-686', }, }
-		include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))