Merge pull request #84 from osallou/python

Integration of the *raw* provider with additional plugins

base/manifest-schema.json

@@ -10,8 +10,10 @@
 			"type": "object",
 			"properties": {
 				"mount_dir": { "type": "string" },
+                                "image_file": { "type": "string" },
 				"tarball": { "type": "boolean" },
-				"tarball_dir": { "type": "string" }
+				"tarball_dir": { "type": "string" },
+				"device" : { "type" : "string" }
 			},
 			"required": ["mount_dir"]
 		},
@@ -28,7 +30,8 @@
 				},
 				"timezone": { "type": "string" },
 				"locale":   { "type": "string" },
-				"charmap":  { "type": "string" }
+				"charmap":  { "type": "string" },
+				"mirror":   { "type": "string" }
 			},
 			"required": ["release", "architecture", "timezone", "locale", "charmap"]
 		},

common/tools.py

@@ -1,14 +1,14 @@
 
 
-def log_check_call(command):
+def log_check_call(command, input=None):
-	status, stdout, stderr = log_call(command)
+	status, stdout, stderr = log_call(command, input)
 	if status != 0:
 		from subprocess import CalledProcessError
 		raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr))
 	return stdout
 
 
-def log_call(command):
+def log_call(command, input=None):
 	import subprocess
 	import select
 
@@ -17,7 +17,15 @@ def log_call(command):
 	command_log = realpath(command[0]).replace('/', '.')
 	log = logging.getLogger(__name__ + command_log)
 
-	process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+	if input is not None:
+		process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+		process.stdin.write(input+"\n")
+		process.stdin.flush()
+		process.stdin.close()
+		# (stdout,stderr) = process.communicate(input+"\n")
+		# return process.returncode, stdout, stderr
+	else:
+		process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 	stdout = []
 	stderr = []
 	while True:

manifests/one-ide.manifest

@@ -0,0 +1,54 @@
+{
+	"provider"      : "raw",
+	"virtualization": "ide",
+	"credentials"   : {
+		"access-key": null,
+		"secret-key": null,
+		"root": "test"
+	},
+
+	"bootstrapper": {
+		"mount_dir": "/mnt/target",
+		"image_file": "/tmp/one.img",
+		"tarball": true
+	},
+	"image": {
+		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} ({virtualization})"
+	},
+	"system": {
+		"release"     : "wheezy",
+		"architecture": "amd64",
+		"timezone"    : "UTC",
+		"locale"      : "en_US",
+		"charmap"     : "UTF-8",
+		"mirror"      : "ftp://ftp.fr.debian.org/debian/"
+	},
+	"volume": {
+		"backing"   : "raw",
+		"filesystem": "ext4",
+		"size"      : 1024
+	},
+	"plugins": {
+		"admin_user": {
+			"enabled": false
+		},
+		"build_metadata": {
+			"enabled": false,
+			"path"   : "/root/build-metadata-{ami_name}"
+		},
+		"prebootstrapped": {
+			"enabled": false,
+			"snapshot": ""
+		},
+		"opennebula": {
+			"enabled": true
+		},
+                "user_packages": {
+                        "enabled": true,
+                        "repo": [ "apache2" ],
+                        "local": []
+                }
+
+	}
+}

manifests/one-virtio.manifest

@@ -0,0 +1,54 @@
+{
+	"provider"      : "raw",
+	"virtualization": "virtio",
+	"credentials"   : {
+		"access-key": null,
+		"secret-key": null,
+		"root": "test"
+	},
+
+	"bootstrapper": {
+		"mount_dir": "/mnt/target",
+		"image_file": "/tmp/one.img",
+		"tarball": true
+	},
+	"image": {
+		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} ({virtualization})"
+	},
+	"system": {
+		"release"     : "wheezy",
+		"architecture": "amd64",
+		"timezone"    : "UTC",
+		"locale"      : "en_US",
+		"charmap"     : "UTF-8",
+		"mirror"      : "ftp://ftp.fr.debian.org/debian/"
+	},
+	"volume": {
+		"backing"   : "raw",
+		"filesystem": "ext4",
+		"size"      : 1024
+	},
+	"plugins": {
+		"admin_user": {
+			"enabled": false
+		},
+		"build_metadata": {
+			"enabled": false,
+			"path"   : "/root/build-metadata-{ami_name}"
+		},
+		"prebootstrapped": {
+			"enabled": false,
+			"snapshot": ""
+		},
+		"opennebula": {
+			"enabled": true
+		},
+                "user_packages": {
+                        "enabled": true,
+                        "repo": [ "apache2" ],
+                        "local": []
+                }
+
+	}
+}

plugins/opennebula/README.md

@@ -0,0 +1,17 @@
+# Open Nebula context plugin
+
+This plugin adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong).
+
+It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like:
+
+    ETH0_DNS            $NETWORK[DNS, NETWORK_ID=2]
+    ETH0_GATEWAY        $NETWORK[GATEWAY, NETWORK_ID=2]
+    ETH0_IP             $NIC[IP, NETWORK_ID=2]
+    ETH0_MASK	        $NETWORK[MASK, NETWORK_ID=2]
+    ETH0_NETWORK        $NETWORK[NETWORK, NETWORK_ID=2]
+    FILES               path_to_my_ssh_public_key.pub
+
+Plugin will install all *.pub* files in the root authorized_keys file.
+
+In case of an EC2 start, if the USER_EC2_DATA element is a script, the plugin will execute it.
+

plugins/opennebula/__init__.py

@@ -0,0 +1,5 @@
+
+
+def tasks(tasklist, manifest):
+	from opennebula import OpenNebulaContext
+	tasklist.add(OpenNebulaContext())

plugins/opennebula/assets/one-ec2.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ -n "$EC2_USER_DATA" ]; then
+  # Check if EC2 user data is a script, if yes, execute
+  if [[ $EC2_USER_DATA =~ ^#! ]]; then
+    echo "EC2 data is an executable script, so execute it now"
+    TMPFILE=$(mktemp /tmp/output.XXXXXXXXXX)
+    chmod 755 $TMPFILE
+    $TMPFILE
+    cat $TMPFILE
+  else
+    print "Not an executable"
+  fi
+fi

plugins/opennebula/assets/one-pubkey.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+echo "Reconfigure host ssh keys"
+dpkg-reconfigure openssh-server
+
+if [ ! -e /root/.ssh ]; then
+  mkdir /root/.ssh
+  touch /root/.ssh/authorized_keys
+  chmod 600 /root/.ssh/authorized_keys
+fi
+
+echo "Copy public ssh keys to authorized_keys"
+for f in /mnt/*.pub
+do
+  cat $f >> /root/.ssh/authorized_keys
+
+done

plugins/opennebula/opennebula.py

@@ -0,0 +1,42 @@
+from base import Task
+from common import phases
+import os
+from providers.raw.tasks.locale import GenerateLocale
+
+
+class OpenNebulaContext(Task):
+	description = 'Setup OpenNebula init context'
+	phase = phases.system_modification
+        after = [GenerateLocale]
+
+	def run(self, info):
+		import stat
+		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+		             stat.S_IRGRP                | stat.S_IXGRP |
+		             stat.S_IROTH                | stat.S_IXOTH)
+
+		from shutil import copy
+		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-context_3.8.1.deb'))
+		script_dst = os.path.join(info.root, 'tmp/one-context_3.8.1.deb')
+		copy(script_src, script_dst)
+		os.chmod(script_dst, rwxr_xr_x)
+
+                from common.tools import log_check_call
+		log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb'])
+		# Fix start
+                from common.tools import sed_i
+                vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext')
+                sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5')
+		os.chmod(vmcontext_def, rwxr_xr_x)
+		log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', '90', '2', '3', '4', '5', 'stop', '90', '0', '6'])
+
+		# Load all pubkeys in root authorized_keys
+                script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-pubkey.sh'))
+                script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh')
+                copy(script_src, script_dst)
+
+                # If USER_EC2_DATA is a script, execute it
+                script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-ec2.sh'))
+                script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh')
+                copy(script_src, script_dst)
+

plugins/user_packages/README.md

@@ -0,0 +1,15 @@
+# User package plugin
+
+This plugin gives the possibility to the user to install Debian packages in the virtual image.
+
+Plugin is defined in the manifest file, plugin section with:
+
+    "user_packages": {
+        "enabled": true,
+        "repo": [ "apache2" ],
+        "local": [ "/tmp/mypackage.deb" ]
+    }
+
+The *repo* element refers to packages available in Debian repository (apt-get).
+
+The *local* element will copy the specified .deb files and install them in the image with a dpkg command. Packages are installed in the order of their declaration.

plugins/user_packages/__init__.py

@@ -0,0 +1,6 @@
+
+
+def tasks(tasklist, manifest):
+	from user_packages import AddUserPackages, AddLocalUserPackages
+	tasklist.add(AddUserPackages())
+	tasklist.add(AddLocalUserPackages())

plugins/user_packages/user_packages.py

@@ -0,0 +1,45 @@
+from base import Task
+from common import phases
+import os
+from providers.raw.tasks.packages import ImagePackages
+from providers.raw.tasks.host import CheckPackages
+from providers.raw.tasks.filesystem import MountVolume
+
+
+class AddUserPackages(Task):
+	description = 'Adding user defined packages to the image packages'
+	phase = phases.preparation
+	after = [ImagePackages]
+	before = [CheckPackages]
+
+	def run(self, info):
+		if 'repo' not in info.manifest.plugins['user_packages']:
+			return
+		for pkg in info.manifest.plugins['user_packages']['repo']:
+			info.img_packages[0].add(pkg)
+
+class AddLocalUserPackages(Task):
+        description = 'Adding user local packages to the image packages'
+        phase = phases.system_modification
+        after = [MountVolume]
+
+        def run(self, info):
+                if 'local' not in info.manifest.plugins['user_packages']:
+                        return
+
+                import stat
+                rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                             stat.S_IRGRP                | stat.S_IXGRP |
+                             stat.S_IROTH                | stat.S_IXOTH)
+
+                from shutil import copy
+                from common.tools import log_check_call
+
+		for pkg in info.manifest.plugins['user_packages']['local']:
+                	script_src = os.path.normpath(pkg)
+                	script_dst = os.path.join(info.root, 'tmp/'+os.path.basename(script_src))
+                	copy(script_src, script_dst)
+                	os.chmod(script_dst, rwxr_xr_x)
+
+                	log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/'+os.path.basename(script_src)])
+

providers/raw/REAME.md

@@ -0,0 +1,16 @@
+# RAW provider
+
+This provider creates a raw image file. It can be combined with  opennebula plugin to add OpenNebula contextualization.
+
+By default, it creates a network interface configured with DHCP.
+
+# Configuration
+
+## provider
+
+*raw* : use this provider
+
+##virtualization
+
+* ide: basic disk emulation (/dev/sda)
+* virtio: Virtio emulation (for KVM), provides better disk performances (/dev/vda)

providers/raw/__init__.py

@@ -0,0 +1,77 @@
+from manifest import Manifest
+import logging
+from tasks import packages
+from tasks import host
+from tasks import filesystem
+from tasks import bootstrap
+from tasks import locale
+from tasks import apt
+from tasks import boot
+from tasks import security
+from tasks import network
+from tasks import initd
+from tasks import cleanup
+from tasks import fake
+
+
+def initialize():
+	# Regardless of of loglevel, we don't want boto debug stuff, it's very noisy
+	logging.getLogger('boto').setLevel(logging.INFO)
+
+
+def tasks(tasklist, manifest):
+	tasklist.add(packages.HostPackages(),
+	             packages.ImagePackages(),
+	             host.CheckPackages(),
+	             host.GetInfo())
+	tasklist.add(filesystem.FormatVolume())
+	if manifest.volume['filesystem'].lower() == 'xfs':
+		tasklist.add(filesystem.AddXFSProgs())
+	if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
+		tasklist.add(filesystem.TuneVolumeFS())
+	tasklist.add(filesystem.CreateMountDir(),
+	             filesystem.MountVolume())
+	if manifest.bootstrapper['tarball']:
+		tasklist.add(bootstrap.MakeTarball())
+	tasklist.add(bootstrap.Bootstrap(),
+	             filesystem.MountSpecials(),
+	             locale.GenerateLocale(),
+	             locale.SetTimezone(),
+	             apt.DisableDaemonAutostart(),
+	             apt.AptSources(),
+		     #No network for the moment, skip
+	             #apt.AptUpgrade(),
+	             boot.ConfigureGrub(),
+	             filesystem.ModifyFstab(),
+	             boot.BlackListModules(),
+	             boot.DisableGetTTYs(),
+	             security.EnableShadowConfig(),
+		     security.SetRootPassword(),
+	             security.DisableSSHPasswordAuthentication(),
+	             security.DisableSSHDNSLookup(),
+	             network.RemoveDNSInfo(),
+	             network.ConfigureNetworkIF(),
+	             network.ConfigureDHCP(),
+	             initd.ResolveInitScripts(),
+	             initd.InstallInitScripts(),
+	             cleanup.ClearMOTD(),
+	             cleanup.ShredHostkeys(),
+	             cleanup.CleanTMP(),
+	             apt.PurgeUnusedPackages(),
+	             apt.AptClean(),
+	             apt.EnableDaemonAutostart(),
+	             filesystem.UnmountSpecials(),
+	             filesystem.UnmountVolume(),
+	             filesystem.DeleteMountDir())
+
+
+def rollback_tasks(tasklist, tasks_completed, manifest):
+	completed = [type(task) for task in tasks_completed]
+
+	def counter_task(task, counter):
+		if task in completed and counter not in completed:
+			tasklist.add(counter())
+
+	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
+	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
+	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)

providers/raw/assets/grub.d/40_custom

@@ -0,0 +1,93 @@
+#!/bin/sh
+
+# This file generates the old menu.lst configuration with grub2
+# It was copied from tomheadys github repo:
+# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
+
+prefix=/usr
+exec_prefix=${prefix}
+bindir=${exec_prefix}/bin
+libdir=${exec_prefix}/lib
+. ${libdir}/grub/grub-mkconfig_lib
+
+export TEXTDOMAIN=grub
+export TEXTDOMAINDIR=${prefix}/share/locale
+
+GRUB_DEVICE=/dev/sda1
+
+
+cat << EOF
+set default=${GRUB_DEFAULT}
+set timeout=${GRUB_TIMEOUT}
+insmod part_msdos
+insmod ext2
+insmod gettext
+set menu_color_normal=cyan/blue
+set menu_color_highlight=white/blue
+set root='(hd0,msdos1)'
+EOF
+
+if ${GRUB_HIDDEN_TIMEOUT:-false}; then
+	printf "hiddenmenu\n"
+fi
+
+linux_entry ()
+{
+	os="$1"
+	version="$2"
+	args="$4"
+
+	title="$(gettext_quoted "%s, with Linux %s")"
+
+	cat << EOF
+menuentry 'Debian GNU/Linux for OpenNebula, ${version}' --class debian --class gnu-linux --class os {
+        insmod part_msdos
+        insmod ext2
+	set timeout=${GRUB_TIMEOUT}
+	set root='(hd0,msdos1)'
+        echo 'Loading Linux ${version}'
+	linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
+        echo 'Loading initial ramdisk ...'
+	initrd ${rel_dirname}/${initrd}
+}
+EOF
+}
+
+list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
+		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
+		done`
+prepare_boot_cache=
+
+while [ "x$list" != "x" ] ; do
+	linux=`version_find_latest $list`
+	basename=`basename $linux`
+	dirname=`dirname $linux`
+	rel_dirname=`make_system_path_relative_to_its_root $dirname`
+	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
+	alt_version=`echo $version | sed -e "s,\.old$,,g"`
+	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
+
+	initrd=
+	for i in "initrd.img-${version}" "initrd-${version}.img" \
+		"initrd-${version}" "initramfs-${version}.img" \
+		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
+		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
+	if test -e "${dirname}/${i}" ; then
+		initrd="$i"
+		break
+	fi
+	done
+
+	initramfs=
+	for i in "config-${version}" "config-${alt_version}"; do
+	if test -e "${dirname}/${i}" ; then
+		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"`
+		break
+	fi
+	done
+
+	linux_entry "${OS}" "${version}" \
+		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
+
+	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
+done

providers/raw/assets/grub.d/40_custom.orig

@@ -0,0 +1,81 @@
+#!/bin/sh
+
+# This file generates the old menu.lst configuration with grub2
+# It was copied from tomheadys github repo:
+# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
+
+prefix=/usr
+exec_prefix=${prefix}
+bindir=${exec_prefix}/bin
+libdir=${exec_prefix}/lib
+. ${libdir}/grub/grub-mkconfig_lib
+
+export TEXTDOMAIN=grub
+export TEXTDOMAINDIR=${prefix}/share/locale
+
+GRUB_DEVICE=/dev/sda1
+
+
+cat << EOF
+default ${GRUB_DEFAULT}
+timeout ${GRUB_TIMEOUT}
+EOF
+
+if ${GRUB_HIDDEN_TIMEOUT:-false}; then
+	printf "hiddenmenu\n"
+fi
+
+linux_entry ()
+{
+	os="$1"
+	version="$2"
+	args="$4"
+
+	title="$(gettext_quoted "%s, with Linux %s")"
+
+	cat << EOF
+title ${version}
+	root (hd0)
+	kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
+	initrd ${rel_dirname}/${initrd}
+EOF
+}
+
+list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
+		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
+		done`
+prepare_boot_cache=
+
+while [ "x$list" != "x" ] ; do
+	linux=`version_find_latest $list`
+	basename=`basename $linux`
+	dirname=`dirname $linux`
+	rel_dirname=`make_system_path_relative_to_its_root $dirname`
+	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
+	alt_version=`echo $version | sed -e "s,\.old$,,g"`
+	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
+
+	initrd=
+	for i in "initrd.img-${version}" "initrd-${version}.img" \
+		"initrd-${version}" "initramfs-${version}.img" \
+		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
+		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
+	if test -e "${dirname}/${i}" ; then
+		initrd="$i"
+		break
+	fi
+	done
+
+	initramfs=
+	for i in "config-${version}" "config-${alt_version}"; do
+	if test -e "${dirname}/${i}" ; then
+		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"`
+		break
+	fi
+	done
+
+	linux_entry "${OS}" "${version}" \
+		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
+
+	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
+done

providers/raw/assets/init.d/ec2-get-credentials

@@ -0,0 +1,45 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:       ec2-get-credentials
+# Required-Start: $network
+# Required-Stop:  
+# Should-Start:   
+# Should-Stop:    
+# Default-Start:  2 3 4 5
+# Default-Stop:   
+# Description:    Retrieve the ssh credentials and add to authorized_keys
+### END INIT INFO
+#
+# ec2-get-credentials - Retrieve the ssh credentials and add to authorized_keys
+#
+# Based on /usr/local/sbin/ec2-get-credentials from Amazon's ami-20b65349
+#
+
+prog=$(basename $0)
+logger="logger -t $prog"
+
+public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
+username='root'
+# A little bit of nastyness to get the homedir, when the username is a variable
+ssh_dir="`eval printf ~$username`/.ssh"
+authorized_keys="$ssh_dir/authorized_keys"
+
+# Try to get the ssh public key from instance data.
+public_key=`wget -qO - $public_key_url`
+if [ -n "$public_key" ]; then
+	if [ ! -f $authorized_keys ]; then
+		if [ ! -d $ssh_dir ]; then
+			mkdir -m 700 $ssh_dir
+			chown $username:$username $ssh_dir
+		fi
+		touch $authorized_keys
+		chown $username:$username $authorized_keys
+	fi
+	
+	if ! grep -s -q "$public_key" $authorized_keys; then
+		printf "\n%s" -- "$public_key" >> $authorized_keys
+		$logger "New ssh key added to $authorized_keys from $public_key_url"
+		chmod 600 $authorized_keys
+		chown $username:$username $authorized_keys
+	fi
+fi

providers/raw/assets/init.d/ec2-run-user-data

@@ -0,0 +1,46 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:       ec2-run-user-data
+# Required-Start: ec2-get-credentials
+# Required-Stop:  
+# Should-Start:   
+# Should-Stop:    
+# Default-Start:  2 3 4 5
+# Default-Stop:   
+# Description:    Run instance user-data if it looks like a script.
+### END INIT INFO
+#
+# Only retrieves and runs the user-data script once per instance.  If
+# you want the user-data script to run again (e.g., on the next boot)
+# then readd this script with insserv:
+#   insserv -d ec2-run-user-data
+#
+prog=$(basename $0)
+logger="logger -t $prog"
+instance_data_url="http://169.254.169.254/2008-02-01"
+
+
+# Retrieve the instance user-data and run it if it looks like a script
+user_data_file=$(tempfile --prefix ec2 --suffix .user-data --mode 700)
+$logger "Retrieving user-data"
+wget -qO $user_data_file $instance_data_url/user-data 2>&1 | $logger
+
+if [ $(file -b --mime-type $user_data_file) = 'application/x-gzip' ]; then
+	$logger "Uncompressing gzip'd user-data"
+	mv $user_data_file $user_data_file.gz
+	gunzip $user_data_file.gz
+fi
+
+if [ ! -s $user_data_file ]; then
+	$logger "No user-data available"
+elif head -1 $user_data_file | egrep -v '^#!'; then
+	$logger "Skipping user-data as it does not begin with #!"
+else
+	$logger "Running user-data"
+	$user_data_file 2>&1 | logger -t "user-data"
+	$logger "user-data exit code: $?"
+fi
+rm -f $user_data_file
+
+# Disable this script, it may only run once
+insserv -r $0

providers/raw/assets/init.d/expand-volume

@@ -0,0 +1,26 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:       expand-volume
+# Required-Start: 
+# Required-Stop:  
+# Should-Start:   
+# Should-Stop:    
+# Default-Start:  2 3 4 5
+# Default-Stop:   
+# Description:    Expand the filesystem of the mounted root volume to its maximum possible size
+### END INIT INFO
+
+prog=$(basename $0)
+logger="logger -t $prog"
+
+device_path="/dev/xvda1"
+
+filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'`
+
+case $filesystem in
+	xfs)  xfs_growfs / ;;
+	ext2) resize2fs $device_path ;;
+	ext3) resize2fs $device_path ;;
+	ext4) resize2fs $device_path ;;
+	*)    $logger "The filesystem $filesystem was not recognized. Unable to expand size." ;;
+esac

providers/raw/assets/init.d/generate-ssh-hostkeys

@@ -0,0 +1,36 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:       generate-ssh-hostkeys
+# Required-Start: $local_fs
+# Required-Stop:  
+# Should-Start:   
+# Should-Stop:    
+# Default-Start:  S
+# Default-Stop:   
+# Description:    Generate ssh host keys if they do not exist
+### END INIT INFO
+
+prog=$(basename $0)
+logger="logger -t $prog"
+
+rsa_key="/etc/ssh/ssh_host_rsa_key"
+dsa_key="/etc/ssh/ssh_host_dsa_key"
+ecdsa_key="/etc/ssh/ssh_host_ecdsa_key"
+
+# Exit if the hostkeys already exist
+if [ -f $rsa_key -a -f $dsa_key -a -f $ecdsa_key ]; then
+	exit
+fi
+
+# Generate the ssh host keys
+[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N ''
+[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N ''
+[ -f $ecdsa_key ] || ssh-keygen -f $ecdsa_key -t ecdsa -C 'host' -N ''
+
+# Output the public keys to the console
+# This allows user to get host keys securely through console log
+echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger
+ssh-keygen -l -f $rsa_key.pub | $logger
+ssh-keygen -l -f $dsa_key.pub | $logger
+ssh-keygen -l -f $ecdsa_key.pub | $logger
+echo "------END SSH HOST KEY FINGERPRINTS------" | $logger

providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys

@@ -0,0 +1,33 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:       generate-ssh-hostkeys
+# Required-Start: $local_fs
+# Required-Stop:  
+# Should-Start:   
+# Should-Stop:    
+# Default-Start:  S
+# Default-Stop:   
+# Description:    Generate ssh host keys if they do not exist
+### END INIT INFO
+
+prog=$(basename $0)
+logger="logger -t $prog"
+
+rsa_key="/etc/ssh/ssh_host_rsa_key"
+dsa_key="/etc/ssh/ssh_host_dsa_key"
+
+# Exit if the hostkeys already exist
+if [ -f $rsa_key -a -f $dsa_key ]; then
+	exit
+fi
+
+# Generate the ssh host keys
+[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N ''
+[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N ''
+
+# Output the public keys to the console
+# This allows user to get host keys securely through console log
+echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger
+ssh-keygen -l -f $rsa_key.pub | $logger
+ssh-keygen -l -f $dsa_key.pub | $logger
+echo "------END SSH HOST KEY FINGERPRINTS------" | $logger

providers/raw/manifest-schema.json

@@ -0,0 +1,22 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "OpenNebula manifest",
+	"type": "object",
+	"properties": {
+		"volume": {
+			"type": "object",
+			"properties": {
+				"backing": {
+					"type": "string",
+					"enum": ["raw", "qcow2"]
+				},
+				"filesystem": {
+					"type": "string",
+					"enum": ["ext2", "ext3", "ext4", "xfs"]
+				}
+			},
+			"required": ["backing", "filesystem"]
+		}
+	},
+	"required": ["volume"]
+}

providers/raw/manifest.py

@@ -0,0 +1,15 @@
+import base
+
+
+class Manifest(base.Manifest):
+	def validate(self, data):
+		super(Manifest, self).validate(data)
+		from os import path
+		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
+		self.schema_validate(data, schema_path)
+
+	def parse(self, data):
+		super(Manifest, self).parse(data)
+		self.credentials    = data['credentials']
+		self.virtualization = data['virtualization']
+		self.image          = data['image']

providers/raw/tasks/__init__.py

@@ -0,0 +1 @@
+__all__ = ['packages', 'connection', 'host', 'ec2']

providers/raw/tasks/apt.py

@@ -0,0 +1,81 @@
+from base import Task
+from common import phases
+from common.tools import log_check_call
+import os
+from locale import GenerateLocale
+
+
+class AptSources(Task):
+	description = 'Adding aptitude sources'
+	phase = phases.system_modification
+
+	def run(self, info):
+		mirror = 'http://http.debian.net/debian'
+                if info.manifest.system['mirror']:
+			mirror = info.manifest.system['mirror']
+		sources_path = os.path.join(info.root, 'etc/apt/sources.list')
+		with open(sources_path, 'w') as apt_sources:
+			apt_sources.write(('deb     {apt_mirror} {release} main\n'
+			                   'deb-src {apt_mirror} {release} main\n'
+			                   .format(apt_mirror=mirror,
+			                           release=info.manifest.system['release'])))
+			apt_sources.write(('deb     {apt_mirror} {release}/updates main\n'
+			                   'deb-src {apt_mirror} {release}/updates main\n'
+			                   .format(apt_mirror='http://security.debian.org/',
+			                           release=info.manifest.system['release'])))
+
+
+class DisableDaemonAutostart(Task):
+	description = 'Disabling daemon autostart'
+	phase = phases.system_modification
+
+	def run(self, info):
+		rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d')
+		with open(rc_policy_path, 'w') as rc_policy:
+			rc_policy.write(('#!/bin/sh\n'
+			                 'exit 101'))
+		import stat
+		os.chmod(rc_policy_path,
+		         stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+		         stat.S_IRGRP                | stat.S_IXGRP |
+		         stat.S_IROTH                | stat.S_IXOTH)
+
+
+class AptUpgrade(Task):
+	description = 'Upgrading packages and fixing broken dependencies'
+	phase = phases.system_modification
+	after = [GenerateLocale, AptSources, DisableDaemonAutostart]
+
+	def run(self, info):
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-f', '-y', 'install'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-y', 'upgrade'])
+
+
+class PurgeUnusedPackages(Task):
+	description = 'Removing unused packages'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'autoremove', '--purge'])
+
+
+class AptClean(Task):
+	description = 'Clearing the aptitude cache'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'clean'])
+
+		lists = os.path.join(info.root, 'var/lib/apt/lists')
+		for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]:
+			if os.path.isfile(list_file):
+				os.remove(list_file)
+
+
+class EnableDaemonAutostart(Task):
+	description = 'Re-enabling daemon autostart after installation'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		os.remove(os.path.join(info.root, 'usr/sbin/policy-rc.d'))

providers/raw/tasks/boot.py

@@ -0,0 +1,81 @@
+from base import Task
+from common import phases
+import os
+
+
+class ConfigureGrub(Task):
+	description = 'Configuring grub'
+	phase = phases.system_modification
+
+	def run(self, info):
+		import stat
+		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+		             stat.S_IRGRP                | stat.S_IXGRP |
+		             stat.S_IROTH                | stat.S_IXOTH)
+		x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+
+		grubd = os.path.join(info.root, 'etc/grub.d')
+		for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]:
+			os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all)
+
+		from common.tools import log_check_call
+		from shutil import copy
+
+		script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom'))
+		script_dst = os.path.join(info.root, 'etc/grub.d/40_custom')
+		copy(script_src, script_dst)
+		os.chmod(script_dst, rwxr_xr_x)
+
+                if info.manifest.virtualization == 'virtio':
+                        modules_path = os.path.join(info.root,
+                                        'etc/initramfs-tools/modules')
+                        with open(modules_path, 'a') as modules:
+                                modules.write("\nvirtio_pci\nvirtio_blk\n")
+
+
+		grub_def = os.path.join(info.root, 'etc/default/grub')
+
+		log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst'])
+
+		log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u'])
+		log_check_call(['grub-install', '--boot-directory='+info.root+"/boot/", '/dev/loop0'])
+
+                log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+
+                from common.tools import sed_i
+                if info.manifest.virtualization == 'virtio':
+			grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg')
+                        sed_i(grub_cfg, 'sda', 'vda')
+                        device_map = os.path.join(info.root,
+                                        'boot/grub/device.map')
+                        sed_i(device_map, 'sda', 'vda')
+			#log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+
+
+
+class BlackListModules(Task):
+	description = 'Blacklisting kernel modules'
+	phase = phases.system_modification
+
+	def run(self, info):
+		blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf')
+		with open(blacklist_path, 'a') as blacklist:
+			blacklist.write(('# disable pc speaker\n'
+			                 'blacklist pcspkr'))
+
+
+class DisableGetTTYs(Task):
+	description = 'Disabling getty processes'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import sed_i
+		inittab_path = os.path.join(info.root, 'etc/inittab')
+		tty1 = '1:2345:respawn:/sbin/getty 38400 tty1'
+		sed_i(inittab_path, '^'+tty1, '#'+tty1)
+		ttyx = ':23:respawn:/sbin/getty 38400 tty'
+		for i in range(2, 6):
+			i = str(i)
+			sed_i(inittab_path, '^'+i+ttyx+i, '#'+i+ttyx+i)

providers/raw/tasks/bootstrap.py

@@ -0,0 +1,58 @@
+from base import Task
+from common import phases
+from common.exceptions import TaskError
+import logging
+log = logging.getLogger(__name__)
+
+
+def get_bootstrap_args(info):
+        mirror = 'http://http.debian.net/debian'
+        if info.manifest.system['mirror']:
+	        mirror = info.manifest.system['mirror']
+
+	executable = ['/usr/sbin/debootstrap']
+	options = ['--arch=' + info.manifest.system['architecture']]
+	include, exclude = info.img_packages
+	if len(include) > 0:
+		options.append('--include=' + ','.join(include))
+	if len(exclude) > 0:
+		options.append('--exclude=' + ','.join(exclude))
+	arguments = [info.manifest.system['release'], info.root, mirror]
+	return executable, options, arguments
+
+
+class MakeTarball(Task):
+	description = 'Creating bootstrap tarball'
+	phase = phases.os_installation
+
+	def run(self, info):
+		from hashlib import sha1
+		import os.path
+		executable, options, arguments = get_bootstrap_args(info)
+		# Filter info.root which points at /target/volume-id, we won't ever hit anything with that in there.
+		hash_args = [arg for arg in arguments if arg != info.root]
+		tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8]
+		tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id)
+		info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename)
+		if os.path.isfile(info.tarball):
+			log.debug('Found matching tarball, skipping download')
+		else:
+			from common.tools import log_call
+			status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments)
+			if status != 1:
+				msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status)
+				raise TaskError(msg)
+
+
+class Bootstrap(Task):
+	description = 'Installing Debian'
+	phase = phases.os_installation
+	after = [MakeTarball]
+
+	def run(self, info):
+		executable, options, arguments = get_bootstrap_args(info)
+		if hasattr(info, 'tarball'):
+			options.extend(['--unpack-tarball=' + info.tarball])
+
+		from common.tools import log_check_call
+		log_check_call(executable + options + arguments)

providers/raw/tasks/cleanup.py

@@ -0,0 +1,43 @@
+from base import Task
+from common import phases
+import os
+
+
+class ClearMOTD(Task):
+	description = 'Clearing the MOTD'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		with open('/var/run/motd', 'w'):
+			pass
+
+
+class ShredHostkeys(Task):
+	description = 'Securely deleting ssh hostkeys'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		ssh_hostkeys = ['ssh_host_dsa_key',
+		                'ssh_host_rsa_key']
+		if info.manifest.system['release'] != 'squeeze':
+			ssh_hostkeys.append('ssh_host_ecdsa_key')
+
+		private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
+		public = [path + '.pub' for path in private]
+
+		from common.tools import log_check_call
+		log_check_call(['/usr/bin/shred', '--remove'] + private + public)
+
+
+class CleanTMP(Task):
+	description = 'Removing temporary files'
+	phase = phases.system_cleaning
+
+	def run(self, info):
+		tmp = os.path.join(info.root, 'tmp')
+		for tmp_file in [os.path.join(tmp, f) for f in os.listdir(tmp)]:
+			os.remove(tmp_file)
+
+		log = os.path.join(info.root, 'var/log/')
+		os.remove(os.path.join(log, 'bootstrap.log'))
+		os.remove(os.path.join(log, 'dpkg.log'))

providers/raw/tasks/fake.py

@@ -0,0 +1,13 @@
+from base import Task
+from common import phases
+import os.path
+
+class Fake(Task):
+	description = 'create fake file'
+	phase = phases.system_modification
+
+	def run(self, info):
+		fake_path = os.path.join(info.root, 'fake.txt')
+		with open(fake_path, 'a') as fakefile:
+			fakefile.write("fake file")
+

providers/raw/tasks/filesystem.py

@@ -0,0 +1,147 @@
+from base import Task
+from common import phases
+from common.exceptions import TaskError
+from common.tools import log_check_call
+from bootstrap import Bootstrap
+import os
+
+class FormatVolume(Task):
+	description = 'Formatting the volume'
+	phase = phases.volume_preparation
+
+	def run(self, info):
+                mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M']
+                log_check_call(mkmount)
+
+                loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']]
+                log_check_call(loopcmd)
+
+		# parted
+		log_check_call(['parted','-a', 'optimal', '-s', '/dev/loop0', "mklabel", "msdos"])
+                log_check_call(['parted', '-a', 'optimal', '-s', '/dev/loop0', "--", "mkpart", "primary", "ext4", "32k", "-1"])
+		log_check_call(['parted','-s', '/dev/loop0', "--", "set", "1", "boot", "on"])
+
+
+                #log_check_call(['kpartx','-a','-v', info.manifest.bootstrapper['image_file']])
+		log_check_call(['kpartx','-a', '-v', '/dev/loop0'])
+                mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/mapper/loop0p1']
+                log_check_call(mkfs)
+
+
+
+
+class TuneVolumeFS(Task):
+	description = 'Tuning the bootstrap volume filesystem'
+	phase = phases.volume_preparation
+	after = [FormatVolume]
+
+	def run(self, info):
+		#dev_path = info.bootstrap_device['path']
+                #dev_path = info.manifest.bootstrapper['image_file']
+		dev_path = '/dev/mapper/loop0p1'
+		# Disable the time based filesystem check
+		log_check_call(['/sbin/tune2fs', '-i', '0', dev_path])
+
+
+class AddXFSProgs(Task):
+	description = 'Adding `xfsprogs\' to the image packages'
+	phase = phases.preparation
+
+	def run(self, info):
+		include, exclude = info.img_packages
+		include.add('xfsprogs')
+
+
+class CreateMountDir(Task):
+	description = 'Creating mountpoint for the bootstrap volume'
+	phase = phases.volume_mounting
+
+	def run(self, info):
+		import os
+		mount_dir = info.manifest.bootstrapper['mount_dir']
+		info.root = mount_dir
+		# Works recursively, fails if last part exists, which is exaclty what we want.
+		os.makedirs(mount_dir)
+
+
+class MountVolume(Task):
+	description = 'Mounting the bootstrap volume'
+	phase = phases.volume_mounting
+	after = [CreateMountDir]
+
+	def run(self, info):
+		with open('/proc/mounts') as mounts:
+			for mount in mounts:
+				if info.root in mount:
+					msg = 'Something is already mounted at {root}'.format(root=info.root)
+					raise TaskError(msg)
+
+		log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/mapper/loop0p1', info.root])
+
+class MountSpecials(Task):
+	description = 'Mounting special block devices'
+	phase = phases.os_installation
+	after = [Bootstrap]
+
+	def run(self, info):
+		log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)])
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'proc', 'none', '/proc'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'sysfs', 'none', '/sys'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'devpts', 'none', '/dev/pts'])
+
+
+class UnmountSpecials(Task):
+	description = 'Unmunting special block devices'
+	phase = phases.volume_unmounting
+
+	def run(self, info):
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys'])
+		log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc'])
+		log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)])
+
+
+class UnmountVolume(Task):
+	description = 'Unmounting the bootstrap volume'
+	phase = phases.volume_unmounting
+	after = [UnmountSpecials]
+
+	def run(self, info):
+		log_check_call(['/bin/umount', info.root])
+		#log_check_call(['partx','-d','/dev/loop0'])
+                #log_check_call(['/sbin/losetup', '-d', '/dev/loop0'])
+		log_check_call(['kpartx','-d', info.manifest.bootstrapper['image_file']])
+
+
+
+class DeleteMountDir(Task):
+	description = 'Deleting mountpoint for the bootstrap volume'
+	phase = phases.volume_unmounting
+	after = [UnmountVolume]
+
+	def run(self, info):
+		import os
+		os.rmdir(info.root)
+		del info.root
+
+
+class ModifyFstab(Task):
+	description = 'Adding root volume to the fstab'
+	phase = phases.system_modification
+
+	def run(self, info):
+		import os.path
+		mount_opts = ['defaults']
+		if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
+			mount_opts.append('barrier=0')
+		if info.manifest.volume['filesystem'].lower() == 'xfs':
+			mount_opts.append('nobarrier')
+		fstab_path = os.path.join(info.root, 'etc/fstab')
+		with open(fstab_path, 'a') as fstab:
+			device = '/dev/sda1'
+	                if info.manifest.virtualization == 'virtio':
+				device = '/dev/vda1'
+
+			fstab.write((device+' /     {filesystem}    {mount_opts} 1 1\n'
+			             .format(filesystem=info.manifest.volume['filesystem'].lower(),
+			                     mount_opts=','.join(mount_opts))))

providers/raw/tasks/host.py

@@ -0,0 +1,29 @@
+from base import Task
+from common import phases
+from common.exceptions import TaskError
+import packages
+
+
+class CheckPackages(Task):
+	description = 'Checking installed host packages'
+	phase = phases.preparation
+	after = [packages.HostPackages, packages.ImagePackages]
+
+	def run(self, info):
+		from common.tools import log_check_call
+		from subprocess import CalledProcessError
+		for package in info.host_packages:
+			try:
+				log_check_call(['/usr/bin/dpkg', '-s', package])
+			except CalledProcessError:
+				msg = "The package ``{0}\'\' is not installed".format(package)
+				raise TaskError(msg)
+
+
+class GetInfo(Task):
+	description = 'Retrieving instance metadata'
+	phase = phases.preparation
+
+	def run(self, info):
+		info.host = {}
+		return info

providers/raw/tasks/initd.py

@@ -0,0 +1,49 @@
+from base import Task
+from common import phases
+import os.path
+
+
+class ResolveInitScripts(Task):
+	description = 'Determining which startup scripts to install or disable'
+	phase = phases.system_modification
+
+	def run(self, info):
+		init_scripts = {'ec2-get-credentials': 'ec2-get-credentials',
+		                'ec2-run-user-data': 'ec2-run-user-data',
+		                'expand-volume': 'expand-volume'}
+
+		init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys'
+		if info.manifest.system['release'] == 'squeeze':
+			init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys'
+
+		disable_scripts = ['hwclock.sh']
+		if info.manifest.system['release'] == 'squeeze':
+			disable_scripts.append('hwclockfirst.sh')
+
+		for name, path in init_scripts.iteritems():
+			init_scripts[name] = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d', path))
+
+		info.initd = {'install': init_scripts,
+		              'disable': disable_scripts}
+
+
+class InstallInitScripts(Task):
+	description = 'Installing startup scripts'
+	phase = phases.system_modification
+	after = [ResolveInitScripts]
+
+	def run(self, info):
+		import stat
+		rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+		             stat.S_IRGRP                | stat.S_IXGRP |
+		             stat.S_IROTH                | stat.S_IXOTH)
+		from shutil import copy
+		from common.tools import log_check_call
+		for name, src in info.initd['install'].iteritems():
+			dst = os.path.join(info.root, 'etc/init.d', name)
+			copy(src, dst)
+			os.chmod(dst, rwxr_xr_x)
+			log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-d', name])
+
+		for name in info.initd['disable']:
+			log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-r', name])

providers/raw/tasks/locale.py

@@ -0,0 +1,35 @@
+from base import Task
+from common import phases
+import os.path
+
+
+class GenerateLocale(Task):
+	description = 'Generating the selected locale'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import sed_i
+		from common.tools import log_check_call
+		locale_gen = os.path.join(info.root, 'etc/locale.gen')
+		locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'],
+		                                                   charmap=info.manifest.system['charmap'])
+		search = '# ' + locale_str
+		sed_i(locale_gen, search, locale_str)
+
+		command = ['/usr/sbin/chroot', info.root, '/usr/sbin/locale-gen']
+		log_check_call(command)
+
+
+class SetTimezone(Task):
+	description = 'Setting the selected timezone'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from shutil import copy
+		tz_path = os.path.join(info.root, 'etc/timezone')
+		timezone = info.manifest.system['timezone']
+		with open(tz_path, 'w') as tz_file:
+			tz_file.write(timezone)
+		zoneinfo_path = os.path.join(info.root, '/usr/share/zoneinfo', timezone)
+		localtime_path = os.path.join(info.root, 'etc/localtime')
+		copy(zoneinfo_path, localtime_path)

providers/raw/tasks/network.py

@@ -0,0 +1,38 @@
+from base import Task
+from common import phases
+import os.path
+
+
+class RemoveDNSInfo(Task):
+	description = 'Removing resolv.conf'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from os import remove
+		remove(os.path.join(info.root, 'etc/resolv.conf'))
+
+
+class ConfigureNetworkIF(Task):
+	description = 'Configuring network interfaces'
+	phase = phases.system_modification
+
+	def run(self, info):
+		interfaces_path = os.path.join(info.root, 'etc/network/interfaces')
+		if_config = {'squeeze': ('auto lo\n'
+		                         'iface lo inet loopback\n'
+		                         'auto eth0\n'
+		                         'iface eth0 inet dhcp\n'),
+		             'wheezy':  'auto eth0\n'
+		                        'iface eth0 inet dhcp\n'}
+		with open(interfaces_path, 'a') as interfaces:
+			interfaces.write(if_config.get(info.manifest.system['release']))
+
+
+class ConfigureDHCP(Task):
+	description = 'Configuring the DHCP client'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import sed_i
+		dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
+		sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')

providers/raw/tasks/packages.py

@@ -0,0 +1,49 @@
+from base import Task
+from common import phases
+
+
+class HostPackages(Task):
+	description = 'Determining required host packages'
+	phase = phases.preparation
+
+	def run(self, info):
+		packages = set(['debootstrap', 'qemu-utils', 'parted', 'grub2', 'sysv-rc'])
+		if info.manifest.volume['filesystem'] == 'xfs':
+			packages.add('xfsprogs')
+
+		info.host_packages = packages
+
+
+class ImagePackages(Task):
+	description = 'Determining required image packages'
+	phase = phases.preparation
+
+	def run(self, info):
+		manifest = info.manifest
+		# Add some basic packages we are going to need
+		include = set(['udev',
+			       'parted',
+		               'openssh-server',
+		               # We could bootstrap without locales, but things just suck without them, error messages etc.
+		               'locales',
+		               # Needed for the init scripts
+		               'file',
+		               # isc-dhcp-client doesn't work properly with ec2
+		               'dhcpcd',
+			       'grub2',
+			       'chkconfig',
+			       'openssh-client'
+		               ])
+
+		exclude = set(['isc-dhcp-client',
+		               'isc-dhcp-common',
+		               ])
+
+		# In squeeze, we need a special kernel flavor for xen
+		kernels = {'squeeze': {'amd64': 'linux-image-amd64',
+		                       'i386':  'linux-image-686', },
+		           'wheezy':  {'amd64': 'linux-image-amd64',
+		                       'i386':  'linux-image-686', }, }
+		include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))
+
+		info.img_packages = include, exclude

providers/raw/tasks/security.py

@@ -0,0 +1,42 @@
+from base import Task
+from common import phases
+import os.path
+
+
+class EnableShadowConfig(Task):
+	description = 'Enabling shadowconfig'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import log_check_call
+		log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on'])
+
+class SetRootPassword(Task):
+	description = 'Set password for root'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import log_check_call
+		if info.manifest.credentials['root']:
+			log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root'])
+
+class DisableSSHPasswordAuthentication(Task):
+	description = 'Disabling SSH password authentication'
+	phase = phases.system_modification
+
+	def run(self, info):
+		from common.tools import sed_i
+		if 'root' not in info.manifest.credentials:
+			# If no password set for root
+			sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
+			sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no')
+
+
+class DisableSSHDNSLookup(Task):
+	description = 'Disabling sshd remote host name lookup'
+	phase = phases.system_modification
+
+	def run(self, info):
+		sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
+		with open(sshd_config_path, 'a') as sshd_config:
+			sshd_config.write('UseDNS no')