add kvm provider with virtio support

manifests/kvm-virtio.manifest.json

@@ -0,0 +1,40 @@
+{
+	"provider"      : "kvm",
+	"virtualization": "virtio",
+
+	"bootstrapper": {
+		"mount_dir": "/mnt/target",
+		"mirror"      : "http://ftp.fr.debian.org/debian/"
+	},
+	"image": {
+		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} ({virtualization})"
+	},
+	"system": {
+		"release"     : "wheezy",
+		"architecture": "amd64",
+		"timezone"    : "UTC",
+		"locale"      : "en_US",
+		"charmap"     : "UTF-8"
+	},
+	"volume": {
+		"backing"   : "raw",
+		"filesystem": "ext4",
+		"size"      : 1024,
+		"loopback_dir" : "/tmp"
+	},
+	"plugins": {
+		"user_packages": {
+			"enabled": true,
+			"repo": [ "apache2" ],
+			"local": []
+		},
+		"root_password": {
+			"enabled": true,
+			"password": "test"
+                },
+		"opennebula": {
+			"enabled": true
+		}
+	}
+}

providers/kvm/README.md

@@ -0,0 +1,9 @@
+this provider creates images for KVM.
+
+It is possible to add opennebula plugin for OpenNebula images.
+
+Disk virtuzalition is specified by the virtualization field in the manifest.
+Allowed values are:
+
+* ide: basic disk emulation on /dev/sda1
+* virtio: enhanced performance on /dev/vda1. It adds virtio modules in the kernel and needs configuration update in VM template: <target dev='vda' bus='virtio'/>

providers/kvm/__init__.py

@@ -0,0 +1,92 @@
+from manifest import Manifest
+from tasks import packages
+from common.tasks import packages as common_packages
+from common.tasks import host
+from common.tasks import loopback
+from common.tasks import parted
+from common.tasks import filesystem
+from common.tasks import bootstrap
+from common.tasks import locale
+from common.tasks import apt
+from tasks import boot
+from common.tasks import boot as common_boot
+from common.tasks import security
+from common.tasks import network
+from common.tasks import initd
+from common.tasks import cleanup
+from common.tasks import loopback
+
+
+def initialize():
+	pass
+
+
+def tasks(tasklist, manifest):
+	tasklist.add(packages.HostPackages(),
+	             common_packages.HostPackages(),
+	             packages.ImagePackages(),
+	             common_packages.ImagePackages(),
+	             host.CheckPackages(),
+
+	             loopback.CreateQemuImg(),
+	             loopback.Attach(),
+	             parted.PartitionVolume(),
+	             parted.MapPartitions(),
+	             parted.FormatPartitions(),
+	             filesystem.CreateMountDir(),
+	             filesystem.MountVolume(),
+
+	             bootstrap.Bootstrap(),
+	             filesystem.MountSpecials(),
+	             locale.GenerateLocale(),
+	             locale.SetTimezone(),
+	             apt.DisableDaemonAutostart(),
+	             apt.AptSources(),
+	             apt.AptUpgrade(),
+	             boot.ConfigureGrub(),
+	             filesystem.ModifyFstab(),
+	             common_boot.BlackListModules(),
+	             common_boot.DisableGetTTYs(),
+	             security.EnableShadowConfig(),
+	             security.DisableSSHPasswordAuthentication(),
+	             security.DisableSSHDNSLookup(),
+	             network.RemoveDNSInfo(),
+	             network.ConfigureNetworkIF(),
+	             network.ConfigureDHCP(),
+	             initd.ResolveInitScripts(),
+	             initd.InstallInitScripts(),
+	             cleanup.ClearMOTD(),
+	             cleanup.ShredHostkeys(),
+	             cleanup.CleanTMP(),
+	             apt.PurgeUnusedPackages(),
+	             apt.AptClean(),
+	             apt.EnableDaemonAutostart(),
+	             filesystem.UnmountSpecials(),
+
+	             filesystem.UnmountVolume(),
+	             parted.UnmapPartitions(),
+	             loopback.Detach(),
+	             filesystem.DeleteMountDir())
+
+	if manifest.bootstrapper['tarball']:
+		tasklist.add(bootstrap.MakeTarball())
+
+	filesystem_specific_tasks = {'xfs': [filesystem.AddXFSProgs()],
+	                             'ext2': [filesystem.TuneVolumeFS()],
+	                             'ext3': [filesystem.TuneVolumeFS()],
+	                             'ext4': [filesystem.TuneVolumeFS()]}
+	tasklist.add(*filesystem_specific_tasks.get(manifest.volume['filesystem'].lower()))
+
+
+def rollback_tasks(tasklist, tasks_completed, manifest):
+	completed = [type(task) for task in tasks_completed]
+
+	def counter_task(task, counter):
+		if task in completed and counter not in completed:
+			tasklist.add(counter())
+
+	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
+	counter_task(parted.MapPartitions, parted.UnmapPartitions)
+	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
+	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
+	counter_task(loopback.Attach, loopback.Detach)

providers/kvm/assets/grub.d/00_header

@@ -0,0 +1,4 @@
+#! /bin/sh
+set -e
+
+# nothing to do, skip grub mkconfig for this

providers/kvm/assets/grub.d/10_linux

@@ -0,0 +1,93 @@
+#!/bin/sh
+
+# This file generates the old menu.lst configuration with grub2
+# It was copied from tomheadys github repo:
+# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
+
+prefix=/usr
+exec_prefix=${prefix}
+bindir=${exec_prefix}/bin
+libdir=${exec_prefix}/lib
+. ${libdir}/grub/grub-mkconfig_lib
+
+export TEXTDOMAIN=grub
+export TEXTDOMAINDIR=${prefix}/share/locale
+
+GRUB_DEVICE=/dev/sda1
+
+
+cat << EOF
+set default=${GRUB_DEFAULT}
+set timeout=${GRUB_TIMEOUT}
+insmod part_msdos
+insmod ext2
+insmod gettext
+set menu_color_normal=cyan/blue
+set menu_color_highlight=white/blue
+set root='(hd0,msdos1)'
+EOF
+
+if ${GRUB_HIDDEN_TIMEOUT:-false}; then
+	printf "hiddenmenu\n"
+fi
+
+linux_entry ()
+{
+	os="$1"
+	version="$2"
+	args="$4"
+
+	title="$(gettext_quoted "%s, with Linux %s")"
+
+	cat << EOF
+menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os {
+        insmod part_msdos
+        insmod ext2
+	set timeout=${GRUB_TIMEOUT}
+	set root='(hd0,msdos1)'
+        echo 'Loading Linux ${version}'
+	linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
+        echo 'Loading initial ramdisk ...'
+	initrd ${rel_dirname}/${initrd}
+}
+EOF
+}
+
+list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
+		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
+		done`
+prepare_boot_cache=
+
+while [ "x$list" != "x" ] ; do
+	linux=`version_find_latest $list`
+	basename=`basename $linux`
+	dirname=`dirname $linux`
+	rel_dirname=`make_system_path_relative_to_its_root $dirname`
+	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
+	alt_version=`echo $version | sed -e "s,\.old$,,g"`
+	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
+
+	initrd=
+	for i in "initrd.img-${version}" "initrd-${version}.img" \
+		"initrd-${version}" "initramfs-${version}.img" \
+		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
+		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
+	if test -e "${dirname}/${i}" ; then
+		initrd="$i"
+		break
+	fi
+	done
+
+	initramfs=
+	for i in "config-${version}" "config-${alt_version}"; do
+	if test -e "${dirname}/${i}" ; then
+		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut   -f2 -d= | tr -d \"`
+		break
+	fi
+	done
+
+	linux_entry "${OS}" "${version}" \
+		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
+
+	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
+done

providers/kvm/manifest-schema.json

@@ -0,0 +1,22 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Kvm manifest",
+	"type": "object",
+	"properties": {
+		"volume": {
+			"type": "object",
+			"properties": {
+				"backing": {
+					"type": "string",
+					"enum": ["raw", "qcow2"]
+				},
+				"filesystem": {
+					"type": "string",
+					"enum": ["ext2", "ext3", "ext4", "xfs"]
+				}
+			},
+			"required": ["backing", "filesystem"]
+		}
+	},
+	"required": ["volume"]
+}

providers/kvm/manifest.py

@@ -0,0 +1,16 @@
+import base
+
+
+class Manifest(base.Manifest):
+	def validate(self, data):
+		super(Manifest, self).validate(data)
+		from os import path
+		schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
+		self.schema_validate(data, schema_path)
+
+	def parse(self, data):
+		super(Manifest, self).parse(data)
+		self.image          = data['image']
+		self.virtualization = data['virtualization']
+		if 'loopback_dir' not in self.volume:
+			self.volume['loopback_dir'] = '/tmp'

providers/kvm/tasks/boot.py

@@ -0,0 +1,55 @@
+from base import Task
+from common import phases
+
+
+class ConfigureGrub(Task):
+	description = 'Configuring grub for KVM'
+	phase = phases.system_modification
+
+	def run(self, info):
+		import stat
+	        rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                     stat.S_IRGRP                | stat.S_IXGRP |
+                     stat.S_IROTH                | stat.S_IXOTH)
+        	x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+		import os.path
+		device_map_path = os.path.join(info.root, 'boot/grub/device.map')
+		with open(device_map_path, 'w') as device_map:
+			device_map.write('(hd0)   /dev/sda\n')
+
+		from common.tools import log_check_call
+
+		from shutil import copy
+        	script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux'))
+        	script_dst = os.path.join(info.root, 'etc/grub.d/10_linux')
+        	copy(script_src, script_dst)
+        	os.chmod(script_dst, rwxr_xr_x)
+                script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header'))
+                script_dst = os.path.join(info.root, 'etc/grub.d/00_header')
+                copy(script_src, script_dst)
+                os.chmod(script_dst, rwxr_xr_x)
+
+                if info.manifest.virtualization == 'virtio':
+			print "Using virtio"
+                        modules_path = os.path.join(info.root,
+                                        'etc/initramfs-tools/modules')
+                        with open(modules_path, 'a') as modules:
+                                modules.write("\nvirtio_pci\nvirtio_blk\n")
+
+		log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u'])
+		# Install grub in mbr
+		log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']])
+
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
+
+                if info.manifest.virtualization == 'virtio':
+			from common.tools import sed_i
+                        grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg')
+                        sed_i(grub_cfg, 'sda', 'vda')
+                        device_map = os.path.join(info.root,
+                                        'boot/grub/device.map')
+                        sed_i(device_map, 'sda', 'vda')
+			fstab_file =  os.path.join(info.root,
+                                        'etc/fstab')
+			sed_i(fstab_file, 'sda', 'vda')
+

providers/kvm/tasks/packages.py

@@ -0,0 +1,48 @@
+from base import Task
+from common import phases
+from common.tasks import packages
+from common.tasks.host import CheckPackages
+
+
+class HostPackages(Task):
+	description = 'Determining required host packages'
+	phase = phases.preparation
+	before = [CheckPackages]
+	after = [packages.HostPackages]
+
+	def run(self, info):
+		info.host_packages.update(['qemu-utils', 'parted', 'grub2', 'sysv-rc'])
+		if info.manifest.volume['filesystem'] == 'xfs':
+			info.host_packages.add('xfsprogs')
+
+
+class ImagePackages(Task):
+	description = 'Determining required image packages'
+	phase = phases.preparation
+	after = [packages.ImagePackages]
+
+	def run(self, info):
+		manifest = info.manifest
+		include, exclude = info.img_packages
+		# Add some basic packages we are going to need
+		include.update(['parted',
+		                'kpartx',
+		               # Needed for the init scripts
+		               'file',
+		               # isc-dhcp-client doesn't work properly with ec2
+		               'dhcpcd',
+		               'chkconfig',
+		               'openssh-client',
+			       'grub2'
+		                ])
+
+		exclude.update(['isc-dhcp-client',
+		               'isc-dhcp-common',
+		                ])
+
+		# In squeeze, we need a special kernel flavor for xen
+		kernels = {'squeeze': {'amd64': 'linux-image-amd64',
+		                       'i386':  'linux-image-686', },
+		           'wheezy':  {'amd64': 'linux-image-amd64',
+		                       'i386':  'linux-image-686', }, }
+		include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))