Merge pull request #86 from osallou/virtualbox

Fixes for virtualbox provider

base/manifest-schema.json

@@ -13,9 +13,10 @@
                                 "image_file": { "type": "string" },
 				"tarball": { "type": "boolean" },
 				"tarball_dir": { "type": "string" },
-				"device" : { "type" : "string" }
+				"device" : { "type" : "string" },
+				"mirror":   { "type": "string" }
 			},
-			"required": ["mount_dir"]
+			"required": ["mount_dir", "mirror"]
 		},
 		"system": {
 			"type": "object",
@@ -30,8 +31,7 @@
 				},
 				"timezone": { "type": "string" },
 				"locale":   { "type": "string" },
-				"charmap":  { "type": "string" },
-				"mirror":   { "type": "string" }
+				"charmap":  { "type": "string" }
 			},
 			"required": ["release", "architecture", "timezone", "locale", "charmap"]
 		},

common/phases.py

@@ -9,6 +9,7 @@ system_modification = Phase('System modification', 'Installing software, modifyi
 system_cleaning = Phase('System cleaning', 'Removing sensitive data, temporary files and other leftovers')
 volume_unmounting = Phase('Volume unmounting', 'Unmounting the bootstrap volume')
 image_registration = Phase('Image registration', 'Uploading/Registering with the provider')
+image_conversion = Phase('Image conversion', 'Conversion/Compression of the image result file')
 cleaning = Phase('Cleaning', 'Removing temporary files')
 
 order = [preparation,
@@ -20,5 +21,6 @@ order = [preparation,
          system_cleaning,
          volume_unmounting,
          image_registration,
+	 image_conversion,
          cleaning,
          ]

common/tasks/bootstrap.py

@@ -13,7 +13,8 @@ def get_bootstrap_args(info):
 		options.append('--include=' + ','.join(include))
 	if len(exclude) > 0:
 		options.append('--exclude=' + ','.join(exclude))
-	arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian']
+	mirror =  info.manifest.bootstrapper['mirror']
+	arguments = [info.manifest.system['release'], info.root, mirror]
 	return executable, options, arguments
 
 

common/tasks/parted.py

@@ -25,13 +25,9 @@ class MapPartitions(Task):
 	after = [PartitionVolume]
 
 	def run(self, info):
-		log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']])
-		root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper')+'p1'
-
-		[root_loopback_path] = log_check_call(['/sbin/losetup', '--find'])
-		log_check_call(['/sbin/losetup', root_loopback_path, root_partition_path])
-
-		info.bootstrap_device['partitions'] = {'root_path': root_loopback_path}
+		root_partition_path =  info.bootstrap_device['path'].replace('/dev', '/dev/mapper')+'p1'
+                log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']])
+		info.bootstrap_device['partitions'] = {'root_path': root_partition_path}
 
 
 class FormatPartitions(Task):
@@ -52,7 +48,5 @@ class UnmapPartitions(Task):
 	after = [filesystem.UnmountVolume]
 
 	def run(self, info):
-		log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['partitions']['root_path']])
-		del info.bootstrap_device['partitions']['root_path']
-
 		log_check_call(['kpartx', '-d', info.bootstrap_device['path']])
+		del info.bootstrap_device['partitions']['root_path']

manifests/virtualbox.json

@@ -0,0 +1,41 @@
+{
+	"provider"      : "virtualbox",
+	"virtualization": "ide",
+
+	"bootstrapper": {
+		"mount_dir": "/mnt/target",
+		"mirror"      : "http://ftp.fr.debian.org/debian/"
+	},
+	"image": {
+		"name"       : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
+		"description": "Debian {release} {architecture} ({virtualization})"
+	},
+	"system": {
+		"release"     : "wheezy",
+		"architecture": "amd64",
+		"timezone"    : "UTC",
+		"locale"      : "en_US",
+		"charmap"     : "UTF-8"
+	},
+	"volume": {
+		"backing"   : "raw",
+		"filesystem": "ext4",
+		"size"      : 1024,
+		"loopback_dir" : "/tmp"
+	},
+	"plugins": {
+		"user_packages": {
+			"enabled": true,
+			"repo": [ "apache2" ],
+			"local": []
+		},
+		"root_password": {
+			"enabled": true,
+			"password": "test"
+                },
+		"convert_image": {
+			"enabled": true,
+			"format": "vdi"
+		}
+	}
+}

plugins/convert_image/__init__.py

@@ -0,0 +1,11 @@
+
+
+def tasks(tasklist, manifest):
+	from tasks import ConvertImage
+	tasklist.add(ConvertImage())
+
+
+def validate_manifest(data, schema_validate):
+	from os import path
+	schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json'))
+	schema_validate(data, schema_path)

plugins/convert_image/manifest-schema.json

@@ -0,0 +1,23 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Convert image",
+	"type": "object",
+	"properties": {
+		"plugins": {
+			"type": "object",
+			"properties": {
+				"convert_image": {
+					"type": "object",
+					"properties": {
+						"format": {
+							"type": "string"
+						}
+					},
+					"required": ["format"]
+				}
+			},
+			"required": ["convert_image"]
+		}
+	},
+	"required": ["plugins"]
+}

plugins/convert_image/tasks.py

@@ -0,0 +1,16 @@
+from base import Task
+from common import phases
+from common.tasks.filesystem import DeleteMountDir
+
+
+class ConvertImage(Task):
+	description = 'Convert raw image'
+	phase = phases.image_conversion
+
+	def run(self, info):
+		from common.tools import log_check_call
+		converted_file = info.loopback_file.replace('img',info.manifest.plugins['convert_image']['format'])
+		log_check_call(['/usr/bin/qemu-img', 'convert', '-O', info.manifest.plugins['convert_image']['format'], info.loopback_file, converted_file])
+		import os
+		os.remove(info.loopback_file)
+		

plugins/root_password/__init__.py

@@ -1,7 +1,7 @@
 
 
 def tasks(tasklist, manifest):
-	from common.tasks import DisableSSHPasswordAuthentication
+	from common.tasks.security import DisableSSHPasswordAuthentication
 	from tasks import SetRootPassword
 	tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword())
 

plugins/user_packages/user_packages.py

@@ -3,7 +3,7 @@ from common import phases
 import os
 from common.tasks.packages import ImagePackages
 from common.tasks.host import CheckPackages
-from providers.raw.tasks.filesystem import MountVolume
+from common.tasks.filesystem import MountVolume
 
 
 class AddUserPackages(Task):

providers/virtualbox/__init__.py

@@ -14,6 +14,7 @@ from common.tasks import security
 from common.tasks import network
 from common.tasks import initd
 from common.tasks import cleanup
+from common.tasks import loopback
 
 
 def initialize():
@@ -88,3 +89,4 @@ def rollback_tasks(tasklist, tasks_completed, manifest):
 	counter_task(parted.MapPartitions, parted.UnmapPartitions)
 	counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
 	counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)
+	counter_task(loopback.Attach, loopback.Detach)

providers/virtualbox/assets/grub.d/00_header

@@ -0,0 +1,4 @@
+#! /bin/sh
+set -e
+
+# nothing to do, skip grub mkconfig for this

providers/virtualbox/assets/grub.d/10_linux

@@ -0,0 +1,93 @@
+#!/bin/sh
+
+# This file generates the old menu.lst configuration with grub2
+# It was copied from tomheadys github repo:
+# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
+
+prefix=/usr
+exec_prefix=${prefix}
+bindir=${exec_prefix}/bin
+libdir=${exec_prefix}/lib
+. ${libdir}/grub/grub-mkconfig_lib
+
+export TEXTDOMAIN=grub
+export TEXTDOMAINDIR=${prefix}/share/locale
+
+GRUB_DEVICE=/dev/sda1
+
+
+cat << EOF
+set default=${GRUB_DEFAULT}
+set timeout=${GRUB_TIMEOUT}
+insmod part_msdos
+insmod ext2
+insmod gettext
+set menu_color_normal=cyan/blue
+set menu_color_highlight=white/blue
+set root='(hd0,msdos1)'
+EOF
+
+if ${GRUB_HIDDEN_TIMEOUT:-false}; then
+	printf "hiddenmenu\n"
+fi
+
+linux_entry ()
+{
+	os="$1"
+	version="$2"
+	args="$4"
+
+	title="$(gettext_quoted "%s, with Linux %s")"
+
+	cat << EOF
+menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os {
+        insmod part_msdos
+        insmod ext2
+	set timeout=${GRUB_TIMEOUT}
+	set root='(hd0,msdos1)'
+        echo 'Loading Linux ${version}'
+	linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
+        echo 'Loading initial ramdisk ...'
+	initrd ${rel_dirname}/${initrd}
+}
+EOF
+}
+
+list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
+		if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
+		done`
+prepare_boot_cache=
+
+while [ "x$list" != "x" ] ; do
+	linux=`version_find_latest $list`
+	basename=`basename $linux`
+	dirname=`dirname $linux`
+	rel_dirname=`make_system_path_relative_to_its_root $dirname`
+	version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
+	alt_version=`echo $version | sed -e "s,\.old$,,g"`
+	linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
+
+	initrd=
+	for i in "initrd.img-${version}" "initrd-${version}.img" \
+		"initrd-${version}" "initramfs-${version}.img" \
+		"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
+		"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
+	if test -e "${dirname}/${i}" ; then
+		initrd="$i"
+		break
+	fi
+	done
+
+	initramfs=
+	for i in "config-${version}" "config-${alt_version}"; do
+	if test -e "${dirname}/${i}" ; then
+		initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut   -f2 -d= | tr -d \"`
+		break
+	fi
+	done
+
+	linux_entry "${OS}" "${version}" \
+		"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
+
+	list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
+done

providers/virtualbox/manifest-schema.json

@@ -1,6 +1,6 @@
 {
 	"$schema": "http://json-schema.org/draft-04/schema#",
-	"title": "OpenNebula manifest",
+	"title": "VirtualBox manifest",
 	"type": "object",
 	"properties": {
 		"volume": {

providers/virtualbox/tasks/boot.py

@@ -7,19 +7,29 @@ class ConfigureGrub(Task):
 	phase = phases.system_modification
 
 	def run(self, info):
+		import stat
+	        rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
+                     stat.S_IRGRP                | stat.S_IXGRP |
+                     stat.S_IROTH                | stat.S_IXOTH)
+        	x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
 		import os.path
 		device_map_path = os.path.join(info.root, 'boot/grub/device.map')
 		with open(device_map_path, 'w') as device_map:
-			device_map.write(('(hd0)   {dev_path}\n'
-			                  '(hd0,1) {root_path}'
-			                  .format(dev_path=info.bootstrap_device['path'],
-			                          root_path=info.bootstrap_device['partitions']['root_path'])))
+			device_map.write('(hd0)   /dev/sda\n')
 
 		from common.tools import log_check_call
+
+		from shutil import copy
+        	script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux'))
+        	script_dst = os.path.join(info.root, 'etc/grub.d/10_linux')
+        	copy(script_src, script_dst)
+        	os.chmod(script_dst, rwxr_xr_x)
+                script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header'))
+                script_dst = os.path.join(info.root, 'etc/grub.d/00_header')
+                copy(script_src, script_dst)
+                os.chmod(script_dst, rwxr_xr_x)
 		log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u'])
+		# Install grub in mbr
+		log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']])
 
-		log_check_call(['/usr/sbin/chroot', info.root,
-		                '/usr/sbin/grub-mkconfig', '-o', '/boot/grub/grub.cfg'])
-
-		log_check_call(['/usr/sbin/chroot', info.root,
-		                '/usr/sbin/grub-install', info.bootstrap_device['path']])
+		log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])

providers/virtualbox/tasks/packages.py

@@ -32,7 +32,8 @@ class ImagePackages(Task):
 		               # isc-dhcp-client doesn't work properly with ec2
 		               'dhcpcd',
 		               'chkconfig',
-		               'openssh-client'
+		               'openssh-client',
+		               'grub2'
 		                ])
 
 		exclude.update(['isc-dhcp-client',