diff --git a/base/manifest-schema.json b/base/manifest-schema.json index 21eb276..a8065b0 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -13,9 +13,10 @@ "image_file": { "type": "string" }, "tarball": { "type": "boolean" }, "tarball_dir": { "type": "string" }, - "device" : { "type" : "string" } + "device" : { "type" : "string" }, + "mirror": { "type": "string" } }, - "required": ["mount_dir"] + "required": ["mount_dir", "mirror"] }, "system": { "type": "object", @@ -30,8 +31,7 @@ }, "timezone": { "type": "string" }, "locale": { "type": "string" }, - "charmap": { "type": "string" }, - "mirror": { "type": "string" } + "charmap": { "type": "string" } }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, diff --git a/common/phases.py b/common/phases.py index 3b44eb2..83743d8 100644 --- a/common/phases.py +++ b/common/phases.py @@ -9,6 +9,7 @@ system_modification = Phase('System modification', 'Installing software, modifyi system_cleaning = Phase('System cleaning', 'Removing sensitive data, temporary files and other leftovers') volume_unmounting = Phase('Volume unmounting', 'Unmounting the bootstrap volume') image_registration = Phase('Image registration', 'Uploading/Registering with the provider') +image_conversion = Phase('Image conversion', 'Conversion/Compression of the image result file') cleaning = Phase('Cleaning', 'Removing temporary files') order = [preparation, @@ -20,5 +21,6 @@ order = [preparation, system_cleaning, volume_unmounting, image_registration, + image_conversion, cleaning, ] diff --git a/common/tasks/bootstrap.py b/common/tasks/bootstrap.py index 9e27f36..a41d697 100644 --- a/common/tasks/bootstrap.py +++ b/common/tasks/bootstrap.py @@ -13,7 +13,8 @@ def get_bootstrap_args(info): options.append('--include=' + ','.join(include)) if len(exclude) > 0: options.append('--exclude=' + ','.join(exclude)) - arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian'] + mirror = info.manifest.bootstrapper['mirror'] + arguments = [info.manifest.system['release'], info.root, mirror] return executable, options, arguments diff --git a/common/tasks/parted.py b/common/tasks/parted.py index c8d1190..caade0a 100644 --- a/common/tasks/parted.py +++ b/common/tasks/parted.py @@ -25,13 +25,9 @@ class MapPartitions(Task): after = [PartitionVolume] def run(self, info): - log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) - root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper')+'p1' - - [root_loopback_path] = log_check_call(['/sbin/losetup', '--find']) - log_check_call(['/sbin/losetup', root_loopback_path, root_partition_path]) - - info.bootstrap_device['partitions'] = {'root_path': root_loopback_path} + root_partition_path = info.bootstrap_device['path'].replace('/dev', '/dev/mapper')+'p1' + log_check_call(['kpartx', '-a', '-v', info.bootstrap_device['path']]) + info.bootstrap_device['partitions'] = {'root_path': root_partition_path} class FormatPartitions(Task): @@ -52,7 +48,5 @@ class UnmapPartitions(Task): after = [filesystem.UnmountVolume] def run(self, info): - log_check_call(['/sbin/losetup', '-d', info.bootstrap_device['partitions']['root_path']]) - del info.bootstrap_device['partitions']['root_path'] - log_check_call(['kpartx', '-d', info.bootstrap_device['path']]) + del info.bootstrap_device['partitions']['root_path'] diff --git a/manifests/virtualbox.json b/manifests/virtualbox.json new file mode 100644 index 0000000..238ef98 --- /dev/null +++ b/manifests/virtualbox.json @@ -0,0 +1,41 @@ +{ + "provider" : "virtualbox", + "virtualization": "ide", + + "bootstrapper": { + "mount_dir": "/mnt/target", + "mirror" : "http://ftp.fr.debian.org/debian/" + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024, + "loopback_dir" : "/tmp" + }, + "plugins": { + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + }, + "root_password": { + "enabled": true, + "password": "test" + }, + "convert_image": { + "enabled": true, + "format": "vdi" + } + } +} diff --git a/plugins/convert_image/__init__.py b/plugins/convert_image/__init__.py new file mode 100644 index 0000000..a36d8f4 --- /dev/null +++ b/plugins/convert_image/__init__.py @@ -0,0 +1,11 @@ + + +def tasks(tasklist, manifest): + from tasks import ConvertImage + tasklist.add(ConvertImage()) + + +def validate_manifest(data, schema_validate): + from os import path + schema_path = path.normpath(path.join(path.dirname(__file__), 'manifest-schema.json')) + schema_validate(data, schema_path) diff --git a/plugins/convert_image/manifest-schema.json b/plugins/convert_image/manifest-schema.json new file mode 100644 index 0000000..6c7fd8d --- /dev/null +++ b/plugins/convert_image/manifest-schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Convert image", + "type": "object", + "properties": { + "plugins": { + "type": "object", + "properties": { + "convert_image": { + "type": "object", + "properties": { + "format": { + "type": "string" + } + }, + "required": ["format"] + } + }, + "required": ["convert_image"] + } + }, + "required": ["plugins"] +} diff --git a/plugins/convert_image/tasks.py b/plugins/convert_image/tasks.py new file mode 100644 index 0000000..91f02fc --- /dev/null +++ b/plugins/convert_image/tasks.py @@ -0,0 +1,16 @@ +from base import Task +from common import phases +from common.tasks.filesystem import DeleteMountDir + + +class ConvertImage(Task): + description = 'Convert raw image' + phase = phases.image_conversion + + def run(self, info): + from common.tools import log_check_call + converted_file = info.loopback_file.replace('img',info.manifest.plugins['convert_image']['format']) + log_check_call(['/usr/bin/qemu-img', 'convert', '-O', info.manifest.plugins['convert_image']['format'], info.loopback_file, converted_file]) + import os + os.remove(info.loopback_file) + diff --git a/plugins/root_password/__init__.py b/plugins/root_password/__init__.py index f9590f5..f3d4e87 100644 --- a/plugins/root_password/__init__.py +++ b/plugins/root_password/__init__.py @@ -1,7 +1,7 @@ def tasks(tasklist, manifest): - from common.tasks import DisableSSHPasswordAuthentication + from common.tasks.security import DisableSSHPasswordAuthentication from tasks import SetRootPassword tasklist.replace(DisableSSHPasswordAuthentication, SetRootPassword()) diff --git a/plugins/user_packages/user_packages.py b/plugins/user_packages/user_packages.py index 82ab462..3ad8c1b 100644 --- a/plugins/user_packages/user_packages.py +++ b/plugins/user_packages/user_packages.py @@ -3,7 +3,7 @@ from common import phases import os from common.tasks.packages import ImagePackages from common.tasks.host import CheckPackages -from providers.raw.tasks.filesystem import MountVolume +from common.tasks.filesystem import MountVolume class AddUserPackages(Task): diff --git a/providers/virtualbox/__init__.py b/providers/virtualbox/__init__.py index 2e7893e..6a2603c 100644 --- a/providers/virtualbox/__init__.py +++ b/providers/virtualbox/__init__.py @@ -14,6 +14,7 @@ from common.tasks import security from common.tasks import network from common.tasks import initd from common.tasks import cleanup +from common.tasks import loopback def initialize(): @@ -88,3 +89,4 @@ def rollback_tasks(tasklist, tasks_completed, manifest): counter_task(parted.MapPartitions, parted.UnmapPartitions) counter_task(filesystem.MountVolume, filesystem.UnmountVolume) counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) + counter_task(loopback.Attach, loopback.Detach) diff --git a/providers/virtualbox/assets/grub.d/00_header b/providers/virtualbox/assets/grub.d/00_header new file mode 100644 index 0000000..43927bc --- /dev/null +++ b/providers/virtualbox/assets/grub.d/00_header @@ -0,0 +1,4 @@ +#! /bin/sh +set -e + +# nothing to do, skip grub mkconfig for this diff --git a/providers/virtualbox/assets/grub.d/10_linux b/providers/virtualbox/assets/grub.d/10_linux new file mode 100644 index 0000000..0b08346 --- /dev/null +++ b/providers/virtualbox/assets/grub.d/10_linux @@ -0,0 +1,93 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/sda1 + + +cat << EOF +set default=${GRUB_DEFAULT} +set timeout=${GRUB_TIMEOUT} +insmod part_msdos +insmod ext2 +insmod gettext +set menu_color_normal=cyan/blue +set menu_color_highlight=white/blue +set root='(hd0,msdos1)' +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { + insmod part_msdos + insmod ext2 + set timeout=${GRUB_TIMEOUT} + set root='(hd0,msdos1)' + echo 'Loading Linux ${version}' + linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + echo 'Loading initial ramdisk ...' + initrd ${rel_dirname}/${initrd} +} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/virtualbox/manifest-schema.json b/providers/virtualbox/manifest-schema.json index 8c9e8f5..52ac2ee 100644 --- a/providers/virtualbox/manifest-schema.json +++ b/providers/virtualbox/manifest-schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "OpenNebula manifest", + "title": "VirtualBox manifest", "type": "object", "properties": { "volume": { diff --git a/providers/virtualbox/tasks/boot.py b/providers/virtualbox/tasks/boot.py index fcd25f9..02943b6 100644 --- a/providers/virtualbox/tasks/boot.py +++ b/providers/virtualbox/tasks/boot.py @@ -7,19 +7,29 @@ class ConfigureGrub(Task): phase = phases.system_modification def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH import os.path device_map_path = os.path.join(info.root, 'boot/grub/device.map') with open(device_map_path, 'w') as device_map: - device_map.write(('(hd0) {dev_path}\n' - '(hd0,1) {root_path}' - .format(dev_path=info.bootstrap_device['path'], - root_path=info.bootstrap_device['partitions']['root_path']))) + device_map.write('(hd0) /dev/sda\n') from common.tools import log_check_call + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/10_linux')) + script_dst = os.path.join(info.root, 'etc/grub.d/10_linux') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/00_header')) + script_dst = os.path.join(info.root, 'etc/grub.d/00_header') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) + # Install grub in mbr + log_check_call(['/usr/sbin/grub-install', '--boot-directory='+info.root+"/boot/", info.bootstrap_device['path']]) - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/grub-mkconfig', '-o', '/boot/grub/grub.cfg']) - - log_check_call(['/usr/sbin/chroot', info.root, - '/usr/sbin/grub-install', info.bootstrap_device['path']]) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) diff --git a/providers/virtualbox/tasks/packages.py b/providers/virtualbox/tasks/packages.py index 43fe477..8282122 100644 --- a/providers/virtualbox/tasks/packages.py +++ b/providers/virtualbox/tasks/packages.py @@ -32,7 +32,8 @@ class ImagePackages(Task): # isc-dhcp-client doesn't work properly with ec2 'dhcpcd', 'chkconfig', - 'openssh-client' + 'openssh-client', + 'grub2' ]) exclude.update(['isc-dhcp-client',