diff --git a/base/manifest-schema.json b/base/manifest-schema.json index a98907b..21eb276 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -10,8 +10,10 @@ "type": "object", "properties": { "mount_dir": { "type": "string" }, + "image_file": { "type": "string" }, "tarball": { "type": "boolean" }, - "tarball_dir": { "type": "string" } + "tarball_dir": { "type": "string" }, + "device" : { "type" : "string" } }, "required": ["mount_dir"] }, @@ -28,7 +30,8 @@ }, "timezone": { "type": "string" }, "locale": { "type": "string" }, - "charmap": { "type": "string" } + "charmap": { "type": "string" }, + "mirror": { "type": "string" } }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, diff --git a/common/tools.py b/common/tools.py index d5f5ae2..4a89c47 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,14 +1,14 @@ -def log_check_call(command): - status, stdout, stderr = log_call(command) +def log_check_call(command, input=None): + status, stdout, stderr = log_call(command, input) if status != 0: from subprocess import CalledProcessError raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr)) return stdout -def log_call(command): +def log_call(command, input=None): import subprocess import select @@ -17,7 +17,15 @@ def log_call(command): command_log = realpath(command[0]).replace('/', '.') log = logging.getLogger(__name__ + command_log) - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if input is not None: + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.stdin.write(input+"\n") + process.stdin.flush() + process.stdin.close() + # (stdout,stderr) = process.communicate(input+"\n") + # return process.returncode, stdout, stderr + else: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = [] stderr = [] while True: diff --git a/manifests/one-ide.manifest b/manifests/one-ide.manifest new file mode 100644 index 0000000..26db2a9 --- /dev/null +++ b/manifests/one-ide.manifest @@ -0,0 +1,54 @@ +{ + "provider" : "raw", + "virtualization": "ide", + "credentials" : { + "access-key": null, + "secret-key": null, + "root": "test" + }, + + "bootstrapper": { + "mount_dir": "/mnt/target", + "image_file": "/tmp/one.img", + "tarball": true + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8", + "mirror" : "ftp://ftp.fr.debian.org/debian/" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024 + }, + "plugins": { + "admin_user": { + "enabled": false + }, + "build_metadata": { + "enabled": false, + "path" : "/root/build-metadata-{ami_name}" + }, + "prebootstrapped": { + "enabled": false, + "snapshot": "" + }, + "opennebula": { + "enabled": true + }, + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + } + + } +} diff --git a/manifests/one-virtio.manifest b/manifests/one-virtio.manifest new file mode 100644 index 0000000..02ce685 --- /dev/null +++ b/manifests/one-virtio.manifest @@ -0,0 +1,54 @@ +{ + "provider" : "raw", + "virtualization": "virtio", + "credentials" : { + "access-key": null, + "secret-key": null, + "root": "test" + }, + + "bootstrapper": { + "mount_dir": "/mnt/target", + "image_file": "/tmp/one.img", + "tarball": true + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8", + "mirror" : "ftp://ftp.fr.debian.org/debian/" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024 + }, + "plugins": { + "admin_user": { + "enabled": false + }, + "build_metadata": { + "enabled": false, + "path" : "/root/build-metadata-{ami_name}" + }, + "prebootstrapped": { + "enabled": false, + "snapshot": "" + }, + "opennebula": { + "enabled": true + }, + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + } + + } +} diff --git a/plugins/opennebula/README.md b/plugins/opennebula/README.md new file mode 100644 index 0000000..c36f2b3 --- /dev/null +++ b/plugins/opennebula/README.md @@ -0,0 +1,17 @@ +# Open Nebula context plugin + +This plugin adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong). + +It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like: + + ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] + ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] + ETH0_IP $NIC[IP, NETWORK_ID=2] + ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] + ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] + FILES path_to_my_ssh_public_key.pub + +Plugin will install all *.pub* files in the root authorized_keys file. + +In case of an EC2 start, if the USER_EC2_DATA element is a script, the plugin will execute it. + diff --git a/plugins/opennebula/__init__.py b/plugins/opennebula/__init__.py new file mode 100644 index 0000000..cf8567c --- /dev/null +++ b/plugins/opennebula/__init__.py @@ -0,0 +1,5 @@ + + +def tasks(tasklist, manifest): + from opennebula import OpenNebulaContext + tasklist.add(OpenNebulaContext()) diff --git a/plugins/opennebula/assets/one-context_3.8.1.deb b/plugins/opennebula/assets/one-context_3.8.1.deb new file mode 100644 index 0000000..2e81188 Binary files /dev/null and b/plugins/opennebula/assets/one-context_3.8.1.deb differ diff --git a/plugins/opennebula/assets/one-ec2.sh b/plugins/opennebula/assets/one-ec2.sh new file mode 100755 index 0000000..e52c72d --- /dev/null +++ b/plugins/opennebula/assets/one-ec2.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ -n "$EC2_USER_DATA" ]; then + # Check if EC2 user data is a script, if yes, execute + if [[ $EC2_USER_DATA =~ ^#! ]]; then + echo "EC2 data is an executable script, so execute it now" + TMPFILE=$(mktemp /tmp/output.XXXXXXXXXX) + chmod 755 $TMPFILE + $TMPFILE + cat $TMPFILE + else + print "Not an executable" + fi +fi diff --git a/plugins/opennebula/assets/one-pubkey.sh b/plugins/opennebula/assets/one-pubkey.sh new file mode 100755 index 0000000..7d7a209 --- /dev/null +++ b/plugins/opennebula/assets/one-pubkey.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "Reconfigure host ssh keys" +dpkg-reconfigure openssh-server + +if [ ! -e /root/.ssh ]; then + mkdir /root/.ssh + touch /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys +fi + +echo "Copy public ssh keys to authorized_keys" +for f in /mnt/*.pub +do + cat $f >> /root/.ssh/authorized_keys + +done diff --git a/plugins/opennebula/opennebula.py b/plugins/opennebula/opennebula.py new file mode 100644 index 0000000..df50a4b --- /dev/null +++ b/plugins/opennebula/opennebula.py @@ -0,0 +1,42 @@ +from base import Task +from common import phases +import os +from providers.raw.tasks.locale import GenerateLocale + + +class OpenNebulaContext(Task): + description = 'Setup OpenNebula init context' + phase = phases.system_modification + after = [GenerateLocale] + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-context_3.8.1.deb')) + script_dst = os.path.join(info.root, 'tmp/one-context_3.8.1.deb') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb']) + # Fix start + from common.tools import sed_i + vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext') + sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5') + os.chmod(vmcontext_def, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', '90', '2', '3', '4', '5', 'stop', '90', '0', '6']) + + # Load all pubkeys in root authorized_keys + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-pubkey.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') + copy(script_src, script_dst) + + # If USER_EC2_DATA is a script, execute it + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-ec2.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh') + copy(script_src, script_dst) + diff --git a/plugins/user_packages/README.md b/plugins/user_packages/README.md new file mode 100644 index 0000000..059bf63 --- /dev/null +++ b/plugins/user_packages/README.md @@ -0,0 +1,15 @@ +# User package plugin + +This plugin gives the possibility to the user to install Debian packages in the virtual image. + +Plugin is defined in the manifest file, plugin section with: + + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [ "/tmp/mypackage.deb" ] + } + +The *repo* element refers to packages available in Debian repository (apt-get). + +The *local* element will copy the specified .deb files and install them in the image with a dpkg command. Packages are installed in the order of their declaration. diff --git a/plugins/user_packages/__init__.py b/plugins/user_packages/__init__.py new file mode 100644 index 0000000..e223e0a --- /dev/null +++ b/plugins/user_packages/__init__.py @@ -0,0 +1,6 @@ + + +def tasks(tasklist, manifest): + from user_packages import AddUserPackages, AddLocalUserPackages + tasklist.add(AddUserPackages()) + tasklist.add(AddLocalUserPackages()) diff --git a/plugins/user_packages/user_packages.py b/plugins/user_packages/user_packages.py new file mode 100644 index 0000000..e1093ca --- /dev/null +++ b/plugins/user_packages/user_packages.py @@ -0,0 +1,45 @@ +from base import Task +from common import phases +import os +from providers.raw.tasks.packages import ImagePackages +from providers.raw.tasks.host import CheckPackages +from providers.raw.tasks.filesystem import MountVolume + + +class AddUserPackages(Task): + description = 'Adding user defined packages to the image packages' + phase = phases.preparation + after = [ImagePackages] + before = [CheckPackages] + + def run(self, info): + if 'repo' not in info.manifest.plugins['user_packages']: + return + for pkg in info.manifest.plugins['user_packages']['repo']: + info.img_packages[0].add(pkg) + +class AddLocalUserPackages(Task): + description = 'Adding user local packages to the image packages' + phase = phases.system_modification + after = [MountVolume] + + def run(self, info): + if 'local' not in info.manifest.plugins['user_packages']: + return + + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + from shutil import copy + from common.tools import log_check_call + + for pkg in info.manifest.plugins['user_packages']['local']: + script_src = os.path.normpath(pkg) + script_dst = os.path.join(info.root, 'tmp/'+os.path.basename(script_src)) + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/'+os.path.basename(script_src)]) + diff --git a/providers/raw/REAME.md b/providers/raw/REAME.md new file mode 100644 index 0000000..efbd58b --- /dev/null +++ b/providers/raw/REAME.md @@ -0,0 +1,16 @@ +# RAW provider + +This provider creates a raw image file. It can be combined with opennebula plugin to add OpenNebula contextualization. + +By default, it creates a network interface configured with DHCP. + +# Configuration + +## provider + +*raw* : use this provider + +##virtualization + +* ide: basic disk emulation (/dev/sda) +* virtio: Virtio emulation (for KVM), provides better disk performances (/dev/vda) diff --git a/providers/raw/__init__.py b/providers/raw/__init__.py new file mode 100644 index 0000000..2cd2f36 --- /dev/null +++ b/providers/raw/__init__.py @@ -0,0 +1,77 @@ +from manifest import Manifest +import logging +from tasks import packages +from tasks import host +from tasks import filesystem +from tasks import bootstrap +from tasks import locale +from tasks import apt +from tasks import boot +from tasks import security +from tasks import network +from tasks import initd +from tasks import cleanup +from tasks import fake + + +def initialize(): + # Regardless of of loglevel, we don't want boto debug stuff, it's very noisy + logging.getLogger('boto').setLevel(logging.INFO) + + +def tasks(tasklist, manifest): + tasklist.add(packages.HostPackages(), + packages.ImagePackages(), + host.CheckPackages(), + host.GetInfo()) + tasklist.add(filesystem.FormatVolume()) + if manifest.volume['filesystem'].lower() == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + tasklist.add(filesystem.TuneVolumeFS()) + tasklist.add(filesystem.CreateMountDir(), + filesystem.MountVolume()) + if manifest.bootstrapper['tarball']: + tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.Bootstrap(), + filesystem.MountSpecials(), + locale.GenerateLocale(), + locale.SetTimezone(), + apt.DisableDaemonAutostart(), + apt.AptSources(), + #No network for the moment, skip + #apt.AptUpgrade(), + boot.ConfigureGrub(), + filesystem.ModifyFstab(), + boot.BlackListModules(), + boot.DisableGetTTYs(), + security.EnableShadowConfig(), + security.SetRootPassword(), + security.DisableSSHPasswordAuthentication(), + security.DisableSSHDNSLookup(), + network.RemoveDNSInfo(), + network.ConfigureNetworkIF(), + network.ConfigureDHCP(), + initd.ResolveInitScripts(), + initd.InstallInitScripts(), + cleanup.ClearMOTD(), + cleanup.ShredHostkeys(), + cleanup.CleanTMP(), + apt.PurgeUnusedPackages(), + apt.AptClean(), + apt.EnableDaemonAutostart(), + filesystem.UnmountSpecials(), + filesystem.UnmountVolume(), + filesystem.DeleteMountDir()) + + +def rollback_tasks(tasklist, tasks_completed, manifest): + completed = [type(task) for task in tasks_completed] + + def counter_task(task, counter): + if task in completed and counter not in completed: + tasklist.add(counter()) + + counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) + counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/raw/assets/grub.d/40_custom b/providers/raw/assets/grub.d/40_custom new file mode 100644 index 0000000..cb035a2 --- /dev/null +++ b/providers/raw/assets/grub.d/40_custom @@ -0,0 +1,93 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/sda1 + + +cat << EOF +set default=${GRUB_DEFAULT} +set timeout=${GRUB_TIMEOUT} +insmod part_msdos +insmod ext2 +insmod gettext +set menu_color_normal=cyan/blue +set menu_color_highlight=white/blue +set root='(hd0,msdos1)' +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +menuentry 'Debian GNU/Linux for OpenNebula, ${version}' --class debian --class gnu-linux --class os { + insmod part_msdos + insmod ext2 + set timeout=${GRUB_TIMEOUT} + set root='(hd0,msdos1)' + echo 'Loading Linux ${version}' + linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + echo 'Loading initial ramdisk ...' + initrd ${rel_dirname}/${initrd} +} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/raw/assets/grub.d/40_custom.orig b/providers/raw/assets/grub.d/40_custom.orig new file mode 100644 index 0000000..0e74a8e --- /dev/null +++ b/providers/raw/assets/grub.d/40_custom.orig @@ -0,0 +1,81 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/sda1 + + +cat << EOF +default ${GRUB_DEFAULT} +timeout ${GRUB_TIMEOUT} +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +title ${version} + root (hd0) + kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + initrd ${rel_dirname}/${initrd} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/raw/assets/init.d/ec2-get-credentials b/providers/raw/assets/init.d/ec2-get-credentials new file mode 100644 index 0000000..b304ae2 --- /dev/null +++ b/providers/raw/assets/init.d/ec2-get-credentials @@ -0,0 +1,45 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-get-credentials +# Required-Start: $network +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Retrieve the ssh credentials and add to authorized_keys +### END INIT INFO +# +# ec2-get-credentials - Retrieve the ssh credentials and add to authorized_keys +# +# Based on /usr/local/sbin/ec2-get-credentials from Amazon's ami-20b65349 +# + +prog=$(basename $0) +logger="logger -t $prog" + +public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key +username='root' +# A little bit of nastyness to get the homedir, when the username is a variable +ssh_dir="`eval printf ~$username`/.ssh" +authorized_keys="$ssh_dir/authorized_keys" + +# Try to get the ssh public key from instance data. +public_key=`wget -qO - $public_key_url` +if [ -n "$public_key" ]; then + if [ ! -f $authorized_keys ]; then + if [ ! -d $ssh_dir ]; then + mkdir -m 700 $ssh_dir + chown $username:$username $ssh_dir + fi + touch $authorized_keys + chown $username:$username $authorized_keys + fi + + if ! grep -s -q "$public_key" $authorized_keys; then + printf "\n%s" -- "$public_key" >> $authorized_keys + $logger "New ssh key added to $authorized_keys from $public_key_url" + chmod 600 $authorized_keys + chown $username:$username $authorized_keys + fi +fi diff --git a/providers/raw/assets/init.d/ec2-run-user-data b/providers/raw/assets/init.d/ec2-run-user-data new file mode 100644 index 0000000..17b8b6f --- /dev/null +++ b/providers/raw/assets/init.d/ec2-run-user-data @@ -0,0 +1,46 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-run-user-data +# Required-Start: ec2-get-credentials +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Run instance user-data if it looks like a script. +### END INIT INFO +# +# Only retrieves and runs the user-data script once per instance. If +# you want the user-data script to run again (e.g., on the next boot) +# then readd this script with insserv: +# insserv -d ec2-run-user-data +# +prog=$(basename $0) +logger="logger -t $prog" +instance_data_url="http://169.254.169.254/2008-02-01" + + +# Retrieve the instance user-data and run it if it looks like a script +user_data_file=$(tempfile --prefix ec2 --suffix .user-data --mode 700) +$logger "Retrieving user-data" +wget -qO $user_data_file $instance_data_url/user-data 2>&1 | $logger + +if [ $(file -b --mime-type $user_data_file) = 'application/x-gzip' ]; then + $logger "Uncompressing gzip'd user-data" + mv $user_data_file $user_data_file.gz + gunzip $user_data_file.gz +fi + +if [ ! -s $user_data_file ]; then + $logger "No user-data available" +elif head -1 $user_data_file | egrep -v '^#!'; then + $logger "Skipping user-data as it does not begin with #!" +else + $logger "Running user-data" + $user_data_file 2>&1 | logger -t "user-data" + $logger "user-data exit code: $?" +fi +rm -f $user_data_file + +# Disable this script, it may only run once +insserv -r $0 diff --git a/providers/raw/assets/init.d/expand-volume b/providers/raw/assets/init.d/expand-volume new file mode 100644 index 0000000..3b2d2a6 --- /dev/null +++ b/providers/raw/assets/init.d/expand-volume @@ -0,0 +1,26 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: expand-volume +# Required-Start: +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Expand the filesystem of the mounted root volume to its maximum possible size +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +device_path="/dev/xvda1" + +filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'` + +case $filesystem in + xfs) xfs_growfs / ;; + ext2) resize2fs $device_path ;; + ext3) resize2fs $device_path ;; + ext4) resize2fs $device_path ;; + *) $logger "The filesystem $filesystem was not recognized. Unable to expand size." ;; +esac diff --git a/providers/raw/assets/init.d/generate-ssh-hostkeys b/providers/raw/assets/init.d/generate-ssh-hostkeys new file mode 100644 index 0000000..c9efb12 --- /dev/null +++ b/providers/raw/assets/init.d/generate-ssh-hostkeys @@ -0,0 +1,36 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" +ecdsa_key="/etc/ssh/ssh_host_ecdsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key -a -f $ecdsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' +[ -f $ecdsa_key ] || ssh-keygen -f $ecdsa_key -t ecdsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +ssh-keygen -l -f $ecdsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys b/providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys new file mode 100644 index 0000000..148b87d --- /dev/null +++ b/providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys @@ -0,0 +1,33 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/raw/manifest-schema.json b/providers/raw/manifest-schema.json new file mode 100644 index 0000000..8c9e8f5 --- /dev/null +++ b/providers/raw/manifest-schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "OpenNebula manifest", + "type": "object", + "properties": { + "volume": { + "type": "object", + "properties": { + "backing": { + "type": "string", + "enum": ["raw", "qcow2"] + }, + "filesystem": { + "type": "string", + "enum": ["ext2", "ext3", "ext4", "xfs"] + } + }, + "required": ["backing", "filesystem"] + } + }, + "required": ["volume"] +} diff --git a/providers/raw/manifest.py b/providers/raw/manifest.py new file mode 100644 index 0000000..d2e0f2c --- /dev/null +++ b/providers/raw/manifest.py @@ -0,0 +1,15 @@ +import base + + +class Manifest(base.Manifest): + def validate(self, data): + super(Manifest, self).validate(data) + from os import path + schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') + self.schema_validate(data, schema_path) + + def parse(self, data): + super(Manifest, self).parse(data) + self.credentials = data['credentials'] + self.virtualization = data['virtualization'] + self.image = data['image'] diff --git a/providers/raw/tasks/__init__.py b/providers/raw/tasks/__init__.py new file mode 100644 index 0000000..873a99d --- /dev/null +++ b/providers/raw/tasks/__init__.py @@ -0,0 +1 @@ +__all__ = ['packages', 'connection', 'host', 'ec2'] diff --git a/providers/raw/tasks/apt.py b/providers/raw/tasks/apt.py new file mode 100644 index 0000000..d2d86d3 --- /dev/null +++ b/providers/raw/tasks/apt.py @@ -0,0 +1,81 @@ +from base import Task +from common import phases +from common.tools import log_check_call +import os +from locale import GenerateLocale + + +class AptSources(Task): + description = 'Adding aptitude sources' + phase = phases.system_modification + + def run(self, info): + mirror = 'http://http.debian.net/debian' + if info.manifest.system['mirror']: + mirror = info.manifest.system['mirror'] + sources_path = os.path.join(info.root, 'etc/apt/sources.list') + with open(sources_path, 'w') as apt_sources: + apt_sources.write(('deb {apt_mirror} {release} main\n' + 'deb-src {apt_mirror} {release} main\n' + .format(apt_mirror=mirror, + release=info.manifest.system['release']))) + apt_sources.write(('deb {apt_mirror} {release}/updates main\n' + 'deb-src {apt_mirror} {release}/updates main\n' + .format(apt_mirror='http://security.debian.org/', + release=info.manifest.system['release']))) + + +class DisableDaemonAutostart(Task): + description = 'Disabling daemon autostart' + phase = phases.system_modification + + def run(self, info): + rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d') + with open(rc_policy_path, 'w') as rc_policy: + rc_policy.write(('#!/bin/sh\n' + 'exit 101')) + import stat + os.chmod(rc_policy_path, + stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + +class AptUpgrade(Task): + description = 'Upgrading packages and fixing broken dependencies' + phase = phases.system_modification + after = [GenerateLocale, AptSources, DisableDaemonAutostart] + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-f', '-y', 'install']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-y', 'upgrade']) + + +class PurgeUnusedPackages(Task): + description = 'Removing unused packages' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'autoremove', '--purge']) + + +class AptClean(Task): + description = 'Clearing the aptitude cache' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'clean']) + + lists = os.path.join(info.root, 'var/lib/apt/lists') + for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]: + if os.path.isfile(list_file): + os.remove(list_file) + + +class EnableDaemonAutostart(Task): + description = 'Re-enabling daemon autostart after installation' + phase = phases.system_cleaning + + def run(self, info): + os.remove(os.path.join(info.root, 'usr/sbin/policy-rc.d')) diff --git a/providers/raw/tasks/boot.py b/providers/raw/tasks/boot.py new file mode 100644 index 0000000..6ae840c --- /dev/null +++ b/providers/raw/tasks/boot.py @@ -0,0 +1,81 @@ +from base import Task +from common import phases +import os + + +class ConfigureGrub(Task): + description = 'Configuring grub' + phase = phases.system_modification + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + grubd = os.path.join(info.root, 'etc/grub.d') + for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]: + os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all) + + from common.tools import log_check_call + from shutil import copy + + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom')) + script_dst = os.path.join(info.root, 'etc/grub.d/40_custom') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + if info.manifest.virtualization == 'virtio': + modules_path = os.path.join(info.root, + 'etc/initramfs-tools/modules') + with open(modules_path, 'a') as modules: + modules.write("\nvirtio_pci\nvirtio_blk\n") + + + grub_def = os.path.join(info.root, 'etc/default/grub') + + log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) + + log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) + log_check_call(['grub-install', '--boot-directory='+info.root+"/boot/", '/dev/loop0']) + + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + + from common.tools import sed_i + if info.manifest.virtualization == 'virtio': + grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg') + sed_i(grub_cfg, 'sda', 'vda') + device_map = os.path.join(info.root, + 'boot/grub/device.map') + sed_i(device_map, 'sda', 'vda') + #log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + + + +class BlackListModules(Task): + description = 'Blacklisting kernel modules' + phase = phases.system_modification + + def run(self, info): + blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf') + with open(blacklist_path, 'a') as blacklist: + blacklist.write(('# disable pc speaker\n' + 'blacklist pcspkr')) + + +class DisableGetTTYs(Task): + description = 'Disabling getty processes' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + inittab_path = os.path.join(info.root, 'etc/inittab') + tty1 = '1:2345:respawn:/sbin/getty 38400 tty1' + sed_i(inittab_path, '^'+tty1, '#'+tty1) + ttyx = ':23:respawn:/sbin/getty 38400 tty' + for i in range(2, 6): + i = str(i) + sed_i(inittab_path, '^'+i+ttyx+i, '#'+i+ttyx+i) diff --git a/providers/raw/tasks/bootstrap.py b/providers/raw/tasks/bootstrap.py new file mode 100644 index 0000000..9b405b0 --- /dev/null +++ b/providers/raw/tasks/bootstrap.py @@ -0,0 +1,58 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import logging +log = logging.getLogger(__name__) + + +def get_bootstrap_args(info): + mirror = 'http://http.debian.net/debian' + if info.manifest.system['mirror']: + mirror = info.manifest.system['mirror'] + + executable = ['/usr/sbin/debootstrap'] + options = ['--arch=' + info.manifest.system['architecture']] + include, exclude = info.img_packages + if len(include) > 0: + options.append('--include=' + ','.join(include)) + if len(exclude) > 0: + options.append('--exclude=' + ','.join(exclude)) + arguments = [info.manifest.system['release'], info.root, mirror] + return executable, options, arguments + + +class MakeTarball(Task): + description = 'Creating bootstrap tarball' + phase = phases.os_installation + + def run(self, info): + from hashlib import sha1 + import os.path + executable, options, arguments = get_bootstrap_args(info) + # Filter info.root which points at /target/volume-id, we won't ever hit anything with that in there. + hash_args = [arg for arg in arguments if arg != info.root] + tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] + tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) + info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + if os.path.isfile(info.tarball): + log.debug('Found matching tarball, skipping download') + else: + from common.tools import log_call + status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments) + if status != 1: + msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status) + raise TaskError(msg) + + +class Bootstrap(Task): + description = 'Installing Debian' + phase = phases.os_installation + after = [MakeTarball] + + def run(self, info): + executable, options, arguments = get_bootstrap_args(info) + if hasattr(info, 'tarball'): + options.extend(['--unpack-tarball=' + info.tarball]) + + from common.tools import log_check_call + log_check_call(executable + options + arguments) diff --git a/providers/raw/tasks/cleanup.py b/providers/raw/tasks/cleanup.py new file mode 100644 index 0000000..7b793a4 --- /dev/null +++ b/providers/raw/tasks/cleanup.py @@ -0,0 +1,43 @@ +from base import Task +from common import phases +import os + + +class ClearMOTD(Task): + description = 'Clearing the MOTD' + phase = phases.system_cleaning + + def run(self, info): + with open('/var/run/motd', 'w'): + pass + + +class ShredHostkeys(Task): + description = 'Securely deleting ssh hostkeys' + phase = phases.system_cleaning + + def run(self, info): + ssh_hostkeys = ['ssh_host_dsa_key', + 'ssh_host_rsa_key'] + if info.manifest.system['release'] != 'squeeze': + ssh_hostkeys.append('ssh_host_ecdsa_key') + + private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys] + public = [path + '.pub' for path in private] + + from common.tools import log_check_call + log_check_call(['/usr/bin/shred', '--remove'] + private + public) + + +class CleanTMP(Task): + description = 'Removing temporary files' + phase = phases.system_cleaning + + def run(self, info): + tmp = os.path.join(info.root, 'tmp') + for tmp_file in [os.path.join(tmp, f) for f in os.listdir(tmp)]: + os.remove(tmp_file) + + log = os.path.join(info.root, 'var/log/') + os.remove(os.path.join(log, 'bootstrap.log')) + os.remove(os.path.join(log, 'dpkg.log')) diff --git a/providers/raw/tasks/fake.py b/providers/raw/tasks/fake.py new file mode 100644 index 0000000..33648ac --- /dev/null +++ b/providers/raw/tasks/fake.py @@ -0,0 +1,13 @@ +from base import Task +from common import phases +import os.path + +class Fake(Task): + description = 'create fake file' + phase = phases.system_modification + + def run(self, info): + fake_path = os.path.join(info.root, 'fake.txt') + with open(fake_path, 'a') as fakefile: + fakefile.write("fake file") + diff --git a/providers/raw/tasks/filesystem.py b/providers/raw/tasks/filesystem.py new file mode 100644 index 0000000..48dc473 --- /dev/null +++ b/providers/raw/tasks/filesystem.py @@ -0,0 +1,147 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from common.tools import log_check_call +from bootstrap import Bootstrap +import os + +class FormatVolume(Task): + description = 'Formatting the volume' + phase = phases.volume_preparation + + def run(self, info): + mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] + log_check_call(mkmount) + + loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] + log_check_call(loopcmd) + + # parted + log_check_call(['parted','-a', 'optimal', '-s', '/dev/loop0', "mklabel", "msdos"]) + log_check_call(['parted', '-a', 'optimal', '-s', '/dev/loop0', "--", "mkpart", "primary", "ext4", "32k", "-1"]) + log_check_call(['parted','-s', '/dev/loop0', "--", "set", "1", "boot", "on"]) + + + #log_check_call(['kpartx','-a','-v', info.manifest.bootstrapper['image_file']]) + log_check_call(['kpartx','-a', '-v', '/dev/loop0']) + mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/mapper/loop0p1'] + log_check_call(mkfs) + + + + +class TuneVolumeFS(Task): + description = 'Tuning the bootstrap volume filesystem' + phase = phases.volume_preparation + after = [FormatVolume] + + def run(self, info): + #dev_path = info.bootstrap_device['path'] + #dev_path = info.manifest.bootstrapper['image_file'] + dev_path = '/dev/mapper/loop0p1' + # Disable the time based filesystem check + log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) + + +class AddXFSProgs(Task): + description = 'Adding `xfsprogs\' to the image packages' + phase = phases.preparation + + def run(self, info): + include, exclude = info.img_packages + include.add('xfsprogs') + + +class CreateMountDir(Task): + description = 'Creating mountpoint for the bootstrap volume' + phase = phases.volume_mounting + + def run(self, info): + import os + mount_dir = info.manifest.bootstrapper['mount_dir'] + info.root = mount_dir + # Works recursively, fails if last part exists, which is exaclty what we want. + os.makedirs(mount_dir) + + +class MountVolume(Task): + description = 'Mounting the bootstrap volume' + phase = phases.volume_mounting + after = [CreateMountDir] + + def run(self, info): + with open('/proc/mounts') as mounts: + for mount in mounts: + if info.root in mount: + msg = 'Something is already mounted at {root}'.format(root=info.root) + raise TaskError(msg) + + log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/mapper/loop0p1', info.root]) + +class MountSpecials(Task): + description = 'Mounting special block devices' + phase = phases.os_installation + after = [Bootstrap] + + def run(self, info): + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'devpts', 'none', '/dev/pts']) + + +class UnmountSpecials(Task): + description = 'Unmunting special block devices' + phase = phases.volume_unmounting + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) + + +class UnmountVolume(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountSpecials] + + def run(self, info): + log_check_call(['/bin/umount', info.root]) + #log_check_call(['partx','-d','/dev/loop0']) + #log_check_call(['/sbin/losetup', '-d', '/dev/loop0']) + log_check_call(['kpartx','-d', info.manifest.bootstrapper['image_file']]) + + + +class DeleteMountDir(Task): + description = 'Deleting mountpoint for the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountVolume] + + def run(self, info): + import os + os.rmdir(info.root) + del info.root + + +class ModifyFstab(Task): + description = 'Adding root volume to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + mount_opts = ['defaults'] + if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if info.manifest.volume['filesystem'].lower() == 'xfs': + mount_opts.append('nobarrier') + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'a') as fstab: + device = '/dev/sda1' + if info.manifest.virtualization == 'virtio': + device = '/dev/vda1' + + fstab.write((device+' / {filesystem} {mount_opts} 1 1\n' + .format(filesystem=info.manifest.volume['filesystem'].lower(), + mount_opts=','.join(mount_opts)))) diff --git a/providers/raw/tasks/host.py b/providers/raw/tasks/host.py new file mode 100644 index 0000000..61af65a --- /dev/null +++ b/providers/raw/tasks/host.py @@ -0,0 +1,29 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import packages + + +class CheckPackages(Task): + description = 'Checking installed host packages' + phase = phases.preparation + after = [packages.HostPackages, packages.ImagePackages] + + def run(self, info): + from common.tools import log_check_call + from subprocess import CalledProcessError + for package in info.host_packages: + try: + log_check_call(['/usr/bin/dpkg', '-s', package]) + except CalledProcessError: + msg = "The package ``{0}\'\' is not installed".format(package) + raise TaskError(msg) + + +class GetInfo(Task): + description = 'Retrieving instance metadata' + phase = phases.preparation + + def run(self, info): + info.host = {} + return info diff --git a/providers/raw/tasks/initd.py b/providers/raw/tasks/initd.py new file mode 100644 index 0000000..5a479fe --- /dev/null +++ b/providers/raw/tasks/initd.py @@ -0,0 +1,49 @@ +from base import Task +from common import phases +import os.path + + +class ResolveInitScripts(Task): + description = 'Determining which startup scripts to install or disable' + phase = phases.system_modification + + def run(self, info): + init_scripts = {'ec2-get-credentials': 'ec2-get-credentials', + 'ec2-run-user-data': 'ec2-run-user-data', + 'expand-volume': 'expand-volume'} + + init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' + if info.manifest.system['release'] == 'squeeze': + init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + + disable_scripts = ['hwclock.sh'] + if info.manifest.system['release'] == 'squeeze': + disable_scripts.append('hwclockfirst.sh') + + for name, path in init_scripts.iteritems(): + init_scripts[name] = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d', path)) + + info.initd = {'install': init_scripts, + 'disable': disable_scripts} + + +class InstallInitScripts(Task): + description = 'Installing startup scripts' + phase = phases.system_modification + after = [ResolveInitScripts] + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + from shutil import copy + from common.tools import log_check_call + for name, src in info.initd['install'].iteritems(): + dst = os.path.join(info.root, 'etc/init.d', name) + copy(src, dst) + os.chmod(dst, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-d', name]) + + for name in info.initd['disable']: + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-r', name]) diff --git a/providers/raw/tasks/locale.py b/providers/raw/tasks/locale.py new file mode 100644 index 0000000..a0005b4 --- /dev/null +++ b/providers/raw/tasks/locale.py @@ -0,0 +1,35 @@ +from base import Task +from common import phases +import os.path + + +class GenerateLocale(Task): + description = 'Generating the selected locale' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + from common.tools import log_check_call + locale_gen = os.path.join(info.root, 'etc/locale.gen') + locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'], + charmap=info.manifest.system['charmap']) + search = '# ' + locale_str + sed_i(locale_gen, search, locale_str) + + command = ['/usr/sbin/chroot', info.root, '/usr/sbin/locale-gen'] + log_check_call(command) + + +class SetTimezone(Task): + description = 'Setting the selected timezone' + phase = phases.system_modification + + def run(self, info): + from shutil import copy + tz_path = os.path.join(info.root, 'etc/timezone') + timezone = info.manifest.system['timezone'] + with open(tz_path, 'w') as tz_file: + tz_file.write(timezone) + zoneinfo_path = os.path.join(info.root, '/usr/share/zoneinfo', timezone) + localtime_path = os.path.join(info.root, 'etc/localtime') + copy(zoneinfo_path, localtime_path) diff --git a/providers/raw/tasks/network.py b/providers/raw/tasks/network.py new file mode 100644 index 0000000..0910608 --- /dev/null +++ b/providers/raw/tasks/network.py @@ -0,0 +1,38 @@ +from base import Task +from common import phases +import os.path + + +class RemoveDNSInfo(Task): + description = 'Removing resolv.conf' + phase = phases.system_modification + + def run(self, info): + from os import remove + remove(os.path.join(info.root, 'etc/resolv.conf')) + + +class ConfigureNetworkIF(Task): + description = 'Configuring network interfaces' + phase = phases.system_modification + + def run(self, info): + interfaces_path = os.path.join(info.root, 'etc/network/interfaces') + if_config = {'squeeze': ('auto lo\n' + 'iface lo inet loopback\n' + 'auto eth0\n' + 'iface eth0 inet dhcp\n'), + 'wheezy': 'auto eth0\n' + 'iface eth0 inet dhcp\n'} + with open(interfaces_path, 'a') as interfaces: + interfaces.write(if_config.get(info.manifest.system['release'])) + + +class ConfigureDHCP(Task): + description = 'Configuring the DHCP client' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') + sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/raw/tasks/packages.py b/providers/raw/tasks/packages.py new file mode 100644 index 0000000..89c1589 --- /dev/null +++ b/providers/raw/tasks/packages.py @@ -0,0 +1,49 @@ +from base import Task +from common import phases + + +class HostPackages(Task): + description = 'Determining required host packages' + phase = phases.preparation + + def run(self, info): + packages = set(['debootstrap', 'qemu-utils', 'parted', 'grub2', 'sysv-rc']) + if info.manifest.volume['filesystem'] == 'xfs': + packages.add('xfsprogs') + + info.host_packages = packages + + +class ImagePackages(Task): + description = 'Determining required image packages' + phase = phases.preparation + + def run(self, info): + manifest = info.manifest + # Add some basic packages we are going to need + include = set(['udev', + 'parted', + 'openssh-server', + # We could bootstrap without locales, but things just suck without them, error messages etc. + 'locales', + # Needed for the init scripts + 'file', + # isc-dhcp-client doesn't work properly with ec2 + 'dhcpcd', + 'grub2', + 'chkconfig', + 'openssh-client' + ]) + + exclude = set(['isc-dhcp-client', + 'isc-dhcp-common', + ]) + + # In squeeze, we need a special kernel flavor for xen + kernels = {'squeeze': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, + 'wheezy': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, } + include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + + info.img_packages = include, exclude diff --git a/providers/raw/tasks/security.py b/providers/raw/tasks/security.py new file mode 100644 index 0000000..77f986c --- /dev/null +++ b/providers/raw/tasks/security.py @@ -0,0 +1,42 @@ +from base import Task +from common import phases +import os.path + + +class EnableShadowConfig(Task): + description = 'Enabling shadowconfig' + phase = phases.system_modification + + def run(self, info): + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on']) + +class SetRootPassword(Task): + description = 'Set password for root' + phase = phases.system_modification + + def run(self, info): + from common.tools import log_check_call + if info.manifest.credentials['root']: + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) + +class DisableSSHPasswordAuthentication(Task): + description = 'Disabling SSH password authentication' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + if 'root' not in info.manifest.credentials: + # If no password set for root + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') + + +class DisableSSHDNSLookup(Task): + description = 'Disabling sshd remote host name lookup' + phase = phases.system_modification + + def run(self, info): + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + with open(sshd_config_path, 'a') as sshd_config: + sshd_config.write('UseDNS no')