From 778ec8c27055da5e4ae5690df87cfc471ef352fe Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 10:49:45 +0200 Subject: [PATCH] add open nebula management --- manifests/one-raw-virtio.manifest.json | 43 ++++++ providers/one/__init__.py | 77 +++++++++++ providers/one/assets/grub.d/40_custom | 81 ++++++++++++ .../one/assets/init.d/ec2-get-credentials | 45 +++++++ providers/one/assets/init.d/ec2-run-user-data | 46 +++++++ providers/one/assets/init.d/expand-volume | 26 ++++ .../one/assets/init.d/generate-ssh-hostkeys | 36 +++++ .../init.d/squeeze/generate-ssh-hostkeys | 33 +++++ providers/one/manifest-schema.json | 22 ++++ providers/one/manifest.py | 15 +++ providers/one/tasks/__init__.py | 1 + providers/one/tasks/ami.py | 76 +++++++++++ providers/one/tasks/apt.py | 78 +++++++++++ providers/one/tasks/boot.py | 60 +++++++++ providers/one/tasks/bootstrap.py | 54 ++++++++ providers/one/tasks/cleanup.py | 43 ++++++ providers/one/tasks/connection.py | 40 ++++++ providers/one/tasks/ebs.py | 89 +++++++++++++ providers/one/tasks/filesystem.py | 123 ++++++++++++++++++ providers/one/tasks/host.py | 29 +++++ providers/one/tasks/initd.py | 49 +++++++ providers/one/tasks/locale.py | 35 +++++ providers/one/tasks/network.py | 38 ++++++ providers/one/tasks/packages.py | 48 +++++++ providers/one/tasks/security.py | 32 +++++ 25 files changed, 1219 insertions(+) create mode 100644 manifests/one-raw-virtio.manifest.json create mode 100644 providers/one/__init__.py create mode 100644 providers/one/assets/grub.d/40_custom create mode 100644 providers/one/assets/init.d/ec2-get-credentials create mode 100644 providers/one/assets/init.d/ec2-run-user-data create mode 100644 providers/one/assets/init.d/expand-volume create mode 100644 providers/one/assets/init.d/generate-ssh-hostkeys create mode 100644 providers/one/assets/init.d/squeeze/generate-ssh-hostkeys create mode 100644 providers/one/manifest-schema.json create mode 100644 providers/one/manifest.py create mode 100644 providers/one/tasks/__init__.py create mode 100644 providers/one/tasks/ami.py create mode 100644 providers/one/tasks/apt.py create mode 100644 providers/one/tasks/boot.py create mode 100644 providers/one/tasks/bootstrap.py create mode 100644 providers/one/tasks/cleanup.py create mode 100644 providers/one/tasks/connection.py create mode 100644 providers/one/tasks/ebs.py create mode 100644 providers/one/tasks/filesystem.py create mode 100644 providers/one/tasks/host.py create mode 100644 providers/one/tasks/initd.py create mode 100644 providers/one/tasks/locale.py create mode 100644 providers/one/tasks/network.py create mode 100644 providers/one/tasks/packages.py create mode 100644 providers/one/tasks/security.py diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json new file mode 100644 index 0000000..96d5003 --- /dev/null +++ b/manifests/one-raw-virtio.manifest.json @@ -0,0 +1,43 @@ +{ + "provider" : "one", + "virtualization": "virtio", + "credentials" : { + "access-key": null, + "secret-key": null + }, + + "bootstrapper": { + "mount_dir": "/target", + "tarball": true, + "device": "/dev/sda" + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024 + }, + "plugins": { + "admin_user": { + "enabled": false + }, + "build_metadata": { + "enabled": false, + "path" : "/root/build-metadata-{ami_name}" + }, + "prebootstrapped": { + "enabled": false, + "snapshot": "" + } + } +} diff --git a/providers/one/__init__.py b/providers/one/__init__.py new file mode 100644 index 0000000..3372c1f --- /dev/null +++ b/providers/one/__init__.py @@ -0,0 +1,77 @@ +from manifest import Manifest +import logging +from tasks import packages +from tasks import connection +from tasks import host +from tasks import ami +from tasks import ebs +from tasks import filesystem +from tasks import bootstrap +from tasks import locale +from tasks import apt +from tasks import boot +from tasks import security +from tasks import network +from tasks import initd +from tasks import cleanup + + +def initialize(): + # Regardless of of loglevel, we don't want boto debug stuff, it's very noisy + logging.getLogger('boto').setLevel(logging.INFO) + + +def tasks(tasklist, manifest): + tasklist.add(packages.HostPackages(), + packages.ImagePackages(), + host.CheckPackages(), + host.GetInfo()) + tasklist.add(filesystem.FormatVolume()) + if manifest.volume['filesystem'].lower() == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + tasklist.add(filesystem.TuneVolumeFS()) + tasklist.add(filesystem.CreateMountDir(), + filesystem.MountVolume()) + if manifest.bootstrapper['tarball']: + tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.Bootstrap(), + filesystem.MountSpecials(), + locale.GenerateLocale(), + locale.SetTimezone(), + apt.DisableDaemonAutostart(), + apt.AptSources(), + apt.AptUpgrade(), + boot.ConfigureGrub(), + filesystem.ModifyFstab(), + boot.BlackListModules(), + boot.DisableGetTTYs(), + security.EnableShadowConfig(), + security.DisableSSHPasswordAuthentication(), + security.DisableSSHDNSLookup(), + network.RemoveDNSInfo(), + network.ConfigureNetworkIF(), + network.ConfigureDHCP(), + initd.ResolveInitScripts(), + initd.InstallInitScripts(), + cleanup.ClearMOTD(), + cleanup.ShredHostkeys(), + cleanup.CleanTMP(), + apt.PurgeUnusedPackages(), + apt.AptClean(), + apt.EnableDaemonAutostart(), + filesystem.UnmountSpecials(), + filesystem.UnmountVolume(), + filesystem.DeleteMountDir()) + + +def rollback_tasks(tasklist, tasks_completed, manifest): + completed = [type(task) for task in tasks_completed] + + def counter_task(task, counter): + if task in completed and counter not in completed: + tasklist.add(counter()) + + counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) + counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom new file mode 100644 index 0000000..799e887 --- /dev/null +++ b/providers/one/assets/grub.d/40_custom @@ -0,0 +1,81 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/xvda1 + + +cat << EOF +default ${GRUB_DEFAULT} +timeout ${GRUB_TIMEOUT} +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +title ${version} + root (hd0) + kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + initrd ${rel_dirname}/${initrd} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/one/assets/init.d/ec2-get-credentials b/providers/one/assets/init.d/ec2-get-credentials new file mode 100644 index 0000000..b304ae2 --- /dev/null +++ b/providers/one/assets/init.d/ec2-get-credentials @@ -0,0 +1,45 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-get-credentials +# Required-Start: $network +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Retrieve the ssh credentials and add to authorized_keys +### END INIT INFO +# +# ec2-get-credentials - Retrieve the ssh credentials and add to authorized_keys +# +# Based on /usr/local/sbin/ec2-get-credentials from Amazon's ami-20b65349 +# + +prog=$(basename $0) +logger="logger -t $prog" + +public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key +username='root' +# A little bit of nastyness to get the homedir, when the username is a variable +ssh_dir="`eval printf ~$username`/.ssh" +authorized_keys="$ssh_dir/authorized_keys" + +# Try to get the ssh public key from instance data. +public_key=`wget -qO - $public_key_url` +if [ -n "$public_key" ]; then + if [ ! -f $authorized_keys ]; then + if [ ! -d $ssh_dir ]; then + mkdir -m 700 $ssh_dir + chown $username:$username $ssh_dir + fi + touch $authorized_keys + chown $username:$username $authorized_keys + fi + + if ! grep -s -q "$public_key" $authorized_keys; then + printf "\n%s" -- "$public_key" >> $authorized_keys + $logger "New ssh key added to $authorized_keys from $public_key_url" + chmod 600 $authorized_keys + chown $username:$username $authorized_keys + fi +fi diff --git a/providers/one/assets/init.d/ec2-run-user-data b/providers/one/assets/init.d/ec2-run-user-data new file mode 100644 index 0000000..17b8b6f --- /dev/null +++ b/providers/one/assets/init.d/ec2-run-user-data @@ -0,0 +1,46 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-run-user-data +# Required-Start: ec2-get-credentials +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Run instance user-data if it looks like a script. +### END INIT INFO +# +# Only retrieves and runs the user-data script once per instance. If +# you want the user-data script to run again (e.g., on the next boot) +# then readd this script with insserv: +# insserv -d ec2-run-user-data +# +prog=$(basename $0) +logger="logger -t $prog" +instance_data_url="http://169.254.169.254/2008-02-01" + + +# Retrieve the instance user-data and run it if it looks like a script +user_data_file=$(tempfile --prefix ec2 --suffix .user-data --mode 700) +$logger "Retrieving user-data" +wget -qO $user_data_file $instance_data_url/user-data 2>&1 | $logger + +if [ $(file -b --mime-type $user_data_file) = 'application/x-gzip' ]; then + $logger "Uncompressing gzip'd user-data" + mv $user_data_file $user_data_file.gz + gunzip $user_data_file.gz +fi + +if [ ! -s $user_data_file ]; then + $logger "No user-data available" +elif head -1 $user_data_file | egrep -v '^#!'; then + $logger "Skipping user-data as it does not begin with #!" +else + $logger "Running user-data" + $user_data_file 2>&1 | logger -t "user-data" + $logger "user-data exit code: $?" +fi +rm -f $user_data_file + +# Disable this script, it may only run once +insserv -r $0 diff --git a/providers/one/assets/init.d/expand-volume b/providers/one/assets/init.d/expand-volume new file mode 100644 index 0000000..3b2d2a6 --- /dev/null +++ b/providers/one/assets/init.d/expand-volume @@ -0,0 +1,26 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: expand-volume +# Required-Start: +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Expand the filesystem of the mounted root volume to its maximum possible size +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +device_path="/dev/xvda1" + +filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'` + +case $filesystem in + xfs) xfs_growfs / ;; + ext2) resize2fs $device_path ;; + ext3) resize2fs $device_path ;; + ext4) resize2fs $device_path ;; + *) $logger "The filesystem $filesystem was not recognized. Unable to expand size." ;; +esac diff --git a/providers/one/assets/init.d/generate-ssh-hostkeys b/providers/one/assets/init.d/generate-ssh-hostkeys new file mode 100644 index 0000000..c9efb12 --- /dev/null +++ b/providers/one/assets/init.d/generate-ssh-hostkeys @@ -0,0 +1,36 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" +ecdsa_key="/etc/ssh/ssh_host_ecdsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key -a -f $ecdsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' +[ -f $ecdsa_key ] || ssh-keygen -f $ecdsa_key -t ecdsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +ssh-keygen -l -f $ecdsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys b/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys new file mode 100644 index 0000000..148b87d --- /dev/null +++ b/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys @@ -0,0 +1,33 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/one/manifest-schema.json b/providers/one/manifest-schema.json new file mode 100644 index 0000000..0a6a038 --- /dev/null +++ b/providers/one/manifest-schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "EC2 manifest", + "type": "object", + "properties": { + "volume": { + "type": "object", + "properties": { + "backing": { + "type": "string", + "enum": ["raw", "qcow2"] + }, + "filesystem": { + "type": "string", + "enum": ["ext2", "ext3", "ext4", "xfs"] + } + }, + "required": ["backing", "filesystem"] + } + }, + "required": ["volume"] +} diff --git a/providers/one/manifest.py b/providers/one/manifest.py new file mode 100644 index 0000000..d2e0f2c --- /dev/null +++ b/providers/one/manifest.py @@ -0,0 +1,15 @@ +import base + + +class Manifest(base.Manifest): + def validate(self, data): + super(Manifest, self).validate(data) + from os import path + schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') + self.schema_validate(data, schema_path) + + def parse(self, data): + super(Manifest, self).parse(data) + self.credentials = data['credentials'] + self.virtualization = data['virtualization'] + self.image = data['image'] diff --git a/providers/one/tasks/__init__.py b/providers/one/tasks/__init__.py new file mode 100644 index 0000000..873a99d --- /dev/null +++ b/providers/one/tasks/__init__.py @@ -0,0 +1 @@ +__all__ = ['packages', 'connection', 'host', 'ec2'] diff --git a/providers/one/tasks/ami.py b/providers/one/tasks/ami.py new file mode 100644 index 0000000..fbabbf3 --- /dev/null +++ b/providers/one/tasks/ami.py @@ -0,0 +1,76 @@ +from base import Task +from common import phases +from ebs import CreateSnapshot +from connection import Connect +from common.exceptions import TaskError + + +class AMIName(Task): + description = 'Determining the AMI name' + phase = phases.preparation + after = [Connect] + + def run(self, info): + image_vars = {'release': info.manifest.system['release'], + 'architecture': info.manifest.system['architecture'], + 'virtualization': info.manifest.virtualization, + 'backing': info.manifest.volume['backing']} + from datetime import datetime + now = datetime.now() + time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', + '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', + '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] + for var in time_vars: + image_vars[var] = now.strftime(var) + + ami_name = info.manifest.image['name'].format(**image_vars) + ami_description = info.manifest.image['description'].format(**image_vars) + + images = info.connection.get_all_images() + for image in images: + if ami_name == image.name: + msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name) + raise TaskError(msg) + info.ami_name = ami_name + info.ami_description = ami_description + + +class RegisterAMI(Task): + description = 'Registering the image as an AMI' + phase = phases.image_registration + after = [CreateSnapshot] + + def run(self, info): + arch = {'i386': 'i386', + 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', + 'i386': 'aki-b6aa75df'}, + 'us-west-1': {'amd64': 'aki-f77e26b2', + 'i386': 'aki-f57e26b0'}, + 'us-west-2': {'amd64': 'aki-fc37bacc', + 'i386': 'aki-fa37baca'}, + 'eu-west-1': {'amd64': 'aki-71665e05', + 'i386': 'aki-75665e01'}, + 'ap-southeast-1': {'amd64': 'aki-fe1354ac', + 'i386': 'aki-f81354aa'}, + 'ap-southeast-2': {'amd64': 'aki-31990e0b', + 'i386': 'aki-33990e09'}, + 'ap-northeast-1': {'amd64': 'aki-44992845', + 'i386': 'aki-42992843'}, + 'sa-east-1': {'amd64': 'aki-c48f51d9', + 'i386': 'aki-ca8f51d7'}, + 'us-gov-west-1': {'amd64': 'aki-79a4c05a', + 'i386': 'aki-7ba4c058'}} + kernel_id = kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) + + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=int(info.manifest.volume['size']/1024)) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda1'] = block_device + + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='/dev/sda1', + block_device_map=block_device_map) diff --git a/providers/one/tasks/apt.py b/providers/one/tasks/apt.py new file mode 100644 index 0000000..e039df4 --- /dev/null +++ b/providers/one/tasks/apt.py @@ -0,0 +1,78 @@ +from base import Task +from common import phases +from common.tools import log_check_call +import os +from locale import GenerateLocale + + +class AptSources(Task): + description = 'Adding aptitude sources' + phase = phases.system_modification + + def run(self, info): + sources_path = os.path.join(info.root, 'etc/apt/sources.list') + with open(sources_path, 'w') as apt_sources: + apt_sources.write(('deb {apt_mirror} {release} main\n' + 'deb-src {apt_mirror} {release} main\n' + .format(apt_mirror='http://http.debian.net/debian', + release=info.manifest.system['release']))) + apt_sources.write(('deb {apt_mirror} {release}/updates main\n' + 'deb-src {apt_mirror} {release}/updates main\n' + .format(apt_mirror='http://security.debian.org/', + release=info.manifest.system['release']))) + + +class DisableDaemonAutostart(Task): + description = 'Disabling daemon autostart' + phase = phases.system_modification + + def run(self, info): + rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d') + with open(rc_policy_path, 'w') as rc_policy: + rc_policy.write(('#!/bin/sh\n' + 'exit 101')) + import stat + os.chmod(rc_policy_path, + stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + +class AptUpgrade(Task): + description = 'Upgrading packages and fixing broken dependencies' + phase = phases.system_modification + after = [GenerateLocale, AptSources, DisableDaemonAutostart] + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-f', '-y', 'install']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-y', 'upgrade']) + + +class PurgeUnusedPackages(Task): + description = 'Removing unused packages' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'autoremove', '--purge']) + + +class AptClean(Task): + description = 'Clearing the aptitude cache' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'clean']) + + lists = os.path.join(info.root, 'var/lib/apt/lists') + for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]: + if os.path.isfile(list_file): + os.remove(list_file) + + +class EnableDaemonAutostart(Task): + description = 'Re-enabling daemon autostart after installation' + phase = phases.system_cleaning + + def run(self, info): + os.remove(os.path.join(info.root, 'usr/sbin/policy-rc.d')) diff --git a/providers/one/tasks/boot.py b/providers/one/tasks/boot.py new file mode 100644 index 0000000..4265b9d --- /dev/null +++ b/providers/one/tasks/boot.py @@ -0,0 +1,60 @@ +from base import Task +from common import phases +import os + + +class ConfigureGrub(Task): + description = 'Configuring grub' + phase = phases.system_modification + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + grubd = os.path.join(info.root, 'etc/grub.d') + for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]: + os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all) + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom')) + script_dst = os.path.join(info.root, 'etc/grub.d/40_custom') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + from common.tools import sed_i + grub_def = os.path.join(info.root, 'etc/default/grub') + sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' + 'GRUB_HIDDEN_TIMEOUT=true') + + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) + + +class BlackListModules(Task): + description = 'Blacklisting kernel modules' + phase = phases.system_modification + + def run(self, info): + blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf') + with open(blacklist_path, 'a') as blacklist: + blacklist.write(('# disable pc speaker\n' + 'blacklist pcspkr')) + + +class DisableGetTTYs(Task): + description = 'Disabling getty processes' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + inittab_path = os.path.join(info.root, 'etc/inittab') + tty1 = '1:2345:respawn:/sbin/getty 38400 tty1' + sed_i(inittab_path, '^'+tty1, '#'+tty1) + ttyx = ':23:respawn:/sbin/getty 38400 tty' + for i in range(2, 6): + i = str(i) + sed_i(inittab_path, '^'+i+ttyx+i, '#'+i+ttyx+i) diff --git a/providers/one/tasks/bootstrap.py b/providers/one/tasks/bootstrap.py new file mode 100644 index 0000000..9e27f36 --- /dev/null +++ b/providers/one/tasks/bootstrap.py @@ -0,0 +1,54 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import logging +log = logging.getLogger(__name__) + + +def get_bootstrap_args(info): + executable = ['/usr/sbin/debootstrap'] + options = ['--arch=' + info.manifest.system['architecture']] + include, exclude = info.img_packages + if len(include) > 0: + options.append('--include=' + ','.join(include)) + if len(exclude) > 0: + options.append('--exclude=' + ','.join(exclude)) + arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian'] + return executable, options, arguments + + +class MakeTarball(Task): + description = 'Creating bootstrap tarball' + phase = phases.os_installation + + def run(self, info): + from hashlib import sha1 + import os.path + executable, options, arguments = get_bootstrap_args(info) + # Filter info.root which points at /target/volume-id, we won't ever hit anything with that in there. + hash_args = [arg for arg in arguments if arg != info.root] + tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] + tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) + info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + if os.path.isfile(info.tarball): + log.debug('Found matching tarball, skipping download') + else: + from common.tools import log_call + status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments) + if status != 1: + msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status) + raise TaskError(msg) + + +class Bootstrap(Task): + description = 'Installing Debian' + phase = phases.os_installation + after = [MakeTarball] + + def run(self, info): + executable, options, arguments = get_bootstrap_args(info) + if hasattr(info, 'tarball'): + options.extend(['--unpack-tarball=' + info.tarball]) + + from common.tools import log_check_call + log_check_call(executable + options + arguments) diff --git a/providers/one/tasks/cleanup.py b/providers/one/tasks/cleanup.py new file mode 100644 index 0000000..7b793a4 --- /dev/null +++ b/providers/one/tasks/cleanup.py @@ -0,0 +1,43 @@ +from base import Task +from common import phases +import os + + +class ClearMOTD(Task): + description = 'Clearing the MOTD' + phase = phases.system_cleaning + + def run(self, info): + with open('/var/run/motd', 'w'): + pass + + +class ShredHostkeys(Task): + description = 'Securely deleting ssh hostkeys' + phase = phases.system_cleaning + + def run(self, info): + ssh_hostkeys = ['ssh_host_dsa_key', + 'ssh_host_rsa_key'] + if info.manifest.system['release'] != 'squeeze': + ssh_hostkeys.append('ssh_host_ecdsa_key') + + private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys] + public = [path + '.pub' for path in private] + + from common.tools import log_check_call + log_check_call(['/usr/bin/shred', '--remove'] + private + public) + + +class CleanTMP(Task): + description = 'Removing temporary files' + phase = phases.system_cleaning + + def run(self, info): + tmp = os.path.join(info.root, 'tmp') + for tmp_file in [os.path.join(tmp, f) for f in os.listdir(tmp)]: + os.remove(tmp_file) + + log = os.path.join(info.root, 'var/log/') + os.remove(os.path.join(log, 'bootstrap.log')) + os.remove(os.path.join(log, 'dpkg.log')) diff --git a/providers/one/tasks/connection.py b/providers/one/tasks/connection.py new file mode 100644 index 0000000..2c2ce19 --- /dev/null +++ b/providers/one/tasks/connection.py @@ -0,0 +1,40 @@ +from base import Task +from common import phases +import host + + +class GetCredentials(Task): + description = 'Getting AWS credentials' + phase = phases.preparation + + def run(self, info): + info.credentials = self.get_credentials(info.manifest) + + def get_credentials(self, manifest): + from os import getenv + # manifest overrides environment + if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): + return {'access_key': manifest.credentials['access-key'], + 'secret_key': manifest.credentials['secret-key']} + if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')): + return {'access_key': getenv('EC2_ACCESS_KEY'), + 'secret_key': getenv('EC2_SECRET_KEY')} + + if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])): + raise RuntimeError('Both the access key and secret key must be specified in the manifest.') + if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))): + raise RuntimeError('Both the access key and secret key must be specified as environment variables.') + + raise RuntimeError('No ec2 credentials found.') + + +class Connect(Task): + description = 'Connecting to EC2' + phase = phases.preparation + after = [GetCredentials, host.GetInfo] + + def run(self, info): + from boto.ec2 import connect_to_region + info.connection = connect_to_region(info.host['region'], + aws_access_key_id=info.credentials['access_key'], + aws_secret_access_key=info.credentials['secret_key']) diff --git a/providers/one/tasks/ebs.py b/providers/one/tasks/ebs.py new file mode 100644 index 0000000..a07b1a8 --- /dev/null +++ b/providers/one/tasks/ebs.py @@ -0,0 +1,89 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from connection import Connect +from filesystem import UnmountVolume +import time + + +class CreateVolume(Task): + phase = phases.volume_creation + after = [Connect] + + description = 'Creating an EBS volume for bootstrapping' + + def run(self, info): + volume_size = int(info.manifest.volume['size']/1024) + + info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone']) + while info.volume.volume_state() != 'available': + time.sleep(5) + info.volume.update() + + +class AttachVolume(Task): + phase = phases.volume_creation + after = [CreateVolume] + + description = 'Attaching the EBS volume' + + def run(self, info): + def char_range(c1, c2): + """Generates the characters from `c1` to `c2`, inclusive.""" + for c in xrange(ord(c1), ord(c2)+1): + yield chr(c) + + import os.path + info.bootstrap_device = {} + for letter in char_range('f', 'z'): + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + info.bootstrap_device['path'] = dev_path + info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) + break + if 'path' not in info.bootstrap_device: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) + while info.volume.attachment_state() != 'attached': + time.sleep(2) + info.volume.update() + + +class DetachVolume(Task): + phase = phases.volume_unmounting + after = [UnmountVolume] + + description = 'Detaching the EBS volume' + + def run(self, info): + info.volume.detach() + while info.volume.attachment_state() is not None: + time.sleep(2) + info.volume.update() + + +class CreateSnapshot(Task): + description = 'Creating a snapshot of the EBS volume' + phase = phases.image_registration + + def run(self, info): + info.snapshot = info.volume.create_snapshot() + while info.snapshot.status != 'completed': + time.sleep(2) + info.snapshot.update() + + +class DeleteVolume(Task): + phase = phases.cleaning + after = [] + + description = 'Deleting the EBS volume' + + def run(self, info): + info.volume.delete() + del info.volume + + +class VolumeError(TaskError): + pass diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py new file mode 100644 index 0000000..50481f4 --- /dev/null +++ b/providers/one/tasks/filesystem.py @@ -0,0 +1,123 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from common.tools import log_check_call +from bootstrap import Bootstrap + + +class FormatVolume(Task): + description = 'Formatting the volume' + phase = phases.volume_preparation + + def run(self, info): + dev_path = info.manifest.bootstrapper['device'] + mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) + log_check_call([mkfs, dev_path]) + + +class TuneVolumeFS(Task): + description = 'Tuning the bootstrap volume filesystem' + phase = phases.volume_preparation + after = [FormatVolume] + + def run(self, info): + dev_path = info.bootstrap_device['path'] + # Disable the time based filesystem check + log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) + + +class AddXFSProgs(Task): + description = 'Adding `xfsprogs\' to the image packages' + phase = phases.preparation + + def run(self, info): + include, exclude = info.img_packages + include.add('xfsprogs') + + +class CreateMountDir(Task): + description = 'Creating mountpoint for the bootstrap volume' + phase = phases.volume_mounting + + def run(self, info): + import os + mount_dir = info.manifest.bootstrapper['mount_dir'] + info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id) + # Works recursively, fails if last part exists, which is exaclty what we want. + os.makedirs(info.root) + + +class MountVolume(Task): + description = 'Mounting the bootstrap volume' + phase = phases.volume_mounting + after = [CreateMountDir] + + def run(self, info): + with open('/proc/mounts') as mounts: + for mount in mounts: + if info.root in mount: + msg = 'Something is already mounted at {root}'.format(root=info.root) + raise TaskError(msg) + + log_check_call(['/bin/mount', info.bootstrap_device['path'], info.root]) + + +class MountSpecials(Task): + description = 'Mounting special block devices' + phase = phases.os_installation + after = [Bootstrap] + + def run(self, info): + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'devpts', 'none', '/dev/pts']) + + +class UnmountSpecials(Task): + description = 'Unmunting special block devices' + phase = phases.volume_unmounting + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) + + +class UnmountVolume(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountSpecials] + + def run(self, info): + log_check_call(['/bin/umount', info.root]) + + +class DeleteMountDir(Task): + description = 'Deleting mountpoint for the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountVolume] + + def run(self, info): + import os + os.rmdir(info.root) + del info.root + + +class ModifyFstab(Task): + description = 'Adding root volume to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + mount_opts = ['defaults'] + if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if info.manifest.volume['filesystem'].lower() == 'xfs': + mount_opts.append('nobarrier') + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'a') as fstab: + fstab.write(('/dev/xvda1 / {filesystem} {mount_opts} 1 1\n' + .format(filesystem=info.manifest.volume['filesystem'].lower(), + mount_opts=','.join(mount_opts)))) diff --git a/providers/one/tasks/host.py b/providers/one/tasks/host.py new file mode 100644 index 0000000..61af65a --- /dev/null +++ b/providers/one/tasks/host.py @@ -0,0 +1,29 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import packages + + +class CheckPackages(Task): + description = 'Checking installed host packages' + phase = phases.preparation + after = [packages.HostPackages, packages.ImagePackages] + + def run(self, info): + from common.tools import log_check_call + from subprocess import CalledProcessError + for package in info.host_packages: + try: + log_check_call(['/usr/bin/dpkg', '-s', package]) + except CalledProcessError: + msg = "The package ``{0}\'\' is not installed".format(package) + raise TaskError(msg) + + +class GetInfo(Task): + description = 'Retrieving instance metadata' + phase = phases.preparation + + def run(self, info): + info.host = {} + return info diff --git a/providers/one/tasks/initd.py b/providers/one/tasks/initd.py new file mode 100644 index 0000000..5a479fe --- /dev/null +++ b/providers/one/tasks/initd.py @@ -0,0 +1,49 @@ +from base import Task +from common import phases +import os.path + + +class ResolveInitScripts(Task): + description = 'Determining which startup scripts to install or disable' + phase = phases.system_modification + + def run(self, info): + init_scripts = {'ec2-get-credentials': 'ec2-get-credentials', + 'ec2-run-user-data': 'ec2-run-user-data', + 'expand-volume': 'expand-volume'} + + init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' + if info.manifest.system['release'] == 'squeeze': + init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + + disable_scripts = ['hwclock.sh'] + if info.manifest.system['release'] == 'squeeze': + disable_scripts.append('hwclockfirst.sh') + + for name, path in init_scripts.iteritems(): + init_scripts[name] = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d', path)) + + info.initd = {'install': init_scripts, + 'disable': disable_scripts} + + +class InstallInitScripts(Task): + description = 'Installing startup scripts' + phase = phases.system_modification + after = [ResolveInitScripts] + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + from shutil import copy + from common.tools import log_check_call + for name, src in info.initd['install'].iteritems(): + dst = os.path.join(info.root, 'etc/init.d', name) + copy(src, dst) + os.chmod(dst, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-d', name]) + + for name in info.initd['disable']: + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-r', name]) diff --git a/providers/one/tasks/locale.py b/providers/one/tasks/locale.py new file mode 100644 index 0000000..de3e434 --- /dev/null +++ b/providers/one/tasks/locale.py @@ -0,0 +1,35 @@ +from base import Task +from common import phases +import os.path + + +class GenerateLocale(Task): + description = 'Generating the selected locale' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + from common.tools import log_check_call + locale_gen = os.path.join(info.root, 'etc/locale.gen') + locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'], + charmap=info.manifest.system['charmap']) + search = '# ' + locale_str + sed_i(locale_gen, search, locale_str) + + command = ['/usr/sbin/chroot', info.root, '/usr/sbin/dpkg-reconfigure', '--priority=critical', 'locales'] + log_check_call(command) + + +class SetTimezone(Task): + description = 'Setting the selected timezone' + phase = phases.system_modification + + def run(self, info): + from shutil import copy + tz_path = os.path.join(info.root, 'etc/timezone') + timezone = info.manifest.system['timezone'] + with open(tz_path, 'w') as tz_file: + tz_file.write(timezone) + zoneinfo_path = os.path.join(info.root, '/usr/share/zoneinfo', timezone) + localtime_path = os.path.join(info.root, 'etc/localtime') + copy(zoneinfo_path, localtime_path) diff --git a/providers/one/tasks/network.py b/providers/one/tasks/network.py new file mode 100644 index 0000000..0910608 --- /dev/null +++ b/providers/one/tasks/network.py @@ -0,0 +1,38 @@ +from base import Task +from common import phases +import os.path + + +class RemoveDNSInfo(Task): + description = 'Removing resolv.conf' + phase = phases.system_modification + + def run(self, info): + from os import remove + remove(os.path.join(info.root, 'etc/resolv.conf')) + + +class ConfigureNetworkIF(Task): + description = 'Configuring network interfaces' + phase = phases.system_modification + + def run(self, info): + interfaces_path = os.path.join(info.root, 'etc/network/interfaces') + if_config = {'squeeze': ('auto lo\n' + 'iface lo inet loopback\n' + 'auto eth0\n' + 'iface eth0 inet dhcp\n'), + 'wheezy': 'auto eth0\n' + 'iface eth0 inet dhcp\n'} + with open(interfaces_path, 'a') as interfaces: + interfaces.write(if_config.get(info.manifest.system['release'])) + + +class ConfigureDHCP(Task): + description = 'Configuring the DHCP client' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') + sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py new file mode 100644 index 0000000..b5474b4 --- /dev/null +++ b/providers/one/tasks/packages.py @@ -0,0 +1,48 @@ +from base import Task +from common import phases + + +class HostPackages(Task): + description = 'Determining required host packages' + phase = phases.preparation + + def run(self, info): + packages = set(['debootstrap']) + if info.manifest.volume['filesystem'] == 'xfs': + packages.add('xfsprogs') + + info.host_packages = packages + + +class ImagePackages(Task): + description = 'Determining required image packages' + phase = phases.preparation + + def run(self, info): + manifest = info.manifest + # Add some basic packages we are going to need + include = set(['udev', + 'openssh-server', + # We could bootstrap without locales, but things just suck without them, error messages etc. + 'locales', + # Needed for the init scripts + 'file', + # isc-dhcp-client doesn't work properly with ec2 + 'dhcpcd', + ]) + + if manifest.virtualization == 'pvm': + include.add('grub-pc') + + exclude = set(['isc-dhcp-client', + 'isc-dhcp-common', + ]) + + # In squeeze, we need a special kernel flavor for xen + kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64', + 'i386': 'linux-image-xen-686', }, + 'wheezy': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, } + include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + + info.img_packages = include, exclude diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py new file mode 100644 index 0000000..50d8db6 --- /dev/null +++ b/providers/one/tasks/security.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import os.path + + +class EnableShadowConfig(Task): + description = 'Enabling shadowconfig' + phase = phases.system_modification + + def run(self, info): + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on']) + + +class DisableSSHPasswordAuthentication(Task): + description = 'Disabling SSH password authentication' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') + + +class DisableSSHDNSLookup(Task): + description = 'Disabling sshd remote host name lookup' + phase = phases.system_modification + + def run(self, info): + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + with open(sshd_config_path, 'a') as sshd_config: + sshd_config.write('UseDNS no')