kevinheyer/bootstrap-vz
1
0
Fork 0
mirror of https://github.com/kevingruesser/bootstrap-vz.git synced 2025-08-22 18:00:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add open nebula management

Browse source
This commit is contained in:
Olivier Sallou 2013-07-10 10:49:45 +02:00
parent 311db52f7b
commit 778ec8c270
25 changed files with 1219 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
manifests/one-raw-virtio.manifest.json Normal file
View file

@ -0,0 +1,43 @@
{
"provider" : "one",
"virtualization": "virtio",
"credentials" : {
"access-key": null,
"secret-key": null
},
"bootstrapper": {
"mount_dir": "/target",
"tarball": true,
"device": "/dev/sda"
},
"image": {
"name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}",
"description": "Debian {release} {architecture} AMI ({virtualization})"
},
"system": {
"release" : "wheezy",
"architecture": "amd64",
"timezone" : "UTC",
"locale" : "en_US",
"charmap" : "UTF-8"
},
"volume": {
"backing" : "raw",
"filesystem": "ext4",
"size" : 1024
},
"plugins": {
"admin_user": {
"enabled": false
},
"build_metadata": {
"enabled": false,
"path" : "/root/build-metadata-{ami_name}"
},
"prebootstrapped": {
"enabled": false,
"snapshot": ""
}
}
}

77
providers/one/__init__.py Normal file
View file

@ -0,0 +1,77 @@
from manifest import Manifest
import logging
from tasks import packages
from tasks import connection
from tasks import host
from tasks import ami
from tasks import ebs
from tasks import filesystem
from tasks import bootstrap
from tasks import locale
from tasks import apt
from tasks import boot
from tasks import security
from tasks import network
from tasks import initd
from tasks import cleanup
def initialize():
# Regardless of of loglevel, we don't want boto debug stuff, it's very noisy
logging.getLogger('boto').setLevel(logging.INFO)
def tasks(tasklist, manifest):
tasklist.add(packages.HostPackages(),
packages.ImagePackages(),
host.CheckPackages(),
host.GetInfo())
tasklist.add(filesystem.FormatVolume())
if manifest.volume['filesystem'].lower() == 'xfs':
tasklist.add(filesystem.AddXFSProgs())
if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
tasklist.add(filesystem.TuneVolumeFS())
tasklist.add(filesystem.CreateMountDir(),
filesystem.MountVolume())
if manifest.bootstrapper['tarball']:
tasklist.add(bootstrap.MakeTarball())
tasklist.add(bootstrap.Bootstrap(),
filesystem.MountSpecials(),
locale.GenerateLocale(),
locale.SetTimezone(),
apt.DisableDaemonAutostart(),
apt.AptSources(),
apt.AptUpgrade(),
boot.ConfigureGrub(),
filesystem.ModifyFstab(),
boot.BlackListModules(),
boot.DisableGetTTYs(),
security.EnableShadowConfig(),
security.DisableSSHPasswordAuthentication(),
security.DisableSSHDNSLookup(),
network.RemoveDNSInfo(),
network.ConfigureNetworkIF(),
network.ConfigureDHCP(),
initd.ResolveInitScripts(),
initd.InstallInitScripts(),
cleanup.ClearMOTD(),
cleanup.ShredHostkeys(),
cleanup.CleanTMP(),
apt.PurgeUnusedPackages(),
apt.AptClean(),
apt.EnableDaemonAutostart(),
filesystem.UnmountSpecials(),
filesystem.UnmountVolume(),
filesystem.DeleteMountDir())
def rollback_tasks(tasklist, tasks_completed, manifest):
completed = [type(task) for task in tasks_completed]
def counter_task(task, counter):
if task in completed and counter not in completed:
tasklist.add(counter())
counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
counter_task(filesystem.MountVolume, filesystem.UnmountVolume)
counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials)

81
providers/one/assets/grub.d/40_custom Normal file
View file

@ -0,0 +1,81 @@
#!/bin/sh
# This file generates the old menu.lst configuration with grub2
# It was copied from tomheadys github repo:
# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom
prefix=/usr
exec_prefix=${prefix}
bindir=${exec_prefix}/bin
libdir=${exec_prefix}/lib
. ${libdir}/grub/grub-mkconfig_lib
export TEXTDOMAIN=grub
export TEXTDOMAINDIR=${prefix}/share/locale
GRUB_DEVICE=/dev/xvda1
cat << EOF
default ${GRUB_DEFAULT}
timeout ${GRUB_TIMEOUT}
EOF
if ${GRUB_HIDDEN_TIMEOUT:-false}; then
printf "hiddenmenu\n"
fi
linux_entry ()
{
os="$1"
version="$2"
args="$4"
title="$(gettext_quoted "%s, with Linux %s")"
cat << EOF
title ${version}
root (hd0)
kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args}
initrd ${rel_dirname}/${initrd}
EOF
}
list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do
if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi
done`
prepare_boot_cache=
while [ "x$list" != "x" ] ; do
linux=`version_find_latest $list`
basename=`basename $linux`
dirname=`dirname $linux`
rel_dirname=`make_system_path_relative_to_its_root $dirname`
version=`echo $basename | sed -e "s,^[^0-9]*-,,g"`
alt_version=`echo $version | sed -e "s,\.old$,,g"`
linux_root_device_thisversion="${LINUX_ROOT_DEVICE}"
initrd=
for i in "initrd.img-${version}" "initrd-${version}.img" \
"initrd-${version}" "initramfs-${version}.img" \
"initrd.img-${alt_version}" "initrd-${alt_version}.img" \
"initrd-${alt_version}" "initramfs-${alt_version}.img"; do
if test -e "${dirname}/${i}" ; then
initrd="$i"
break
fi
done
initramfs=
for i in "config-${version}" "config-${alt_version}"; do
if test -e "${dirname}/${i}" ; then
initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"`
break
fi
done
linux_entry "${OS}" "${version}" \
"${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}"
list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '`
done

45
providers/one/assets/init.d/ec2-get-credentials Normal file
View file

@ -0,0 +1,45 @@
#!/bin/bash
### BEGIN INIT INFO
# Provides: ec2-get-credentials
# Required-Start: $network
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start: 2 3 4 5
# Default-Stop:
# Description: Retrieve the ssh credentials and add to authorized_keys
### END INIT INFO
#
# ec2-get-credentials - Retrieve the ssh credentials and add to authorized_keys
#
# Based on /usr/local/sbin/ec2-get-credentials from Amazon's ami-20b65349
#
prog=$(basename $0)
logger="logger -t $prog"
public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key
username='root'
# A little bit of nastyness to get the homedir, when the username is a variable
ssh_dir="`eval printf ~$username`/.ssh"
authorized_keys="$ssh_dir/authorized_keys"
# Try to get the ssh public key from instance data.
public_key=`wget -qO - $public_key_url`
if [ -n "$public_key" ]; then
if [ ! -f $authorized_keys ]; then
if [ ! -d $ssh_dir ]; then
mkdir -m 700 $ssh_dir
chown $username:$username $ssh_dir
fi
touch $authorized_keys
chown $username:$username $authorized_keys
fi
if ! grep -s -q "$public_key" $authorized_keys; then
printf "\n%s" -- "$public_key" >> $authorized_keys
$logger "New ssh key added to $authorized_keys from $public_key_url"
chmod 600 $authorized_keys
chown $username:$username $authorized_keys
fi
fi

46
providers/one/assets/init.d/ec2-run-user-data Normal file
View file

@ -0,0 +1,46 @@
#!/bin/bash
### BEGIN INIT INFO
# Provides: ec2-run-user-data
# Required-Start: ec2-get-credentials
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start: 2 3 4 5
# Default-Stop:
# Description: Run instance user-data if it looks like a script.
### END INIT INFO
#
# Only retrieves and runs the user-data script once per instance. If
# you want the user-data script to run again (e.g., on the next boot)
# then readd this script with insserv:
# insserv -d ec2-run-user-data
#
prog=$(basename $0)
logger="logger -t $prog"
instance_data_url="http://169.254.169.254/2008-02-01"
# Retrieve the instance user-data and run it if it looks like a script
user_data_file=$(tempfile --prefix ec2 --suffix .user-data --mode 700)
$logger "Retrieving user-data"
wget -qO $user_data_file $instance_data_url/user-data 2>&1 | $logger
if [ $(file -b --mime-type $user_data_file) = 'application/x-gzip' ]; then
$logger "Uncompressing gzip'd user-data"
mv $user_data_file $user_data_file.gz
gunzip $user_data_file.gz
fi
if [ ! -s $user_data_file ]; then
$logger "No user-data available"
elif head -1 $user_data_file | egrep -v '^#!'; then
$logger "Skipping user-data as it does not begin with #!"
else
$logger "Running user-data"
$user_data_file 2>&1 | logger -t "user-data"
$logger "user-data exit code: $?"
fi
rm -f $user_data_file
# Disable this script, it may only run once
insserv -r $0

26
providers/one/assets/init.d/expand-volume Normal file
View file

@ -0,0 +1,26 @@
#!/bin/bash
### BEGIN INIT INFO
# Provides: expand-volume
# Required-Start:
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start: 2 3 4 5
# Default-Stop:
# Description: Expand the filesystem of the mounted root volume to its maximum possible size
### END INIT INFO
prog=$(basename $0)
logger="logger -t $prog"
device_path="/dev/xvda1"
filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'`
case $filesystem in
xfs) xfs_growfs / ;;
ext2) resize2fs $device_path ;;
ext3) resize2fs $device_path ;;
ext4) resize2fs $device_path ;;
*) $logger "The filesystem $filesystem was not recognized. Unable to expand size." ;;
esac

36
providers/one/assets/init.d/generate-ssh-hostkeys Normal file
View file

@ -0,0 +1,36 @@
#!/bin/sh
### BEGIN INIT INFO
# Provides: generate-ssh-hostkeys
# Required-Start: $local_fs
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start: S
# Default-Stop:
# Description: Generate ssh host keys if they do not exist
### END INIT INFO
prog=$(basename $0)
logger="logger -t $prog"
rsa_key="/etc/ssh/ssh_host_rsa_key"
dsa_key="/etc/ssh/ssh_host_dsa_key"
ecdsa_key="/etc/ssh/ssh_host_ecdsa_key"
# Exit if the hostkeys already exist
if [ -f $rsa_key -a -f $dsa_key -a -f $ecdsa_key ]; then
exit
fi
# Generate the ssh host keys
[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N ''
[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N ''
[ -f $ecdsa_key ] || ssh-keygen -f $ecdsa_key -t ecdsa -C 'host' -N ''
# Output the public keys to the console
# This allows user to get host keys securely through console log
echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger
ssh-keygen -l -f $rsa_key.pub | $logger
ssh-keygen -l -f $dsa_key.pub | $logger
ssh-keygen -l -f $ecdsa_key.pub | $logger
echo "------END SSH HOST KEY FINGERPRINTS------" | $logger

33
providers/one/assets/init.d/squeeze/generate-ssh-hostkeys Normal file
View file

@ -0,0 +1,33 @@
#!/bin/sh
### BEGIN INIT INFO
# Provides: generate-ssh-hostkeys
# Required-Start: $local_fs
# Required-Stop:
# Should-Start:
# Should-Stop:
# Default-Start: S
# Default-Stop:
# Description: Generate ssh host keys if they do not exist
### END INIT INFO
prog=$(basename $0)
logger="logger -t $prog"
rsa_key="/etc/ssh/ssh_host_rsa_key"
dsa_key="/etc/ssh/ssh_host_dsa_key"
# Exit if the hostkeys already exist
if [ -f $rsa_key -a -f $dsa_key ]; then
exit
fi
# Generate the ssh host keys
[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N ''
[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N ''
# Output the public keys to the console
# This allows user to get host keys securely through console log
echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger
ssh-keygen -l -f $rsa_key.pub | $logger
ssh-keygen -l -f $dsa_key.pub | $logger
echo "------END SSH HOST KEY FINGERPRINTS------" | $logger

22
providers/one/manifest-schema.json Normal file
View file

@ -0,0 +1,22 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "EC2 manifest",
"type": "object",
"properties": {
"volume": {
"type": "object",
"properties": {
"backing": {
"type": "string",
"enum": ["raw", "qcow2"]
},
"filesystem": {
"type": "string",
"enum": ["ext2", "ext3", "ext4", "xfs"]
}
},
"required": ["backing", "filesystem"]
}
},
"required": ["volume"]
}

15
providers/one/manifest.py Normal file
View file

@ -0,0 +1,15 @@
import base
class Manifest(base.Manifest):
def validate(self, data):
super(Manifest, self).validate(data)
from os import path
schema_path = path.join(path.dirname(__file__), 'manifest-schema.json')
self.schema_validate(data, schema_path)
def parse(self, data):
super(Manifest, self).parse(data)
self.credentials = data['credentials']
self.virtualization = data['virtualization']
self.image = data['image']

1
providers/one/tasks/__init__.py Normal file
View file

@ -0,0 +1 @@
__all__ = ['packages', 'connection', 'host', 'ec2']

76
providers/one/tasks/ami.py Normal file
View file

@ -0,0 +1,76 @@
from base import Task
from common import phases
from ebs import CreateSnapshot
from connection import Connect
from common.exceptions import TaskError
class AMIName(Task):
description = 'Determining the AMI name'
phase = phases.preparation
after = [Connect]
def run(self, info):
image_vars = {'release': info.manifest.system['release'],
'architecture': info.manifest.system['architecture'],
'virtualization': info.manifest.virtualization,
'backing': info.manifest.volume['backing']}
from datetime import datetime
now = datetime.now()
time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H',
'%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w',
'%W', '%x', '%X', '%y', '%Y', '%z', '%Z']
for var in time_vars:
image_vars[var] = now.strftime(var)
ami_name = info.manifest.image['name'].format(**image_vars)
ami_description = info.manifest.image['description'].format(**image_vars)
images = info.connection.get_all_images()
for image in images:
if ami_name == image.name:
msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name)
raise TaskError(msg)
info.ami_name = ami_name
info.ami_description = ami_description
class RegisterAMI(Task):
description = 'Registering the image as an AMI'
phase = phases.image_registration
after = [CreateSnapshot]
def run(self, info):
arch = {'i386': 'i386',
'amd64': 'x86_64'}.get(info.manifest.system['architecture'])
kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1',
'i386': 'aki-b6aa75df'},
'us-west-1': {'amd64': 'aki-f77e26b2',
'i386': 'aki-f57e26b0'},
'us-west-2': {'amd64': 'aki-fc37bacc',
'i386': 'aki-fa37baca'},
'eu-west-1': {'amd64': 'aki-71665e05',
'i386': 'aki-75665e01'},
'ap-southeast-1': {'amd64': 'aki-fe1354ac',
'i386': 'aki-f81354aa'},
'ap-southeast-2': {'amd64': 'aki-31990e0b',
'i386': 'aki-33990e09'},
'ap-northeast-1': {'amd64': 'aki-44992845',
'i386': 'aki-42992843'},
'sa-east-1': {'amd64': 'aki-c48f51d9',
'i386': 'aki-ca8f51d7'},
'us-gov-west-1': {'amd64': 'aki-79a4c05a',
'i386': 'aki-7ba4c058'}}
kernel_id = kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture'])
from boto.ec2.blockdevicemapping import BlockDeviceType
from boto.ec2.blockdevicemapping import BlockDeviceMapping
block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True,
size=int(info.manifest.volume['size']/1024))
block_device_map = BlockDeviceMapping()
block_device_map['/dev/sda1'] = block_device
info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description,
architecture=arch, kernel_id=kernel_id,
root_device_name='/dev/sda1',
block_device_map=block_device_map)

78
providers/one/tasks/apt.py Normal file
View file

@ -0,0 +1,78 @@
from base import Task
from common import phases
from common.tools import log_check_call
import os
from locale import GenerateLocale
class AptSources(Task):
description = 'Adding aptitude sources'
phase = phases.system_modification
def run(self, info):
sources_path = os.path.join(info.root, 'etc/apt/sources.list')
with open(sources_path, 'w') as apt_sources:
apt_sources.write(('deb {apt_mirror} {release} main\n'
'deb-src {apt_mirror} {release} main\n'
.format(apt_mirror='http://http.debian.net/debian',
release=info.manifest.system['release'])))
apt_sources.write(('deb {apt_mirror} {release}/updates main\n'
'deb-src {apt_mirror} {release}/updates main\n'
.format(apt_mirror='http://security.debian.org/',
release=info.manifest.system['release'])))
class DisableDaemonAutostart(Task):
description = 'Disabling daemon autostart'
phase = phases.system_modification
def run(self, info):
rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d')
with open(rc_policy_path, 'w') as rc_policy:
rc_policy.write(('#!/bin/sh\n'
'exit 101'))
import stat
os.chmod(rc_policy_path,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
class AptUpgrade(Task):
description = 'Upgrading packages and fixing broken dependencies'
phase = phases.system_modification
after = [GenerateLocale, AptSources, DisableDaemonAutostart]
def run(self, info):
log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update'])
log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-f', '-y', 'install'])
log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-y', 'upgrade'])
class PurgeUnusedPackages(Task):
description = 'Removing unused packages'
phase = phases.system_cleaning
def run(self, info):
log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'autoremove', '--purge'])
class AptClean(Task):
description = 'Clearing the aptitude cache'
phase = phases.system_cleaning
def run(self, info):
log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'clean'])
lists = os.path.join(info.root, 'var/lib/apt/lists')
for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]:
if os.path.isfile(list_file):
os.remove(list_file)
class EnableDaemonAutostart(Task):
description = 'Re-enabling daemon autostart after installation'
phase = phases.system_cleaning
def run(self, info):
os.remove(os.path.join(info.root, 'usr/sbin/policy-rc.d'))

60
providers/one/tasks/boot.py Normal file
View file

@ -0,0 +1,60 @@
from base import Task
from common import phases
import os
class ConfigureGrub(Task):
description = 'Configuring grub'
phase = phases.system_modification
def run(self, info):
import stat
rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
grubd = os.path.join(info.root, 'etc/grub.d')
for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]:
os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all)
from shutil import copy
script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom'))
script_dst = os.path.join(info.root, 'etc/grub.d/40_custom')
copy(script_src, script_dst)
os.chmod(script_dst, rwxr_xr_x)
from common.tools import sed_i
grub_def = os.path.join(info.root, 'etc/default/grub')
sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n'
'GRUB_HIDDEN_TIMEOUT=true')
from common.tools import log_check_call
log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub'])
log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst'])
class BlackListModules(Task):
description = 'Blacklisting kernel modules'
phase = phases.system_modification
def run(self, info):
blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf')
with open(blacklist_path, 'a') as blacklist:
blacklist.write(('# disable pc speaker\n'
'blacklist pcspkr'))
class DisableGetTTYs(Task):
description = 'Disabling getty processes'
phase = phases.system_modification
def run(self, info):
from common.tools import sed_i
inittab_path = os.path.join(info.root, 'etc/inittab')
tty1 = '1:2345:respawn:/sbin/getty 38400 tty1'
sed_i(inittab_path, '^'+tty1, '#'+tty1)
ttyx = ':23:respawn:/sbin/getty 38400 tty'
for i in range(2, 6):
i = str(i)
sed_i(inittab_path, '^'+i+ttyx+i, '#'+i+ttyx+i)

54
providers/one/tasks/bootstrap.py Normal file
View file

@ -0,0 +1,54 @@
from base import Task
from common import phases
from common.exceptions import TaskError
import logging
log = logging.getLogger(__name__)
def get_bootstrap_args(info):
executable = ['/usr/sbin/debootstrap']
options = ['--arch=' + info.manifest.system['architecture']]
include, exclude = info.img_packages
if len(include) > 0:
options.append('--include=' + ','.join(include))
if len(exclude) > 0:
options.append('--exclude=' + ','.join(exclude))
arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian']
return executable, options, arguments
class MakeTarball(Task):
description = 'Creating bootstrap tarball'
phase = phases.os_installation
def run(self, info):
from hashlib import sha1
import os.path
executable, options, arguments = get_bootstrap_args(info)
# Filter info.root which points at /target/volume-id, we won't ever hit anything with that in there.
hash_args = [arg for arg in arguments if arg != info.root]
tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8]
tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id)
info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename)
if os.path.isfile(info.tarball):
log.debug('Found matching tarball, skipping download')
else:
from common.tools import log_call
status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments)
if status != 1:
msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status)
raise TaskError(msg)
class Bootstrap(Task):
description = 'Installing Debian'
phase = phases.os_installation
after = [MakeTarball]
def run(self, info):
executable, options, arguments = get_bootstrap_args(info)
if hasattr(info, 'tarball'):
options.extend(['--unpack-tarball=' + info.tarball])
from common.tools import log_check_call
log_check_call(executable + options + arguments)

43
providers/one/tasks/cleanup.py Normal file
View file

@ -0,0 +1,43 @@
from base import Task
from common import phases
import os
class ClearMOTD(Task):
description = 'Clearing the MOTD'
phase = phases.system_cleaning
def run(self, info):
with open('/var/run/motd', 'w'):
pass
class ShredHostkeys(Task):
description = 'Securely deleting ssh hostkeys'
phase = phases.system_cleaning
def run(self, info):
ssh_hostkeys = ['ssh_host_dsa_key',
'ssh_host_rsa_key']
if info.manifest.system['release'] != 'squeeze':
ssh_hostkeys.append('ssh_host_ecdsa_key')
private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys]
public = [path + '.pub' for path in private]
from common.tools import log_check_call
log_check_call(['/usr/bin/shred', '--remove'] + private + public)
class CleanTMP(Task):
description = 'Removing temporary files'
phase = phases.system_cleaning
def run(self, info):
tmp = os.path.join(info.root, 'tmp')
for tmp_file in [os.path.join(tmp, f) for f in os.listdir(tmp)]:
os.remove(tmp_file)
log = os.path.join(info.root, 'var/log/')
os.remove(os.path.join(log, 'bootstrap.log'))
os.remove(os.path.join(log, 'dpkg.log'))

40
providers/one/tasks/connection.py Normal file
View file

@ -0,0 +1,40 @@
from base import Task
from common import phases
import host
class GetCredentials(Task):
description = 'Getting AWS credentials'
phase = phases.preparation
def run(self, info):
info.credentials = self.get_credentials(info.manifest)
def get_credentials(self, manifest):
from os import getenv
# manifest overrides environment
if(manifest.credentials['access-key'] and manifest.credentials['secret-key']):
return {'access_key': manifest.credentials['access-key'],
'secret_key': manifest.credentials['secret-key']}
if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')):
return {'access_key': getenv('EC2_ACCESS_KEY'),
'secret_key': getenv('EC2_SECRET_KEY')}
if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])):
raise RuntimeError('Both the access key and secret key must be specified in the manifest.')
if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))):
raise RuntimeError('Both the access key and secret key must be specified as environment variables.')
raise RuntimeError('No ec2 credentials found.')
class Connect(Task):
description = 'Connecting to EC2'
phase = phases.preparation
after = [GetCredentials, host.GetInfo]
def run(self, info):
from boto.ec2 import connect_to_region
info.connection = connect_to_region(info.host['region'],
aws_access_key_id=info.credentials['access_key'],
aws_secret_access_key=info.credentials['secret_key'])

89
providers/one/tasks/ebs.py Normal file
View file

@ -0,0 +1,89 @@
from base import Task
from common import phases
from common.exceptions import TaskError
from connection import Connect
from filesystem import UnmountVolume
import time
class CreateVolume(Task):
phase = phases.volume_creation
after = [Connect]
description = 'Creating an EBS volume for bootstrapping'
def run(self, info):
volume_size = int(info.manifest.volume['size']/1024)
info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone'])
while info.volume.volume_state() != 'available':
time.sleep(5)
info.volume.update()
class AttachVolume(Task):
phase = phases.volume_creation
after = [CreateVolume]
description = 'Attaching the EBS volume'
def run(self, info):
def char_range(c1, c2):
"""Generates the characters from `c1` to `c2`, inclusive."""
for c in xrange(ord(c1), ord(c2)+1):
yield chr(c)
import os.path
info.bootstrap_device = {}
for letter in char_range('f', 'z'):
dev_path = os.path.join('/dev', 'xvd' + letter)
if not os.path.exists(dev_path):
info.bootstrap_device['path'] = dev_path
info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter)
break
if 'path' not in info.bootstrap_device:
raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume')
info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path'])
while info.volume.attachment_state() != 'attached':
time.sleep(2)
info.volume.update()
class DetachVolume(Task):
phase = phases.volume_unmounting
after = [UnmountVolume]
description = 'Detaching the EBS volume'
def run(self, info):
info.volume.detach()
while info.volume.attachment_state() is not None:
time.sleep(2)
info.volume.update()
class CreateSnapshot(Task):
description = 'Creating a snapshot of the EBS volume'
phase = phases.image_registration
def run(self, info):
info.snapshot = info.volume.create_snapshot()
while info.snapshot.status != 'completed':
time.sleep(2)
info.snapshot.update()
class DeleteVolume(Task):
phase = phases.cleaning
after = []
description = 'Deleting the EBS volume'
def run(self, info):
info.volume.delete()
del info.volume
class VolumeError(TaskError):
pass

123
providers/one/tasks/filesystem.py Normal file
View file

@ -0,0 +1,123 @@
from base import Task
from common import phases
from common.exceptions import TaskError
from common.tools import log_check_call
from bootstrap import Bootstrap
class FormatVolume(Task):
description = 'Formatting the volume'
phase = phases.volume_preparation
def run(self, info):
dev_path = info.manifest.bootstrapper['device']
mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem'])
log_check_call([mkfs, dev_path])
class TuneVolumeFS(Task):
description = 'Tuning the bootstrap volume filesystem'
phase = phases.volume_preparation
after = [FormatVolume]
def run(self, info):
dev_path = info.bootstrap_device['path']
# Disable the time based filesystem check
log_check_call(['/sbin/tune2fs', '-i', '0', dev_path])
class AddXFSProgs(Task):
description = 'Adding `xfsprogs\' to the image packages'
phase = phases.preparation
def run(self, info):
include, exclude = info.img_packages
include.add('xfsprogs')
class CreateMountDir(Task):
description = 'Creating mountpoint for the bootstrap volume'
phase = phases.volume_mounting
def run(self, info):
import os
mount_dir = info.manifest.bootstrapper['mount_dir']
info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id)
# Works recursively, fails if last part exists, which is exaclty what we want.
os.makedirs(info.root)
class MountVolume(Task):
description = 'Mounting the bootstrap volume'
phase = phases.volume_mounting
after = [CreateMountDir]
def run(self, info):
with open('/proc/mounts') as mounts:
for mount in mounts:
if info.root in mount:
msg = 'Something is already mounted at {root}'.format(root=info.root)
raise TaskError(msg)
log_check_call(['/bin/mount', info.bootstrap_device['path'], info.root])
class MountSpecials(Task):
description = 'Mounting special block devices'
phase = phases.os_installation
after = [Bootstrap]
def run(self, info):
log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)])
log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'proc', 'none', '/proc'])
log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'sysfs', 'none', '/sys'])
log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'devpts', 'none', '/dev/pts'])
class UnmountSpecials(Task):
description = 'Unmunting special block devices'
phase = phases.volume_unmounting
def run(self, info):
log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts'])
log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys'])
log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc'])
log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)])
class UnmountVolume(Task):
description = 'Unmounting the bootstrap volume'
phase = phases.volume_unmounting
after = [UnmountSpecials]
def run(self, info):
log_check_call(['/bin/umount', info.root])
class DeleteMountDir(Task):
description = 'Deleting mountpoint for the bootstrap volume'
phase = phases.volume_unmounting
after = [UnmountVolume]
def run(self, info):
import os
os.rmdir(info.root)
del info.root
class ModifyFstab(Task):
description = 'Adding root volume to the fstab'
phase = phases.system_modification
def run(self, info):
import os.path
mount_opts = ['defaults']
if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']:
mount_opts.append('barrier=0')
if info.manifest.volume['filesystem'].lower() == 'xfs':
mount_opts.append('nobarrier')
fstab_path = os.path.join(info.root, 'etc/fstab')
with open(fstab_path, 'a') as fstab:
fstab.write(('/dev/xvda1 / {filesystem} {mount_opts} 1 1\n'
.format(filesystem=info.manifest.volume['filesystem'].lower(),
mount_opts=','.join(mount_opts))))

29
providers/one/tasks/host.py Normal file
View file

@ -0,0 +1,29 @@
from base import Task
from common import phases
from common.exceptions import TaskError
import packages
class CheckPackages(Task):
description = 'Checking installed host packages'
phase = phases.preparation
after = [packages.HostPackages, packages.ImagePackages]
def run(self, info):
from common.tools import log_check_call
from subprocess import CalledProcessError
for package in info.host_packages:
try:
log_check_call(['/usr/bin/dpkg', '-s', package])
except CalledProcessError:
msg = "The package ``{0}\'\' is not installed".format(package)
raise TaskError(msg)
class GetInfo(Task):
description = 'Retrieving instance metadata'
phase = phases.preparation
def run(self, info):
info.host = {}
return info

49
providers/one/tasks/initd.py Normal file
View file

@ -0,0 +1,49 @@
from base import Task
from common import phases
import os.path
class ResolveInitScripts(Task):
description = 'Determining which startup scripts to install or disable'
phase = phases.system_modification
def run(self, info):
init_scripts = {'ec2-get-credentials': 'ec2-get-credentials',
'ec2-run-user-data': 'ec2-run-user-data',
'expand-volume': 'expand-volume'}
init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys'
if info.manifest.system['release'] == 'squeeze':
init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys'
disable_scripts = ['hwclock.sh']
if info.manifest.system['release'] == 'squeeze':
disable_scripts.append('hwclockfirst.sh')
for name, path in init_scripts.iteritems():
init_scripts[name] = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d', path))
info.initd = {'install': init_scripts,
'disable': disable_scripts}
class InstallInitScripts(Task):
description = 'Installing startup scripts'
phase = phases.system_modification
after = [ResolveInitScripts]
def run(self, info):
import stat
rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
from shutil import copy
from common.tools import log_check_call
for name, src in info.initd['install'].iteritems():
dst = os.path.join(info.root, 'etc/init.d', name)
copy(src, dst)
os.chmod(dst, rwxr_xr_x)
log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-d', name])
for name in info.initd['disable']:
log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-r', name])

35
providers/one/tasks/locale.py Normal file
View file

@ -0,0 +1,35 @@
from base import Task
from common import phases
import os.path
class GenerateLocale(Task):
description = 'Generating the selected locale'
phase = phases.system_modification
def run(self, info):
from common.tools import sed_i
from common.tools import log_check_call
locale_gen = os.path.join(info.root, 'etc/locale.gen')
locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'],
charmap=info.manifest.system['charmap'])
search = '# ' + locale_str
sed_i(locale_gen, search, locale_str)
command = ['/usr/sbin/chroot', info.root, '/usr/sbin/dpkg-reconfigure', '--priority=critical', 'locales']
log_check_call(command)
class SetTimezone(Task):
description = 'Setting the selected timezone'
phase = phases.system_modification
def run(self, info):
from shutil import copy
tz_path = os.path.join(info.root, 'etc/timezone')
timezone = info.manifest.system['timezone']
with open(tz_path, 'w') as tz_file:
tz_file.write(timezone)
zoneinfo_path = os.path.join(info.root, '/usr/share/zoneinfo', timezone)
localtime_path = os.path.join(info.root, 'etc/localtime')
copy(zoneinfo_path, localtime_path)

38
providers/one/tasks/network.py Normal file
View file

@ -0,0 +1,38 @@
from base import Task
from common import phases
import os.path
class RemoveDNSInfo(Task):
description = 'Removing resolv.conf'
phase = phases.system_modification
def run(self, info):
from os import remove
remove(os.path.join(info.root, 'etc/resolv.conf'))
class ConfigureNetworkIF(Task):
description = 'Configuring network interfaces'
phase = phases.system_modification
def run(self, info):
interfaces_path = os.path.join(info.root, 'etc/network/interfaces')
if_config = {'squeeze': ('auto lo\n'
'iface lo inet loopback\n'
'auto eth0\n'
'iface eth0 inet dhcp\n'),
'wheezy': 'auto eth0\n'
'iface eth0 inet dhcp\n'}
with open(interfaces_path, 'a') as interfaces:
interfaces.write(if_config.get(info.manifest.system['release']))
class ConfigureDHCP(Task):
description = 'Configuring the DHCP client'
phase = phases.system_modification
def run(self, info):
from common.tools import sed_i
dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd')
sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'')

48
providers/one/tasks/packages.py Normal file
View file

@ -0,0 +1,48 @@
from base import Task
from common import phases
class HostPackages(Task):
description = 'Determining required host packages'
phase = phases.preparation
def run(self, info):
packages = set(['debootstrap'])
if info.manifest.volume['filesystem'] == 'xfs':
packages.add('xfsprogs')
info.host_packages = packages
class ImagePackages(Task):
description = 'Determining required image packages'
phase = phases.preparation
def run(self, info):
manifest = info.manifest
# Add some basic packages we are going to need
include = set(['udev',
'openssh-server',
# We could bootstrap without locales, but things just suck without them, error messages etc.
'locales',
# Needed for the init scripts
'file',
# isc-dhcp-client doesn't work properly with ec2
'dhcpcd',
])
if manifest.virtualization == 'pvm':
include.add('grub-pc')
exclude = set(['isc-dhcp-client',
'isc-dhcp-common',
])
# In squeeze, we need a special kernel flavor for xen
kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64',
'i386': 'linux-image-xen-686', },
'wheezy': {'amd64': 'linux-image-amd64',
'i386': 'linux-image-686', }, }
include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture']))
info.img_packages = include, exclude

32
providers/one/tasks/security.py Normal file
View file

@ -0,0 +1,32 @@
from base import Task
from common import phases
import os.path
class EnableShadowConfig(Task):
description = 'Enabling shadowconfig'
phase = phases.system_modification
def run(self, info):
from common.tools import log_check_call
log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on'])
class DisableSSHPasswordAuthentication(Task):
description = 'Disabling SSH password authentication'
phase = phases.system_modification
def run(self, info):
from common.tools import sed_i
sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no')
class DisableSSHDNSLookup(Task):
description = 'Disabling sshd remote host name lookup'
phase = phases.system_modification
def run(self, info):
sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config')
with open(sshd_config_path, 'a') as sshd_config:
sshd_config.write('UseDNS no')