From 778ec8c27055da5e4ae5690df87cfc471ef352fe Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 10:49:45 +0200 Subject: [PATCH 01/22] add open nebula management --- manifests/one-raw-virtio.manifest.json | 43 ++++++ providers/one/__init__.py | 77 +++++++++++ providers/one/assets/grub.d/40_custom | 81 ++++++++++++ .../one/assets/init.d/ec2-get-credentials | 45 +++++++ providers/one/assets/init.d/ec2-run-user-data | 46 +++++++ providers/one/assets/init.d/expand-volume | 26 ++++ .../one/assets/init.d/generate-ssh-hostkeys | 36 +++++ .../init.d/squeeze/generate-ssh-hostkeys | 33 +++++ providers/one/manifest-schema.json | 22 ++++ providers/one/manifest.py | 15 +++ providers/one/tasks/__init__.py | 1 + providers/one/tasks/ami.py | 76 +++++++++++ providers/one/tasks/apt.py | 78 +++++++++++ providers/one/tasks/boot.py | 60 +++++++++ providers/one/tasks/bootstrap.py | 54 ++++++++ providers/one/tasks/cleanup.py | 43 ++++++ providers/one/tasks/connection.py | 40 ++++++ providers/one/tasks/ebs.py | 89 +++++++++++++ providers/one/tasks/filesystem.py | 123 ++++++++++++++++++ providers/one/tasks/host.py | 29 +++++ providers/one/tasks/initd.py | 49 +++++++ providers/one/tasks/locale.py | 35 +++++ providers/one/tasks/network.py | 38 ++++++ providers/one/tasks/packages.py | 48 +++++++ providers/one/tasks/security.py | 32 +++++ 25 files changed, 1219 insertions(+) create mode 100644 manifests/one-raw-virtio.manifest.json create mode 100644 providers/one/__init__.py create mode 100644 providers/one/assets/grub.d/40_custom create mode 100644 providers/one/assets/init.d/ec2-get-credentials create mode 100644 providers/one/assets/init.d/ec2-run-user-data create mode 100644 providers/one/assets/init.d/expand-volume create mode 100644 providers/one/assets/init.d/generate-ssh-hostkeys create mode 100644 providers/one/assets/init.d/squeeze/generate-ssh-hostkeys create mode 100644 providers/one/manifest-schema.json create mode 100644 providers/one/manifest.py create mode 100644 providers/one/tasks/__init__.py create mode 100644 providers/one/tasks/ami.py create mode 100644 providers/one/tasks/apt.py create mode 100644 providers/one/tasks/boot.py create mode 100644 providers/one/tasks/bootstrap.py create mode 100644 providers/one/tasks/cleanup.py create mode 100644 providers/one/tasks/connection.py create mode 100644 providers/one/tasks/ebs.py create mode 100644 providers/one/tasks/filesystem.py create mode 100644 providers/one/tasks/host.py create mode 100644 providers/one/tasks/initd.py create mode 100644 providers/one/tasks/locale.py create mode 100644 providers/one/tasks/network.py create mode 100644 providers/one/tasks/packages.py create mode 100644 providers/one/tasks/security.py diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json new file mode 100644 index 0000000..96d5003 --- /dev/null +++ b/manifests/one-raw-virtio.manifest.json @@ -0,0 +1,43 @@ +{ + "provider" : "one", + "virtualization": "virtio", + "credentials" : { + "access-key": null, + "secret-key": null + }, + + "bootstrapper": { + "mount_dir": "/target", + "tarball": true, + "device": "/dev/sda" + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} AMI ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024 + }, + "plugins": { + "admin_user": { + "enabled": false + }, + "build_metadata": { + "enabled": false, + "path" : "/root/build-metadata-{ami_name}" + }, + "prebootstrapped": { + "enabled": false, + "snapshot": "" + } + } +} diff --git a/providers/one/__init__.py b/providers/one/__init__.py new file mode 100644 index 0000000..3372c1f --- /dev/null +++ b/providers/one/__init__.py @@ -0,0 +1,77 @@ +from manifest import Manifest +import logging +from tasks import packages +from tasks import connection +from tasks import host +from tasks import ami +from tasks import ebs +from tasks import filesystem +from tasks import bootstrap +from tasks import locale +from tasks import apt +from tasks import boot +from tasks import security +from tasks import network +from tasks import initd +from tasks import cleanup + + +def initialize(): + # Regardless of of loglevel, we don't want boto debug stuff, it's very noisy + logging.getLogger('boto').setLevel(logging.INFO) + + +def tasks(tasklist, manifest): + tasklist.add(packages.HostPackages(), + packages.ImagePackages(), + host.CheckPackages(), + host.GetInfo()) + tasklist.add(filesystem.FormatVolume()) + if manifest.volume['filesystem'].lower() == 'xfs': + tasklist.add(filesystem.AddXFSProgs()) + if manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + tasklist.add(filesystem.TuneVolumeFS()) + tasklist.add(filesystem.CreateMountDir(), + filesystem.MountVolume()) + if manifest.bootstrapper['tarball']: + tasklist.add(bootstrap.MakeTarball()) + tasklist.add(bootstrap.Bootstrap(), + filesystem.MountSpecials(), + locale.GenerateLocale(), + locale.SetTimezone(), + apt.DisableDaemonAutostart(), + apt.AptSources(), + apt.AptUpgrade(), + boot.ConfigureGrub(), + filesystem.ModifyFstab(), + boot.BlackListModules(), + boot.DisableGetTTYs(), + security.EnableShadowConfig(), + security.DisableSSHPasswordAuthentication(), + security.DisableSSHDNSLookup(), + network.RemoveDNSInfo(), + network.ConfigureNetworkIF(), + network.ConfigureDHCP(), + initd.ResolveInitScripts(), + initd.InstallInitScripts(), + cleanup.ClearMOTD(), + cleanup.ShredHostkeys(), + cleanup.CleanTMP(), + apt.PurgeUnusedPackages(), + apt.AptClean(), + apt.EnableDaemonAutostart(), + filesystem.UnmountSpecials(), + filesystem.UnmountVolume(), + filesystem.DeleteMountDir()) + + +def rollback_tasks(tasklist, tasks_completed, manifest): + completed = [type(task) for task in tasks_completed] + + def counter_task(task, counter): + if task in completed and counter not in completed: + tasklist.add(counter()) + + counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) + counter_task(filesystem.MountVolume, filesystem.UnmountVolume) + counter_task(filesystem.MountSpecials, filesystem.UnmountSpecials) diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom new file mode 100644 index 0000000..799e887 --- /dev/null +++ b/providers/one/assets/grub.d/40_custom @@ -0,0 +1,81 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/xvda1 + + +cat << EOF +default ${GRUB_DEFAULT} +timeout ${GRUB_TIMEOUT} +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +title ${version} + root (hd0) + kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + initrd ${rel_dirname}/${initrd} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/one/assets/init.d/ec2-get-credentials b/providers/one/assets/init.d/ec2-get-credentials new file mode 100644 index 0000000..b304ae2 --- /dev/null +++ b/providers/one/assets/init.d/ec2-get-credentials @@ -0,0 +1,45 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-get-credentials +# Required-Start: $network +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Retrieve the ssh credentials and add to authorized_keys +### END INIT INFO +# +# ec2-get-credentials - Retrieve the ssh credentials and add to authorized_keys +# +# Based on /usr/local/sbin/ec2-get-credentials from Amazon's ami-20b65349 +# + +prog=$(basename $0) +logger="logger -t $prog" + +public_key_url=http://169.254.169.254/1.0/meta-data/public-keys/0/openssh-key +username='root' +# A little bit of nastyness to get the homedir, when the username is a variable +ssh_dir="`eval printf ~$username`/.ssh" +authorized_keys="$ssh_dir/authorized_keys" + +# Try to get the ssh public key from instance data. +public_key=`wget -qO - $public_key_url` +if [ -n "$public_key" ]; then + if [ ! -f $authorized_keys ]; then + if [ ! -d $ssh_dir ]; then + mkdir -m 700 $ssh_dir + chown $username:$username $ssh_dir + fi + touch $authorized_keys + chown $username:$username $authorized_keys + fi + + if ! grep -s -q "$public_key" $authorized_keys; then + printf "\n%s" -- "$public_key" >> $authorized_keys + $logger "New ssh key added to $authorized_keys from $public_key_url" + chmod 600 $authorized_keys + chown $username:$username $authorized_keys + fi +fi diff --git a/providers/one/assets/init.d/ec2-run-user-data b/providers/one/assets/init.d/ec2-run-user-data new file mode 100644 index 0000000..17b8b6f --- /dev/null +++ b/providers/one/assets/init.d/ec2-run-user-data @@ -0,0 +1,46 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: ec2-run-user-data +# Required-Start: ec2-get-credentials +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Run instance user-data if it looks like a script. +### END INIT INFO +# +# Only retrieves and runs the user-data script once per instance. If +# you want the user-data script to run again (e.g., on the next boot) +# then readd this script with insserv: +# insserv -d ec2-run-user-data +# +prog=$(basename $0) +logger="logger -t $prog" +instance_data_url="http://169.254.169.254/2008-02-01" + + +# Retrieve the instance user-data and run it if it looks like a script +user_data_file=$(tempfile --prefix ec2 --suffix .user-data --mode 700) +$logger "Retrieving user-data" +wget -qO $user_data_file $instance_data_url/user-data 2>&1 | $logger + +if [ $(file -b --mime-type $user_data_file) = 'application/x-gzip' ]; then + $logger "Uncompressing gzip'd user-data" + mv $user_data_file $user_data_file.gz + gunzip $user_data_file.gz +fi + +if [ ! -s $user_data_file ]; then + $logger "No user-data available" +elif head -1 $user_data_file | egrep -v '^#!'; then + $logger "Skipping user-data as it does not begin with #!" +else + $logger "Running user-data" + $user_data_file 2>&1 | logger -t "user-data" + $logger "user-data exit code: $?" +fi +rm -f $user_data_file + +# Disable this script, it may only run once +insserv -r $0 diff --git a/providers/one/assets/init.d/expand-volume b/providers/one/assets/init.d/expand-volume new file mode 100644 index 0000000..3b2d2a6 --- /dev/null +++ b/providers/one/assets/init.d/expand-volume @@ -0,0 +1,26 @@ +#!/bin/bash +### BEGIN INIT INFO +# Provides: expand-volume +# Required-Start: +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: +# Description: Expand the filesystem of the mounted root volume to its maximum possible size +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +device_path="/dev/xvda1" + +filesystem=`blkid | grep $device_path | sed 's#\(.*\):.*TYPE="\(.*\)".*#\2#'` + +case $filesystem in + xfs) xfs_growfs / ;; + ext2) resize2fs $device_path ;; + ext3) resize2fs $device_path ;; + ext4) resize2fs $device_path ;; + *) $logger "The filesystem $filesystem was not recognized. Unable to expand size." ;; +esac diff --git a/providers/one/assets/init.d/generate-ssh-hostkeys b/providers/one/assets/init.d/generate-ssh-hostkeys new file mode 100644 index 0000000..c9efb12 --- /dev/null +++ b/providers/one/assets/init.d/generate-ssh-hostkeys @@ -0,0 +1,36 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" +ecdsa_key="/etc/ssh/ssh_host_ecdsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key -a -f $ecdsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' +[ -f $ecdsa_key ] || ssh-keygen -f $ecdsa_key -t ecdsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +ssh-keygen -l -f $ecdsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys b/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys new file mode 100644 index 0000000..148b87d --- /dev/null +++ b/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys @@ -0,0 +1,33 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: generate-ssh-hostkeys +# Required-Start: $local_fs +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: S +# Default-Stop: +# Description: Generate ssh host keys if they do not exist +### END INIT INFO + +prog=$(basename $0) +logger="logger -t $prog" + +rsa_key="/etc/ssh/ssh_host_rsa_key" +dsa_key="/etc/ssh/ssh_host_dsa_key" + +# Exit if the hostkeys already exist +if [ -f $rsa_key -a -f $dsa_key ]; then + exit +fi + +# Generate the ssh host keys +[ -f $rsa_key ] || ssh-keygen -f $rsa_key -t rsa -C 'host' -N '' +[ -f $dsa_key ] || ssh-keygen -f $dsa_key -t dsa -C 'host' -N '' + +# Output the public keys to the console +# This allows user to get host keys securely through console log +echo "-----BEGIN SSH HOST KEY FINGERPRINTS-----" | $logger +ssh-keygen -l -f $rsa_key.pub | $logger +ssh-keygen -l -f $dsa_key.pub | $logger +echo "------END SSH HOST KEY FINGERPRINTS------" | $logger diff --git a/providers/one/manifest-schema.json b/providers/one/manifest-schema.json new file mode 100644 index 0000000..0a6a038 --- /dev/null +++ b/providers/one/manifest-schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "EC2 manifest", + "type": "object", + "properties": { + "volume": { + "type": "object", + "properties": { + "backing": { + "type": "string", + "enum": ["raw", "qcow2"] + }, + "filesystem": { + "type": "string", + "enum": ["ext2", "ext3", "ext4", "xfs"] + } + }, + "required": ["backing", "filesystem"] + } + }, + "required": ["volume"] +} diff --git a/providers/one/manifest.py b/providers/one/manifest.py new file mode 100644 index 0000000..d2e0f2c --- /dev/null +++ b/providers/one/manifest.py @@ -0,0 +1,15 @@ +import base + + +class Manifest(base.Manifest): + def validate(self, data): + super(Manifest, self).validate(data) + from os import path + schema_path = path.join(path.dirname(__file__), 'manifest-schema.json') + self.schema_validate(data, schema_path) + + def parse(self, data): + super(Manifest, self).parse(data) + self.credentials = data['credentials'] + self.virtualization = data['virtualization'] + self.image = data['image'] diff --git a/providers/one/tasks/__init__.py b/providers/one/tasks/__init__.py new file mode 100644 index 0000000..873a99d --- /dev/null +++ b/providers/one/tasks/__init__.py @@ -0,0 +1 @@ +__all__ = ['packages', 'connection', 'host', 'ec2'] diff --git a/providers/one/tasks/ami.py b/providers/one/tasks/ami.py new file mode 100644 index 0000000..fbabbf3 --- /dev/null +++ b/providers/one/tasks/ami.py @@ -0,0 +1,76 @@ +from base import Task +from common import phases +from ebs import CreateSnapshot +from connection import Connect +from common.exceptions import TaskError + + +class AMIName(Task): + description = 'Determining the AMI name' + phase = phases.preparation + after = [Connect] + + def run(self, info): + image_vars = {'release': info.manifest.system['release'], + 'architecture': info.manifest.system['architecture'], + 'virtualization': info.manifest.virtualization, + 'backing': info.manifest.volume['backing']} + from datetime import datetime + now = datetime.now() + time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', + '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', + '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] + for var in time_vars: + image_vars[var] = now.strftime(var) + + ami_name = info.manifest.image['name'].format(**image_vars) + ami_description = info.manifest.image['description'].format(**image_vars) + + images = info.connection.get_all_images() + for image in images: + if ami_name == image.name: + msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name) + raise TaskError(msg) + info.ami_name = ami_name + info.ami_description = ami_description + + +class RegisterAMI(Task): + description = 'Registering the image as an AMI' + phase = phases.image_registration + after = [CreateSnapshot] + + def run(self, info): + arch = {'i386': 'i386', + 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) + kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', + 'i386': 'aki-b6aa75df'}, + 'us-west-1': {'amd64': 'aki-f77e26b2', + 'i386': 'aki-f57e26b0'}, + 'us-west-2': {'amd64': 'aki-fc37bacc', + 'i386': 'aki-fa37baca'}, + 'eu-west-1': {'amd64': 'aki-71665e05', + 'i386': 'aki-75665e01'}, + 'ap-southeast-1': {'amd64': 'aki-fe1354ac', + 'i386': 'aki-f81354aa'}, + 'ap-southeast-2': {'amd64': 'aki-31990e0b', + 'i386': 'aki-33990e09'}, + 'ap-northeast-1': {'amd64': 'aki-44992845', + 'i386': 'aki-42992843'}, + 'sa-east-1': {'amd64': 'aki-c48f51d9', + 'i386': 'aki-ca8f51d7'}, + 'us-gov-west-1': {'amd64': 'aki-79a4c05a', + 'i386': 'aki-7ba4c058'}} + kernel_id = kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) + + from boto.ec2.blockdevicemapping import BlockDeviceType + from boto.ec2.blockdevicemapping import BlockDeviceMapping + block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, + size=int(info.manifest.volume['size']/1024)) + block_device_map = BlockDeviceMapping() + block_device_map['/dev/sda1'] = block_device + + info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, + architecture=arch, kernel_id=kernel_id, + root_device_name='/dev/sda1', + block_device_map=block_device_map) diff --git a/providers/one/tasks/apt.py b/providers/one/tasks/apt.py new file mode 100644 index 0000000..e039df4 --- /dev/null +++ b/providers/one/tasks/apt.py @@ -0,0 +1,78 @@ +from base import Task +from common import phases +from common.tools import log_check_call +import os +from locale import GenerateLocale + + +class AptSources(Task): + description = 'Adding aptitude sources' + phase = phases.system_modification + + def run(self, info): + sources_path = os.path.join(info.root, 'etc/apt/sources.list') + with open(sources_path, 'w') as apt_sources: + apt_sources.write(('deb {apt_mirror} {release} main\n' + 'deb-src {apt_mirror} {release} main\n' + .format(apt_mirror='http://http.debian.net/debian', + release=info.manifest.system['release']))) + apt_sources.write(('deb {apt_mirror} {release}/updates main\n' + 'deb-src {apt_mirror} {release}/updates main\n' + .format(apt_mirror='http://security.debian.org/', + release=info.manifest.system['release']))) + + +class DisableDaemonAutostart(Task): + description = 'Disabling daemon autostart' + phase = phases.system_modification + + def run(self, info): + rc_policy_path = os.path.join(info.root, 'usr/sbin/policy-rc.d') + with open(rc_policy_path, 'w') as rc_policy: + rc_policy.write(('#!/bin/sh\n' + 'exit 101')) + import stat + os.chmod(rc_policy_path, + stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + +class AptUpgrade(Task): + description = 'Upgrading packages and fixing broken dependencies' + phase = phases.system_modification + after = [GenerateLocale, AptSources, DisableDaemonAutostart] + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'update']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-f', '-y', 'install']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', '-y', 'upgrade']) + + +class PurgeUnusedPackages(Task): + description = 'Removing unused packages' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'autoremove', '--purge']) + + +class AptClean(Task): + description = 'Clearing the aptitude cache' + phase = phases.system_cleaning + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/usr/bin/apt-get', 'clean']) + + lists = os.path.join(info.root, 'var/lib/apt/lists') + for list_file in [os.path.join(lists, f) for f in os.listdir(lists)]: + if os.path.isfile(list_file): + os.remove(list_file) + + +class EnableDaemonAutostart(Task): + description = 'Re-enabling daemon autostart after installation' + phase = phases.system_cleaning + + def run(self, info): + os.remove(os.path.join(info.root, 'usr/sbin/policy-rc.d')) diff --git a/providers/one/tasks/boot.py b/providers/one/tasks/boot.py new file mode 100644 index 0000000..4265b9d --- /dev/null +++ b/providers/one/tasks/boot.py @@ -0,0 +1,60 @@ +from base import Task +from common import phases +import os + + +class ConfigureGrub(Task): + description = 'Configuring grub' + phase = phases.system_modification + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + x_all = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + grubd = os.path.join(info.root, 'etc/grub.d') + for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]: + os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all) + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom')) + script_dst = os.path.join(info.root, 'etc/grub.d/40_custom') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + from common.tools import sed_i + grub_def = os.path.join(info.root, 'etc/default/grub') + sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' + 'GRUB_HIDDEN_TIMEOUT=true') + + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) + + +class BlackListModules(Task): + description = 'Blacklisting kernel modules' + phase = phases.system_modification + + def run(self, info): + blacklist_path = os.path.join(info.root, 'etc/modprobe.d/blacklist.conf') + with open(blacklist_path, 'a') as blacklist: + blacklist.write(('# disable pc speaker\n' + 'blacklist pcspkr')) + + +class DisableGetTTYs(Task): + description = 'Disabling getty processes' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + inittab_path = os.path.join(info.root, 'etc/inittab') + tty1 = '1:2345:respawn:/sbin/getty 38400 tty1' + sed_i(inittab_path, '^'+tty1, '#'+tty1) + ttyx = ':23:respawn:/sbin/getty 38400 tty' + for i in range(2, 6): + i = str(i) + sed_i(inittab_path, '^'+i+ttyx+i, '#'+i+ttyx+i) diff --git a/providers/one/tasks/bootstrap.py b/providers/one/tasks/bootstrap.py new file mode 100644 index 0000000..9e27f36 --- /dev/null +++ b/providers/one/tasks/bootstrap.py @@ -0,0 +1,54 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import logging +log = logging.getLogger(__name__) + + +def get_bootstrap_args(info): + executable = ['/usr/sbin/debootstrap'] + options = ['--arch=' + info.manifest.system['architecture']] + include, exclude = info.img_packages + if len(include) > 0: + options.append('--include=' + ','.join(include)) + if len(exclude) > 0: + options.append('--exclude=' + ','.join(exclude)) + arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian'] + return executable, options, arguments + + +class MakeTarball(Task): + description = 'Creating bootstrap tarball' + phase = phases.os_installation + + def run(self, info): + from hashlib import sha1 + import os.path + executable, options, arguments = get_bootstrap_args(info) + # Filter info.root which points at /target/volume-id, we won't ever hit anything with that in there. + hash_args = [arg for arg in arguments if arg != info.root] + tarball_id = sha1(repr(frozenset(options + hash_args))).hexdigest()[0:8] + tarball_filename = 'debootstrap-{id}.tar'.format(id=tarball_id) + info.tarball = os.path.join(info.manifest.bootstrapper['tarball_dir'], tarball_filename) + if os.path.isfile(info.tarball): + log.debug('Found matching tarball, skipping download') + else: + from common.tools import log_call + status, out, err = log_call(executable + options + ['--make-tarball=' + info.tarball] + arguments) + if status != 1: + msg = 'debootstrap exited with status {status}, it should exit with status 1'.format(status=status) + raise TaskError(msg) + + +class Bootstrap(Task): + description = 'Installing Debian' + phase = phases.os_installation + after = [MakeTarball] + + def run(self, info): + executable, options, arguments = get_bootstrap_args(info) + if hasattr(info, 'tarball'): + options.extend(['--unpack-tarball=' + info.tarball]) + + from common.tools import log_check_call + log_check_call(executable + options + arguments) diff --git a/providers/one/tasks/cleanup.py b/providers/one/tasks/cleanup.py new file mode 100644 index 0000000..7b793a4 --- /dev/null +++ b/providers/one/tasks/cleanup.py @@ -0,0 +1,43 @@ +from base import Task +from common import phases +import os + + +class ClearMOTD(Task): + description = 'Clearing the MOTD' + phase = phases.system_cleaning + + def run(self, info): + with open('/var/run/motd', 'w'): + pass + + +class ShredHostkeys(Task): + description = 'Securely deleting ssh hostkeys' + phase = phases.system_cleaning + + def run(self, info): + ssh_hostkeys = ['ssh_host_dsa_key', + 'ssh_host_rsa_key'] + if info.manifest.system['release'] != 'squeeze': + ssh_hostkeys.append('ssh_host_ecdsa_key') + + private = [os.path.join(info.root, 'etc/ssh', name) for name in ssh_hostkeys] + public = [path + '.pub' for path in private] + + from common.tools import log_check_call + log_check_call(['/usr/bin/shred', '--remove'] + private + public) + + +class CleanTMP(Task): + description = 'Removing temporary files' + phase = phases.system_cleaning + + def run(self, info): + tmp = os.path.join(info.root, 'tmp') + for tmp_file in [os.path.join(tmp, f) for f in os.listdir(tmp)]: + os.remove(tmp_file) + + log = os.path.join(info.root, 'var/log/') + os.remove(os.path.join(log, 'bootstrap.log')) + os.remove(os.path.join(log, 'dpkg.log')) diff --git a/providers/one/tasks/connection.py b/providers/one/tasks/connection.py new file mode 100644 index 0000000..2c2ce19 --- /dev/null +++ b/providers/one/tasks/connection.py @@ -0,0 +1,40 @@ +from base import Task +from common import phases +import host + + +class GetCredentials(Task): + description = 'Getting AWS credentials' + phase = phases.preparation + + def run(self, info): + info.credentials = self.get_credentials(info.manifest) + + def get_credentials(self, manifest): + from os import getenv + # manifest overrides environment + if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): + return {'access_key': manifest.credentials['access-key'], + 'secret_key': manifest.credentials['secret-key']} + if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')): + return {'access_key': getenv('EC2_ACCESS_KEY'), + 'secret_key': getenv('EC2_SECRET_KEY')} + + if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])): + raise RuntimeError('Both the access key and secret key must be specified in the manifest.') + if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))): + raise RuntimeError('Both the access key and secret key must be specified as environment variables.') + + raise RuntimeError('No ec2 credentials found.') + + +class Connect(Task): + description = 'Connecting to EC2' + phase = phases.preparation + after = [GetCredentials, host.GetInfo] + + def run(self, info): + from boto.ec2 import connect_to_region + info.connection = connect_to_region(info.host['region'], + aws_access_key_id=info.credentials['access_key'], + aws_secret_access_key=info.credentials['secret_key']) diff --git a/providers/one/tasks/ebs.py b/providers/one/tasks/ebs.py new file mode 100644 index 0000000..a07b1a8 --- /dev/null +++ b/providers/one/tasks/ebs.py @@ -0,0 +1,89 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from connection import Connect +from filesystem import UnmountVolume +import time + + +class CreateVolume(Task): + phase = phases.volume_creation + after = [Connect] + + description = 'Creating an EBS volume for bootstrapping' + + def run(self, info): + volume_size = int(info.manifest.volume['size']/1024) + + info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone']) + while info.volume.volume_state() != 'available': + time.sleep(5) + info.volume.update() + + +class AttachVolume(Task): + phase = phases.volume_creation + after = [CreateVolume] + + description = 'Attaching the EBS volume' + + def run(self, info): + def char_range(c1, c2): + """Generates the characters from `c1` to `c2`, inclusive.""" + for c in xrange(ord(c1), ord(c2)+1): + yield chr(c) + + import os.path + info.bootstrap_device = {} + for letter in char_range('f', 'z'): + dev_path = os.path.join('/dev', 'xvd' + letter) + if not os.path.exists(dev_path): + info.bootstrap_device['path'] = dev_path + info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) + break + if 'path' not in info.bootstrap_device: + raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') + + info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) + while info.volume.attachment_state() != 'attached': + time.sleep(2) + info.volume.update() + + +class DetachVolume(Task): + phase = phases.volume_unmounting + after = [UnmountVolume] + + description = 'Detaching the EBS volume' + + def run(self, info): + info.volume.detach() + while info.volume.attachment_state() is not None: + time.sleep(2) + info.volume.update() + + +class CreateSnapshot(Task): + description = 'Creating a snapshot of the EBS volume' + phase = phases.image_registration + + def run(self, info): + info.snapshot = info.volume.create_snapshot() + while info.snapshot.status != 'completed': + time.sleep(2) + info.snapshot.update() + + +class DeleteVolume(Task): + phase = phases.cleaning + after = [] + + description = 'Deleting the EBS volume' + + def run(self, info): + info.volume.delete() + del info.volume + + +class VolumeError(TaskError): + pass diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py new file mode 100644 index 0000000..50481f4 --- /dev/null +++ b/providers/one/tasks/filesystem.py @@ -0,0 +1,123 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +from common.tools import log_check_call +from bootstrap import Bootstrap + + +class FormatVolume(Task): + description = 'Formatting the volume' + phase = phases.volume_preparation + + def run(self, info): + dev_path = info.manifest.bootstrapper['device'] + mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) + log_check_call([mkfs, dev_path]) + + +class TuneVolumeFS(Task): + description = 'Tuning the bootstrap volume filesystem' + phase = phases.volume_preparation + after = [FormatVolume] + + def run(self, info): + dev_path = info.bootstrap_device['path'] + # Disable the time based filesystem check + log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) + + +class AddXFSProgs(Task): + description = 'Adding `xfsprogs\' to the image packages' + phase = phases.preparation + + def run(self, info): + include, exclude = info.img_packages + include.add('xfsprogs') + + +class CreateMountDir(Task): + description = 'Creating mountpoint for the bootstrap volume' + phase = phases.volume_mounting + + def run(self, info): + import os + mount_dir = info.manifest.bootstrapper['mount_dir'] + info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id) + # Works recursively, fails if last part exists, which is exaclty what we want. + os.makedirs(info.root) + + +class MountVolume(Task): + description = 'Mounting the bootstrap volume' + phase = phases.volume_mounting + after = [CreateMountDir] + + def run(self, info): + with open('/proc/mounts') as mounts: + for mount in mounts: + if info.root in mount: + msg = 'Something is already mounted at {root}'.format(root=info.root) + raise TaskError(msg) + + log_check_call(['/bin/mount', info.bootstrap_device['path'], info.root]) + + +class MountSpecials(Task): + description = 'Mounting special block devices' + phase = phases.os_installation + after = [Bootstrap] + + def run(self, info): + log_check_call(['/bin/mount', '--bind', '/dev', '{root}/dev'.format(root=info.root)]) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'proc', 'none', '/proc']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'sysfs', 'none', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/mount', '-t', 'devpts', 'none', '/dev/pts']) + + +class UnmountSpecials(Task): + description = 'Unmunting special block devices' + phase = phases.volume_unmounting + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/dev/pts']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/sys']) + log_check_call(['/usr/sbin/chroot', info.root, '/bin/umount', '/proc']) + log_check_call(['/bin/umount', '{root}/dev'.format(root=info.root)]) + + +class UnmountVolume(Task): + description = 'Unmounting the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountSpecials] + + def run(self, info): + log_check_call(['/bin/umount', info.root]) + + +class DeleteMountDir(Task): + description = 'Deleting mountpoint for the bootstrap volume' + phase = phases.volume_unmounting + after = [UnmountVolume] + + def run(self, info): + import os + os.rmdir(info.root) + del info.root + + +class ModifyFstab(Task): + description = 'Adding root volume to the fstab' + phase = phases.system_modification + + def run(self, info): + import os.path + mount_opts = ['defaults'] + if info.manifest.volume['filesystem'].lower() in ['ext2', 'ext3', 'ext4']: + mount_opts.append('barrier=0') + if info.manifest.volume['filesystem'].lower() == 'xfs': + mount_opts.append('nobarrier') + fstab_path = os.path.join(info.root, 'etc/fstab') + with open(fstab_path, 'a') as fstab: + fstab.write(('/dev/xvda1 / {filesystem} {mount_opts} 1 1\n' + .format(filesystem=info.manifest.volume['filesystem'].lower(), + mount_opts=','.join(mount_opts)))) diff --git a/providers/one/tasks/host.py b/providers/one/tasks/host.py new file mode 100644 index 0000000..61af65a --- /dev/null +++ b/providers/one/tasks/host.py @@ -0,0 +1,29 @@ +from base import Task +from common import phases +from common.exceptions import TaskError +import packages + + +class CheckPackages(Task): + description = 'Checking installed host packages' + phase = phases.preparation + after = [packages.HostPackages, packages.ImagePackages] + + def run(self, info): + from common.tools import log_check_call + from subprocess import CalledProcessError + for package in info.host_packages: + try: + log_check_call(['/usr/bin/dpkg', '-s', package]) + except CalledProcessError: + msg = "The package ``{0}\'\' is not installed".format(package) + raise TaskError(msg) + + +class GetInfo(Task): + description = 'Retrieving instance metadata' + phase = phases.preparation + + def run(self, info): + info.host = {} + return info diff --git a/providers/one/tasks/initd.py b/providers/one/tasks/initd.py new file mode 100644 index 0000000..5a479fe --- /dev/null +++ b/providers/one/tasks/initd.py @@ -0,0 +1,49 @@ +from base import Task +from common import phases +import os.path + + +class ResolveInitScripts(Task): + description = 'Determining which startup scripts to install or disable' + phase = phases.system_modification + + def run(self, info): + init_scripts = {'ec2-get-credentials': 'ec2-get-credentials', + 'ec2-run-user-data': 'ec2-run-user-data', + 'expand-volume': 'expand-volume'} + + init_scripts['generate-ssh-hostkeys'] = 'generate-ssh-hostkeys' + if info.manifest.system['release'] == 'squeeze': + init_scripts['generate-ssh-hostkeys'] = 'squeeze/generate-ssh-hostkeys' + + disable_scripts = ['hwclock.sh'] + if info.manifest.system['release'] == 'squeeze': + disable_scripts.append('hwclockfirst.sh') + + for name, path in init_scripts.iteritems(): + init_scripts[name] = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/init.d', path)) + + info.initd = {'install': init_scripts, + 'disable': disable_scripts} + + +class InstallInitScripts(Task): + description = 'Installing startup scripts' + phase = phases.system_modification + after = [ResolveInitScripts] + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + from shutil import copy + from common.tools import log_check_call + for name, src in info.initd['install'].iteritems(): + dst = os.path.join(info.root, 'etc/init.d', name) + copy(src, dst) + os.chmod(dst, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-d', name]) + + for name in info.initd['disable']: + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/insserv', '-r', name]) diff --git a/providers/one/tasks/locale.py b/providers/one/tasks/locale.py new file mode 100644 index 0000000..de3e434 --- /dev/null +++ b/providers/one/tasks/locale.py @@ -0,0 +1,35 @@ +from base import Task +from common import phases +import os.path + + +class GenerateLocale(Task): + description = 'Generating the selected locale' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + from common.tools import log_check_call + locale_gen = os.path.join(info.root, 'etc/locale.gen') + locale_str = '{locale}.{charmap} {charmap}'.format(locale=info.manifest.system['locale'], + charmap=info.manifest.system['charmap']) + search = '# ' + locale_str + sed_i(locale_gen, search, locale_str) + + command = ['/usr/sbin/chroot', info.root, '/usr/sbin/dpkg-reconfigure', '--priority=critical', 'locales'] + log_check_call(command) + + +class SetTimezone(Task): + description = 'Setting the selected timezone' + phase = phases.system_modification + + def run(self, info): + from shutil import copy + tz_path = os.path.join(info.root, 'etc/timezone') + timezone = info.manifest.system['timezone'] + with open(tz_path, 'w') as tz_file: + tz_file.write(timezone) + zoneinfo_path = os.path.join(info.root, '/usr/share/zoneinfo', timezone) + localtime_path = os.path.join(info.root, 'etc/localtime') + copy(zoneinfo_path, localtime_path) diff --git a/providers/one/tasks/network.py b/providers/one/tasks/network.py new file mode 100644 index 0000000..0910608 --- /dev/null +++ b/providers/one/tasks/network.py @@ -0,0 +1,38 @@ +from base import Task +from common import phases +import os.path + + +class RemoveDNSInfo(Task): + description = 'Removing resolv.conf' + phase = phases.system_modification + + def run(self, info): + from os import remove + remove(os.path.join(info.root, 'etc/resolv.conf')) + + +class ConfigureNetworkIF(Task): + description = 'Configuring network interfaces' + phase = phases.system_modification + + def run(self, info): + interfaces_path = os.path.join(info.root, 'etc/network/interfaces') + if_config = {'squeeze': ('auto lo\n' + 'iface lo inet loopback\n' + 'auto eth0\n' + 'iface eth0 inet dhcp\n'), + 'wheezy': 'auto eth0\n' + 'iface eth0 inet dhcp\n'} + with open(interfaces_path, 'a') as interfaces: + interfaces.write(if_config.get(info.manifest.system['release'])) + + +class ConfigureDHCP(Task): + description = 'Configuring the DHCP client' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + dhcpcd = os.path.join(info.root, 'etc/default/dhcpcd') + sed_i(dhcpcd, '^#*SET_DNS=.*', 'SET_DNS=\'yes\'') diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py new file mode 100644 index 0000000..b5474b4 --- /dev/null +++ b/providers/one/tasks/packages.py @@ -0,0 +1,48 @@ +from base import Task +from common import phases + + +class HostPackages(Task): + description = 'Determining required host packages' + phase = phases.preparation + + def run(self, info): + packages = set(['debootstrap']) + if info.manifest.volume['filesystem'] == 'xfs': + packages.add('xfsprogs') + + info.host_packages = packages + + +class ImagePackages(Task): + description = 'Determining required image packages' + phase = phases.preparation + + def run(self, info): + manifest = info.manifest + # Add some basic packages we are going to need + include = set(['udev', + 'openssh-server', + # We could bootstrap without locales, but things just suck without them, error messages etc. + 'locales', + # Needed for the init scripts + 'file', + # isc-dhcp-client doesn't work properly with ec2 + 'dhcpcd', + ]) + + if manifest.virtualization == 'pvm': + include.add('grub-pc') + + exclude = set(['isc-dhcp-client', + 'isc-dhcp-common', + ]) + + # In squeeze, we need a special kernel flavor for xen + kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64', + 'i386': 'linux-image-xen-686', }, + 'wheezy': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, } + include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) + + info.img_packages = include, exclude diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py new file mode 100644 index 0000000..50d8db6 --- /dev/null +++ b/providers/one/tasks/security.py @@ -0,0 +1,32 @@ +from base import Task +from common import phases +import os.path + + +class EnableShadowConfig(Task): + description = 'Enabling shadowconfig' + phase = phases.system_modification + + def run(self, info): + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on']) + + +class DisableSSHPasswordAuthentication(Task): + description = 'Disabling SSH password authentication' + phase = phases.system_modification + + def run(self, info): + from common.tools import sed_i + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') + + +class DisableSSHDNSLookup(Task): + description = 'Disabling sshd remote host name lookup' + phase = phases.system_modification + + def run(self, info): + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + with open(sshd_config_path, 'a') as sshd_config: + sshd_config.write('UseDNS no') From 0854f7920e0c92991e03a8d0719d5434373660a8 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 11:27:49 +0200 Subject: [PATCH 02/22] add raw image file creation --- manifests/one-raw-virtio.manifest.json | 2 +- providers/one/tasks/filesystem.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json index 96d5003..999e8a1 100644 --- a/manifests/one-raw-virtio.manifest.json +++ b/manifests/one-raw-virtio.manifest.json @@ -7,7 +7,7 @@ }, "bootstrapper": { - "mount_dir": "/target", + "mount_dir": "/tmp/target", "tarball": true, "device": "/dev/sda" }, diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index 50481f4..30a8c36 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -10,7 +10,9 @@ class FormatVolume(Task): phase = phases.volume_preparation def run(self, info): - dev_path = info.manifest.bootstrapper['device'] + mkmount = 'create -f raw "'+info.manifest.bootstrapper['mount_dir']+'" "'+str(info.manifest.volume['size'])+'"' + log_check_call(['/usr/bin/qemu-img',mkmount]) + dev_path = info.manifest.bootstrapper['mount_dir'] mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) log_check_call([mkfs, dev_path]) From ed76870ec929d64f682dbbc40e368d817aaee791 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Jul 2013 14:42:03 +0200 Subject: [PATCH 03/22] fix raw image creation --- base/manifest-schema.json | 4 +++- manifests/one-raw-virtio.manifest.json | 3 ++- providers/one/tasks/filesystem.py | 27 ++++++++++++++++---------- providers/one/tasks/packages.py | 2 +- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index a98907b..cc723a4 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -10,8 +10,10 @@ "type": "object", "properties": { "mount_dir": { "type": "string" }, + "image_file": { "type": "string" }, "tarball": { "type": "boolean" }, - "tarball_dir": { "type": "string" } + "tarball_dir": { "type": "string" }, + "device" : { "type" : "string" } }, "required": ["mount_dir"] }, diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json index 999e8a1..94f65cc 100644 --- a/manifests/one-raw-virtio.manifest.json +++ b/manifests/one-raw-virtio.manifest.json @@ -7,7 +7,8 @@ }, "bootstrapper": { - "mount_dir": "/tmp/target", + "mount_dir": "/mnt/target", + "image_file": "/tmp/target", "tarball": true, "device": "/dev/sda" }, diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index 30a8c36..d3d9a4f 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -3,18 +3,22 @@ from common import phases from common.exceptions import TaskError from common.tools import log_check_call from bootstrap import Bootstrap - +import os class FormatVolume(Task): description = 'Formatting the volume' phase = phases.volume_preparation def run(self, info): - mkmount = 'create -f raw "'+info.manifest.bootstrapper['mount_dir']+'" "'+str(info.manifest.volume['size'])+'"' - log_check_call(['/usr/bin/qemu-img',mkmount]) - dev_path = info.manifest.bootstrapper['mount_dir'] - mkfs = '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']) - log_check_call([mkfs, dev_path]) + mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] + log_check_call(mkmount) + ddcmd = ['/bin/dd', 'if=/dev/zero', 'bs=1024', 'conv=notrunc', 'count='+str(info.manifest.volume['size']), 'of='+info.manifest.bootstrapper['image_file']] + log_check_call(ddcmd) + loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] + log_check_call(loopcmd) + mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/loop0'] + log_check_call(mkfs) + class TuneVolumeFS(Task): @@ -23,7 +27,8 @@ class TuneVolumeFS(Task): after = [FormatVolume] def run(self, info): - dev_path = info.bootstrap_device['path'] + #dev_path = info.bootstrap_device['path'] + dev_path = info.manifest.bootstrapper['image_file'] # Disable the time based filesystem check log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) @@ -44,9 +49,9 @@ class CreateMountDir(Task): def run(self, info): import os mount_dir = info.manifest.bootstrapper['mount_dir'] - info.root = '{mount_dir}/{vol_id}'.format(mount_dir=mount_dir, vol_id=info.volume.id) + info.root = mount_dir # Works recursively, fails if last part exists, which is exaclty what we want. - os.makedirs(info.root) + os.makedirs(mount_dir) class MountVolume(Task): @@ -61,7 +66,7 @@ class MountVolume(Task): msg = 'Something is already mounted at {root}'.format(root=info.root) raise TaskError(msg) - log_check_call(['/bin/mount', info.bootstrap_device['path'], info.root]) + log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/loop0', info.root]) class MountSpecials(Task): @@ -94,6 +99,8 @@ class UnmountVolume(Task): def run(self, info): log_check_call(['/bin/umount', info.root]) + log_check_call(['/sbin/losetup', '-d', '/dev/loop0']) + class DeleteMountDir(Task): diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index b5474b4..eb2260a 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -7,7 +7,7 @@ class HostPackages(Task): phase = phases.preparation def run(self, info): - packages = set(['debootstrap']) + packages = set(['debootstrap', 'qemu-utils']) if info.manifest.volume['filesystem'] == 'xfs': packages.add('xfsprogs') From f9fafe71fbc8ec8480b46495ab7cb7038a63a0e7 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 14:55:27 +0200 Subject: [PATCH 04/22] install grub --- providers/one/tasks/packages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index eb2260a..45359a1 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -29,6 +29,7 @@ class ImagePackages(Task): 'file', # isc-dhcp-client doesn't work properly with ec2 'dhcpcd', + 'grub2', ]) if manifest.virtualization == 'pvm': From 17876070a5c227cb85f33653f855551440b2d539 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 15:43:39 +0200 Subject: [PATCH 05/22] add mirror attribute --- base/manifest-schema.json | 3 ++- providers/one/__init__.py | 3 ++- providers/one/tasks/apt.py | 5 ++++- providers/one/tasks/bootstrap.py | 6 +++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/base/manifest-schema.json b/base/manifest-schema.json index cc723a4..21eb276 100644 --- a/base/manifest-schema.json +++ b/base/manifest-schema.json @@ -30,7 +30,8 @@ }, "timezone": { "type": "string" }, "locale": { "type": "string" }, - "charmap": { "type": "string" } + "charmap": { "type": "string" }, + "mirror": { "type": "string" } }, "required": ["release", "architecture", "timezone", "locale", "charmap"] }, diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 3372c1f..426607f 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -41,7 +41,8 @@ def tasks(tasklist, manifest): locale.SetTimezone(), apt.DisableDaemonAutostart(), apt.AptSources(), - apt.AptUpgrade(), + #No network for the moment, skip + #apt.AptUpgrade(), boot.ConfigureGrub(), filesystem.ModifyFstab(), boot.BlackListModules(), diff --git a/providers/one/tasks/apt.py b/providers/one/tasks/apt.py index e039df4..d2d86d3 100644 --- a/providers/one/tasks/apt.py +++ b/providers/one/tasks/apt.py @@ -10,11 +10,14 @@ class AptSources(Task): phase = phases.system_modification def run(self, info): + mirror = 'http://http.debian.net/debian' + if info.manifest.system['mirror']: + mirror = info.manifest.system['mirror'] sources_path = os.path.join(info.root, 'etc/apt/sources.list') with open(sources_path, 'w') as apt_sources: apt_sources.write(('deb {apt_mirror} {release} main\n' 'deb-src {apt_mirror} {release} main\n' - .format(apt_mirror='http://http.debian.net/debian', + .format(apt_mirror=mirror, release=info.manifest.system['release']))) apt_sources.write(('deb {apt_mirror} {release}/updates main\n' 'deb-src {apt_mirror} {release}/updates main\n' diff --git a/providers/one/tasks/bootstrap.py b/providers/one/tasks/bootstrap.py index 9e27f36..9b405b0 100644 --- a/providers/one/tasks/bootstrap.py +++ b/providers/one/tasks/bootstrap.py @@ -6,6 +6,10 @@ log = logging.getLogger(__name__) def get_bootstrap_args(info): + mirror = 'http://http.debian.net/debian' + if info.manifest.system['mirror']: + mirror = info.manifest.system['mirror'] + executable = ['/usr/sbin/debootstrap'] options = ['--arch=' + info.manifest.system['architecture']] include, exclude = info.img_packages @@ -13,7 +17,7 @@ def get_bootstrap_args(info): options.append('--include=' + ','.join(include)) if len(exclude) > 0: options.append('--exclude=' + ','.join(exclude)) - arguments = [info.manifest.system['release'], info.root, 'http://http.debian.net/debian'] + arguments = [info.manifest.system['release'], info.root, mirror] return executable, options, arguments From 6c81ee2f8c43c6ba214a5daf2a16762b60552d2c Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 16:33:35 +0200 Subject: [PATCH 06/22] allow to PIPE some input to command --- common/tools.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/common/tools.py b/common/tools.py index 546f697..fee632b 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,13 +1,13 @@ -def log_check_call(command): - status, stdout, stderr = log_call(command) +def log_check_call(command, input=None): + status, stdout, stderr = log_call(command, input) if status != 0: from subprocess import CalledProcessError raise CalledProcessError(status, ' '.join(command), '\n'.join(stderr)) -def log_call(command): +def log_call(command, input=None): import subprocess import select @@ -16,7 +16,15 @@ def log_call(command): command_log = realpath(command[0]).replace('/', '.') log = logging.getLogger(__name__ + command_log) - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if input is not None: + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.stdin.write(input+"\n") + process.stdin.flush() + process.stdin.close() + # (stdout,stderr) = process.communicate(input+"\n") + # return process.returncode, stdout, stderr + else: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = [] stderr = [] while True: From 5e6ac546cd3c9b9106f26023fe9a40113583c019 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 16:34:12 +0200 Subject: [PATCH 07/22] add root password setting --- providers/one/__init__.py | 4 +++- providers/one/manifest-schema.json | 2 +- providers/one/tasks/security.py | 7 +++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 426607f..067a964 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -48,7 +48,9 @@ def tasks(tasklist, manifest): boot.BlackListModules(), boot.DisableGetTTYs(), security.EnableShadowConfig(), - security.DisableSSHPasswordAuthentication(), + security.SetRootPassword(), + # Disable for the time of debugging + #security.DisableSSHPasswordAuthentication(), security.DisableSSHDNSLookup(), network.RemoveDNSInfo(), network.ConfigureNetworkIF(), diff --git a/providers/one/manifest-schema.json b/providers/one/manifest-schema.json index 0a6a038..8c9e8f5 100644 --- a/providers/one/manifest-schema.json +++ b/providers/one/manifest-schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "EC2 manifest", + "title": "OpenNebula manifest", "type": "object", "properties": { "volume": { diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py index 50d8db6..0178af2 100644 --- a/providers/one/tasks/security.py +++ b/providers/one/tasks/security.py @@ -11,6 +11,13 @@ class EnableShadowConfig(Task): from common.tools import log_check_call log_check_call(['/usr/sbin/chroot', info.root, '/sbin/shadowconfig', 'on']) +class SetRootPassword(Task): + description = 'Set password for root' + phase = phases.system_modification + + def run(self, info): + from common.tools import log_check_call + log_check_call(['/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) class DisableSSHPasswordAuthentication(Task): description = 'Disabling SSH password authentication' From 1bfff5221abd5630ac35443242e6b41729ecfa61 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 16:34:28 +0200 Subject: [PATCH 08/22] add sample root password --- manifests/one-raw-virtio.manifest.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json index 94f65cc..6bf2579 100644 --- a/manifests/one-raw-virtio.manifest.json +++ b/manifests/one-raw-virtio.manifest.json @@ -3,7 +3,8 @@ "virtualization": "virtio", "credentials" : { "access-key": null, - "secret-key": null + "secret-key": null, + "root": "test" }, "bootstrapper": { @@ -21,7 +22,8 @@ "architecture": "amd64", "timezone" : "UTC", "locale" : "en_US", - "charmap" : "UTF-8" + "charmap" : "UTF-8", + "mirror" : "ftp://ftp.fr.debian.org/debian/" }, "volume": { "backing" : "raw", From a4e8f082605f4c0a7c5ee8b5a50282425467be29 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 16:34:48 +0200 Subject: [PATCH 09/22] =?UTF-8?q?set=20sda=20for=20default=20grub=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- providers/one/assets/grub.d/40_custom | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom index 799e887..0e74a8e 100644 --- a/providers/one/assets/grub.d/40_custom +++ b/providers/one/assets/grub.d/40_custom @@ -13,7 +13,7 @@ libdir=${exec_prefix}/lib export TEXTDOMAIN=grub export TEXTDOMAINDIR=${prefix}/share/locale -GRUB_DEVICE=/dev/xvda1 +GRUB_DEVICE=/dev/sda1 cat << EOF From a2e8d0838e63b3c48e59d34aac85d1c8519ed270 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 10 Jul 2013 16:35:40 +0200 Subject: [PATCH 10/22] set root password only if set in manifest --- providers/one/tasks/security.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py index 0178af2..cdf519b 100644 --- a/providers/one/tasks/security.py +++ b/providers/one/tasks/security.py @@ -17,7 +17,8 @@ class SetRootPassword(Task): def run(self, info): from common.tools import log_check_call - log_check_call(['/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) + if info.manifest.credentials['root']: + log_check_call(['/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) class DisableSSHPasswordAuthentication(Task): description = 'Disabling SSH password authentication' From 1fe0d56bc3369cc2041f449bdee21f20add76708 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Fri, 26 Jul 2013 08:50:53 +0200 Subject: [PATCH 11/22] updates --- manifests/one-raw-virtio.manifest.json | 2 +- providers/one/__init__.py | 2 ++ providers/one/tasks/boot.py | 6 ++++-- providers/one/tasks/fake.py | 13 +++++++++++++ providers/one/tasks/filesystem.py | 14 +++++++++++--- providers/one/tasks/packages.py | 1 + 6 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 providers/one/tasks/fake.py diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json index 6bf2579..33cff2a 100644 --- a/manifests/one-raw-virtio.manifest.json +++ b/manifests/one-raw-virtio.manifest.json @@ -15,7 +15,7 @@ }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} AMI ({virtualization})" + "description": "Debian {release} {architecture} Open Nebula ({virtualization})" }, "system": { "release" : "wheezy", diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 067a964..2e64e1c 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -14,6 +14,7 @@ from tasks import security from tasks import network from tasks import initd from tasks import cleanup +from tasks import fake def initialize(): @@ -55,6 +56,7 @@ def tasks(tasklist, manifest): network.RemoveDNSInfo(), network.ConfigureNetworkIF(), network.ConfigureDHCP(), + fake.Fake(), initd.ResolveInitScripts(), initd.InstallInitScripts(), cleanup.ClearMOTD(), diff --git a/providers/one/tasks/boot.py b/providers/one/tasks/boot.py index 4265b9d..58c0918 100644 --- a/providers/one/tasks/boot.py +++ b/providers/one/tasks/boot.py @@ -26,11 +26,13 @@ class ConfigureGrub(Task): from common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') - sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' - 'GRUB_HIDDEN_TIMEOUT=true') + #sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' + # 'GRUB_HIDDEN_TIMEOUT=true') from common.tools import log_check_call log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/grub.cfg']) + log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/device.map']) log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) diff --git a/providers/one/tasks/fake.py b/providers/one/tasks/fake.py new file mode 100644 index 0000000..33648ac --- /dev/null +++ b/providers/one/tasks/fake.py @@ -0,0 +1,13 @@ +from base import Task +from common import phases +import os.path + +class Fake(Task): + description = 'create fake file' + phase = phases.system_modification + + def run(self, info): + fake_path = os.path.join(info.root, 'fake.txt') + with open(fake_path, 'a') as fakefile: + fakefile.write("fake file") + diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index d3d9a4f..070d3cb 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -12,8 +12,8 @@ class FormatVolume(Task): def run(self, info): mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] log_check_call(mkmount) - ddcmd = ['/bin/dd', 'if=/dev/zero', 'bs=1024', 'conv=notrunc', 'count='+str(info.manifest.volume['size']), 'of='+info.manifest.bootstrapper['image_file']] - log_check_call(ddcmd) + #ddcmd = ['/bin/dd', 'if=/dev/zero', 'bs=1024', 'conv=notrunc', 'count='+str(info.manifest.volume['size']), 'of='+info.manifest.bootstrapper['image_file']] + #log_check_call(ddcmd) loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] log_check_call(loopcmd) mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/loop0'] @@ -127,6 +127,14 @@ class ModifyFstab(Task): mount_opts.append('nobarrier') fstab_path = os.path.join(info.root, 'etc/fstab') with open(fstab_path, 'a') as fstab: - fstab.write(('/dev/xvda1 / {filesystem} {mount_opts} 1 1\n' + fstab.write(('/dev/sda1 / {filesystem} {mount_opts} 1 1\n' .format(filesystem=info.manifest.volume['filesystem'].lower(), mount_opts=','.join(mount_opts)))) + log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/etc/fstab']) + +class InstallMbr(Task): + description = 'Install MBR' + phase = phases.system_modification + + def run(self, info): + log_check_call(['/usr/sbin/chroot', info.root, 'install-mbr', '/dev/sda']) diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index 45359a1..3f8ff36 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -22,6 +22,7 @@ class ImagePackages(Task): manifest = info.manifest # Add some basic packages we are going to need include = set(['udev', + 'mbr', 'openssh-server', # We could bootstrap without locales, but things just suck without them, error messages etc. 'locales', From cb4b2aad19cc225b16f2c8cca4b759a5e961b2f3 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Fri, 26 Jul 2013 14:16:40 +0200 Subject: [PATCH 12/22] fixes for loopback and filesystem setup --- providers/one/__init__.py | 1 + providers/one/tasks/filesystem.py | 29 +++++++++++++++++++---------- providers/one/tasks/packages.py | 1 + 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 2e64e1c..a7c16ec 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -46,6 +46,7 @@ def tasks(tasklist, manifest): #apt.AptUpgrade(), boot.ConfigureGrub(), filesystem.ModifyFstab(), + #filesystem.InstallMbr(), boot.BlackListModules(), boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index 070d3cb..08f149a 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -12,12 +12,19 @@ class FormatVolume(Task): def run(self, info): mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] log_check_call(mkmount) - #ddcmd = ['/bin/dd', 'if=/dev/zero', 'bs=1024', 'conv=notrunc', 'count='+str(info.manifest.volume['size']), 'of='+info.manifest.bootstrapper['image_file']] - #log_check_call(ddcmd) - loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] - log_check_call(loopcmd) - mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/loop0'] - log_check_call(mkfs) + + # parted + log_check_call(['parted','-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "mklabel", "msdos"]) + log_check_call(['parted', '-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "--", "mkpart", "primary", "ext4", "1", "-1"]) + log_check_call(['parted','-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "--", "set", "1", "boot", "on"]) + log_check_call(['kpartx','-a','-v', info.manifest.bootstrapper['image_file']]) + + + #loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] + #log_check_call(loopcmd) + mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/mapper/loop0p1'] + log_check_call(mkfs) + @@ -28,7 +35,8 @@ class TuneVolumeFS(Task): def run(self, info): #dev_path = info.bootstrap_device['path'] - dev_path = info.manifest.bootstrapper['image_file'] + #dev_path = info.manifest.bootstrapper['image_file'] + dev_path = '/dev/mapper/loop0p1' # Disable the time based filesystem check log_check_call(['/sbin/tune2fs', '-i', '0', dev_path]) @@ -66,8 +74,7 @@ class MountVolume(Task): msg = 'Something is already mounted at {root}'.format(root=info.root) raise TaskError(msg) - log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/loop0', info.root]) - + log_check_call(['/bin/mount', '-t', info.manifest.volume['filesystem'], '/dev/mapper/loop0p1', info.root]) class MountSpecials(Task): description = 'Mounting special block devices' @@ -99,7 +106,8 @@ class UnmountVolume(Task): def run(self, info): log_check_call(['/bin/umount', info.root]) - log_check_call(['/sbin/losetup', '-d', '/dev/loop0']) + #log_check_call(['/sbin/losetup', '-d', '/dev/loop0']) + log_check_call(['kpartx','-d', info.manifest.bootstrapper['image_file']]) @@ -138,3 +146,4 @@ class InstallMbr(Task): def run(self, info): log_check_call(['/usr/sbin/chroot', info.root, 'install-mbr', '/dev/sda']) + log_check_call(['/usr/sbin/chroot', info.root, 'fdisk', '-l', '/dev/sda']) diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index 3f8ff36..53df61d 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -23,6 +23,7 @@ class ImagePackages(Task): # Add some basic packages we are going to need include = set(['udev', 'mbr', + 'parted', 'openssh-server', # We could bootstrap without locales, but things just suck without them, error messages etc. 'locales', From af7dcebb25e13eebc170178b7278318afc3c44a7 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Fri, 26 Jul 2013 15:33:16 +0200 Subject: [PATCH 13/22] udpate grub to add menu and add MBR --- providers/one/__init__.py | 2 +- providers/one/assets/grub.d/40_custom | 21 ++++-- providers/one/assets/grub.d/40_custom.orig | 81 ++++++++++++++++++++++ providers/one/tasks/filesystem.py | 3 +- 4 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 providers/one/assets/grub.d/40_custom.orig diff --git a/providers/one/__init__.py b/providers/one/__init__.py index a7c16ec..7cc01a7 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -46,7 +46,7 @@ def tasks(tasklist, manifest): #apt.AptUpgrade(), boot.ConfigureGrub(), filesystem.ModifyFstab(), - #filesystem.InstallMbr(), + filesystem.InstallMbr(), boot.BlackListModules(), boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom index 0e74a8e..9977def 100644 --- a/providers/one/assets/grub.d/40_custom +++ b/providers/one/assets/grub.d/40_custom @@ -17,8 +17,14 @@ GRUB_DEVICE=/dev/sda1 cat << EOF -default ${GRUB_DEFAULT} -timeout ${GRUB_TIMEOUT} +set default ${GRUB_DEFAULT} +set timeout ${GRUB_TIMEOUT} +insmod part_msdos +insmod ext4 +insmod gettext +set menu_color_normal=cyan/blue +set meu_color_highlight=white/blue +set root='(hd0,msdos1)' EOF if ${GRUB_HIDDEN_TIMEOUT:-false}; then @@ -34,10 +40,15 @@ linux_entry () title="$(gettext_quoted "%s, with Linux %s")" cat << EOF -title ${version} - root (hd0) - kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} +menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { + insmod part_msdos + insmod ext4 + set root='(hd0,msdos1)' + echo 'Loading Linux ${version}' + linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + echo 'Loading initial ramdisk ...' initrd ${rel_dirname}/${initrd} +} EOF } diff --git a/providers/one/assets/grub.d/40_custom.orig b/providers/one/assets/grub.d/40_custom.orig new file mode 100644 index 0000000..0e74a8e --- /dev/null +++ b/providers/one/assets/grub.d/40_custom.orig @@ -0,0 +1,81 @@ +#!/bin/sh + +# This file generates the old menu.lst configuration with grub2 +# It was copied from tomheadys github repo: +# https://github.com/tomheady/ec2debian/blob/master/src/root/etc/grub.d/40_custom + +prefix=/usr +exec_prefix=${prefix} +bindir=${exec_prefix}/bin +libdir=${exec_prefix}/lib +. ${libdir}/grub/grub-mkconfig_lib + +export TEXTDOMAIN=grub +export TEXTDOMAINDIR=${prefix}/share/locale + +GRUB_DEVICE=/dev/sda1 + + +cat << EOF +default ${GRUB_DEFAULT} +timeout ${GRUB_TIMEOUT} +EOF + +if ${GRUB_HIDDEN_TIMEOUT:-false}; then + printf "hiddenmenu\n" +fi + +linux_entry () +{ + os="$1" + version="$2" + args="$4" + + title="$(gettext_quoted "%s, with Linux %s")" + + cat << EOF +title ${version} + root (hd0) + kernel ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} + initrd ${rel_dirname}/${initrd} +EOF +} + +list=`for i in /boot/vmlinuz-* /boot/vmlinux-* /vmlinuz-* /vmlinux-* ; do + if grub_file_is_not_garbage "$i" ; then echo -n "$i " ; fi + done` +prepare_boot_cache= + +while [ "x$list" != "x" ] ; do + linux=`version_find_latest $list` + basename=`basename $linux` + dirname=`dirname $linux` + rel_dirname=`make_system_path_relative_to_its_root $dirname` + version=`echo $basename | sed -e "s,^[^0-9]*-,,g"` + alt_version=`echo $version | sed -e "s,\.old$,,g"` + linux_root_device_thisversion="${LINUX_ROOT_DEVICE}" + + initrd= + for i in "initrd.img-${version}" "initrd-${version}.img" \ + "initrd-${version}" "initramfs-${version}.img" \ + "initrd.img-${alt_version}" "initrd-${alt_version}.img" \ + "initrd-${alt_version}" "initramfs-${alt_version}.img"; do + if test -e "${dirname}/${i}" ; then + initrd="$i" + break + fi + done + + initramfs= + for i in "config-${version}" "config-${alt_version}"; do + if test -e "${dirname}/${i}" ; then + initramfs=`grep CONFIG_INITRAMFS_SOURCE= "${dirname}/${i}" | cut -f2 -d= | tr -d \"` + break + fi + done + + linux_entry "${OS}" "${version}" \ + "${GRUB_CMDLINE_LINUX} ${GRUB_CMDLINE_LINUX_DEFAULT}" + + list=`echo $list | tr ' ' '\n' | grep -vx $linux | tr '\n' ' '` +done diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index 08f149a..618d8e1 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -145,5 +145,4 @@ class InstallMbr(Task): phase = phases.system_modification def run(self, info): - log_check_call(['/usr/sbin/chroot', info.root, 'install-mbr', '/dev/sda']) - log_check_call(['/usr/sbin/chroot', info.root, 'fdisk', '-l', '/dev/sda']) + log_check_call(['install-mbr', info.manifest.bootstrapper['image_file']]) From 6be6879db7933d8111531bb11a6ee4aae4a9fade Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Tue, 30 Jul 2013 11:21:09 +0200 Subject: [PATCH 14/22] fix grub install --- providers/one/__init__.py | 2 +- providers/one/assets/grub.d/40_custom | 5 +++-- providers/one/tasks/boot.py | 5 +++++ providers/one/tasks/filesystem.py | 17 ++++++++++------- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 7cc01a7..ebabe0e 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -34,6 +34,7 @@ def tasks(tasklist, manifest): tasklist.add(filesystem.TuneVolumeFS()) tasklist.add(filesystem.CreateMountDir(), filesystem.MountVolume()) + #tasklist.add(filesystem.InstallMbr()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) tasklist.add(bootstrap.Bootstrap(), @@ -46,7 +47,6 @@ def tasks(tasklist, manifest): #apt.AptUpgrade(), boot.ConfigureGrub(), filesystem.ModifyFstab(), - filesystem.InstallMbr(), boot.BlackListModules(), boot.DisableGetTTYs(), security.EnableShadowConfig(), diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom index 9977def..56419c7 100644 --- a/providers/one/assets/grub.d/40_custom +++ b/providers/one/assets/grub.d/40_custom @@ -20,7 +20,7 @@ cat << EOF set default ${GRUB_DEFAULT} set timeout ${GRUB_TIMEOUT} insmod part_msdos -insmod ext4 +insmod ext2 insmod gettext set menu_color_normal=cyan/blue set meu_color_highlight=white/blue @@ -42,7 +42,8 @@ linux_entry () cat << EOF menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { insmod part_msdos - insmod ext4 + insmod ext2 + set timeout ${GRUB_TIMEOUT} set root='(hd0,msdos1)' echo 'Loading Linux ${version}' linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} diff --git a/providers/one/tasks/boot.py b/providers/one/tasks/boot.py index 58c0918..abedc65 100644 --- a/providers/one/tasks/boot.py +++ b/providers/one/tasks/boot.py @@ -35,6 +35,11 @@ class ConfigureGrub(Task): log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/device.map']) log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) + log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) + log_check_call(['/usr/sbin/chroot', info.root, 'ls', '-l', '/boot/']) + #sed_i(info.root+'/boot/grub/device.map','/dev/sda','/dev/mapper/loop0p1;') + log_check_call(['grub-install', '--boot-directory='+info.root+"/boot/", '/dev/loop0']) + #sed_i(info.root+'/boot/grub/device.map','/dev/mapper/loop0p1;','/dev/sda') class BlackListModules(Task): description = 'Blacklisting kernel modules' diff --git a/providers/one/tasks/filesystem.py b/providers/one/tasks/filesystem.py index 618d8e1..5b255c7 100644 --- a/providers/one/tasks/filesystem.py +++ b/providers/one/tasks/filesystem.py @@ -13,15 +13,17 @@ class FormatVolume(Task): mkmount = ['/usr/bin/qemu-img', 'create', '-f', 'raw', info.manifest.bootstrapper['image_file'], str(info.manifest.volume['size'])+'M'] log_check_call(mkmount) + loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] + log_check_call(loopcmd) + # parted - log_check_call(['parted','-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "mklabel", "msdos"]) - log_check_call(['parted', '-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "--", "mkpart", "primary", "ext4", "1", "-1"]) - log_check_call(['parted','-a', 'optimal', '-s', info.manifest.bootstrapper['image_file'], "--", "set", "1", "boot", "on"]) - log_check_call(['kpartx','-a','-v', info.manifest.bootstrapper['image_file']]) + log_check_call(['parted','-a', 'optimal', '-s', '/dev/loop0', "mklabel", "msdos"]) + log_check_call(['parted', '-a', 'optimal', '-s', '/dev/loop0', "--", "mkpart", "primary", "ext4", "32k", "-1"]) + log_check_call(['parted','-s', '/dev/loop0', "--", "set", "1", "boot", "on"]) - #loopcmd = ['/sbin/losetup', '/dev/loop0', info.manifest.bootstrapper['image_file']] - #log_check_call(loopcmd) + #log_check_call(['kpartx','-a','-v', info.manifest.bootstrapper['image_file']]) + log_check_call(['kpartx','-a', '-v', '/dev/loop0']) mkfs = [ '/sbin/mkfs.{fs}'.format(fs=info.manifest.volume['filesystem']), '-m', '1', '-v', '/dev/mapper/loop0p1'] log_check_call(mkfs) @@ -106,6 +108,7 @@ class UnmountVolume(Task): def run(self, info): log_check_call(['/bin/umount', info.root]) + #log_check_call(['partx','-d','/dev/loop0']) #log_check_call(['/sbin/losetup', '-d', '/dev/loop0']) log_check_call(['kpartx','-d', info.manifest.bootstrapper['image_file']]) @@ -145,4 +148,4 @@ class InstallMbr(Task): phase = phases.system_modification def run(self, info): - log_check_call(['install-mbr', info.manifest.bootstrapper['image_file']]) + log_check_call(['install-mbr', '-v', info.manifest.bootstrapper['image_file']]) From ac6870e37d3f167e94a150004f9daf89b2e4067d Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Tue, 30 Jul 2013 13:03:08 +0200 Subject: [PATCH 15/22] fix root auth and grub settings --- providers/one/assets/grub.d/40_custom | 10 +++++----- providers/one/tasks/boot.py | 10 ++++------ providers/one/tasks/security.py | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/providers/one/assets/grub.d/40_custom b/providers/one/assets/grub.d/40_custom index 56419c7..cb035a2 100644 --- a/providers/one/assets/grub.d/40_custom +++ b/providers/one/assets/grub.d/40_custom @@ -17,13 +17,13 @@ GRUB_DEVICE=/dev/sda1 cat << EOF -set default ${GRUB_DEFAULT} -set timeout ${GRUB_TIMEOUT} +set default=${GRUB_DEFAULT} +set timeout=${GRUB_TIMEOUT} insmod part_msdos insmod ext2 insmod gettext set menu_color_normal=cyan/blue -set meu_color_highlight=white/blue +set menu_color_highlight=white/blue set root='(hd0,msdos1)' EOF @@ -40,10 +40,10 @@ linux_entry () title="$(gettext_quoted "%s, with Linux %s")" cat << EOF -menuentry 'Debian GNU/Linux, ${version}' --class debian --class gnu-linux --class os { +menuentry 'Debian GNU/Linux for OpenNebula, ${version}' --class debian --class gnu-linux --class os { insmod part_msdos insmod ext2 - set timeout ${GRUB_TIMEOUT} + set timeout=${GRUB_TIMEOUT} set root='(hd0,msdos1)' echo 'Loading Linux ${version}' linux ${rel_dirname}/${basename} root=${GRUB_DEVICE} ro ${args} diff --git a/providers/one/tasks/boot.py b/providers/one/tasks/boot.py index abedc65..4d7c7f9 100644 --- a/providers/one/tasks/boot.py +++ b/providers/one/tasks/boot.py @@ -30,16 +30,14 @@ class ConfigureGrub(Task): # 'GRUB_HIDDEN_TIMEOUT=true') from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/grub.cfg']) - log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/device.map']) + #log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + #log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/grub.cfg']) + #log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/device.map']) log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) - log_check_call(['/usr/sbin/chroot', info.root, 'ls', '-l', '/boot/']) - #sed_i(info.root+'/boot/grub/device.map','/dev/sda','/dev/mapper/loop0p1;') log_check_call(['grub-install', '--boot-directory='+info.root+"/boot/", '/dev/loop0']) - #sed_i(info.root+'/boot/grub/device.map','/dev/mapper/loop0p1;','/dev/sda') + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) class BlackListModules(Task): description = 'Blacklisting kernel modules' diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py index cdf519b..af1aa8c 100644 --- a/providers/one/tasks/security.py +++ b/providers/one/tasks/security.py @@ -18,7 +18,7 @@ class SetRootPassword(Task): def run(self, info): from common.tools import log_check_call if info.manifest.credentials['root']: - log_check_call(['/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/chpasswd'], 'root:'+info.manifest.credentials['root']) class DisableSSHPasswordAuthentication(Task): description = 'Disabling SSH password authentication' From 74398c099ad7642c1fd46c8a5a150bd6c89a1bb0 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Tue, 30 Jul 2013 14:13:30 +0200 Subject: [PATCH 16/22] add scripts to load ssh keys and execute ec2 user data if it is a script --- providers/one/__init__.py | 3 +- providers/one/assets/one-context_3.8.1.deb | Bin 0 -> 3790 bytes providers/one/assets/one-ec2.sh | 14 ++++++++++ providers/one/assets/one-pubkey.sh | 8 ++++++ providers/one/tasks/one.py | 31 +++++++++++++++++++++ providers/one/tasks/packages.py | 10 ++----- 6 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 providers/one/assets/one-context_3.8.1.deb create mode 100755 providers/one/assets/one-ec2.sh create mode 100755 providers/one/assets/one-pubkey.sh create mode 100644 providers/one/tasks/one.py diff --git a/providers/one/__init__.py b/providers/one/__init__.py index ebabe0e..73af652 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -15,6 +15,7 @@ from tasks import network from tasks import initd from tasks import cleanup from tasks import fake +from tasks import one def initialize(): @@ -57,9 +58,9 @@ def tasks(tasklist, manifest): network.RemoveDNSInfo(), network.ConfigureNetworkIF(), network.ConfigureDHCP(), - fake.Fake(), initd.ResolveInitScripts(), initd.InstallInitScripts(), + one.OpenNebulaContext(), cleanup.ClearMOTD(), cleanup.ShredHostkeys(), cleanup.CleanTMP(), diff --git a/providers/one/assets/one-context_3.8.1.deb b/providers/one/assets/one-context_3.8.1.deb new file mode 100644 index 0000000000000000000000000000000000000000..2e811888cea425ba65bfc79e38aaad169ce23bdb GIT binary patch literal 3790 zcmai$XD}R$_QplYvP5U~5-pcgH`_DV~KX>l^be}o%%=vK6`SQ&C=I|>!1~|K^0H3%xxjXtuIJx^c20RDz z@kzl!K;&g*`S||S{|gS3mV^MEeSLxge7z)t90Mc~q2T|y zFC{Id@UH`KAa4>S$>3F*F$vkOg}3jVJpC|&@XBVqM3+@JazKCERm*P9=5=c@y`pHe zBc)p+DzU#z`;mX4H!(uT+9R97x`AI8YCxA}#h5y>uvzt`?|6!bTdt>b(f%aq?8rCR z&UYd>WE*ff6v|YLpY}|an+dxbotk>&T}Ao~Xi3y=w&&AgKFHhDIdY`_iurO0-9o+q zc^Zc`Cg@ddf#Xc1BQ<#)s*r8g7<1x%+nOyRxk5#!P$7!-K_^!j^NbE`IB(qc6|3w5 z$fuCzSx2mPL9j*)d(2B47!!luWVBJ+fdI{hW@Z!6b-u2n3~1=im3f*)9fSPj1&F_A z$oSz~!Mnx0)~L4_;fN@$1ZUQJDwy6OaDQ*TlTohFh8Ft8*@FTKW>i8&J%8UTee4j5 zf~~gn=YwJEjK8I1gx#MP^?Z&t9M5ZgWT>0Fsdn&0O9QUF#*tyZm3C+TW8O_50v|Q9 z+wkBn8$TBE-i?7|=B4CxP;@fmc$!@l{ALv;KP7)XUY}E#ri%GiKp?A3pzpS+Bd1hx zg*;=7FMEi1S*z{mrf%b0(&(>`M3c|u{<7}8t2%$Tv&bz|IJk0QPW0Je78ImW;wbYd zfOesz7(2_!>+LX8FwbObf^FDAzuHx)C@bsv`behR!?uB5aux+Yyndgw_8lD)xxQ+L z57+wPp2fqTo>ZO&(JH+%CMl|2mL9}p?M55&Ro6T%1f3lPSS;6Gb6;DogeFeBTmzgv zQ)z!kd^5DrW^;rwIVOs9_Cgn9seLt$U#H#jui^ z*{}W;2g0<2k<1njDio2bybZ5gcFj4*<=ZNZJpb8aG#EWBe@@P;?c)6I7q9ST2VN52 zm4xjopH*tGn@O@X@u0H__uV+b7)|KFm>L5?6zZu=NDpP*~bD zld|oON(4tlc|<2|(z>zdQ{bYc?Gn}=ALx4cyv;5ZwAw}8C}O<}Z`*|}ImDyAF?js^ z>pd7UT=pJ!cl^nKZkWQ}X*gS<8bWsGr0<$xcz%p-l&7fs3!&}{8O;cF=o4O(O1Kf5 z0veGR=VP=k6D+_}ZMV~AWYr)hWNcpEdL<;H-T12F>TOI!i zdPbP7&^w^Ft7nUh;gegA9a&LC$u&{V3X(sy)vk_4U|0(xLtu>5enI3l+y0%mT3g?M zfH>s&;rCnoHcbn`v*ZP&@KbC|Cim?RRH@4%LFm;HAk207*u8%vLYF$rCh&u zQ`>4tS$gs12&~F~XJX|k>BjV%fsgfWfL0XZ$|Tn9KW-FqD)Uf`O*pzX6+W-+af}19 z+%>B*F46eUhLCzj_8sN`Ah|7kLqAK|2IQ zz%a`eYMzj-lcXZw`xDPBlS3O!dWlkeiKsVVJsKQy4v;i=dXOp&_o26! zro1+|4{8exkF>8mhaP*Ay*~a}&p{Yj!<^GaJg|#!i(_&({XrfUkfI1FUAqtQJie(M zS9wz<_<BDHFL??$s=>wa_~g6NJfsw7ab3cIl~!35vObb5oBN`6y}*!)EbL-I&zqROoB*GL{{2 zuAgh&CPi$YHbBXC4R~RbpZLuSEH=*7zu4U#GF6$FQxZ9|fd75LoqfIwbvqp{%z*Ul z5JC*dX!iFR#d*o#HZYpOiB=LYi6<9*j}{O0sFs?*R!RupZ|$)-mNaFh_?ISX@&0E0 z*|LIcyUI6Qlm3pL--DsS-ATcixg}AL(~_`@i;V@*aJS;OjbW7wM!YFTaAH-+=>1)u z_NUCM z_BmWN&t83kbVKbgZ)OIrzTnHKbO9x9@1}%F6=1q7MU*DEwk*D52TaX1n1EY z-k6Nr$`KU0(ZLNHILchnmt>Ox2MZXR42*{T@ZvCjM-lN$I}VNlgnD==i0*}0PYawxo z>5DQyfYO+IJ={+B*SRYAYoRy!3`*1R_yn1HGLCkyeFsNdv$MR%^y*K9joGrCELeLd5dd_m z-|yvMGDD(x}kg@m=^8;_m9epPQmBDdZn_Q0DH z2<~`4Ahl_z^t!XaKo0GD2j-WJMuIj&IkXAHmr1Gt{FUlo z(Byqy>0(&Oz4WE48>6*OSJxTvm#WI?cZypxQH_k;XGtEUrG{UowXUn#&K;iK^SY%u z*|UgPE2&2{N|cHOEa0px;}5$_^a!k7kejr3-hl zUMTUXX#I&Pprx=(3m=@HvldVzCm^njPdS~tru96i1-we2Tea?ApJG0W@xK&DTGV6( z%^V{XCuP#(D22sOyu}2y5dFBO*Lw-1RpCDhLP3HvmyaB$>+wh1n);S*1FJ3Y4x<>t zRYPFVU27?ykK!&uM!jJFOzbiEiBeMX&&|VR&$z5M;|DVG&wGS<1FeY3JvOdg4%r4@8 z+aR#+{1zci>ZXyXtTdLCEQo4`qgSBu#w=-+iUEDzE`??ek>dmZc z&XM2Ai-c#4Qqlx5twopK&29|unm?!rln@=r^PYwK zIHgQ?DGv+jWz@qIthpYG6TDXF8%GqF9K{X5FGLS zdniRn=0wZJ0)^2HfQE%hA91;rUZd#=dka;ueJ{%hNwdBTyav7CE8@d83j%TJy_zIJ z{dK`F*cE67d_@?6Y`I+Vl3Da1(Ipq3c5Vr*YVYK}*ve}bth)^!RTvt> /root/.ssh/authorized_keys + +done diff --git a/providers/one/tasks/one.py b/providers/one/tasks/one.py new file mode 100644 index 0000000..b8dbbf7 --- /dev/null +++ b/providers/one/tasks/one.py @@ -0,0 +1,31 @@ +from base import Task +from common import phases +import os + + +class OpenNebulaContext(Task): + description = 'Setup OpenNebula init context' + phase = phases.system_modification + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-context_3.8.1.deb')) + script_dst = os.path.join(info.root, 'tmp/one-context_3.8.1.deb') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb']) + + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-pubkey.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') + copy(script_src, script_dst) + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-ec2.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh') + copy(script_src, script_dst) + diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index 53df61d..4ecdbea 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -7,7 +7,7 @@ class HostPackages(Task): phase = phases.preparation def run(self, info): - packages = set(['debootstrap', 'qemu-utils']) + packages = set(['debootstrap', 'qemu-utils', 'parted', 'grub2']) if info.manifest.volume['filesystem'] == 'xfs': packages.add('xfsprogs') @@ -22,7 +22,6 @@ class ImagePackages(Task): manifest = info.manifest # Add some basic packages we are going to need include = set(['udev', - 'mbr', 'parted', 'openssh-server', # We could bootstrap without locales, but things just suck without them, error messages etc. @@ -34,16 +33,13 @@ class ImagePackages(Task): 'grub2', ]) - if manifest.virtualization == 'pvm': - include.add('grub-pc') - exclude = set(['isc-dhcp-client', 'isc-dhcp-common', ]) # In squeeze, we need a special kernel flavor for xen - kernels = {'squeeze': {'amd64': 'linux-image-xen-amd64', - 'i386': 'linux-image-xen-686', }, + kernels = {'squeeze': {'amd64': 'linux-image-amd64', + 'i386': 'linux-image-686', }, 'wheezy': {'amd64': 'linux-image-amd64', 'i386': 'linux-image-686', }, } include.add(kernels.get(manifest.system['release']).get(manifest.system['architecture'])) From 4fb29212175101a6f497235668b1749b85b3550d Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 31 Jul 2013 08:02:52 +0200 Subject: [PATCH 17/22] fix pub key mngt --- providers/one/assets/one-pubkey.sh | 9 +++++++++ providers/one/tasks/one.py | 6 ++++++ providers/one/tasks/packages.py | 4 +++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/providers/one/assets/one-pubkey.sh b/providers/one/assets/one-pubkey.sh index e28b738..7d7a209 100755 --- a/providers/one/assets/one-pubkey.sh +++ b/providers/one/assets/one-pubkey.sh @@ -1,5 +1,14 @@ #!/bin/bash +echo "Reconfigure host ssh keys" +dpkg-reconfigure openssh-server + +if [ ! -e /root/.ssh ]; then + mkdir /root/.ssh + touch /root/.ssh/authorized_keys + chmod 600 /root/.ssh/authorized_keys +fi + echo "Copy public ssh keys to authorized_keys" for f in /mnt/*.pub do diff --git a/providers/one/tasks/one.py b/providers/one/tasks/one.py index b8dbbf7..745cb43 100644 --- a/providers/one/tasks/one.py +++ b/providers/one/tasks/one.py @@ -21,6 +21,12 @@ class OpenNebulaContext(Task): from common.tools import log_check_call log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb']) + # Fix start + from common.tools import sed_i + vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext') + sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5') + os.chmod(vmcontext_def, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', '90', '2', '3', '4', '5', 'stop', '90', '0', '6']) script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-pubkey.sh')) script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') diff --git a/providers/one/tasks/packages.py b/providers/one/tasks/packages.py index 4ecdbea..89c1589 100644 --- a/providers/one/tasks/packages.py +++ b/providers/one/tasks/packages.py @@ -7,7 +7,7 @@ class HostPackages(Task): phase = phases.preparation def run(self, info): - packages = set(['debootstrap', 'qemu-utils', 'parted', 'grub2']) + packages = set(['debootstrap', 'qemu-utils', 'parted', 'grub2', 'sysv-rc']) if info.manifest.volume['filesystem'] == 'xfs': packages.add('xfsprogs') @@ -31,6 +31,8 @@ class ImagePackages(Task): # isc-dhcp-client doesn't work properly with ec2 'dhcpcd', 'grub2', + 'chkconfig', + 'openssh-client' ]) exclude = set(['isc-dhcp-client', From 237069b9416e0568ed1beab39fe222f9f3b5635b Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Wed, 31 Jul 2013 16:57:29 +0200 Subject: [PATCH 18/22] add plugins to manage user packages, allow root login via ssh when root password is defined in conf, install opennebula context package --- manifests/one-raw-virtio.manifest.json | 8 ++++- plugins/user_packages/__init__.py | 6 ++++ plugins/user_packages/user_packages.py | 45 ++++++++++++++++++++++++++ providers/one/__init__.py | 3 +- providers/one/tasks/locale.py | 2 +- providers/one/tasks/one.py | 2 ++ providers/one/tasks/security.py | 6 ++-- 7 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 plugins/user_packages/__init__.py create mode 100644 plugins/user_packages/user_packages.py diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/one-raw-virtio.manifest.json index 33cff2a..913c8ac 100644 --- a/manifests/one-raw-virtio.manifest.json +++ b/manifests/one-raw-virtio.manifest.json @@ -41,6 +41,12 @@ "prebootstrapped": { "enabled": false, "snapshot": "" - } + }, + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + } + } } diff --git a/plugins/user_packages/__init__.py b/plugins/user_packages/__init__.py new file mode 100644 index 0000000..e223e0a --- /dev/null +++ b/plugins/user_packages/__init__.py @@ -0,0 +1,6 @@ + + +def tasks(tasklist, manifest): + from user_packages import AddUserPackages, AddLocalUserPackages + tasklist.add(AddUserPackages()) + tasklist.add(AddLocalUserPackages()) diff --git a/plugins/user_packages/user_packages.py b/plugins/user_packages/user_packages.py new file mode 100644 index 0000000..296c536 --- /dev/null +++ b/plugins/user_packages/user_packages.py @@ -0,0 +1,45 @@ +from base import Task +from common import phases +import os +from providers.one.tasks.packages import ImagePackages +from providers.one.tasks.host import CheckPackages +from providers.one.tasks.filesystem import MountVolume + + +class AddUserPackages(Task): + description = 'Adding user defined packages to the image packages' + phase = phases.preparation + after = [ImagePackages] + before = [CheckPackages] + + def run(self, info): + if 'repo' not in info.manifest.plugins['user_packages']: + return + for pkg in info.manifest.plugins['user_packages']['repo']: + info.img_packages[0].add(pkg) + +class AddLocalUserPackages(Task): + description = 'Adding user local packages to the image packages' + phase = phases.system_modification + after = [MountVolume] + + def run(self, info): + if 'local' not in info.manifest.plugins['user_packages']: + return + + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + from shutil import copy + from common.tools import log_check_call + + for pkg in info.manifest.plugins['user_packages']['local']: + script_src = os.path.normpath(pkg) + script_dst = os.path.join(info.root, 'tmp/'+os.path.basename(script_src)) + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/'+os.path.basename(script_src)]) + diff --git a/providers/one/__init__.py b/providers/one/__init__.py index 73af652..bdf9fba 100644 --- a/providers/one/__init__.py +++ b/providers/one/__init__.py @@ -52,8 +52,7 @@ def tasks(tasklist, manifest): boot.DisableGetTTYs(), security.EnableShadowConfig(), security.SetRootPassword(), - # Disable for the time of debugging - #security.DisableSSHPasswordAuthentication(), + security.DisableSSHPasswordAuthentication(), security.DisableSSHDNSLookup(), network.RemoveDNSInfo(), network.ConfigureNetworkIF(), diff --git a/providers/one/tasks/locale.py b/providers/one/tasks/locale.py index de3e434..a0005b4 100644 --- a/providers/one/tasks/locale.py +++ b/providers/one/tasks/locale.py @@ -16,7 +16,7 @@ class GenerateLocale(Task): search = '# ' + locale_str sed_i(locale_gen, search, locale_str) - command = ['/usr/sbin/chroot', info.root, '/usr/sbin/dpkg-reconfigure', '--priority=critical', 'locales'] + command = ['/usr/sbin/chroot', info.root, '/usr/sbin/locale-gen'] log_check_call(command) diff --git a/providers/one/tasks/one.py b/providers/one/tasks/one.py index 745cb43..fda5185 100644 --- a/providers/one/tasks/one.py +++ b/providers/one/tasks/one.py @@ -1,11 +1,13 @@ from base import Task from common import phases import os +from providers.one.tasks.locale import GenerateLocale class OpenNebulaContext(Task): description = 'Setup OpenNebula init context' phase = phases.system_modification + after = [GenerateLocale] def run(self, info): import stat diff --git a/providers/one/tasks/security.py b/providers/one/tasks/security.py index af1aa8c..77f986c 100644 --- a/providers/one/tasks/security.py +++ b/providers/one/tasks/security.py @@ -26,8 +26,10 @@ class DisableSSHPasswordAuthentication(Task): def run(self, info): from common.tools import sed_i - sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') - sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') + if 'root' not in info.manifest.credentials: + # If no password set for root + sshd_config_path = os.path.join(info.root, 'etc/ssh/sshd_config') + sed_i(sshd_config_path, '^#PasswordAuthentication yes', 'PasswordAuthentication no') class DisableSSHDNSLookup(Task): From 6ffa601488df79b07d19650050260bc3d97a5df2 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Thu, 1 Aug 2013 08:20:32 +0200 Subject: [PATCH 19/22] change provider to raw, set opennebula as plugin --- ...-virtio.manifest.json => raw-ide.manifest} | 0 plugins/opennebula/__init__.py | 5 +++ .../opennebula}/assets/one-context_3.8.1.deb | Bin .../opennebula}/assets/one-ec2.sh | 0 .../opennebula}/assets/one-pubkey.sh | 0 plugins/opennebula/opennebula.py | 42 ++++++++++++++++++ providers/{one => raw}/__init__.py | 0 .../{one => raw}/assets/grub.d/40_custom | 0 .../{one => raw}/assets/grub.d/40_custom.orig | 0 .../assets/init.d/ec2-get-credentials | 0 .../assets/init.d/ec2-run-user-data | 0 .../{one => raw}/assets/init.d/expand-volume | 0 .../assets/init.d/generate-ssh-hostkeys | 0 .../init.d/squeeze/generate-ssh-hostkeys | 0 providers/raw/assets/one-context_3.8.1.deb | Bin 0 -> 3790 bytes providers/raw/assets/one-ec2.sh | 14 ++++++ providers/raw/assets/one-pubkey.sh | 17 +++++++ providers/{one => raw}/manifest-schema.json | 0 providers/{one => raw}/manifest.py | 0 providers/{one => raw}/tasks/__init__.py | 0 providers/{one => raw}/tasks/ami.py | 0 providers/{one => raw}/tasks/apt.py | 0 providers/{one => raw}/tasks/boot.py | 0 providers/{one => raw}/tasks/bootstrap.py | 0 providers/{one => raw}/tasks/cleanup.py | 0 providers/{one => raw}/tasks/connection.py | 0 providers/{one => raw}/tasks/ebs.py | 0 providers/{one => raw}/tasks/fake.py | 0 providers/{one => raw}/tasks/filesystem.py | 0 providers/{one => raw}/tasks/host.py | 0 providers/{one => raw}/tasks/initd.py | 0 providers/{one => raw}/tasks/locale.py | 0 providers/{one => raw}/tasks/network.py | 0 providers/{one => raw}/tasks/one.py | 0 providers/{one => raw}/tasks/packages.py | 0 providers/{one => raw}/tasks/security.py | 0 36 files changed, 78 insertions(+) rename manifests/{one-raw-virtio.manifest.json => raw-ide.manifest} (100%) create mode 100644 plugins/opennebula/__init__.py rename {providers/one => plugins/opennebula}/assets/one-context_3.8.1.deb (100%) rename {providers/one => plugins/opennebula}/assets/one-ec2.sh (100%) rename {providers/one => plugins/opennebula}/assets/one-pubkey.sh (100%) create mode 100644 plugins/opennebula/opennebula.py rename providers/{one => raw}/__init__.py (100%) rename providers/{one => raw}/assets/grub.d/40_custom (100%) rename providers/{one => raw}/assets/grub.d/40_custom.orig (100%) rename providers/{one => raw}/assets/init.d/ec2-get-credentials (100%) rename providers/{one => raw}/assets/init.d/ec2-run-user-data (100%) rename providers/{one => raw}/assets/init.d/expand-volume (100%) rename providers/{one => raw}/assets/init.d/generate-ssh-hostkeys (100%) rename providers/{one => raw}/assets/init.d/squeeze/generate-ssh-hostkeys (100%) create mode 100644 providers/raw/assets/one-context_3.8.1.deb create mode 100755 providers/raw/assets/one-ec2.sh create mode 100755 providers/raw/assets/one-pubkey.sh rename providers/{one => raw}/manifest-schema.json (100%) rename providers/{one => raw}/manifest.py (100%) rename providers/{one => raw}/tasks/__init__.py (100%) rename providers/{one => raw}/tasks/ami.py (100%) rename providers/{one => raw}/tasks/apt.py (100%) rename providers/{one => raw}/tasks/boot.py (100%) rename providers/{one => raw}/tasks/bootstrap.py (100%) rename providers/{one => raw}/tasks/cleanup.py (100%) rename providers/{one => raw}/tasks/connection.py (100%) rename providers/{one => raw}/tasks/ebs.py (100%) rename providers/{one => raw}/tasks/fake.py (100%) rename providers/{one => raw}/tasks/filesystem.py (100%) rename providers/{one => raw}/tasks/host.py (100%) rename providers/{one => raw}/tasks/initd.py (100%) rename providers/{one => raw}/tasks/locale.py (100%) rename providers/{one => raw}/tasks/network.py (100%) rename providers/{one => raw}/tasks/one.py (100%) rename providers/{one => raw}/tasks/packages.py (100%) rename providers/{one => raw}/tasks/security.py (100%) diff --git a/manifests/one-raw-virtio.manifest.json b/manifests/raw-ide.manifest similarity index 100% rename from manifests/one-raw-virtio.manifest.json rename to manifests/raw-ide.manifest diff --git a/plugins/opennebula/__init__.py b/plugins/opennebula/__init__.py new file mode 100644 index 0000000..cf8567c --- /dev/null +++ b/plugins/opennebula/__init__.py @@ -0,0 +1,5 @@ + + +def tasks(tasklist, manifest): + from opennebula import OpenNebulaContext + tasklist.add(OpenNebulaContext()) diff --git a/providers/one/assets/one-context_3.8.1.deb b/plugins/opennebula/assets/one-context_3.8.1.deb similarity index 100% rename from providers/one/assets/one-context_3.8.1.deb rename to plugins/opennebula/assets/one-context_3.8.1.deb diff --git a/providers/one/assets/one-ec2.sh b/plugins/opennebula/assets/one-ec2.sh similarity index 100% rename from providers/one/assets/one-ec2.sh rename to plugins/opennebula/assets/one-ec2.sh diff --git a/providers/one/assets/one-pubkey.sh b/plugins/opennebula/assets/one-pubkey.sh similarity index 100% rename from providers/one/assets/one-pubkey.sh rename to plugins/opennebula/assets/one-pubkey.sh diff --git a/plugins/opennebula/opennebula.py b/plugins/opennebula/opennebula.py new file mode 100644 index 0000000..1899127 --- /dev/null +++ b/plugins/opennebula/opennebula.py @@ -0,0 +1,42 @@ +from base import Task +from common import phases +import os +from providers.one.tasks.locale import GenerateLocale + + +class OpenNebulaContext(Task): + description = 'Setup OpenNebula init context' + phase = phases.system_modification + after = [GenerateLocale] + + def run(self, info): + import stat + rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) + + from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-context_3.8.1.deb')) + script_dst = os.path.join(info.root, 'tmp/one-context_3.8.1.deb') + copy(script_src, script_dst) + os.chmod(script_dst, rwxr_xr_x) + + from common.tools import log_check_call + log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb']) + # Fix start + from common.tools import sed_i + vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext') + sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5') + os.chmod(vmcontext_def, rwxr_xr_x) + log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', '90', '2', '3', '4', '5', 'stop', '90', '0', '6']) + + # Load all pubkeys in root authorized_keys + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-pubkey.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') + copy(script_src, script_dst) + + # If USER_EC2_DATA is a script, execute it + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets/one-ec2.sh')) + script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh') + copy(script_src, script_dst) + diff --git a/providers/one/__init__.py b/providers/raw/__init__.py similarity index 100% rename from providers/one/__init__.py rename to providers/raw/__init__.py diff --git a/providers/one/assets/grub.d/40_custom b/providers/raw/assets/grub.d/40_custom similarity index 100% rename from providers/one/assets/grub.d/40_custom rename to providers/raw/assets/grub.d/40_custom diff --git a/providers/one/assets/grub.d/40_custom.orig b/providers/raw/assets/grub.d/40_custom.orig similarity index 100% rename from providers/one/assets/grub.d/40_custom.orig rename to providers/raw/assets/grub.d/40_custom.orig diff --git a/providers/one/assets/init.d/ec2-get-credentials b/providers/raw/assets/init.d/ec2-get-credentials similarity index 100% rename from providers/one/assets/init.d/ec2-get-credentials rename to providers/raw/assets/init.d/ec2-get-credentials diff --git a/providers/one/assets/init.d/ec2-run-user-data b/providers/raw/assets/init.d/ec2-run-user-data similarity index 100% rename from providers/one/assets/init.d/ec2-run-user-data rename to providers/raw/assets/init.d/ec2-run-user-data diff --git a/providers/one/assets/init.d/expand-volume b/providers/raw/assets/init.d/expand-volume similarity index 100% rename from providers/one/assets/init.d/expand-volume rename to providers/raw/assets/init.d/expand-volume diff --git a/providers/one/assets/init.d/generate-ssh-hostkeys b/providers/raw/assets/init.d/generate-ssh-hostkeys similarity index 100% rename from providers/one/assets/init.d/generate-ssh-hostkeys rename to providers/raw/assets/init.d/generate-ssh-hostkeys diff --git a/providers/one/assets/init.d/squeeze/generate-ssh-hostkeys b/providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys similarity index 100% rename from providers/one/assets/init.d/squeeze/generate-ssh-hostkeys rename to providers/raw/assets/init.d/squeeze/generate-ssh-hostkeys diff --git a/providers/raw/assets/one-context_3.8.1.deb b/providers/raw/assets/one-context_3.8.1.deb new file mode 100644 index 0000000000000000000000000000000000000000..2e811888cea425ba65bfc79e38aaad169ce23bdb GIT binary patch literal 3790 zcmai$XD}R$_QplYvP5U~5-pcgH`_DV~KX>l^be}o%%=vK6`SQ&C=I|>!1~|K^0H3%xxjXtuIJx^c20RDz z@kzl!K;&g*`S||S{|gS3mV^MEeSLxge7z)t90Mc~q2T|y zFC{Id@UH`KAa4>S$>3F*F$vkOg}3jVJpC|&@XBVqM3+@JazKCERm*P9=5=c@y`pHe zBc)p+DzU#z`;mX4H!(uT+9R97x`AI8YCxA}#h5y>uvzt`?|6!bTdt>b(f%aq?8rCR z&UYd>WE*ff6v|YLpY}|an+dxbotk>&T}Ao~Xi3y=w&&AgKFHhDIdY`_iurO0-9o+q zc^Zc`Cg@ddf#Xc1BQ<#)s*r8g7<1x%+nOyRxk5#!P$7!-K_^!j^NbE`IB(qc6|3w5 z$fuCzSx2mPL9j*)d(2B47!!luWVBJ+fdI{hW@Z!6b-u2n3~1=im3f*)9fSPj1&F_A z$oSz~!Mnx0)~L4_;fN@$1ZUQJDwy6OaDQ*TlTohFh8Ft8*@FTKW>i8&J%8UTee4j5 zf~~gn=YwJEjK8I1gx#MP^?Z&t9M5ZgWT>0Fsdn&0O9QUF#*tyZm3C+TW8O_50v|Q9 z+wkBn8$TBE-i?7|=B4CxP;@fmc$!@l{ALv;KP7)XUY}E#ri%GiKp?A3pzpS+Bd1hx zg*;=7FMEi1S*z{mrf%b0(&(>`M3c|u{<7}8t2%$Tv&bz|IJk0QPW0Je78ImW;wbYd zfOesz7(2_!>+LX8FwbObf^FDAzuHx)C@bsv`behR!?uB5aux+Yyndgw_8lD)xxQ+L z57+wPp2fqTo>ZO&(JH+%CMl|2mL9}p?M55&Ro6T%1f3lPSS;6Gb6;DogeFeBTmzgv zQ)z!kd^5DrW^;rwIVOs9_Cgn9seLt$U#H#jui^ z*{}W;2g0<2k<1njDio2bybZ5gcFj4*<=ZNZJpb8aG#EWBe@@P;?c)6I7q9ST2VN52 zm4xjopH*tGn@O@X@u0H__uV+b7)|KFm>L5?6zZu=NDpP*~bD zld|oON(4tlc|<2|(z>zdQ{bYc?Gn}=ALx4cyv;5ZwAw}8C}O<}Z`*|}ImDyAF?js^ z>pd7UT=pJ!cl^nKZkWQ}X*gS<8bWsGr0<$xcz%p-l&7fs3!&}{8O;cF=o4O(O1Kf5 z0veGR=VP=k6D+_}ZMV~AWYr)hWNcpEdL<;H-T12F>TOI!i zdPbP7&^w^Ft7nUh;gegA9a&LC$u&{V3X(sy)vk_4U|0(xLtu>5enI3l+y0%mT3g?M zfH>s&;rCnoHcbn`v*ZP&@KbC|Cim?RRH@4%LFm;HAk207*u8%vLYF$rCh&u zQ`>4tS$gs12&~F~XJX|k>BjV%fsgfWfL0XZ$|Tn9KW-FqD)Uf`O*pzX6+W-+af}19 z+%>B*F46eUhLCzj_8sN`Ah|7kLqAK|2IQ zz%a`eYMzj-lcXZw`xDPBlS3O!dWlkeiKsVVJsKQy4v;i=dXOp&_o26! zro1+|4{8exkF>8mhaP*Ay*~a}&p{Yj!<^GaJg|#!i(_&({XrfUkfI1FUAqtQJie(M zS9wz<_<BDHFL??$s=>wa_~g6NJfsw7ab3cIl~!35vObb5oBN`6y}*!)EbL-I&zqROoB*GL{{2 zuAgh&CPi$YHbBXC4R~RbpZLuSEH=*7zu4U#GF6$FQxZ9|fd75LoqfIwbvqp{%z*Ul z5JC*dX!iFR#d*o#HZYpOiB=LYi6<9*j}{O0sFs?*R!RupZ|$)-mNaFh_?ISX@&0E0 z*|LIcyUI6Qlm3pL--DsS-ATcixg}AL(~_`@i;V@*aJS;OjbW7wM!YFTaAH-+=>1)u z_NUCM z_BmWN&t83kbVKbgZ)OIrzTnHKbO9x9@1}%F6=1q7MU*DEwk*D52TaX1n1EY z-k6Nr$`KU0(ZLNHILchnmt>Ox2MZXR42*{T@ZvCjM-lN$I}VNlgnD==i0*}0PYawxo z>5DQyfYO+IJ={+B*SRYAYoRy!3`*1R_yn1HGLCkyeFsNdv$MR%^y*K9joGrCELeLd5dd_m z-|yvMGDD(x}kg@m=^8;_m9epPQmBDdZn_Q0DH z2<~`4Ahl_z^t!XaKo0GD2j-WJMuIj&IkXAHmr1Gt{FUlo z(Byqy>0(&Oz4WE48>6*OSJxTvm#WI?cZypxQH_k;XGtEUrG{UowXUn#&K;iK^SY%u z*|UgPE2&2{N|cHOEa0px;}5$_^a!k7kejr3-hl zUMTUXX#I&Pprx=(3m=@HvldVzCm^njPdS~tru96i1-we2Tea?ApJG0W@xK&DTGV6( z%^V{XCuP#(D22sOyu}2y5dFBO*Lw-1RpCDhLP3HvmyaB$>+wh1n);S*1FJ3Y4x<>t zRYPFVU27?ykK!&uM!jJFOzbiEiBeMX&&|VR&$z5M;|DVG&wGS<1FeY3JvOdg4%r4@8 z+aR#+{1zci>ZXyXtTdLCEQo4`qgSBu#w=-+iUEDzE`??ek>dmZc z&XM2Ai-c#4Qqlx5twopK&29|unm?!rln@=r^PYwK zIHgQ?DGv+jWz@qIthpYG6TDXF8%GqF9K{X5FGLS zdniRn=0wZJ0)^2HfQE%hA91;rUZd#=dka;ueJ{%hNwdBTyav7CE8@d83j%TJy_zIJ z{dK`F*cE67d_@?6Y`I+Vl3Da1(Ipq3c5Vr*YVYK}*ve}bth)^!RTvt> /root/.ssh/authorized_keys + +done diff --git a/providers/one/manifest-schema.json b/providers/raw/manifest-schema.json similarity index 100% rename from providers/one/manifest-schema.json rename to providers/raw/manifest-schema.json diff --git a/providers/one/manifest.py b/providers/raw/manifest.py similarity index 100% rename from providers/one/manifest.py rename to providers/raw/manifest.py diff --git a/providers/one/tasks/__init__.py b/providers/raw/tasks/__init__.py similarity index 100% rename from providers/one/tasks/__init__.py rename to providers/raw/tasks/__init__.py diff --git a/providers/one/tasks/ami.py b/providers/raw/tasks/ami.py similarity index 100% rename from providers/one/tasks/ami.py rename to providers/raw/tasks/ami.py diff --git a/providers/one/tasks/apt.py b/providers/raw/tasks/apt.py similarity index 100% rename from providers/one/tasks/apt.py rename to providers/raw/tasks/apt.py diff --git a/providers/one/tasks/boot.py b/providers/raw/tasks/boot.py similarity index 100% rename from providers/one/tasks/boot.py rename to providers/raw/tasks/boot.py diff --git a/providers/one/tasks/bootstrap.py b/providers/raw/tasks/bootstrap.py similarity index 100% rename from providers/one/tasks/bootstrap.py rename to providers/raw/tasks/bootstrap.py diff --git a/providers/one/tasks/cleanup.py b/providers/raw/tasks/cleanup.py similarity index 100% rename from providers/one/tasks/cleanup.py rename to providers/raw/tasks/cleanup.py diff --git a/providers/one/tasks/connection.py b/providers/raw/tasks/connection.py similarity index 100% rename from providers/one/tasks/connection.py rename to providers/raw/tasks/connection.py diff --git a/providers/one/tasks/ebs.py b/providers/raw/tasks/ebs.py similarity index 100% rename from providers/one/tasks/ebs.py rename to providers/raw/tasks/ebs.py diff --git a/providers/one/tasks/fake.py b/providers/raw/tasks/fake.py similarity index 100% rename from providers/one/tasks/fake.py rename to providers/raw/tasks/fake.py diff --git a/providers/one/tasks/filesystem.py b/providers/raw/tasks/filesystem.py similarity index 100% rename from providers/one/tasks/filesystem.py rename to providers/raw/tasks/filesystem.py diff --git a/providers/one/tasks/host.py b/providers/raw/tasks/host.py similarity index 100% rename from providers/one/tasks/host.py rename to providers/raw/tasks/host.py diff --git a/providers/one/tasks/initd.py b/providers/raw/tasks/initd.py similarity index 100% rename from providers/one/tasks/initd.py rename to providers/raw/tasks/initd.py diff --git a/providers/one/tasks/locale.py b/providers/raw/tasks/locale.py similarity index 100% rename from providers/one/tasks/locale.py rename to providers/raw/tasks/locale.py diff --git a/providers/one/tasks/network.py b/providers/raw/tasks/network.py similarity index 100% rename from providers/one/tasks/network.py rename to providers/raw/tasks/network.py diff --git a/providers/one/tasks/one.py b/providers/raw/tasks/one.py similarity index 100% rename from providers/one/tasks/one.py rename to providers/raw/tasks/one.py diff --git a/providers/one/tasks/packages.py b/providers/raw/tasks/packages.py similarity index 100% rename from providers/one/tasks/packages.py rename to providers/raw/tasks/packages.py diff --git a/providers/one/tasks/security.py b/providers/raw/tasks/security.py similarity index 100% rename from providers/one/tasks/security.py rename to providers/raw/tasks/security.py From 385eac14a1ed8e7554119b35146c95055fdb0332 Mon Sep 17 00:00:00 2001 From: Olivier Sallou Date: Thu, 1 Aug 2013 08:34:20 +0200 Subject: [PATCH 20/22] remove tasks ot required by raw provider, fix provider name in plugins after renaming from one to raw --- .../{raw-ide.manifest => one-ide.manifest} | 11 ++- plugins/opennebula/opennebula.py | 2 +- plugins/user_packages/user_packages.py | 6 +- providers/raw/__init__.py | 6 -- providers/raw/assets/one-context_3.8.1.deb | Bin 3790 -> 0 bytes providers/raw/assets/one-ec2.sh | 14 --- providers/raw/assets/one-pubkey.sh | 17 ---- providers/raw/tasks/ami.py | 76 --------------- providers/raw/tasks/boot.py | 5 - providers/raw/tasks/connection.py | 40 -------- providers/raw/tasks/ebs.py | 89 ------------------ providers/raw/tasks/one.py | 39 -------- 12 files changed, 11 insertions(+), 294 deletions(-) rename manifests/{raw-ide.manifest => one-ide.manifest} (83%) delete mode 100644 providers/raw/assets/one-context_3.8.1.deb delete mode 100755 providers/raw/assets/one-ec2.sh delete mode 100755 providers/raw/assets/one-pubkey.sh delete mode 100644 providers/raw/tasks/ami.py delete mode 100644 providers/raw/tasks/connection.py delete mode 100644 providers/raw/tasks/ebs.py delete mode 100644 providers/raw/tasks/one.py diff --git a/manifests/raw-ide.manifest b/manifests/one-ide.manifest similarity index 83% rename from manifests/raw-ide.manifest rename to manifests/one-ide.manifest index 913c8ac..7dc267c 100644 --- a/manifests/raw-ide.manifest +++ b/manifests/one-ide.manifest @@ -1,6 +1,6 @@ { - "provider" : "one", - "virtualization": "virtio", + "provider" : "raw", + "virtualization": "ide", "credentials" : { "access-key": null, "secret-key": null, @@ -9,13 +9,13 @@ "bootstrapper": { "mount_dir": "/mnt/target", - "image_file": "/tmp/target", + "image_file": "/tmp/one.img", "tarball": true, "device": "/dev/sda" }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", - "description": "Debian {release} {architecture} Open Nebula ({virtualization})" + "description": "Debian {release} {architecture} ({virtualization})" }, "system": { "release" : "wheezy", @@ -42,6 +42,9 @@ "enabled": false, "snapshot": "" }, + "opennebula": { + "enabled": true + }, "user_packages": { "enabled": true, "repo": [ "apache2" ], diff --git a/plugins/opennebula/opennebula.py b/plugins/opennebula/opennebula.py index 1899127..df50a4b 100644 --- a/plugins/opennebula/opennebula.py +++ b/plugins/opennebula/opennebula.py @@ -1,7 +1,7 @@ from base import Task from common import phases import os -from providers.one.tasks.locale import GenerateLocale +from providers.raw.tasks.locale import GenerateLocale class OpenNebulaContext(Task): diff --git a/plugins/user_packages/user_packages.py b/plugins/user_packages/user_packages.py index 296c536..e1093ca 100644 --- a/plugins/user_packages/user_packages.py +++ b/plugins/user_packages/user_packages.py @@ -1,9 +1,9 @@ from base import Task from common import phases import os -from providers.one.tasks.packages import ImagePackages -from providers.one.tasks.host import CheckPackages -from providers.one.tasks.filesystem import MountVolume +from providers.raw.tasks.packages import ImagePackages +from providers.raw.tasks.host import CheckPackages +from providers.raw.tasks.filesystem import MountVolume class AddUserPackages(Task): diff --git a/providers/raw/__init__.py b/providers/raw/__init__.py index bdf9fba..2cd2f36 100644 --- a/providers/raw/__init__.py +++ b/providers/raw/__init__.py @@ -1,10 +1,7 @@ from manifest import Manifest import logging from tasks import packages -from tasks import connection from tasks import host -from tasks import ami -from tasks import ebs from tasks import filesystem from tasks import bootstrap from tasks import locale @@ -15,7 +12,6 @@ from tasks import network from tasks import initd from tasks import cleanup from tasks import fake -from tasks import one def initialize(): @@ -35,7 +31,6 @@ def tasks(tasklist, manifest): tasklist.add(filesystem.TuneVolumeFS()) tasklist.add(filesystem.CreateMountDir(), filesystem.MountVolume()) - #tasklist.add(filesystem.InstallMbr()) if manifest.bootstrapper['tarball']: tasklist.add(bootstrap.MakeTarball()) tasklist.add(bootstrap.Bootstrap(), @@ -59,7 +54,6 @@ def tasks(tasklist, manifest): network.ConfigureDHCP(), initd.ResolveInitScripts(), initd.InstallInitScripts(), - one.OpenNebulaContext(), cleanup.ClearMOTD(), cleanup.ShredHostkeys(), cleanup.CleanTMP(), diff --git a/providers/raw/assets/one-context_3.8.1.deb b/providers/raw/assets/one-context_3.8.1.deb deleted file mode 100644 index 2e811888cea425ba65bfc79e38aaad169ce23bdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3790 zcmai$XD}R$_QplYvP5U~5-pcgH`_DV~KX>l^be}o%%=vK6`SQ&C=I|>!1~|K^0H3%xxjXtuIJx^c20RDz z@kzl!K;&g*`S||S{|gS3mV^MEeSLxge7z)t90Mc~q2T|y zFC{Id@UH`KAa4>S$>3F*F$vkOg}3jVJpC|&@XBVqM3+@JazKCERm*P9=5=c@y`pHe zBc)p+DzU#z`;mX4H!(uT+9R97x`AI8YCxA}#h5y>uvzt`?|6!bTdt>b(f%aq?8rCR z&UYd>WE*ff6v|YLpY}|an+dxbotk>&T}Ao~Xi3y=w&&AgKFHhDIdY`_iurO0-9o+q zc^Zc`Cg@ddf#Xc1BQ<#)s*r8g7<1x%+nOyRxk5#!P$7!-K_^!j^NbE`IB(qc6|3w5 z$fuCzSx2mPL9j*)d(2B47!!luWVBJ+fdI{hW@Z!6b-u2n3~1=im3f*)9fSPj1&F_A z$oSz~!Mnx0)~L4_;fN@$1ZUQJDwy6OaDQ*TlTohFh8Ft8*@FTKW>i8&J%8UTee4j5 zf~~gn=YwJEjK8I1gx#MP^?Z&t9M5ZgWT>0Fsdn&0O9QUF#*tyZm3C+TW8O_50v|Q9 z+wkBn8$TBE-i?7|=B4CxP;@fmc$!@l{ALv;KP7)XUY}E#ri%GiKp?A3pzpS+Bd1hx zg*;=7FMEi1S*z{mrf%b0(&(>`M3c|u{<7}8t2%$Tv&bz|IJk0QPW0Je78ImW;wbYd zfOesz7(2_!>+LX8FwbObf^FDAzuHx)C@bsv`behR!?uB5aux+Yyndgw_8lD)xxQ+L z57+wPp2fqTo>ZO&(JH+%CMl|2mL9}p?M55&Ro6T%1f3lPSS;6Gb6;DogeFeBTmzgv zQ)z!kd^5DrW^;rwIVOs9_Cgn9seLt$U#H#jui^ z*{}W;2g0<2k<1njDio2bybZ5gcFj4*<=ZNZJpb8aG#EWBe@@P;?c)6I7q9ST2VN52 zm4xjopH*tGn@O@X@u0H__uV+b7)|KFm>L5?6zZu=NDpP*~bD zld|oON(4tlc|<2|(z>zdQ{bYc?Gn}=ALx4cyv;5ZwAw}8C}O<}Z`*|}ImDyAF?js^ z>pd7UT=pJ!cl^nKZkWQ}X*gS<8bWsGr0<$xcz%p-l&7fs3!&}{8O;cF=o4O(O1Kf5 z0veGR=VP=k6D+_}ZMV~AWYr)hWNcpEdL<;H-T12F>TOI!i zdPbP7&^w^Ft7nUh;gegA9a&LC$u&{V3X(sy)vk_4U|0(xLtu>5enI3l+y0%mT3g?M zfH>s&;rCnoHcbn`v*ZP&@KbC|Cim?RRH@4%LFm;HAk207*u8%vLYF$rCh&u zQ`>4tS$gs12&~F~XJX|k>BjV%fsgfWfL0XZ$|Tn9KW-FqD)Uf`O*pzX6+W-+af}19 z+%>B*F46eUhLCzj_8sN`Ah|7kLqAK|2IQ zz%a`eYMzj-lcXZw`xDPBlS3O!dWlkeiKsVVJsKQy4v;i=dXOp&_o26! zro1+|4{8exkF>8mhaP*Ay*~a}&p{Yj!<^GaJg|#!i(_&({XrfUkfI1FUAqtQJie(M zS9wz<_<BDHFL??$s=>wa_~g6NJfsw7ab3cIl~!35vObb5oBN`6y}*!)EbL-I&zqROoB*GL{{2 zuAgh&CPi$YHbBXC4R~RbpZLuSEH=*7zu4U#GF6$FQxZ9|fd75LoqfIwbvqp{%z*Ul z5JC*dX!iFR#d*o#HZYpOiB=LYi6<9*j}{O0sFs?*R!RupZ|$)-mNaFh_?ISX@&0E0 z*|LIcyUI6Qlm3pL--DsS-ATcixg}AL(~_`@i;V@*aJS;OjbW7wM!YFTaAH-+=>1)u z_NUCM z_BmWN&t83kbVKbgZ)OIrzTnHKbO9x9@1}%F6=1q7MU*DEwk*D52TaX1n1EY z-k6Nr$`KU0(ZLNHILchnmt>Ox2MZXR42*{T@ZvCjM-lN$I}VNlgnD==i0*}0PYawxo z>5DQyfYO+IJ={+B*SRYAYoRy!3`*1R_yn1HGLCkyeFsNdv$MR%^y*K9joGrCELeLd5dd_m z-|yvMGDD(x}kg@m=^8;_m9epPQmBDdZn_Q0DH z2<~`4Ahl_z^t!XaKo0GD2j-WJMuIj&IkXAHmr1Gt{FUlo z(Byqy>0(&Oz4WE48>6*OSJxTvm#WI?cZypxQH_k;XGtEUrG{UowXUn#&K;iK^SY%u z*|UgPE2&2{N|cHOEa0px;}5$_^a!k7kejr3-hl zUMTUXX#I&Pprx=(3m=@HvldVzCm^njPdS~tru96i1-we2Tea?ApJG0W@xK&DTGV6( z%^V{XCuP#(D22sOyu}2y5dFBO*Lw-1RpCDhLP3HvmyaB$>+wh1n);S*1FJ3Y4x<>t zRYPFVU27?ykK!&uM!jJFOzbiEiBeMX&&|VR&$z5M;|DVG&wGS<1FeY3JvOdg4%r4@8 z+aR#+{1zci>ZXyXtTdLCEQo4`qgSBu#w=-+iUEDzE`??ek>dmZc z&XM2Ai-c#4Qqlx5twopK&29|unm?!rln@=r^PYwK zIHgQ?DGv+jWz@qIthpYG6TDXF8%GqF9K{X5FGLS zdniRn=0wZJ0)^2HfQE%hA91;rUZd#=dka;ueJ{%hNwdBTyav7CE8@d83j%TJy_zIJ z{dK`F*cE67d_@?6Y`I+Vl3Da1(Ipq3c5Vr*YVYK}*ve}bth)^!RTvt> /root/.ssh/authorized_keys - -done diff --git a/providers/raw/tasks/ami.py b/providers/raw/tasks/ami.py deleted file mode 100644 index fbabbf3..0000000 --- a/providers/raw/tasks/ami.py +++ /dev/null @@ -1,76 +0,0 @@ -from base import Task -from common import phases -from ebs import CreateSnapshot -from connection import Connect -from common.exceptions import TaskError - - -class AMIName(Task): - description = 'Determining the AMI name' - phase = phases.preparation - after = [Connect] - - def run(self, info): - image_vars = {'release': info.manifest.system['release'], - 'architecture': info.manifest.system['architecture'], - 'virtualization': info.manifest.virtualization, - 'backing': info.manifest.volume['backing']} - from datetime import datetime - now = datetime.now() - time_vars = ['%a', '%A', '%b', '%B', '%c', '%d', '%f', '%H', - '%I', '%j', '%m', '%M', '%p', '%S', '%U', '%w', - '%W', '%x', '%X', '%y', '%Y', '%z', '%Z'] - for var in time_vars: - image_vars[var] = now.strftime(var) - - ami_name = info.manifest.image['name'].format(**image_vars) - ami_description = info.manifest.image['description'].format(**image_vars) - - images = info.connection.get_all_images() - for image in images: - if ami_name == image.name: - msg = 'An image by the name {ami_name} already exists.'.format(ami_name=ami_name) - raise TaskError(msg) - info.ami_name = ami_name - info.ami_description = ami_description - - -class RegisterAMI(Task): - description = 'Registering the image as an AMI' - phase = phases.image_registration - after = [CreateSnapshot] - - def run(self, info): - arch = {'i386': 'i386', - 'amd64': 'x86_64'}.get(info.manifest.system['architecture']) - kernel_mapping = {'us-east-1': {'amd64': 'aki-88aa75e1', - 'i386': 'aki-b6aa75df'}, - 'us-west-1': {'amd64': 'aki-f77e26b2', - 'i386': 'aki-f57e26b0'}, - 'us-west-2': {'amd64': 'aki-fc37bacc', - 'i386': 'aki-fa37baca'}, - 'eu-west-1': {'amd64': 'aki-71665e05', - 'i386': 'aki-75665e01'}, - 'ap-southeast-1': {'amd64': 'aki-fe1354ac', - 'i386': 'aki-f81354aa'}, - 'ap-southeast-2': {'amd64': 'aki-31990e0b', - 'i386': 'aki-33990e09'}, - 'ap-northeast-1': {'amd64': 'aki-44992845', - 'i386': 'aki-42992843'}, - 'sa-east-1': {'amd64': 'aki-c48f51d9', - 'i386': 'aki-ca8f51d7'}, - 'us-gov-west-1': {'amd64': 'aki-79a4c05a', - 'i386': 'aki-7ba4c058'}} - kernel_id = kernel_mapping.get(info.host['region']).get(info.manifest.system['architecture']) - - from boto.ec2.blockdevicemapping import BlockDeviceType - from boto.ec2.blockdevicemapping import BlockDeviceMapping - block_device = BlockDeviceType(snapshot_id=info.snapshot.id, delete_on_termination=True, - size=int(info.manifest.volume['size']/1024)) - block_device_map = BlockDeviceMapping() - block_device_map['/dev/sda1'] = block_device - - info.image = info.connection.register_image(name=info.ami_name, description=info.ami_description, - architecture=arch, kernel_id=kernel_id, - root_device_name='/dev/sda1', - block_device_map=block_device_map) diff --git a/providers/raw/tasks/boot.py b/providers/raw/tasks/boot.py index 4d7c7f9..0bf3d28 100644 --- a/providers/raw/tasks/boot.py +++ b/providers/raw/tasks/boot.py @@ -26,13 +26,8 @@ class ConfigureGrub(Task): from common.tools import sed_i grub_def = os.path.join(info.root, 'etc/default/grub') - #sed_i(grub_def, '^GRUB_TIMEOUT=[0-9]+', 'GRUB_TIMEOUT=0\n' - # 'GRUB_HIDDEN_TIMEOUT=true') from common.tools import log_check_call - #log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) - #log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/grub.cfg']) - #log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/boot/grub/device.map']) log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) diff --git a/providers/raw/tasks/connection.py b/providers/raw/tasks/connection.py deleted file mode 100644 index 2c2ce19..0000000 --- a/providers/raw/tasks/connection.py +++ /dev/null @@ -1,40 +0,0 @@ -from base import Task -from common import phases -import host - - -class GetCredentials(Task): - description = 'Getting AWS credentials' - phase = phases.preparation - - def run(self, info): - info.credentials = self.get_credentials(info.manifest) - - def get_credentials(self, manifest): - from os import getenv - # manifest overrides environment - if(manifest.credentials['access-key'] and manifest.credentials['secret-key']): - return {'access_key': manifest.credentials['access-key'], - 'secret_key': manifest.credentials['secret-key']} - if(getenv('EC2_ACCESS_KEY') and getenv('EC2_SECRET_KEY')): - return {'access_key': getenv('EC2_ACCESS_KEY'), - 'secret_key': getenv('EC2_SECRET_KEY')} - - if(bool(manifest.credentials['access-key']) != bool(manifest.credentials['secret-key'])): - raise RuntimeError('Both the access key and secret key must be specified in the manifest.') - if(bool(getenv('EC2_ACCESS_KEY')) != bool(getenv('EC2_SECRET_KEY'))): - raise RuntimeError('Both the access key and secret key must be specified as environment variables.') - - raise RuntimeError('No ec2 credentials found.') - - -class Connect(Task): - description = 'Connecting to EC2' - phase = phases.preparation - after = [GetCredentials, host.GetInfo] - - def run(self, info): - from boto.ec2 import connect_to_region - info.connection = connect_to_region(info.host['region'], - aws_access_key_id=info.credentials['access_key'], - aws_secret_access_key=info.credentials['secret_key']) diff --git a/providers/raw/tasks/ebs.py b/providers/raw/tasks/ebs.py deleted file mode 100644 index a07b1a8..0000000 --- a/providers/raw/tasks/ebs.py +++ /dev/null @@ -1,89 +0,0 @@ -from base import Task -from common import phases -from common.exceptions import TaskError -from connection import Connect -from filesystem import UnmountVolume -import time - - -class CreateVolume(Task): - phase = phases.volume_creation - after = [Connect] - - description = 'Creating an EBS volume for bootstrapping' - - def run(self, info): - volume_size = int(info.manifest.volume['size']/1024) - - info.volume = info.connection.create_volume(volume_size, info.host['availabilityZone']) - while info.volume.volume_state() != 'available': - time.sleep(5) - info.volume.update() - - -class AttachVolume(Task): - phase = phases.volume_creation - after = [CreateVolume] - - description = 'Attaching the EBS volume' - - def run(self, info): - def char_range(c1, c2): - """Generates the characters from `c1` to `c2`, inclusive.""" - for c in xrange(ord(c1), ord(c2)+1): - yield chr(c) - - import os.path - info.bootstrap_device = {} - for letter in char_range('f', 'z'): - dev_path = os.path.join('/dev', 'xvd' + letter) - if not os.path.exists(dev_path): - info.bootstrap_device['path'] = dev_path - info.bootstrap_device['ec2_path'] = os.path.join('/dev', 'sd' + letter) - break - if 'path' not in info.bootstrap_device: - raise VolumeError('Unable to find a free block device path for mounting the bootstrap volume') - - info.volume.attach(info.host['instanceId'], info.bootstrap_device['ec2_path']) - while info.volume.attachment_state() != 'attached': - time.sleep(2) - info.volume.update() - - -class DetachVolume(Task): - phase = phases.volume_unmounting - after = [UnmountVolume] - - description = 'Detaching the EBS volume' - - def run(self, info): - info.volume.detach() - while info.volume.attachment_state() is not None: - time.sleep(2) - info.volume.update() - - -class CreateSnapshot(Task): - description = 'Creating a snapshot of the EBS volume' - phase = phases.image_registration - - def run(self, info): - info.snapshot = info.volume.create_snapshot() - while info.snapshot.status != 'completed': - time.sleep(2) - info.snapshot.update() - - -class DeleteVolume(Task): - phase = phases.cleaning - after = [] - - description = 'Deleting the EBS volume' - - def run(self, info): - info.volume.delete() - del info.volume - - -class VolumeError(TaskError): - pass diff --git a/providers/raw/tasks/one.py b/providers/raw/tasks/one.py deleted file mode 100644 index fda5185..0000000 --- a/providers/raw/tasks/one.py +++ /dev/null @@ -1,39 +0,0 @@ -from base import Task -from common import phases -import os -from providers.one.tasks.locale import GenerateLocale - - -class OpenNebulaContext(Task): - description = 'Setup OpenNebula init context' - phase = phases.system_modification - after = [GenerateLocale] - - def run(self, info): - import stat - rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | - stat.S_IROTH | stat.S_IXOTH) - - from shutil import copy - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-context_3.8.1.deb')) - script_dst = os.path.join(info.root, 'tmp/one-context_3.8.1.deb') - copy(script_src, script_dst) - os.chmod(script_dst, rwxr_xr_x) - - from common.tools import log_check_call - log_check_call(['/usr/sbin/chroot', info.root, 'dpkg', '-i', '/tmp/one-context_3.8.1.deb']) - # Fix start - from common.tools import sed_i - vmcontext_def = os.path.join(info.root, 'etc/init.d/vmcontext') - sed_i(vmcontext_def, '# Default-Start:', '# Default-Start: 2 3 4 5') - os.chmod(vmcontext_def, rwxr_xr_x) - log_check_call(['/usr/sbin/chroot', info.root, 'update-rc.d', 'vmcontext', 'start', '90', '2', '3', '4', '5', 'stop', '90', '0', '6']) - - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-pubkey.sh')) - script_dst = os.path.join(info.root, 'etc/one-context.d/one-pubkey.sh') - copy(script_src, script_dst) - script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/one-ec2.sh')) - script_dst = os.path.join(info.root, 'etc/one-context.d/one-ec2.sh') - copy(script_src, script_dst) - From 61f97ab06898425d7e26e65d1ae9f73e38763190 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Aug 2013 11:19:20 +0200 Subject: [PATCH 21/22] manage use of virtio for disk, add doc --- manifests/one-ide.manifest | 3 +- manifests/one-virtio.manifest | 54 +++++++++++++++++++++++++++++++ providers/raw/REAME.md | 16 +++++++++ providers/raw/tasks/boot.py | 25 ++++++++++++-- providers/raw/tasks/filesystem.py | 14 +++----- 5 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 manifests/one-virtio.manifest create mode 100644 providers/raw/REAME.md diff --git a/manifests/one-ide.manifest b/manifests/one-ide.manifest index 7dc267c..26db2a9 100644 --- a/manifests/one-ide.manifest +++ b/manifests/one-ide.manifest @@ -10,8 +10,7 @@ "bootstrapper": { "mount_dir": "/mnt/target", "image_file": "/tmp/one.img", - "tarball": true, - "device": "/dev/sda" + "tarball": true }, "image": { "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", diff --git a/manifests/one-virtio.manifest b/manifests/one-virtio.manifest new file mode 100644 index 0000000..02ce685 --- /dev/null +++ b/manifests/one-virtio.manifest @@ -0,0 +1,54 @@ +{ + "provider" : "raw", + "virtualization": "virtio", + "credentials" : { + "access-key": null, + "secret-key": null, + "root": "test" + }, + + "bootstrapper": { + "mount_dir": "/mnt/target", + "image_file": "/tmp/one.img", + "tarball": true + }, + "image": { + "name" : "debian-{release}-{architecture}-{virtualization}-{%y}{%m}{%d}", + "description": "Debian {release} {architecture} ({virtualization})" + }, + "system": { + "release" : "wheezy", + "architecture": "amd64", + "timezone" : "UTC", + "locale" : "en_US", + "charmap" : "UTF-8", + "mirror" : "ftp://ftp.fr.debian.org/debian/" + }, + "volume": { + "backing" : "raw", + "filesystem": "ext4", + "size" : 1024 + }, + "plugins": { + "admin_user": { + "enabled": false + }, + "build_metadata": { + "enabled": false, + "path" : "/root/build-metadata-{ami_name}" + }, + "prebootstrapped": { + "enabled": false, + "snapshot": "" + }, + "opennebula": { + "enabled": true + }, + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [] + } + + } +} diff --git a/providers/raw/REAME.md b/providers/raw/REAME.md new file mode 100644 index 0000000..efbd58b --- /dev/null +++ b/providers/raw/REAME.md @@ -0,0 +1,16 @@ +# RAW provider + +This provider creates a raw image file. It can be combined with opennebula plugin to add OpenNebula contextualization. + +By default, it creates a network interface configured with DHCP. + +# Configuration + +## provider + +*raw* : use this provider + +##virtualization + +* ide: basic disk emulation (/dev/sda) +* virtio: Virtio emulation (for KVM), provides better disk performances (/dev/vda) diff --git a/providers/raw/tasks/boot.py b/providers/raw/tasks/boot.py index 0bf3d28..6ae840c 100644 --- a/providers/raw/tasks/boot.py +++ b/providers/raw/tasks/boot.py @@ -18,22 +18,43 @@ class ConfigureGrub(Task): for cfg in [os.path.join(grubd, f) for f in os.listdir(grubd)]: os.chmod(cfg, os.stat(cfg).st_mode & ~ x_all) + from common.tools import log_check_call from shutil import copy + script_src = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets/grub.d/40_custom')) script_dst = os.path.join(info.root, 'etc/grub.d/40_custom') copy(script_src, script_dst) os.chmod(script_dst, rwxr_xr_x) - from common.tools import sed_i + if info.manifest.virtualization == 'virtio': + modules_path = os.path.join(info.root, + 'etc/initramfs-tools/modules') + with open(modules_path, 'a') as modules: + modules.write("\nvirtio_pci\nvirtio_blk\n") + + grub_def = os.path.join(info.root, 'etc/default/grub') - from common.tools import log_check_call log_check_call(['/usr/sbin/chroot', info.root, 'ln', '-s', '/boot/grub/grub.cfg', '/boot/grub/menu.lst']) log_check_call(['/usr/sbin/chroot', info.root, 'update-initramfs', '-u']) log_check_call(['grub-install', '--boot-directory='+info.root+"/boot/", '/dev/loop0']) + + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + from common.tools import sed_i + if info.manifest.virtualization == 'virtio': + grub_cfg = os.path.join(info.root, 'boot/grub/grub.cfg') + sed_i(grub_cfg, 'sda', 'vda') + device_map = os.path.join(info.root, + 'boot/grub/device.map') + sed_i(device_map, 'sda', 'vda') + #log_check_call(['/usr/sbin/chroot', info.root, '/usr/sbin/update-grub']) + + + class BlackListModules(Task): description = 'Blacklisting kernel modules' phase = phases.system_modification diff --git a/providers/raw/tasks/filesystem.py b/providers/raw/tasks/filesystem.py index 5b255c7..48dc473 100644 --- a/providers/raw/tasks/filesystem.py +++ b/providers/raw/tasks/filesystem.py @@ -138,14 +138,10 @@ class ModifyFstab(Task): mount_opts.append('nobarrier') fstab_path = os.path.join(info.root, 'etc/fstab') with open(fstab_path, 'a') as fstab: - fstab.write(('/dev/sda1 / {filesystem} {mount_opts} 1 1\n' + device = '/dev/sda1' + if info.manifest.virtualization == 'virtio': + device = '/dev/vda1' + + fstab.write((device+' / {filesystem} {mount_opts} 1 1\n' .format(filesystem=info.manifest.volume['filesystem'].lower(), mount_opts=','.join(mount_opts)))) - log_check_call(['/usr/sbin/chroot', info.root, 'cat', '/etc/fstab']) - -class InstallMbr(Task): - description = 'Install MBR' - phase = phases.system_modification - - def run(self, info): - log_check_call(['install-mbr', '-v', info.manifest.bootstrapper['image_file']]) From 9745861b460f633f3ba45ae80ac5956678b4b9bb Mon Sep 17 00:00:00 2001 From: root Date: Thu, 1 Aug 2013 11:38:30 +0200 Subject: [PATCH 22/22] add doc for plugins --- plugins/opennebula/README.md | 17 +++++++++++++++++ plugins/user_packages/README.md | 15 +++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 plugins/opennebula/README.md create mode 100644 plugins/user_packages/README.md diff --git a/plugins/opennebula/README.md b/plugins/opennebula/README.md new file mode 100644 index 0000000..c36f2b3 --- /dev/null +++ b/plugins/opennebula/README.md @@ -0,0 +1,17 @@ +# Open Nebula context plugin + +This plugin adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong). + +It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like: + + ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] + ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] + ETH0_IP $NIC[IP, NETWORK_ID=2] + ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] + ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] + FILES path_to_my_ssh_public_key.pub + +Plugin will install all *.pub* files in the root authorized_keys file. + +In case of an EC2 start, if the USER_EC2_DATA element is a script, the plugin will execute it. + diff --git a/plugins/user_packages/README.md b/plugins/user_packages/README.md new file mode 100644 index 0000000..059bf63 --- /dev/null +++ b/plugins/user_packages/README.md @@ -0,0 +1,15 @@ +# User package plugin + +This plugin gives the possibility to the user to install Debian packages in the virtual image. + +Plugin is defined in the manifest file, plugin section with: + + "user_packages": { + "enabled": true, + "repo": [ "apache2" ], + "local": [ "/tmp/mypackage.deb" ] + } + +The *repo* element refers to packages available in Debian repository (apt-get). + +The *local* element will copy the specified .deb files and install them in the image with a dpkg command. Packages are installed in the order of their declaration.