diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index 8d7f407..01cab4d 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -13,8 +13,22 @@ prog=$(basename $0) logger="logger -t $prog" -device_path="/dev/xvda" +growpart="growpart" +hash $growpart 2> /dev/null || { + $logger "$growpart was not found on PATH. Unable to expand size." + exit 1 +} + +root_device_path="/dev/xvda" +root_index="0" + +$growpart $root_device_path $root_index || { + $logger "growpart failed. Unable to expand size." + exit 1 +} + +device_path="${root_device_path}${root_index}" filesystem=$(blkid -s TYPE -o value ${device_path}) case $filesystem in diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py index 5e43fa5..f4537e3 100644 --- a/bootstrapvz/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -57,9 +57,16 @@ class AdjustExpandRootScript(Task): @classmethod def run(cls, info): - import os.path from ..tools import sed_i + from bootstrapvz.common.releases import jessie script = os.path.join(info.root, 'etc/init.d/expand-root') + root_idx = info.volume.partition_map.root.get_index() - device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx) - sed_i(script, '^device_path="/dev/xvda"$', device_path) + root_index_line = 'root_index="{idx}"'.format(idx=root_idx) + sed_i(script, '^root_index="0"$', root_index_line) + + root_device_path = 'root_device_path="{device}"'.format(device=info.volume.device_path) + sed_i(script, '^root_device_path="/dev/xvda"$', root_device_path) + + if info.manifest.release >= jessie: + sed_i(script, '^growpart="growpart"$', 'growpart-workaround') diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py index 2ea5952..21749c6 100644 --- a/bootstrapvz/providers/ec2/__init__.py +++ b/bootstrapvz/providers/ec2/__init__.py @@ -53,7 +53,7 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - from bootstrapvz.common.releases import wheezy + from bootstrapvz.common.releases import wheezy, jessie taskset.update(task_groups.get_standard_groups(manifest)) taskset.update(task_groups.ssh_group) @@ -81,6 +81,9 @@ def resolve_tasks(taskset, manifest): taskset.add(tasks.network.InstallDHCPCD) taskset.add(tasks.network.EnableDHCPCDDNS) + if manifest.release >= jessie: + taskset.add(tasks.packages.AddWorkaroundGrowpart) + if manifest.provider.get('install_init_scripts', True): taskset.add(tasks.initd.AddEC2InitScripts) diff --git a/bootstrapvz/providers/ec2/assets/bin/growpart b/bootstrapvz/providers/ec2/assets/bin/growpart new file mode 100755 index 0000000..85c2d1f --- /dev/null +++ b/bootstrapvz/providers/ec2/assets/bin/growpart @@ -0,0 +1,587 @@ +#!/bin/sh +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Authors: Scott Moser +# Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# the fudge factor. if its within this many 512 byte sectors, dont bother +FUDGE=${GROWPART_FUDGE:-$((20*1024))} +TEMP_D="" +RESTORE_FUNC="" +RESTORE_HUMAN="" +VERBOSITY=0 +DISK="" +PART="" +PT_UPDATE=false +DRY_RUN=0 + +MBR_CHS="" +MBR_BACKUP="" +GPT_BACKUP="" +_capture="" + +error() { + echo "$@" 1>&2 +} + +fail() { + [ $# -eq 0 ] || echo "FAILED:" "$@" + exit 2 +} + +nochange() { + echo "NOCHANGE:" "$@" + exit 1 +} + +changed() { + echo "CHANGED:" "$@" + exit 0 +} + +change() { + echo "CHANGE:" "$@" + exit 0 +} + +cleanup() { + if [ -n "${RESTORE_FUNC}" ]; then + error "***** WARNING: Resize failed, attempting to revert ******" + if ${RESTORE_FUNC} ; then + error "***** Appears to have gone OK ****" + else + error "***** FAILED! or original partition table" \ + "looked like: ****" + cat "${RESTORE_HUMAN}" 1>&2 + fi + fi + [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" +} + +debug() { + local level=${1} + shift + [ "${level}" -gt "${VERBOSITY}" ] && return + if [ "${DEBUG_LOG}" ]; then + echo "$@" >>"${DEBUG_LOG}" + else + error "$@" + fi +} + +debugcat() { + local level="$1" + shift; + [ "${level}" -gt "$VERBOSITY" ] && return + if [ "${DEBUG_LOG}" ]; then + cat "$@" >>"${DEBUG_LOG}" + else + cat "$@" 1>&2 + fi +} + +mktemp_d() { + # just a mktemp -d that doens't need mktemp if its not there. + _RET=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX" 2>/dev/null) && + return + _RET=$(umask 077 && t="${TMPDIR:-/tmp}/${0##*/}.$$" && + mkdir "${t}" && echo "${t}") + return +} + +Usage() { + cat <&2 + error "$@" + exit 2 +} + +mbr_restore() { + sfdisk --no-reread "${DISK}" ${MBR_CHS} -I "${MBR_BACKUP}" +} + +sfdisk_worked_but_blkrrpart_failed() { + local ret="$1" output="$2" + # exit code found was just 1, but dont insist on that + #[ $ret -eq 1 ] || return 1 + # Successfully wrote the new partition table + grep -qi "Success.* wrote.* new.* partition" "$output" && + grep -qi "BLKRRPART: Device or resource busy" "$output" + return +} + +mbr_resize() { + RESTORE_HUMAN="${TEMP_D}/recovery" + MBR_BACKUP="${TEMP_D}/orig.save" + + local change_out=${TEMP_D}/change.out + local dump_out=${TEMP_D}/dump.out + local new_out=${TEMP_D}/new.out + local dump_mod=${TEMP_D}/dump.mod + local tmp="${TEMP_D}/tmp.out" + local err="${TEMP_D}/err.out" + + local _devc cyl _w1 heads _w2 sectors _w3 tot dpart + local pt_start pt_size pt_end max_end new_size change_info + + # --show-pt-geometry outputs something like + # /dev/sda: 164352 cylinders, 4 heads, 32 sectors/track + rqe sfd_geom sfdisk "${DISK}" --show-pt-geometry >"${tmp}" && + read _devc cyl _w1 heads _w2 sectors _w3 <"${tmp}" && + MBR_CHS="-C ${cyl} -H ${heads} -S ${sectors}" || + fail "failed to get CHS from ${DISK}" + + tot=$((${cyl}*${heads}*${sectors})) + + debug 1 "geometry is ${MBR_CHS}. total size=${tot}" + rqe sfd_dump sfdisk ${MBR_CHS} --unit=S --dump "${DISK}" \ + >"${dump_out}" || + fail "failed to dump sfdisk info for ${DISK}" + + { + echo "## sfdisk ${MBR_CHS} --unit=S --dump ${DISK}" + cat "${dump_out}" + } >"${RESTORE_HUMAN}" + [ $? -eq 0 ] || fail "failed to save sfdisk -d output" + + debugcat 1 "${RESTORE_HUMAN}" + + sed -e 's/,//g; s/start=/start /; s/size=/size /' "${dump_out}" \ + >"${dump_mod}" || + fail "sed failed on dump output" + + dpart="${DISK}${PART}" # disk and partition number + if [ -b "${DISK}p${PART}" -a "${DISK%[0-9]}" != "${DISK}" ]; then + # for block devices that end in a number (/dev/nbd0) + # the partition is "p" (/dev/nbd0p1) + dpart="${DISK}p${PART}" + elif [ "${DISK#/dev/loop[0-9]}" != "${DISK}" ]; then + # for /dev/loop devices, sfdisk output will be p + # format also, even though there is not a device there. + dpart="${DISK}p${PART}" + fi + + pt_start=$(awk '$1 == pt { print $4 }' "pt=${dpart}" <"${dump_mod}") && + pt_size=$(awk '$1 == pt { print $6 }' "pt=${dpart}" <"${dump_mod}") && + [ -n "${pt_start}" -a -n "${pt_size}" ] && + pt_end=$((${pt_size}+${pt_start})) || + fail "failed to get start and end for ${dpart} in ${DISK}" + + # find the minimal starting location that is >= pt_end + max_end=$(awk '$3 == "start" { if($4 >= pt_end && $4 < min) + { min = $4 } } END { printf("%s\n",min); }' \ + min=${tot} pt_end=${pt_end} "${dump_mod}") && + [ -n "${max_end}" ] || + fail "failed to get max_end for partition ${PART}" + + debug 1 "max_end=${max_end} tot=${tot} pt_end=${pt_end}" \ + "pt_start=${pt_start} pt_size=${pt_size}" + [ $((${pt_end})) -eq ${max_end} ] && + nochange "partition ${PART} is size ${pt_size}. it cannot be grown" + [ $((${pt_end}+${FUDGE})) -gt ${max_end} ] && + nochange "partition ${PART} could only be grown by" \ + "$((${max_end}-${pt_end})) [fudge=${FUDGE}]" + + # now, change the size for this partition in ${dump_out} to be the + # new size + new_size=$((${max_end}-${pt_start})) + sed "\|^\s*${dpart} |s/${pt_size},/${new_size},/" "${dump_out}" \ + >"${new_out}" || + fail "failed to change size in output" + + change_info="partition=${PART} start=${pt_start} old: size=${pt_size} end=${pt_end} new: size=${new_size},end=${max_end}" + if [ ${DRY_RUN} -ne 0 ]; then + echo "CHANGE: ${change_info}" + { + echo "# === old sfdisk -d ===" + cat "${dump_out}" + echo "# === new sfdisk -d ===" + cat "${new_out}" + } 1>&2 + exit 0 + fi + + LANG=C sfdisk --no-reread "${DISK}" ${MBR_CHS} --force \ + -O "${MBR_BACKUP}" <"${new_out}" >"${change_out}" 2>&1 + ret=$? + [ $ret -eq 0 ] || RESTORE_FUNC="mbr_restore" + + if [ $ret -eq 0 ]; then + : + elif $PT_UPDATE && + sfdisk_worked_but_blkrrpart_failed "$ret" "${change_out}"; then + # if the command failed, but it looks like only because + # the device was busy and we have pt_update, then go on + debug 1 "sfdisk failed, but likely only because of blkrrpart" + else + error "attempt to resize ${DISK} failed. sfdisk output below:" + sed 's,^,| ,' "${change_out}" 1>&2 + fail "failed to resize" + fi + + rq pt_update pt_update "$DISK" "$PART" || + fail "pt_resize failed" + + RESTORE_FUNC="" + + changed "${change_info}" + + # dump_out looks something like: + ## partition table of /tmp/out.img + #unit: sectors + # + #/tmp/out.img1 : start= 1, size= 48194, Id=83 + #/tmp/out.img2 : start= 48195, size= 963900, Id=83 + #/tmp/out.img3 : start= 1012095, size= 305235, Id=82 + #/tmp/out.img4 : start= 1317330, size= 771120, Id= 5 + #/tmp/out.img5 : start= 1317331, size= 642599, Id=83 + #/tmp/out.img6 : start= 1959931, size= 48194, Id=83 + #/tmp/out.img7 : start= 2008126, size= 80324, Id=83 +} + +gpt_restore() { + sgdisk -l "${GPT_BACKUP}" "${DISK}" +} + +gpt_resize() { + GPT_BACKUP="${TEMP_D}/pt.backup" + + local pt_info="${TEMP_D}/pt.info" + local pt_pretend="${TEMP_D}/pt.pretend" + local pt_data="${TEMP_D}/pt.data" + local out="${TEMP_D}/out" + + local dev="disk=${DISK} partition=${PART}" + + local pt_start pt_end pt_size last pt_max code guid name new_size + local old new change_info + + # Dump the original partition information and details to disk. This is + # used in case something goes wrong and human interaction is required + # to revert any changes. + rqe sgd_info sgdisk "--info=${PART}" --print "${DISK}" >"${pt_info}" || + RESTORE_HUMAN="${pt_info}" + + debug 1 "$dev: original sgdisk info:" + debugcat 1 "${pt_info}" + + # Pretend to move the backup GPT header to the end of the disk and dump + # the resulting partition information. We use this info to determine if + # we have to resize the partition. + rqe sgd_pretend sgdisk --pretend --move-second-header \ + --print "${DISK}" >"${pt_pretend}" || + fail "${dev}: failed to dump pretend sgdisk info" + + debug 1 "$dev: pretend sgdisk info" + debugcat 1 "${pt_pretend}" + + # Extract the partition data from the pretend dump + awk 'found { print } ; $1 == "Number" { found = 1 }' \ + "${pt_pretend}" >"${pt_data}" || + fail "${dev}: failed to parse pretend sgdisk info" + + # Get the start and end sectors of the partition to be grown + pt_start=$(awk '$1 == '"${PART}"' { print $2 }' "${pt_data}") && + [ -n "${pt_start}" ] || + fail "${dev}: failed to get start sector" + pt_end=$(awk '$1 == '"${PART}"' { print $3 }' "${pt_data}") && + [ -n "${pt_end}" ] || + fail "${dev}: failed to get end sector" + pt_size="$((${pt_end} - ${pt_start}))" + + # Get the last usable sector + last=$(awk '/last usable sector is/ { print $NF }' \ + "${pt_pretend}") && [ -n "${last}" ] || + fail "${dev}: failed to get last usable sector" + + # Find the minimal start sector that is >= pt_end + pt_max=$(awk '{ if ($2 >= pt_end && $2 < min) { min = $2 } } END \ + { print min }' min="${last}" pt_end="${pt_end}" \ + "${pt_data}") && [ -n "${pt_max}" ] || + fail "${dev}: failed to find max end sector" + + debug 1 "${dev}: pt_start=${pt_start} pt_end=${pt_end}" \ + "pt_size=${pt_size} pt_max=${pt_max} last=${last}" + + # Check if the partition can be grown + [ "${pt_end}" -eq "${pt_max}" ] && + nochange "${dev}: size=${pt_size}, it cannot be grown" + [ "$((${pt_end} + ${FUDGE}))" -gt "${pt_max}" ] && + nochange "${dev}: could only be grown by" \ + "$((${pt_max} - ${pt_end})) [fudge=${FUDGE}]" + + # The partition can be grown if we made it here. Get some more info + # about it so we can do it properly. + # FIXME: Do we care about the attribute flags? + code=$(awk '/^Partition GUID code:/ { print $4 }' "${pt_info}") + guid=$(awk '/^Partition unique GUID:/ { print $4 }' "${pt_info}") + name=$(awk '/^Partition name:/ { gsub(/'"'"'/, "") ; \ + if (NF >= 3) print substr($0, index($0, $3)) }' "${pt_info}") + [ -n "${code}" -a -n "${guid}" ] || + fail "${dev}: failed to parse sgdisk details" + + debug 1 "${dev}: code=${code} guid=${guid} name='${name}'" + + # Calculate the new size of the partition + new_size=$((${pt_max} - ${pt_start})) + old="old: size=${pt_size},end=${pt_end}" + new="new: size=${new_size},end=${pt_max}" + change_info="${dev}: start=${pt_start} ${old} ${new}" + + # Dry run + [ "${DRY_RUN}" -ne 0 ] && change "${change_info}" + + # Backup the current partition table, we're about to modify it + rq sgd_backup sgdisk "--backup=${GPT_BACKUP}" "${DISK}" || + fail "${dev}: failed to backup the partition table" + + # Modify the partition table. We do it all in one go (the order is + # important!): + # - move the GPT backup header to the end of the disk + # - delete the partition + # - recreate the partition with the new size + # - set the partition code + # - set the partition GUID + # - set the partition name + rq sgdisk_mod sgdisk --move-second-header "--delete=${PART}" \ + "--new=${PART}:${pt_start}:${pt_max}" \ + "--typecode=${PART}:${code}" \ + "--partition-guid=${PART}:${guid}" \ + "--change-name=${PART}:${name}" "${DISK}" && + rq pt_update pt_update "$DISK" "$PART" || { + RESTORE_FUNC=gpt_restore + fail "${dev}: failed to repartition" + } + + changed "${change_info}" +} + +kver_to_num() { + local kver="$1" maj="" min="" mic="0" + kver=${kver%%-*} + maj=${kver%%.*} + min=${kver#${maj}.} + min=${min%%.*} + mic=${kver#${maj}.${min}.} + [ "$kver" = "$mic" ] && mic=0 + _RET=$(($maj*1000*1000+$min*1000+$mic)) +} + +kver_cmp() { + local op="$2" n1="" n2="" + kver_to_num "$1" + n1="$_RET" + kver_to_num "$3" + n2="$_RET" + [ $n1 $op $n2 ] +} + +rq() { + # runquieterror(label, command) + # gobble stderr of a command unless it errors + local label="$1" ret="" efile="" + efile="$TEMP_D/$label.err" + shift; + debug 2 "running[$label][$_capture]" "$@" + if [ "${_capture}" = "erronly" ]; then + "$@" 2>"$TEMP_D/$label.err" + ret=$? + else + "$@" >"$TEMP_D/$label.err" 2>&1 + ret=$? + fi + if [ $ret -ne 0 ]; then + error "failed [$label:$ret]" "$@" + cat "$efile" 1>&2 + fi + return $ret +} + +rqe() { + local _capture="erronly" + rq "$@" +} + +verify_ptupdate() { + local input="$1" found="" reason="" kver="" + + # we can always satisfy 'off' + if [ "$input" = "off" ]; then + _RET="false"; + return 0; + fi + + if command -v partx >/dev/null 2>&1; then + partx --help | grep -q -- --update || { + reason="partx has no '--update' flag in usage." + found="off" + } + else + reason="no 'partx' command" + found="off" + fi + + if [ -z "$found" ]; then + if [ "$(uname)" != "Linux" ]; then + reason="Kernel is not Linux per uname." + found="off" + fi + fi + + if [ -z "$found" ]; then + kver=$(uname -r) || debug 1 "uname -r failed!" + + if ! kver_cmp "${kver-0.0.0}" -ge 3.8.0; then + reason="Kernel '$kver' < 3.8.0." + found="off" + fi + fi + + if [ -z "$found" ]; then + _RET="true" + return 0 + fi + + case "$input" in + on) error "$reason"; return 1;; + auto) + _RET="false"; + debug 1 "partition update disabled: $reason" + return 0;; + force) + _RET="true" + error "WARNING: ptupdate forced on even though: $reason" + return 0;; + esac + error "unknown input '$input'"; + return 1; +} + +pt_update() { + local dev="$1" part="$2" update="${3:-$PT_UPDATE}" + if ! $update; then + return 0 + fi + partx --update "$part" "$dev" +} + +has_cmd() { + command -v "${1}" >/dev/null 2>&1 +} + +pt_update="auto" +while [ $# -ne 0 ]; do + cur=${1} + next=${2} + case "$cur" in + -h|--help) + Usage + exit 0 + ;; + --fudge) + FUDGE=${next} + shift + ;; + -N|--dry-run) + DRY_RUN=1 + ;; + -u|--update|--update=*) + if [ "${cur#--update=}" != "$cur" ]; then + next="${cur#--update=}" + else + shift + fi + case "$next" in + off|auto|force|on) pt_update=$next;; + *) fail "unknown --update option: $next";; + esac + ;; + -v|--verbose) + VERBOSITY=$(($VERBOSITY+1)) + ;; + --) + shift + break + ;; + -*) + fail "unknown option ${cur}" + ;; + *) + if [ -z "${DISK}" ]; then + DISK=${cur} + else + [ -z "${PART}" ] || fail "confused by arg ${cur}" + PART=${cur} + fi + ;; + esac + shift +done + +[ -n "${DISK}" ] || bad_Usage "must supply disk and partition-number" +[ -n "${PART}" ] || bad_Usage "must supply partition-number" + +has_cmd "sfdisk" || fail "sfdisk not found" + +[ -e "${DISK}" ] || fail "${DISK}: does not exist" + +[ "${PART#*[!0-9]}" = "${PART}" ] || fail "partition-number must be a number" + +verify_ptupdate "$pt_update" || fail +PT_UPDATE=$_RET + +debug 1 "update-partition set to $PT_UPDATE" + +mktemp_d && TEMP_D="${_RET}" || fail "failed to make temp dir" +trap cleanup EXIT + +# get the ID of the first partition to determine if it's MBR or GPT +id=$(sfdisk --id --force "${DISK}" 1 2>/dev/null) || + fail "unable to determine partition type" + +if [ "${id}" = "ee" ] ; then + has_cmd "sgdisk" || fail "GPT partition found but no sgdisk" + debug 1 "found GPT partition table (id = ${id})" + gpt_resize +else + debug 1 "found MBR partition table (id = ${id})" + mbr_resize +fi + +# vi: ts=4 noexpandtab diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py index bec331c..4d51280 100644 --- a/bootstrapvz/providers/ec2/tasks/packages.py +++ b/bootstrapvz/providers/ec2/tasks/packages.py @@ -1,5 +1,6 @@ from bootstrapvz.base import Task from bootstrapvz.common import phases +import os.path class DefaultPackages(Task): @@ -9,9 +10,22 @@ class DefaultPackages(Task): @classmethod def run(cls, info): info.packages.add('file') # Needed for the init scripts - import os.path + kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml') from bootstrapvz.common.tools import config_get kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename, info.manifest.system['architecture']]) info.packages.add(kernel_package) + + +class AddWorkaroundGrowpart(Task): + description = 'Adding growpart workaround for jessie' + phase = phases.system_modification + + @classmethod + def run(cls, info): + from shutil import copy + from . import assets + src = os.path.join(assets, 'bin/growpart') + dst = os.path.join(info.root, 'usr/bin/growpart-workaround') + copy(src, dst) diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py index d6afbd6..7bdeb78 100644 --- a/tests/integration/manifests/__init__.py +++ b/tests/integration/manifests/__init__.py @@ -1,5 +1,7 @@ import os.path import glob +import random +import string from bootstrapvz.common.tools import load_data partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml')) @@ -13,8 +15,6 @@ for path in partial_json + partial_yaml: raise Exception(msg) partials[key] = load_data(path) -import random -import string pool = string.ascii_uppercase + string.ascii_lowercase + string.digits random_password = ''.join(random.choice(pool) for _ in range(16)) partials['root_password']['plugins']['root_password']['password'] = random_password