From c477c32e4a3ffad94ee383fbd3ce5b264b45e7eb Mon Sep 17 00:00:00 2001 From: Jonh Wendell Date: Tue, 30 Jun 2015 15:14:41 -0300 Subject: [PATCH 1/9] EC2: Added growpart script This is an attempt to fix issue #237 The script "growpart" was added as an asset, because Debian's Jessie version (inside cloud-utils) package is outdated and buggy. Then "expand-root" init script was modified to call growpart before calling resize2fs. In fact, calling resize2fs without resizing the partition first is useless. The task was named 'AddWorkaroundGrowpart' because it must go away in the future in favor of using recent stuff in cloud-utils. Currently even the official images suffer this issue, that makes the system use only 8 GiB, even if the user chooses a bigger storage size inside AWS. --- bootstrapvz/common/assets/init.d/expand-root | 13 +- bootstrapvz/common/tasks/initd.py | 5 +- bootstrapvz/providers/ec2/__init__.py | 1 + bootstrapvz/providers/ec2/assets/bin/growpart | 587 ++++++++++++++++++ bootstrapvz/providers/ec2/tasks/packages.py | 16 +- 5 files changed, 617 insertions(+), 5 deletions(-) create mode 100755 bootstrapvz/providers/ec2/assets/bin/growpart diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index 8d7f407..5baa5bf 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -13,8 +13,19 @@ prog=$(basename $0) logger="logger -t $prog" -device_path="/dev/xvda" +growpart=$(which growpart) || { + $logger "growpart was not found on PATH. Unable to expand size." + exit 0 +} +root_index="0" + +$growpart /dev/xvda $root_index || { + $logger "growpart failed. Unable to expand size." + exit 0 +} + +device_path="/dev/xvda$root_index" filesystem=$(blkid -s TYPE -o value ${device_path}) case $filesystem in diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py index 5e43fa5..5b6101e 100644 --- a/bootstrapvz/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -57,9 +57,8 @@ class AdjustExpandRootScript(Task): @classmethod def run(cls, info): - import os.path from ..tools import sed_i script = os.path.join(info.root, 'etc/init.d/expand-root') root_idx = info.volume.partition_map.root.get_index() - device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx) - sed_i(script, '^device_path="/dev/xvda"$', device_path) + root_index_line = 'root_index="{idx}"'.format(idx=root_idx) + sed_i(script, '^root_index="0"$', root_index_line) diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py index 2ea5952..b057785 100644 --- a/bootstrapvz/providers/ec2/__init__.py +++ b/bootstrapvz/providers/ec2/__init__.py @@ -68,6 +68,7 @@ def resolve_tasks(taskset, manifest): boot.BlackListModules, boot.DisableGetTTYs, initd.AddExpandRoot, + tasks.packages.AddWorkaroundGrowpart, initd.RemoveHWClock, initd.InstallInitScripts, tasks.ami.RegisterAMI, diff --git a/bootstrapvz/providers/ec2/assets/bin/growpart b/bootstrapvz/providers/ec2/assets/bin/growpart new file mode 100755 index 0000000..85c2d1f --- /dev/null +++ b/bootstrapvz/providers/ec2/assets/bin/growpart @@ -0,0 +1,587 @@ +#!/bin/sh +# Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. +# +# Authors: Scott Moser +# Juerg Haefliger +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# the fudge factor. if its within this many 512 byte sectors, dont bother +FUDGE=${GROWPART_FUDGE:-$((20*1024))} +TEMP_D="" +RESTORE_FUNC="" +RESTORE_HUMAN="" +VERBOSITY=0 +DISK="" +PART="" +PT_UPDATE=false +DRY_RUN=0 + +MBR_CHS="" +MBR_BACKUP="" +GPT_BACKUP="" +_capture="" + +error() { + echo "$@" 1>&2 +} + +fail() { + [ $# -eq 0 ] || echo "FAILED:" "$@" + exit 2 +} + +nochange() { + echo "NOCHANGE:" "$@" + exit 1 +} + +changed() { + echo "CHANGED:" "$@" + exit 0 +} + +change() { + echo "CHANGE:" "$@" + exit 0 +} + +cleanup() { + if [ -n "${RESTORE_FUNC}" ]; then + error "***** WARNING: Resize failed, attempting to revert ******" + if ${RESTORE_FUNC} ; then + error "***** Appears to have gone OK ****" + else + error "***** FAILED! or original partition table" \ + "looked like: ****" + cat "${RESTORE_HUMAN}" 1>&2 + fi + fi + [ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" +} + +debug() { + local level=${1} + shift + [ "${level}" -gt "${VERBOSITY}" ] && return + if [ "${DEBUG_LOG}" ]; then + echo "$@" >>"${DEBUG_LOG}" + else + error "$@" + fi +} + +debugcat() { + local level="$1" + shift; + [ "${level}" -gt "$VERBOSITY" ] && return + if [ "${DEBUG_LOG}" ]; then + cat "$@" >>"${DEBUG_LOG}" + else + cat "$@" 1>&2 + fi +} + +mktemp_d() { + # just a mktemp -d that doens't need mktemp if its not there. + _RET=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX" 2>/dev/null) && + return + _RET=$(umask 077 && t="${TMPDIR:-/tmp}/${0##*/}.$$" && + mkdir "${t}" && echo "${t}") + return +} + +Usage() { + cat <&2 + error "$@" + exit 2 +} + +mbr_restore() { + sfdisk --no-reread "${DISK}" ${MBR_CHS} -I "${MBR_BACKUP}" +} + +sfdisk_worked_but_blkrrpart_failed() { + local ret="$1" output="$2" + # exit code found was just 1, but dont insist on that + #[ $ret -eq 1 ] || return 1 + # Successfully wrote the new partition table + grep -qi "Success.* wrote.* new.* partition" "$output" && + grep -qi "BLKRRPART: Device or resource busy" "$output" + return +} + +mbr_resize() { + RESTORE_HUMAN="${TEMP_D}/recovery" + MBR_BACKUP="${TEMP_D}/orig.save" + + local change_out=${TEMP_D}/change.out + local dump_out=${TEMP_D}/dump.out + local new_out=${TEMP_D}/new.out + local dump_mod=${TEMP_D}/dump.mod + local tmp="${TEMP_D}/tmp.out" + local err="${TEMP_D}/err.out" + + local _devc cyl _w1 heads _w2 sectors _w3 tot dpart + local pt_start pt_size pt_end max_end new_size change_info + + # --show-pt-geometry outputs something like + # /dev/sda: 164352 cylinders, 4 heads, 32 sectors/track + rqe sfd_geom sfdisk "${DISK}" --show-pt-geometry >"${tmp}" && + read _devc cyl _w1 heads _w2 sectors _w3 <"${tmp}" && + MBR_CHS="-C ${cyl} -H ${heads} -S ${sectors}" || + fail "failed to get CHS from ${DISK}" + + tot=$((${cyl}*${heads}*${sectors})) + + debug 1 "geometry is ${MBR_CHS}. total size=${tot}" + rqe sfd_dump sfdisk ${MBR_CHS} --unit=S --dump "${DISK}" \ + >"${dump_out}" || + fail "failed to dump sfdisk info for ${DISK}" + + { + echo "## sfdisk ${MBR_CHS} --unit=S --dump ${DISK}" + cat "${dump_out}" + } >"${RESTORE_HUMAN}" + [ $? -eq 0 ] || fail "failed to save sfdisk -d output" + + debugcat 1 "${RESTORE_HUMAN}" + + sed -e 's/,//g; s/start=/start /; s/size=/size /' "${dump_out}" \ + >"${dump_mod}" || + fail "sed failed on dump output" + + dpart="${DISK}${PART}" # disk and partition number + if [ -b "${DISK}p${PART}" -a "${DISK%[0-9]}" != "${DISK}" ]; then + # for block devices that end in a number (/dev/nbd0) + # the partition is "p" (/dev/nbd0p1) + dpart="${DISK}p${PART}" + elif [ "${DISK#/dev/loop[0-9]}" != "${DISK}" ]; then + # for /dev/loop devices, sfdisk output will be p + # format also, even though there is not a device there. + dpart="${DISK}p${PART}" + fi + + pt_start=$(awk '$1 == pt { print $4 }' "pt=${dpart}" <"${dump_mod}") && + pt_size=$(awk '$1 == pt { print $6 }' "pt=${dpart}" <"${dump_mod}") && + [ -n "${pt_start}" -a -n "${pt_size}" ] && + pt_end=$((${pt_size}+${pt_start})) || + fail "failed to get start and end for ${dpart} in ${DISK}" + + # find the minimal starting location that is >= pt_end + max_end=$(awk '$3 == "start" { if($4 >= pt_end && $4 < min) + { min = $4 } } END { printf("%s\n",min); }' \ + min=${tot} pt_end=${pt_end} "${dump_mod}") && + [ -n "${max_end}" ] || + fail "failed to get max_end for partition ${PART}" + + debug 1 "max_end=${max_end} tot=${tot} pt_end=${pt_end}" \ + "pt_start=${pt_start} pt_size=${pt_size}" + [ $((${pt_end})) -eq ${max_end} ] && + nochange "partition ${PART} is size ${pt_size}. it cannot be grown" + [ $((${pt_end}+${FUDGE})) -gt ${max_end} ] && + nochange "partition ${PART} could only be grown by" \ + "$((${max_end}-${pt_end})) [fudge=${FUDGE}]" + + # now, change the size for this partition in ${dump_out} to be the + # new size + new_size=$((${max_end}-${pt_start})) + sed "\|^\s*${dpart} |s/${pt_size},/${new_size},/" "${dump_out}" \ + >"${new_out}" || + fail "failed to change size in output" + + change_info="partition=${PART} start=${pt_start} old: size=${pt_size} end=${pt_end} new: size=${new_size},end=${max_end}" + if [ ${DRY_RUN} -ne 0 ]; then + echo "CHANGE: ${change_info}" + { + echo "# === old sfdisk -d ===" + cat "${dump_out}" + echo "# === new sfdisk -d ===" + cat "${new_out}" + } 1>&2 + exit 0 + fi + + LANG=C sfdisk --no-reread "${DISK}" ${MBR_CHS} --force \ + -O "${MBR_BACKUP}" <"${new_out}" >"${change_out}" 2>&1 + ret=$? + [ $ret -eq 0 ] || RESTORE_FUNC="mbr_restore" + + if [ $ret -eq 0 ]; then + : + elif $PT_UPDATE && + sfdisk_worked_but_blkrrpart_failed "$ret" "${change_out}"; then + # if the command failed, but it looks like only because + # the device was busy and we have pt_update, then go on + debug 1 "sfdisk failed, but likely only because of blkrrpart" + else + error "attempt to resize ${DISK} failed. sfdisk output below:" + sed 's,^,| ,' "${change_out}" 1>&2 + fail "failed to resize" + fi + + rq pt_update pt_update "$DISK" "$PART" || + fail "pt_resize failed" + + RESTORE_FUNC="" + + changed "${change_info}" + + # dump_out looks something like: + ## partition table of /tmp/out.img + #unit: sectors + # + #/tmp/out.img1 : start= 1, size= 48194, Id=83 + #/tmp/out.img2 : start= 48195, size= 963900, Id=83 + #/tmp/out.img3 : start= 1012095, size= 305235, Id=82 + #/tmp/out.img4 : start= 1317330, size= 771120, Id= 5 + #/tmp/out.img5 : start= 1317331, size= 642599, Id=83 + #/tmp/out.img6 : start= 1959931, size= 48194, Id=83 + #/tmp/out.img7 : start= 2008126, size= 80324, Id=83 +} + +gpt_restore() { + sgdisk -l "${GPT_BACKUP}" "${DISK}" +} + +gpt_resize() { + GPT_BACKUP="${TEMP_D}/pt.backup" + + local pt_info="${TEMP_D}/pt.info" + local pt_pretend="${TEMP_D}/pt.pretend" + local pt_data="${TEMP_D}/pt.data" + local out="${TEMP_D}/out" + + local dev="disk=${DISK} partition=${PART}" + + local pt_start pt_end pt_size last pt_max code guid name new_size + local old new change_info + + # Dump the original partition information and details to disk. This is + # used in case something goes wrong and human interaction is required + # to revert any changes. + rqe sgd_info sgdisk "--info=${PART}" --print "${DISK}" >"${pt_info}" || + RESTORE_HUMAN="${pt_info}" + + debug 1 "$dev: original sgdisk info:" + debugcat 1 "${pt_info}" + + # Pretend to move the backup GPT header to the end of the disk and dump + # the resulting partition information. We use this info to determine if + # we have to resize the partition. + rqe sgd_pretend sgdisk --pretend --move-second-header \ + --print "${DISK}" >"${pt_pretend}" || + fail "${dev}: failed to dump pretend sgdisk info" + + debug 1 "$dev: pretend sgdisk info" + debugcat 1 "${pt_pretend}" + + # Extract the partition data from the pretend dump + awk 'found { print } ; $1 == "Number" { found = 1 }' \ + "${pt_pretend}" >"${pt_data}" || + fail "${dev}: failed to parse pretend sgdisk info" + + # Get the start and end sectors of the partition to be grown + pt_start=$(awk '$1 == '"${PART}"' { print $2 }' "${pt_data}") && + [ -n "${pt_start}" ] || + fail "${dev}: failed to get start sector" + pt_end=$(awk '$1 == '"${PART}"' { print $3 }' "${pt_data}") && + [ -n "${pt_end}" ] || + fail "${dev}: failed to get end sector" + pt_size="$((${pt_end} - ${pt_start}))" + + # Get the last usable sector + last=$(awk '/last usable sector is/ { print $NF }' \ + "${pt_pretend}") && [ -n "${last}" ] || + fail "${dev}: failed to get last usable sector" + + # Find the minimal start sector that is >= pt_end + pt_max=$(awk '{ if ($2 >= pt_end && $2 < min) { min = $2 } } END \ + { print min }' min="${last}" pt_end="${pt_end}" \ + "${pt_data}") && [ -n "${pt_max}" ] || + fail "${dev}: failed to find max end sector" + + debug 1 "${dev}: pt_start=${pt_start} pt_end=${pt_end}" \ + "pt_size=${pt_size} pt_max=${pt_max} last=${last}" + + # Check if the partition can be grown + [ "${pt_end}" -eq "${pt_max}" ] && + nochange "${dev}: size=${pt_size}, it cannot be grown" + [ "$((${pt_end} + ${FUDGE}))" -gt "${pt_max}" ] && + nochange "${dev}: could only be grown by" \ + "$((${pt_max} - ${pt_end})) [fudge=${FUDGE}]" + + # The partition can be grown if we made it here. Get some more info + # about it so we can do it properly. + # FIXME: Do we care about the attribute flags? + code=$(awk '/^Partition GUID code:/ { print $4 }' "${pt_info}") + guid=$(awk '/^Partition unique GUID:/ { print $4 }' "${pt_info}") + name=$(awk '/^Partition name:/ { gsub(/'"'"'/, "") ; \ + if (NF >= 3) print substr($0, index($0, $3)) }' "${pt_info}") + [ -n "${code}" -a -n "${guid}" ] || + fail "${dev}: failed to parse sgdisk details" + + debug 1 "${dev}: code=${code} guid=${guid} name='${name}'" + + # Calculate the new size of the partition + new_size=$((${pt_max} - ${pt_start})) + old="old: size=${pt_size},end=${pt_end}" + new="new: size=${new_size},end=${pt_max}" + change_info="${dev}: start=${pt_start} ${old} ${new}" + + # Dry run + [ "${DRY_RUN}" -ne 0 ] && change "${change_info}" + + # Backup the current partition table, we're about to modify it + rq sgd_backup sgdisk "--backup=${GPT_BACKUP}" "${DISK}" || + fail "${dev}: failed to backup the partition table" + + # Modify the partition table. We do it all in one go (the order is + # important!): + # - move the GPT backup header to the end of the disk + # - delete the partition + # - recreate the partition with the new size + # - set the partition code + # - set the partition GUID + # - set the partition name + rq sgdisk_mod sgdisk --move-second-header "--delete=${PART}" \ + "--new=${PART}:${pt_start}:${pt_max}" \ + "--typecode=${PART}:${code}" \ + "--partition-guid=${PART}:${guid}" \ + "--change-name=${PART}:${name}" "${DISK}" && + rq pt_update pt_update "$DISK" "$PART" || { + RESTORE_FUNC=gpt_restore + fail "${dev}: failed to repartition" + } + + changed "${change_info}" +} + +kver_to_num() { + local kver="$1" maj="" min="" mic="0" + kver=${kver%%-*} + maj=${kver%%.*} + min=${kver#${maj}.} + min=${min%%.*} + mic=${kver#${maj}.${min}.} + [ "$kver" = "$mic" ] && mic=0 + _RET=$(($maj*1000*1000+$min*1000+$mic)) +} + +kver_cmp() { + local op="$2" n1="" n2="" + kver_to_num "$1" + n1="$_RET" + kver_to_num "$3" + n2="$_RET" + [ $n1 $op $n2 ] +} + +rq() { + # runquieterror(label, command) + # gobble stderr of a command unless it errors + local label="$1" ret="" efile="" + efile="$TEMP_D/$label.err" + shift; + debug 2 "running[$label][$_capture]" "$@" + if [ "${_capture}" = "erronly" ]; then + "$@" 2>"$TEMP_D/$label.err" + ret=$? + else + "$@" >"$TEMP_D/$label.err" 2>&1 + ret=$? + fi + if [ $ret -ne 0 ]; then + error "failed [$label:$ret]" "$@" + cat "$efile" 1>&2 + fi + return $ret +} + +rqe() { + local _capture="erronly" + rq "$@" +} + +verify_ptupdate() { + local input="$1" found="" reason="" kver="" + + # we can always satisfy 'off' + if [ "$input" = "off" ]; then + _RET="false"; + return 0; + fi + + if command -v partx >/dev/null 2>&1; then + partx --help | grep -q -- --update || { + reason="partx has no '--update' flag in usage." + found="off" + } + else + reason="no 'partx' command" + found="off" + fi + + if [ -z "$found" ]; then + if [ "$(uname)" != "Linux" ]; then + reason="Kernel is not Linux per uname." + found="off" + fi + fi + + if [ -z "$found" ]; then + kver=$(uname -r) || debug 1 "uname -r failed!" + + if ! kver_cmp "${kver-0.0.0}" -ge 3.8.0; then + reason="Kernel '$kver' < 3.8.0." + found="off" + fi + fi + + if [ -z "$found" ]; then + _RET="true" + return 0 + fi + + case "$input" in + on) error "$reason"; return 1;; + auto) + _RET="false"; + debug 1 "partition update disabled: $reason" + return 0;; + force) + _RET="true" + error "WARNING: ptupdate forced on even though: $reason" + return 0;; + esac + error "unknown input '$input'"; + return 1; +} + +pt_update() { + local dev="$1" part="$2" update="${3:-$PT_UPDATE}" + if ! $update; then + return 0 + fi + partx --update "$part" "$dev" +} + +has_cmd() { + command -v "${1}" >/dev/null 2>&1 +} + +pt_update="auto" +while [ $# -ne 0 ]; do + cur=${1} + next=${2} + case "$cur" in + -h|--help) + Usage + exit 0 + ;; + --fudge) + FUDGE=${next} + shift + ;; + -N|--dry-run) + DRY_RUN=1 + ;; + -u|--update|--update=*) + if [ "${cur#--update=}" != "$cur" ]; then + next="${cur#--update=}" + else + shift + fi + case "$next" in + off|auto|force|on) pt_update=$next;; + *) fail "unknown --update option: $next";; + esac + ;; + -v|--verbose) + VERBOSITY=$(($VERBOSITY+1)) + ;; + --) + shift + break + ;; + -*) + fail "unknown option ${cur}" + ;; + *) + if [ -z "${DISK}" ]; then + DISK=${cur} + else + [ -z "${PART}" ] || fail "confused by arg ${cur}" + PART=${cur} + fi + ;; + esac + shift +done + +[ -n "${DISK}" ] || bad_Usage "must supply disk and partition-number" +[ -n "${PART}" ] || bad_Usage "must supply partition-number" + +has_cmd "sfdisk" || fail "sfdisk not found" + +[ -e "${DISK}" ] || fail "${DISK}: does not exist" + +[ "${PART#*[!0-9]}" = "${PART}" ] || fail "partition-number must be a number" + +verify_ptupdate "$pt_update" || fail +PT_UPDATE=$_RET + +debug 1 "update-partition set to $PT_UPDATE" + +mktemp_d && TEMP_D="${_RET}" || fail "failed to make temp dir" +trap cleanup EXIT + +# get the ID of the first partition to determine if it's MBR or GPT +id=$(sfdisk --id --force "${DISK}" 1 2>/dev/null) || + fail "unable to determine partition type" + +if [ "${id}" = "ee" ] ; then + has_cmd "sgdisk" || fail "GPT partition found but no sgdisk" + debug 1 "found GPT partition table (id = ${id})" + gpt_resize +else + debug 1 "found MBR partition table (id = ${id})" + mbr_resize +fi + +# vi: ts=4 noexpandtab diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py index bec331c..262714f 100644 --- a/bootstrapvz/providers/ec2/tasks/packages.py +++ b/bootstrapvz/providers/ec2/tasks/packages.py @@ -1,5 +1,6 @@ from bootstrapvz.base import Task from bootstrapvz.common import phases +import os.path class DefaultPackages(Task): @@ -9,9 +10,22 @@ class DefaultPackages(Task): @classmethod def run(cls, info): info.packages.add('file') # Needed for the init scripts - import os.path + kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml') from bootstrapvz.common.tools import config_get kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename, info.manifest.system['architecture']]) info.packages.add(kernel_package) + + +class AddWorkaroundGrowpart(Task): + description = 'Adding growpart' + phase = phases.system_modification + + @classmethod + def run(cls, info): + from shutil import copy + from . import assets + src = os.path.join(assets, 'bin/growpart') + dst = os.path.join(info.root, 'usr/bin/growpart') + copy(src, dst) From 78b796be5be4a9c11e61632005647455d3a2c17e Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 18:33:18 +0000 Subject: [PATCH 2/9] Exit 1 in expand-root if there was an error Signed-off-by: Andrew Garrett --- bootstrapvz/common/assets/init.d/expand-root | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index 5baa5bf..b4e18d7 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -15,14 +15,14 @@ logger="logger -t $prog" growpart=$(which growpart) || { $logger "growpart was not found on PATH. Unable to expand size." - exit 0 + exit 1 } root_index="0" $growpart /dev/xvda $root_index || { $logger "growpart failed. Unable to expand size." - exit 0 + exit 1 } device_path="/dev/xvda$root_index" From 4429c226a815c616d43005be779cc835778655e7 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 18:33:22 +0000 Subject: [PATCH 3/9] Use `hash` instead of `which` in expand-root Also remove the variable since we established that it is already in the $PATH Signed-off-by: Andrew Garrett --- bootstrapvz/common/assets/init.d/expand-root | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index b4e18d7..43a44f9 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -13,14 +13,14 @@ prog=$(basename $0) logger="logger -t $prog" -growpart=$(which growpart) || { +hash growpart 2> /dev/null || { $logger "growpart was not found on PATH. Unable to expand size." exit 1 } root_index="0" -$growpart /dev/xvda $root_index || { +growpart /dev/xvda $root_index || { $logger "growpart failed. Unable to expand size." exit 1 } From 4e94880b2a17d5c59e2140e9b9eee41c39ecfee8 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 18:33:27 +0000 Subject: [PATCH 4/9] Only add growpart workaround on >= Jessie Works on previous distributions Signed-off-by: Andrew Garrett --- bootstrapvz/providers/ec2/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bootstrapvz/providers/ec2/__init__.py b/bootstrapvz/providers/ec2/__init__.py index b057785..21749c6 100644 --- a/bootstrapvz/providers/ec2/__init__.py +++ b/bootstrapvz/providers/ec2/__init__.py @@ -53,7 +53,7 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): - from bootstrapvz.common.releases import wheezy + from bootstrapvz.common.releases import wheezy, jessie taskset.update(task_groups.get_standard_groups(manifest)) taskset.update(task_groups.ssh_group) @@ -68,7 +68,6 @@ def resolve_tasks(taskset, manifest): boot.BlackListModules, boot.DisableGetTTYs, initd.AddExpandRoot, - tasks.packages.AddWorkaroundGrowpart, initd.RemoveHWClock, initd.InstallInitScripts, tasks.ami.RegisterAMI, @@ -82,6 +81,9 @@ def resolve_tasks(taskset, manifest): taskset.add(tasks.network.InstallDHCPCD) taskset.add(tasks.network.EnableDHCPCDDNS) + if manifest.release >= jessie: + taskset.add(tasks.packages.AddWorkaroundGrowpart) + if manifest.provider.get('install_init_scripts', True): taskset.add(tasks.initd.AddEC2InitScripts) From 2ece5a66e1e36f30bef3a675e8287ab8c1bcc04f Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 18:33:41 +0000 Subject: [PATCH 5/9] sed in root_device_path into expand-root To make it more flexible Signed-off-by: Andrew Garrett --- bootstrapvz/common/assets/init.d/expand-root | 5 +++-- bootstrapvz/common/tasks/initd.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index 43a44f9..cf3053b 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -18,14 +18,15 @@ hash growpart 2> /dev/null || { exit 1 } +root_device_path="/dev/xvda" root_index="0" -growpart /dev/xvda $root_index || { +growpart $root_device_path $root_index || { $logger "growpart failed. Unable to expand size." exit 1 } -device_path="/dev/xvda$root_index" +device_path="${root_device_path}${root_index}" filesystem=$(blkid -s TYPE -o value ${device_path}) case $filesystem in diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py index 5b6101e..5ad65f6 100644 --- a/bootstrapvz/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -59,6 +59,10 @@ class AdjustExpandRootScript(Task): def run(cls, info): from ..tools import sed_i script = os.path.join(info.root, 'etc/init.d/expand-root') + root_idx = info.volume.partition_map.root.get_index() root_index_line = 'root_index="{idx}"'.format(idx=root_idx) sed_i(script, '^root_index="0"$', root_index_line) + + root_device_path = 'root_device_path="{device}"'.format(device=info.volume.device_path) + sed_i(script, '^root_device_path="/dev/xvda"$', root_device_path) From bbcb62c4f3f283e2576417b99bb6f28cf493f063 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 19:09:09 +0000 Subject: [PATCH 6/9] Making growpart workaround description more verbose Signed-off-by: Andrew Garrett --- bootstrapvz/providers/ec2/tasks/packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py index 262714f..56f5b4c 100644 --- a/bootstrapvz/providers/ec2/tasks/packages.py +++ b/bootstrapvz/providers/ec2/tasks/packages.py @@ -19,7 +19,7 @@ class DefaultPackages(Task): class AddWorkaroundGrowpart(Task): - description = 'Adding growpart' + description = 'Adding growpart workaround for jessie' phase = phases.system_modification @classmethod From 7f9ee9d22c1e7dbad8834c82a1901c2d421ca8be Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 19:12:57 +0000 Subject: [PATCH 7/9] Write Jessie growpart workaround script to something other than growpart In case growpart is installed. Sed this workaround path into the init.d script. Signed-off-by: Andrew Garrett --- bootstrapvz/common/assets/init.d/expand-root | 8 +++++--- bootstrapvz/common/tasks/initd.py | 2 ++ bootstrapvz/providers/ec2/tasks/packages.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bootstrapvz/common/assets/init.d/expand-root b/bootstrapvz/common/assets/init.d/expand-root index cf3053b..01cab4d 100644 --- a/bootstrapvz/common/assets/init.d/expand-root +++ b/bootstrapvz/common/assets/init.d/expand-root @@ -13,15 +13,17 @@ prog=$(basename $0) logger="logger -t $prog" -hash growpart 2> /dev/null || { - $logger "growpart was not found on PATH. Unable to expand size." +growpart="growpart" + +hash $growpart 2> /dev/null || { + $logger "$growpart was not found on PATH. Unable to expand size." exit 1 } root_device_path="/dev/xvda" root_index="0" -growpart $root_device_path $root_index || { +$growpart $root_device_path $root_index || { $logger "growpart failed. Unable to expand size." exit 1 } diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py index 5ad65f6..43bf67d 100644 --- a/bootstrapvz/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -66,3 +66,5 @@ class AdjustExpandRootScript(Task): root_device_path = 'root_device_path="{device}"'.format(device=info.volume.device_path) sed_i(script, '^root_device_path="/dev/xvda"$', root_device_path) + + sed_i(script, '^growpart="growpart"$', 'growpart-workaround') diff --git a/bootstrapvz/providers/ec2/tasks/packages.py b/bootstrapvz/providers/ec2/tasks/packages.py index 56f5b4c..4d51280 100644 --- a/bootstrapvz/providers/ec2/tasks/packages.py +++ b/bootstrapvz/providers/ec2/tasks/packages.py @@ -27,5 +27,5 @@ class AddWorkaroundGrowpart(Task): from shutil import copy from . import assets src = os.path.join(assets, 'bin/growpart') - dst = os.path.join(info.root, 'usr/bin/growpart') + dst = os.path.join(info.root, 'usr/bin/growpart-workaround') copy(src, dst) From 673d2a520de5b54382e50c73dfa4df4b83912a04 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Tue, 12 Jan 2016 19:26:48 +0000 Subject: [PATCH 8/9] Fix pep8 errors arising in PR builds PR build is using pep8 1.7.0 but master is using 1.5.7; curious, but this will fix it either way. Signed-off-by: Andrew Garrett --- tests/integration/manifests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/manifests/__init__.py b/tests/integration/manifests/__init__.py index d6afbd6..7bdeb78 100644 --- a/tests/integration/manifests/__init__.py +++ b/tests/integration/manifests/__init__.py @@ -1,5 +1,7 @@ import os.path import glob +import random +import string from bootstrapvz.common.tools import load_data partial_json = glob.glob(os.path.join(os.path.dirname(__file__), '*.yml')) @@ -13,8 +15,6 @@ for path in partial_json + partial_yaml: raise Exception(msg) partials[key] = load_data(path) -import random -import string pool = string.ascii_uppercase + string.ascii_lowercase + string.digits random_password = ''.join(random.choice(pool) for _ in range(16)) partials['root_password']['plugins']['root_password']['password'] = random_password From 433cbf2ae4648fd8411ea587f9fd7e2985911d30 Mon Sep 17 00:00:00 2001 From: Jesse Szwedko Date: Wed, 13 Jan 2016 18:50:41 +0000 Subject: [PATCH 9/9] Only sed in growpart workaround when release is >= jessie This script is only needed/created on jessie or later distros. Signed-off-by: Andrew Garrett --- bootstrapvz/common/tasks/initd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bootstrapvz/common/tasks/initd.py b/bootstrapvz/common/tasks/initd.py index 43bf67d..f4537e3 100644 --- a/bootstrapvz/common/tasks/initd.py +++ b/bootstrapvz/common/tasks/initd.py @@ -58,6 +58,7 @@ class AdjustExpandRootScript(Task): @classmethod def run(cls, info): from ..tools import sed_i + from bootstrapvz.common.releases import jessie script = os.path.join(info.root, 'etc/init.d/expand-root') root_idx = info.volume.partition_map.root.get_index() @@ -67,4 +68,5 @@ class AdjustExpandRootScript(Task): root_device_path = 'root_device_path="{device}"'.format(device=info.volume.device_path) sed_i(script, '^root_device_path="/dev/xvda"$', root_device_path) - sed_i(script, '^growpart="growpart"$', 'growpart-workaround') + if info.manifest.release >= jessie: + sed_i(script, '^growpart="growpart"$', 'growpart-workaround')