EC2: Added growpart script

This is an attempt to fix issue #237

The script "growpart" was added as an asset, because
Debian's Jessie version (inside cloud-utils) package is
outdated and buggy.

Then "expand-root" init script was modified to call growpart
before calling resize2fs. In fact, calling resize2fs without
resizing the partition first is useless.

The task was named 'AddWorkaroundGrowpart' because it must
go away in the future in favor of using recent stuff in cloud-utils.

Currently even the official images suffer this issue, that makes
the system use only 8 GiB, even if the user chooses a bigger
storage size inside AWS.
This commit is contained in:
Jonh Wendell 2015-06-30 15:14:41 -03:00 committed by Jonh
parent 57371794a9
commit c477c32e4a
5 changed files with 617 additions and 5 deletions

View file

@ -13,8 +13,19 @@
prog=$(basename $0)
logger="logger -t $prog"
device_path="/dev/xvda"
growpart=$(which growpart) || {
$logger "growpart was not found on PATH. Unable to expand size."
exit 0
}
root_index="0"
$growpart /dev/xvda $root_index || {
$logger "growpart failed. Unable to expand size."
exit 0
}
device_path="/dev/xvda$root_index"
filesystem=$(blkid -s TYPE -o value ${device_path})
case $filesystem in

View file

@ -57,9 +57,8 @@ class AdjustExpandRootScript(Task):
@classmethod
def run(cls, info):
import os.path
from ..tools import sed_i
script = os.path.join(info.root, 'etc/init.d/expand-root')
root_idx = info.volume.partition_map.root.get_index()
device_path = 'device_path="/dev/xvda{idx}"'.format(idx=root_idx)
sed_i(script, '^device_path="/dev/xvda"$', device_path)
root_index_line = 'root_index="{idx}"'.format(idx=root_idx)
sed_i(script, '^root_index="0"$', root_index_line)

View file

@ -68,6 +68,7 @@ def resolve_tasks(taskset, manifest):
boot.BlackListModules,
boot.DisableGetTTYs,
initd.AddExpandRoot,
tasks.packages.AddWorkaroundGrowpart,
initd.RemoveHWClock,
initd.InstallInitScripts,
tasks.ami.RegisterAMI,

View file

@ -0,0 +1,587 @@
#!/bin/sh
# Copyright (C) 2011 Canonical Ltd.
# Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
#
# Authors: Scott Moser <smoser@canonical.com>
# Juerg Haefliger <juerg.haefliger@hp.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# the fudge factor. if its within this many 512 byte sectors, dont bother
FUDGE=${GROWPART_FUDGE:-$((20*1024))}
TEMP_D=""
RESTORE_FUNC=""
RESTORE_HUMAN=""
VERBOSITY=0
DISK=""
PART=""
PT_UPDATE=false
DRY_RUN=0
MBR_CHS=""
MBR_BACKUP=""
GPT_BACKUP=""
_capture=""
error() {
echo "$@" 1>&2
}
fail() {
[ $# -eq 0 ] || echo "FAILED:" "$@"
exit 2
}
nochange() {
echo "NOCHANGE:" "$@"
exit 1
}
changed() {
echo "CHANGED:" "$@"
exit 0
}
change() {
echo "CHANGE:" "$@"
exit 0
}
cleanup() {
if [ -n "${RESTORE_FUNC}" ]; then
error "***** WARNING: Resize failed, attempting to revert ******"
if ${RESTORE_FUNC} ; then
error "***** Appears to have gone OK ****"
else
error "***** FAILED! or original partition table" \
"looked like: ****"
cat "${RESTORE_HUMAN}" 1>&2
fi
fi
[ -z "${TEMP_D}" -o ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}"
}
debug() {
local level=${1}
shift
[ "${level}" -gt "${VERBOSITY}" ] && return
if [ "${DEBUG_LOG}" ]; then
echo "$@" >>"${DEBUG_LOG}"
else
error "$@"
fi
}
debugcat() {
local level="$1"
shift;
[ "${level}" -gt "$VERBOSITY" ] && return
if [ "${DEBUG_LOG}" ]; then
cat "$@" >>"${DEBUG_LOG}"
else
cat "$@" 1>&2
fi
}
mktemp_d() {
# just a mktemp -d that doens't need mktemp if its not there.
_RET=$(mktemp -d "${TMPDIR:-/tmp}/${0##*/}.XXXXXX" 2>/dev/null) &&
return
_RET=$(umask 077 && t="${TMPDIR:-/tmp}/${0##*/}.$$" &&
mkdir "${t}" && echo "${t}")
return
}
Usage() {
cat <<EOF
${0##*/} disk partition
rewrite partition table so that partition takes up all the space it can
options:
-h | --help print Usage and exit
--fudge F if part could be resized, but change would be
less than 'F', do not resize (default: ${FUDGE})
-N | --dry-run only report what would be done, show new 'sfdisk -d'
-v | --verbose increase verbosity / debug
-u | --update R update the the kernel partition table info after growing
this requires kernel support and 'partx --update'
R is one of:
- 'auto' : [default] update partition if possible
- 'force' : try despite sanity checks (fail on failure)
- 'off' : do not attempt
- 'on' : fail if sanity checks indicate no support
Example:
- ${0##*/} /dev/sda 1
Resize partition 1 on /dev/sda
EOF
}
bad_Usage() {
Usage 1>&2
error "$@"
exit 2
}
mbr_restore() {
sfdisk --no-reread "${DISK}" ${MBR_CHS} -I "${MBR_BACKUP}"
}
sfdisk_worked_but_blkrrpart_failed() {
local ret="$1" output="$2"
# exit code found was just 1, but dont insist on that
#[ $ret -eq 1 ] || return 1
# Successfully wrote the new partition table
grep -qi "Success.* wrote.* new.* partition" "$output" &&
grep -qi "BLKRRPART: Device or resource busy" "$output"
return
}
mbr_resize() {
RESTORE_HUMAN="${TEMP_D}/recovery"
MBR_BACKUP="${TEMP_D}/orig.save"
local change_out=${TEMP_D}/change.out
local dump_out=${TEMP_D}/dump.out
local new_out=${TEMP_D}/new.out
local dump_mod=${TEMP_D}/dump.mod
local tmp="${TEMP_D}/tmp.out"
local err="${TEMP_D}/err.out"
local _devc cyl _w1 heads _w2 sectors _w3 tot dpart
local pt_start pt_size pt_end max_end new_size change_info
# --show-pt-geometry outputs something like
# /dev/sda: 164352 cylinders, 4 heads, 32 sectors/track
rqe sfd_geom sfdisk "${DISK}" --show-pt-geometry >"${tmp}" &&
read _devc cyl _w1 heads _w2 sectors _w3 <"${tmp}" &&
MBR_CHS="-C ${cyl} -H ${heads} -S ${sectors}" ||
fail "failed to get CHS from ${DISK}"
tot=$((${cyl}*${heads}*${sectors}))
debug 1 "geometry is ${MBR_CHS}. total size=${tot}"
rqe sfd_dump sfdisk ${MBR_CHS} --unit=S --dump "${DISK}" \
>"${dump_out}" ||
fail "failed to dump sfdisk info for ${DISK}"
{
echo "## sfdisk ${MBR_CHS} --unit=S --dump ${DISK}"
cat "${dump_out}"
} >"${RESTORE_HUMAN}"
[ $? -eq 0 ] || fail "failed to save sfdisk -d output"
debugcat 1 "${RESTORE_HUMAN}"
sed -e 's/,//g; s/start=/start /; s/size=/size /' "${dump_out}" \
>"${dump_mod}" ||
fail "sed failed on dump output"
dpart="${DISK}${PART}" # disk and partition number
if [ -b "${DISK}p${PART}" -a "${DISK%[0-9]}" != "${DISK}" ]; then
# for block devices that end in a number (/dev/nbd0)
# the partition is "<name>p<partition_number>" (/dev/nbd0p1)
dpart="${DISK}p${PART}"
elif [ "${DISK#/dev/loop[0-9]}" != "${DISK}" ]; then
# for /dev/loop devices, sfdisk output will be <name>p<number>
# format also, even though there is not a device there.
dpart="${DISK}p${PART}"
fi
pt_start=$(awk '$1 == pt { print $4 }' "pt=${dpart}" <"${dump_mod}") &&
pt_size=$(awk '$1 == pt { print $6 }' "pt=${dpart}" <"${dump_mod}") &&
[ -n "${pt_start}" -a -n "${pt_size}" ] &&
pt_end=$((${pt_size}+${pt_start})) ||
fail "failed to get start and end for ${dpart} in ${DISK}"
# find the minimal starting location that is >= pt_end
max_end=$(awk '$3 == "start" { if($4 >= pt_end && $4 < min)
{ min = $4 } } END { printf("%s\n",min); }' \
min=${tot} pt_end=${pt_end} "${dump_mod}") &&
[ -n "${max_end}" ] ||
fail "failed to get max_end for partition ${PART}"
debug 1 "max_end=${max_end} tot=${tot} pt_end=${pt_end}" \
"pt_start=${pt_start} pt_size=${pt_size}"
[ $((${pt_end})) -eq ${max_end} ] &&
nochange "partition ${PART} is size ${pt_size}. it cannot be grown"
[ $((${pt_end}+${FUDGE})) -gt ${max_end} ] &&
nochange "partition ${PART} could only be grown by" \
"$((${max_end}-${pt_end})) [fudge=${FUDGE}]"
# now, change the size for this partition in ${dump_out} to be the
# new size
new_size=$((${max_end}-${pt_start}))
sed "\|^\s*${dpart} |s/${pt_size},/${new_size},/" "${dump_out}" \
>"${new_out}" ||
fail "failed to change size in output"
change_info="partition=${PART} start=${pt_start} old: size=${pt_size} end=${pt_end} new: size=${new_size},end=${max_end}"
if [ ${DRY_RUN} -ne 0 ]; then
echo "CHANGE: ${change_info}"
{
echo "# === old sfdisk -d ==="
cat "${dump_out}"
echo "# === new sfdisk -d ==="
cat "${new_out}"
} 1>&2
exit 0
fi
LANG=C sfdisk --no-reread "${DISK}" ${MBR_CHS} --force \
-O "${MBR_BACKUP}" <"${new_out}" >"${change_out}" 2>&1
ret=$?
[ $ret -eq 0 ] || RESTORE_FUNC="mbr_restore"
if [ $ret -eq 0 ]; then
:
elif $PT_UPDATE &&
sfdisk_worked_but_blkrrpart_failed "$ret" "${change_out}"; then
# if the command failed, but it looks like only because
# the device was busy and we have pt_update, then go on
debug 1 "sfdisk failed, but likely only because of blkrrpart"
else
error "attempt to resize ${DISK} failed. sfdisk output below:"
sed 's,^,| ,' "${change_out}" 1>&2
fail "failed to resize"
fi
rq pt_update pt_update "$DISK" "$PART" ||
fail "pt_resize failed"
RESTORE_FUNC=""
changed "${change_info}"
# dump_out looks something like:
## partition table of /tmp/out.img
#unit: sectors
#
#/tmp/out.img1 : start= 1, size= 48194, Id=83
#/tmp/out.img2 : start= 48195, size= 963900, Id=83
#/tmp/out.img3 : start= 1012095, size= 305235, Id=82
#/tmp/out.img4 : start= 1317330, size= 771120, Id= 5
#/tmp/out.img5 : start= 1317331, size= 642599, Id=83
#/tmp/out.img6 : start= 1959931, size= 48194, Id=83
#/tmp/out.img7 : start= 2008126, size= 80324, Id=83
}
gpt_restore() {
sgdisk -l "${GPT_BACKUP}" "${DISK}"
}
gpt_resize() {
GPT_BACKUP="${TEMP_D}/pt.backup"
local pt_info="${TEMP_D}/pt.info"
local pt_pretend="${TEMP_D}/pt.pretend"
local pt_data="${TEMP_D}/pt.data"
local out="${TEMP_D}/out"
local dev="disk=${DISK} partition=${PART}"
local pt_start pt_end pt_size last pt_max code guid name new_size
local old new change_info
# Dump the original partition information and details to disk. This is
# used in case something goes wrong and human interaction is required
# to revert any changes.
rqe sgd_info sgdisk "--info=${PART}" --print "${DISK}" >"${pt_info}" ||
RESTORE_HUMAN="${pt_info}"
debug 1 "$dev: original sgdisk info:"
debugcat 1 "${pt_info}"
# Pretend to move the backup GPT header to the end of the disk and dump
# the resulting partition information. We use this info to determine if
# we have to resize the partition.
rqe sgd_pretend sgdisk --pretend --move-second-header \
--print "${DISK}" >"${pt_pretend}" ||
fail "${dev}: failed to dump pretend sgdisk info"
debug 1 "$dev: pretend sgdisk info"
debugcat 1 "${pt_pretend}"
# Extract the partition data from the pretend dump
awk 'found { print } ; $1 == "Number" { found = 1 }' \
"${pt_pretend}" >"${pt_data}" ||
fail "${dev}: failed to parse pretend sgdisk info"
# Get the start and end sectors of the partition to be grown
pt_start=$(awk '$1 == '"${PART}"' { print $2 }' "${pt_data}") &&
[ -n "${pt_start}" ] ||
fail "${dev}: failed to get start sector"
pt_end=$(awk '$1 == '"${PART}"' { print $3 }' "${pt_data}") &&
[ -n "${pt_end}" ] ||
fail "${dev}: failed to get end sector"
pt_size="$((${pt_end} - ${pt_start}))"
# Get the last usable sector
last=$(awk '/last usable sector is/ { print $NF }' \
"${pt_pretend}") && [ -n "${last}" ] ||
fail "${dev}: failed to get last usable sector"
# Find the minimal start sector that is >= pt_end
pt_max=$(awk '{ if ($2 >= pt_end && $2 < min) { min = $2 } } END \
{ print min }' min="${last}" pt_end="${pt_end}" \
"${pt_data}") && [ -n "${pt_max}" ] ||
fail "${dev}: failed to find max end sector"
debug 1 "${dev}: pt_start=${pt_start} pt_end=${pt_end}" \
"pt_size=${pt_size} pt_max=${pt_max} last=${last}"
# Check if the partition can be grown
[ "${pt_end}" -eq "${pt_max}" ] &&
nochange "${dev}: size=${pt_size}, it cannot be grown"
[ "$((${pt_end} + ${FUDGE}))" -gt "${pt_max}" ] &&
nochange "${dev}: could only be grown by" \
"$((${pt_max} - ${pt_end})) [fudge=${FUDGE}]"
# The partition can be grown if we made it here. Get some more info
# about it so we can do it properly.
# FIXME: Do we care about the attribute flags?
code=$(awk '/^Partition GUID code:/ { print $4 }' "${pt_info}")
guid=$(awk '/^Partition unique GUID:/ { print $4 }' "${pt_info}")
name=$(awk '/^Partition name:/ { gsub(/'"'"'/, "") ; \
if (NF >= 3) print substr($0, index($0, $3)) }' "${pt_info}")
[ -n "${code}" -a -n "${guid}" ] ||
fail "${dev}: failed to parse sgdisk details"
debug 1 "${dev}: code=${code} guid=${guid} name='${name}'"
# Calculate the new size of the partition
new_size=$((${pt_max} - ${pt_start}))
old="old: size=${pt_size},end=${pt_end}"
new="new: size=${new_size},end=${pt_max}"
change_info="${dev}: start=${pt_start} ${old} ${new}"
# Dry run
[ "${DRY_RUN}" -ne 0 ] && change "${change_info}"
# Backup the current partition table, we're about to modify it
rq sgd_backup sgdisk "--backup=${GPT_BACKUP}" "${DISK}" ||
fail "${dev}: failed to backup the partition table"
# Modify the partition table. We do it all in one go (the order is
# important!):
# - move the GPT backup header to the end of the disk
# - delete the partition
# - recreate the partition with the new size
# - set the partition code
# - set the partition GUID
# - set the partition name
rq sgdisk_mod sgdisk --move-second-header "--delete=${PART}" \
"--new=${PART}:${pt_start}:${pt_max}" \
"--typecode=${PART}:${code}" \
"--partition-guid=${PART}:${guid}" \
"--change-name=${PART}:${name}" "${DISK}" &&
rq pt_update pt_update "$DISK" "$PART" || {
RESTORE_FUNC=gpt_restore
fail "${dev}: failed to repartition"
}
changed "${change_info}"
}
kver_to_num() {
local kver="$1" maj="" min="" mic="0"
kver=${kver%%-*}
maj=${kver%%.*}
min=${kver#${maj}.}
min=${min%%.*}
mic=${kver#${maj}.${min}.}
[ "$kver" = "$mic" ] && mic=0
_RET=$(($maj*1000*1000+$min*1000+$mic))
}
kver_cmp() {
local op="$2" n1="" n2=""
kver_to_num "$1"
n1="$_RET"
kver_to_num "$3"
n2="$_RET"
[ $n1 $op $n2 ]
}
rq() {
# runquieterror(label, command)
# gobble stderr of a command unless it errors
local label="$1" ret="" efile=""
efile="$TEMP_D/$label.err"
shift;
debug 2 "running[$label][$_capture]" "$@"
if [ "${_capture}" = "erronly" ]; then
"$@" 2>"$TEMP_D/$label.err"
ret=$?
else
"$@" >"$TEMP_D/$label.err" 2>&1
ret=$?
fi
if [ $ret -ne 0 ]; then
error "failed [$label:$ret]" "$@"
cat "$efile" 1>&2
fi
return $ret
}
rqe() {
local _capture="erronly"
rq "$@"
}
verify_ptupdate() {
local input="$1" found="" reason="" kver=""
# we can always satisfy 'off'
if [ "$input" = "off" ]; then
_RET="false";
return 0;
fi
if command -v partx >/dev/null 2>&1; then
partx --help | grep -q -- --update || {
reason="partx has no '--update' flag in usage."
found="off"
}
else
reason="no 'partx' command"
found="off"
fi
if [ -z "$found" ]; then
if [ "$(uname)" != "Linux" ]; then
reason="Kernel is not Linux per uname."
found="off"
fi
fi
if [ -z "$found" ]; then
kver=$(uname -r) || debug 1 "uname -r failed!"
if ! kver_cmp "${kver-0.0.0}" -ge 3.8.0; then
reason="Kernel '$kver' < 3.8.0."
found="off"
fi
fi
if [ -z "$found" ]; then
_RET="true"
return 0
fi
case "$input" in
on) error "$reason"; return 1;;
auto)
_RET="false";
debug 1 "partition update disabled: $reason"
return 0;;
force)
_RET="true"
error "WARNING: ptupdate forced on even though: $reason"
return 0;;
esac
error "unknown input '$input'";
return 1;
}
pt_update() {
local dev="$1" part="$2" update="${3:-$PT_UPDATE}"
if ! $update; then
return 0
fi
partx --update "$part" "$dev"
}
has_cmd() {
command -v "${1}" >/dev/null 2>&1
}
pt_update="auto"
while [ $# -ne 0 ]; do
cur=${1}
next=${2}
case "$cur" in
-h|--help)
Usage
exit 0
;;
--fudge)
FUDGE=${next}
shift
;;
-N|--dry-run)
DRY_RUN=1
;;
-u|--update|--update=*)
if [ "${cur#--update=}" != "$cur" ]; then
next="${cur#--update=}"
else
shift
fi
case "$next" in
off|auto|force|on) pt_update=$next;;
*) fail "unknown --update option: $next";;
esac
;;
-v|--verbose)
VERBOSITY=$(($VERBOSITY+1))
;;
--)
shift
break
;;
-*)
fail "unknown option ${cur}"
;;
*)
if [ -z "${DISK}" ]; then
DISK=${cur}
else
[ -z "${PART}" ] || fail "confused by arg ${cur}"
PART=${cur}
fi
;;
esac
shift
done
[ -n "${DISK}" ] || bad_Usage "must supply disk and partition-number"
[ -n "${PART}" ] || bad_Usage "must supply partition-number"
has_cmd "sfdisk" || fail "sfdisk not found"
[ -e "${DISK}" ] || fail "${DISK}: does not exist"
[ "${PART#*[!0-9]}" = "${PART}" ] || fail "partition-number must be a number"
verify_ptupdate "$pt_update" || fail
PT_UPDATE=$_RET
debug 1 "update-partition set to $PT_UPDATE"
mktemp_d && TEMP_D="${_RET}" || fail "failed to make temp dir"
trap cleanup EXIT
# get the ID of the first partition to determine if it's MBR or GPT
id=$(sfdisk --id --force "${DISK}" 1 2>/dev/null) ||
fail "unable to determine partition type"
if [ "${id}" = "ee" ] ; then
has_cmd "sgdisk" || fail "GPT partition found but no sgdisk"
debug 1 "found GPT partition table (id = ${id})"
gpt_resize
else
debug 1 "found MBR partition table (id = ${id})"
mbr_resize
fi
# vi: ts=4 noexpandtab

View file

@ -1,5 +1,6 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
import os.path
class DefaultPackages(Task):
@ -9,9 +10,22 @@ class DefaultPackages(Task):
@classmethod
def run(cls, info):
info.packages.add('file') # Needed for the init scripts
import os.path
kernel_packages_path = os.path.join(os.path.dirname(__file__), 'packages-kernels.yml')
from bootstrapvz.common.tools import config_get
kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
info.manifest.system['architecture']])
info.packages.add(kernel_package)
class AddWorkaroundGrowpart(Task):
description = 'Adding growpart'
phase = phases.system_modification
@classmethod
def run(cls, info):
from shutil import copy
from . import assets
src = os.path.join(assets, 'bin/growpart')
dst = os.path.join(info.root, 'usr/bin/growpart')
copy(src, dst)