Overhaul GCE image build.

This commit is contained in:
Zach Marano 2016-06-02 14:38:47 -07:00
parent 3c1999f809
commit 40ec27ad16
20 changed files with 213 additions and 322 deletions

View file

@ -1,6 +1,17 @@
Changelog
=========
2016-06-02
----------
Zach Marano:
* Fix expand-root script to work with newer version of growpart (in jessie-backports and beyond).
* Overhaul Google Compute Engine image build.
* Add support for Google Cloud repositories.
* Google Cloud SDK install uses a deb package from a Google Cloud repository.
* Google Compute Engine guest software is installed from a Google Cloud repository.
* Google Compute Engine guest software for Debian 8 is updated to new refactor.
* Google Compute Engine wheezy and wheezy-backports manifests are deprecated.
2016-03-03
----------
Anders Ingemann:

View file

@ -23,9 +23,9 @@ hash $growpart 2> /dev/null || {
root_device_path="/dev/xvda"
root_index="0"
# Growpart can fail if the partition is already resized.
$growpart $root_device_path $root_index || {
$logger "growpart failed. Unable to expand size."
exit 1
}
device_path="${root_device_path}${root_index}"

View file

@ -0,0 +1,10 @@
Google Cloud Repo
-----------------
This plugin adds support to use Google Cloud apt repositories for Debian. It adds the public repo key and optionally will add an apt source list file and install a package containing the key in order to maintain the key over time.
Settings
--------
- ``cleanup_bootstrap_key``: Deletes the bootstrap key by removing /etc/apt/trusted.gpg in favor of the package maintained version. This is only to avoid having multiple keys around in the apt-key list. This should only be used with enable_keyring_repo.
- ``enable_keyring_repo``: Add a repository and package to maintain the repo public key over time.

View file

@ -0,0 +1,16 @@
import tasks
import os.path
def validate_manifest(data, validator, error):
schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.yml'))
validator(data, schema_path)
def resolve_tasks(taskset, manifest):
taskset.add(tasks.AddGoogleCloudRepoKey)
if manifest.plugins['google_cloud_repo'].get('enable_keyring_repo', False):
taskset.add(tasks.AddGoogleCloudRepoKeyringRepo)
taskset.add(tasks.InstallGoogleCloudRepoKeyringPackage)
if manifest.plugins['google_cloud_repo'].get('cleanup_bootstrap_key', False):
taskset.add(tasks.CleanupBootstrapRepoKey)

View file

@ -0,0 +1,14 @@
---
$schema: http://json-schema.org/draft-04/schema#
title: Google Cloud repository plugin manifest
type: object
properties:
plugins:
type: object
properties:
google_cloud_repo:
type: object
properties:
cleanup_bootstrap_key: {type: boolean}
enable_keyring_repo: {type: boolean}
additionalProperties: false

View file

@ -0,0 +1,49 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tasks import apt
from bootstrapvz.common.tasks import packages
from bootstrapvz.common.tools import log_check_call
import os
class AddGoogleCloudRepoKey(Task):
description = 'Adding Google Cloud Repo key.'
phase = phases.package_installation
predecessors = [apt.InstallTrustedKeys]
successors = [apt.WriteSources]
@classmethod
def run(cls, info):
key_file = os.path.join(info.root, 'google.gpg.key')
log_check_call(['wget', 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', '-O', key_file])
log_check_call(['chroot', info.root, 'apt-key', 'add', 'google.gpg.key'])
os.remove(key_file)
class AddGoogleCloudRepoKeyringRepo(Task):
description = 'Adding Google Cloud keyring repository.'
phase = phases.preparation
predecessors = [apt.AddManifestSources]
@classmethod
def run(cls, info):
info.source_lists.add('google-cloud', 'deb http://packages.cloud.google.com/apt google-cloud-packages-archive-keyring-{system.release} main')
class InstallGoogleCloudRepoKeyringPackage(Task):
description = 'Installing Google Cloud key package.'
phase = phases.preparation
successors = [packages.AddManifestPackages]
@classmethod
def run(cls, info):
info.packages.add('google-cloud-packages-archive-keyring')
class CleanupBootstrapRepoKey(Task):
description = 'Cleaning up bootstrap repo key.'
phase = phases.system_cleaning
@classmethod
def run(cls, info):
os.remove(os.path.join(info.root, 'etc', 'apt', 'trusted.gpg'))

View file

@ -1,6 +0,0 @@
import tasks
def resolve_tasks(taskset, manifest):
taskset.add(tasks.InstallCloudSDK)
taskset.add(tasks.RemoveCloudSDKTarball)

View file

@ -1,71 +0,0 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tools import log_check_call
import os
class InstallCloudSDK(Task):
description = 'Install Cloud SDK, not yet packaged'
phase = phases.system_modification
@classmethod
def run(cls, info):
import contextlib
import re
import urllib
import urlparse
# The current download URL needs to be determined dynamically via a sha1sum file. Here's the
# necessary logic.
cloudsdk_download_site = 'https://dl.google.com/dl/cloudsdk/release/'
cloudsdk_filelist_url = urlparse.urljoin(cloudsdk_download_site, 'sha1.txt')
cloudsdk_pathname_regexp = r'^packages/google-cloud-sdk-coretools-linux-[0-9]+\.tar\.gz$'
cloudsdk_filename = '' # This is set in the 'with' block below.
with contextlib.closing(urllib.urlopen(cloudsdk_filelist_url)) as cloudsdk_filelist:
# cloudsdk_filelist is in sha1sum format, so <hash><whitespace><pathname>
# pathname is a suffix relative to cloudsdk_download_site
#
# Retrieve the pathname which matches cloudsdk_pathname_regexp. It's currently safe to
# assume that only one pathname will match.
for cloudsdk_filelist_line in cloudsdk_filelist:
_, pathname = cloudsdk_filelist_line.split()
if re.match(cloudsdk_pathname_regexp, pathname):
# Don't use os.path.basename since we're actually parsing a URL
# suffix, not a path. Same probable result, but wrong semantics.
#
# The format of pathname is already known to match
# cloudsdk_pathname_regexp, so this is safe.
_, cloudsdk_filename = pathname.rsplit('/', 1)
break
cloudsdk_download_dest = os.path.join(info.workspace, cloudsdk_filename)
cloudsdk_url = urlparse.urljoin(cloudsdk_download_site, pathname)
urllib.urlretrieve(cloudsdk_url, cloudsdk_download_dest)
# Make a "mental note" of which file to remove in the system cleaning phase.
info._google_cloud_sdk['tarball_pathname'] = cloudsdk_download_dest
cloudsdk_directory = os.path.join(info.root, 'usr/local/share/google')
os.makedirs(cloudsdk_directory)
log_check_call(['tar', 'xaf', cloudsdk_download_dest, '-C', cloudsdk_directory])
# We need to symlink certain programs from the Cloud SDK bin directory into /usr/local/bin.
# Keep a list and do it in a unified way. Naturally this will go away with proper packaging.
gcloud_programs = ['bq', 'gsutil', 'gcutil', 'gcloud', 'git-credential-gcloud.sh']
for prog in gcloud_programs:
src = os.path.join('..', 'share', 'google', 'google-cloud-sdk', 'bin', prog)
dest = os.path.join(info.root, 'usr', 'local', 'bin', prog)
os.symlink(src, dest)
class RemoveCloudSDKTarball(Task):
description = 'Remove tarball for Cloud SDK'
phase = phases.system_cleaning
@classmethod
def run(cls, info):
os.remove(info._google_cloud_sdk['tarball_pathname'])

View file

@ -1,5 +1,4 @@
from bootstrapvz.common import task_groups
import tasks.apt
import tasks.boot
import tasks.configuration
import tasks.image
@ -11,7 +10,6 @@ from bootstrapvz.common.tasks import boot
from bootstrapvz.common.tasks import image
from bootstrapvz.common.tasks import loopback
from bootstrapvz.common.tasks import initd
from bootstrapvz.common.tasks import kernel
from bootstrapvz.common.tasks import ssh
from bootstrapvz.common.tasks import volume
@ -28,18 +26,12 @@ def resolve_tasks(taskset, manifest):
taskset.update([apt.AddBackports,
loopback.AddRequiredCommands,
loopback.Create,
tasks.apt.SetPackageRepositories,
tasks.apt.ImportGoogleKey,
tasks.packages.DefaultPackages,
tasks.packages.ReleasePackages,
tasks.packages.GooglePackages,
tasks.configuration.GatherReleaseInformation,
tasks.host.DisableIPv6,
tasks.host.InstallHostnameHook,
tasks.boot.ConfigureGrub,
initd.AddExpandRoot,
initd.AdjustExpandRootScript,
tasks.initd.AdjustExpandRootDev,
initd.InstallInitScripts,
boot.BlackListModules,
@ -47,20 +39,11 @@ def resolve_tasks(taskset, manifest):
ssh.AddSSHKeyGeneration,
ssh.DisableSSHPasswordAuthentication,
ssh.DisableRootLogin,
tasks.apt.CleanGoogleRepositoriesAndKeys,
image.MoveImage,
tasks.image.CreateTarball,
volume.Delete,
])
if manifest.volume['partitions']['type'] != 'none':
taskset.add(initd.AdjustExpandRootScript)
if manifest.volume['partitions']['type'] != 'mbr':
taskset.update([tasks.initd.AddGrowRootDisable,
kernel.UpdateInitramfs])
if 'gcs_destination' in manifest.provider:
taskset.add(tasks.image.UploadImage)
if 'gce_project' in manifest.provider:

View file

@ -1,77 +0,0 @@
# Selectively disable growroot -*- shell-script -*-
set -e
PREREQ=""
prereqs()
{
echo "$PREREQ"
}
case $1 in
prereqs)
prereqs
exit 0
;;
esac
message() { echo "DISABLE-GROWROOT:" "$@" ; }
error_exit() { message "$@"; exit 1; }
. /scripts/functions
# initramfs-tools exports the following variables, used below:
# $ROOT - Generally "/dev/disk/by-uuid/<id>" which is a link to /dev/sda1
# $ROOTFLAGS - Generally empty
# $ROOTFSTYPE - Generally empty
# $rootmnt - Set to "/root"
# According to the initramfs documentation, it's supposed to wait until
# the disks have been attached and verified before the local-premount scripts
# run. This does not happen, however, and failures can happen below if $ROOT
# is referenced before the disk that it points to is attached. This allows for
# up to 4 seconds for that to happen. In practice, it generally takes less
# than half a second.
for i in $(seq 1 20); do
if [ -e "${ROOT}" ]; then
break
fi
sleep 0.2
done
# Follow link to get real root location
if [ ! -L "${ROOT}" ]; then
real_root=${ROOT}
else
real_root=$(readlink -f "${ROOT}")
fi
# Remove partition number to get disk
disk=$(echo ${real_root} | sed 's/[0-9]*$//')
# Determine number of 512-byte sectors in 2TB
two_tb=$((2*(1024**4)))
max_sectors=$((${two_tb}/512))
# Determine number of sectors on disk
geometry=$(sfdisk ${disk} --show-pt-geometry)
cyl=$(echo $geometry | cut -d " " -f 2)
heads=$(echo $geometry | cut -d " " -f 4)
secs=$(echo $geometry | cut -d " " -f 6)
sectors=$((${cyl}*${heads}*${secs}))
# If disk is >2TB, disable growroot
if [ "$sectors" -gt "$max_sectors" ]; then
message "Disk size >2TB - Not expanding root partition"
# Temporarily mount filesystem
if [ -z "${ROOTFSTYPE}" ]; then
fstype=$(get_fstype "${ROOT}")
else
fstype=${ROOTFSTYPE}
fi
mount -w ${fstype:+-t ${fstype} }${ROOTFLAGS} ${ROOT} ${rootmnt} ||
error_exit "failed to mount ${ROOT}."
# Disable growroot
touch "${rootmnt}/etc/growroot-disabled"
# Unmount filesystem
umount "${rootmnt}" || error_exit "failed to umount ${rootmnt}";
fi

View file

@ -1,3 +0,0 @@
import os.path
assets = os.path.normpath(os.path.join(os.path.dirname(__file__), '../assets'))

View file

@ -1,58 +0,0 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tasks import apt
from bootstrapvz.common.tasks import network
from bootstrapvz.common.tools import log_check_call
import os
class SetPackageRepositories(Task):
description = 'Adding apt sources'
phase = phases.preparation
predecessors = [apt.AddManifestSources, apt.AddBackports]
@classmethod
def run(cls, info):
components = 'main'
if 'components' in info.manifest.system:
components = ' '.join(info.manifest.system['components'])
info.source_lists.add('main', 'deb http://http.debian.net/debian {system.release} ' + components)
info.source_lists.add('main', 'deb-src http://http.debian.net/debian {system.release} ' + components)
info.source_lists.add('backports', 'deb http://http.debian.net/debian {system.release}-backports ' + components)
info.source_lists.add('backports', 'deb-src http://http.debian.net/debian {system.release}-backports ' + components)
info.source_lists.add('google', 'deb http://packages.cloud.google.com/apt google-cloud-compute-legacy-{system.release} main')
class ImportGoogleKey(Task):
description = 'Adding Google key'
phase = phases.package_installation
predecessors = [apt.InstallTrustedKeys]
successors = [apt.WriteSources]
@classmethod
def run(cls, info):
key_file = os.path.join(info.root, 'google.gpg.key')
log_check_call(['wget', 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', '-O', key_file])
log_check_call(['chroot', info.root, 'apt-key', 'add', 'google.gpg.key'])
os.remove(key_file)
class CleanGoogleRepositoriesAndKeys(Task):
description = 'Removing Google key and apt source files'
phase = phases.system_cleaning
predecessors = [apt.AptClean]
successors = [network.RemoveDNSInfo]
@classmethod
def run(cls, info):
keys = log_check_call(['chroot', info.root, 'apt-key',
'adv', '--with-colons', '--list-keys'])
# protect against first lines with debug information,
# not apt-key output
key_id = [key.split(':')[4] for key in keys
if len(key.split(':')) == 13 and
key.split(':')[9].find('@google.com') > 0]
log_check_call(['chroot', info.root, 'apt-key', 'del', key_id[0]])
apt_file = os.path.join(info.root, 'etc/apt/sources.list.d/google.list')
os.remove(apt_file)
log_check_call(['chroot', info.root, 'apt-get', 'update'])

View file

@ -1,7 +1,6 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tasks import network
from bootstrapvz.common.tools import log_check_call
import os.path
@ -16,27 +15,3 @@ class DisableIPv6(Task):
with open(network_configuration_path, 'w') as config_file:
print >>config_file, "net.ipv6.conf.all.disable_ipv6 = 1"
print >>config_file, "net.ipv6.conf.lo.disable_ipv6 = 0"
class InstallHostnameHook(Task):
description = "Installing hostname hook"
phase = phases.system_modification
@classmethod
def run(cls, info):
# There's a surprising amount of software out there which doesn't react well to the system
# hostname being set to a potentially long the fully qualified domain name, including Java 7
# and lower, quite relevant to a lot of cloud use cases such as Hadoop. Since Google Compute
# Engine's out-of-the-box domain names are long but predictable based on project name, we
# install this hook to set the hostname to the short hostname but add a suitable /etc/hosts
# entry.
#
# Since not all operating systems which Google supports on Compute Engine work with the
# /etc/dhcp/dhclient-exit-hooks.d directory, Google's internally-built packaging uses the
# consistent install path of /usr/share/google/set-hostname, and OS-specific build steps are
# used to activate the DHCP hook. In any future Debian-maintained distro-specific packaging,
# the updated deb could handle installing the below symlink or the script itself into
# /etc/dhcp/dhclient-exit-hooks.d.
log_check_call(['chroot', info.root, 'ln', '-s',
'/usr/share/google/set-hostname',
'/etc/dhcp/dhclient-exit-hooks.d/set-hostname'])

View file

@ -1,31 +1,9 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tasks import kernel
from bootstrapvz.common.tasks import initd
from . import assets
import os.path
class AddGrowRootDisable(Task):
description = 'Add script to selectively disable growroot'
phase = phases.system_modification
successors = [kernel.UpdateInitramfs]
@classmethod
def run(cls, info):
import stat
rwxr_xr_x = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
from shutil import copy
script_src = os.path.join(assets,
'initramfs-tools/scripts/local-premount/gce-disable-growroot')
script_dst = os.path.join(info.root,
'etc/initramfs-tools/scripts/local-premount/gce-disable-growroot')
copy(script_src, script_dst)
os.chmod(script_dst, rwxr_xr_x)
class AdjustExpandRootDev(Task):
description = 'Adjusting the expand-root device'
phase = phases.system_modification

View file

@ -1,9 +1,7 @@
from bootstrapvz.base import Task
from bootstrapvz.common import phases
from bootstrapvz.common.tasks import apt
from bootstrapvz.common.tasks import packages
from bootstrapvz.common.tools import config_get
import logging
import os
@ -35,34 +33,3 @@ class DefaultPackages(Task):
kernel_package = config_get(kernel_packages_path, [info.manifest.release.codename,
info.manifest.system['architecture']])
info.packages.add(kernel_package)
class ReleasePackages(Task):
description = 'Adding release-specific packages required for GCE'
phase = phases.preparation
predecessors = [apt.AddBackports, DefaultPackages]
successors = [packages.AddManifestPackages]
@classmethod
def run(cls, info):
# Add release-specific packages, if available.
if (info.source_lists.target_exists('wheezy-backports') or
info.source_lists.target_exists('jessie') or
info.source_lists.target_exists('jessie-backports')):
info.packages.add('cloud-initramfs-growroot')
else:
msg = ('No release-specific packages found for {system.release}').format(**info.manifest_vars)
logging.getLogger(__name__).warning(msg)
class GooglePackages(Task):
description = 'Adding image packages required for GCE from Google repositories'
phase = phases.preparation
predecessors = [DefaultPackages]
successors = [packages.AddManifestPackages]
@classmethod
def run(cls, info):
info.packages.add('google-compute-daemon')
info.packages.add('google-startup-scripts')
info.packages.add('python-gcimagebundle')

View file

@ -1,2 +1,20 @@
Official GCE manifests
======================
These are the official manifests used to build [Google Compute Engine (GCE) Debian images](https://cloud.google.com/compute/docs/images).
The included packages and configuration changes are necessary for Debian to run on GCE as a first class citizen of the platform.
Included GCE software is published on github: [Google Compute Engine guest environment](https://github.com/GoogleCloudPlatform/compute-image-packages)
Debian 8 Jessie Package Notes:
* python-crcmod is pulled in from backports as it provides a compiled crcmod required for the Google Cloud Storage CLI (gsutil).
* cloud-utils and cloud-guest-utils are pulled in from backports as they provide a fixed version of growpart to safely grow the root partition on disks >2TB.
* google-cloud-sdk is pulled from a Google Cloud repository.
* google-compute-engine is pulled from a Google Cloud repository.
* google-config is pulled from a Google Cloud repository.
jessie-minimal Notes:
The only additions are the necessary google-compute-engine and google-config packages. This image is not published on GCE however the manifest is provided here for those wishing a minimal GCE Debian image.
Note: Debian 7 Wheezy and Backports Debian 7 Wheezy are deprecated images on GCE and are no longer supported.
These manifests are provided here for historic purposes.

View file

@ -20,7 +20,15 @@ volume:
filesystem: ext4
size: 10GiB
packages:
sources:
google-cloud:
- deb http://packages.cloud.google.com/apt cloud-sdk-{system.release} main
- deb http://packages.cloud.google.com/apt google-cloud-compute-legacy-{system.release} main
install:
- google-cloud-sdk
- google-compute-daemon
- google-startup-scripts
- python-gcimagebundle
- rsync
- screen
- vim
@ -33,8 +41,14 @@ packages:
- package: init-system-helpers openssh-sftp-server openssh-client openssh-server
pin: release n=wheezy-backports
pin-priority: 500
backport-growroot:
- package: cloud-initramfs-growroot
pin: release n=wheezy-backports
pin-priority: 500
plugins:
google_cloud_sdk: {}
google_cloud_repo:
cleanup_bootstrap_key: True
enable_keyring_repo: True
ntp:
servers:
- metadata.google.internal

View file

@ -0,0 +1,41 @@
---
name: disk
provider:
name: gce
description: Debian {system.release} {system.architecture}
bootstrapper:
workspace: /target
system:
release: wheezy
architecture: amd64
bootloader: grub
charmap: UTF-8
locale: en_US
timezone: UTC
volume:
backing: raw
partitions:
type: msdos
root:
filesystem: ext4
size: 10GiB
packages:
sources:
google-cloud:
- deb http://packages.cloud.google.com/apt cloud-sdk-{system.release} main
- deb http://packages.cloud.google.com/apt google-cloud-compute-legacy-{system.release} main
install:
- google-cloud-sdk
- google-compute-daemon
- google-startup-scripts
- python-gcimagebundle
- rsync
- screen
- vim
plugins:
google_cloud_repo:
cleanup_bootstrap_key: True
enable_keyring_repo: True
ntp:
servers:
- metadata.google.internal

View file

@ -6,7 +6,7 @@ provider:
bootstrapper:
workspace: /target
system:
release: wheezy
release: jessie
architecture: amd64
bootloader: grub
charmap: UTF-8
@ -20,12 +20,16 @@ volume:
filesystem: ext4
size: 10GiB
packages:
sources:
google-cloud:
- deb http://packages.cloud.google.com/apt google-cloud-compute-{system.release} main
install:
- rsync
- screen
- vim
- google-compute-engine-{system.release}
- google-config-{system.release}
plugins:
google_cloud_sdk: {}
google_cloud_repo:
cleanup_bootstrap_key: true
enable_keyring_repo: true
ntp:
servers:
- metadata.google.internal

View file

@ -20,12 +20,38 @@ volume:
filesystem: ext4
size: 10GiB
packages:
sources:
google-cloud:
- deb http://packages.cloud.google.com/apt cloud-sdk-{system.release} main
- deb http://packages.cloud.google.com/apt google-cloud-compute-{system.release} main
install:
- rsync
- cloud-initramfs-growroot
- cloud-utils
- google-cloud-sdk
- google-compute-engine-{system.release}
- google-config-{system.release}
- python-crcmod
- screen
- vim
preferences:
# python-crcmod in backports has a compiled version needed for Google Cloud Storage.
backport-python-crcmod:
- package: python-crcmod
pin: release n=jessie-backports
pin-priority: 500
# cloud-utils in backports has a fixed version of growpart to allow root disk expansion on disks > 2TB.
backport-cloud-utils:
- package: cloud-utils
pin: release n=jessie-backports
pin-priority: 500
plugins:
google_cloud_sdk: {}
google_cloud_repo:
cleanup_bootstrap_key: true
enable_keyring_repo: true
ntp:
servers:
- metadata.google.internal
unattended_upgrades:
download_interval: 1
update_interval: 1
upgrade_interval: 1