diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9bab76a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,4 @@ +2014-05-02: + Tomasz Rybak: + * Added Google Compute Engine Provider + diff --git a/bootstrapvz/base/fs/partitionmaps/abstract.py b/bootstrapvz/base/fs/partitionmaps/abstract.py index 0d7824d..7fad255 100644 --- a/bootstrapvz/base/fs/partitionmaps/abstract.py +++ b/bootstrapvz/base/fs/partitionmaps/abstract.py @@ -78,7 +78,7 @@ class AbstractPartitionMap(FSMProxy): '(?P\d) (?P\d+) ' '{device_path} (?P\d+)$' .format(device_path=volume.device_path)) - log_check_call(['kpartx', '-a', volume.device_path]) + log_check_call(['kpartx', '-as', volume.device_path]) import os.path # Run through the kpartx output and map the paths to the partitions for mapping in mappings: @@ -99,7 +99,7 @@ class AbstractPartitionMap(FSMProxy): for partition in self.partitions: if not partition.fsm.can('unmap'): partition.unmap() - log_check_call(['kpartx', '-d', volume.device_path]) + log_check_call(['kpartx', '-ds', volume.device_path]) raise e def unmap(self, volume): @@ -122,7 +122,7 @@ class AbstractPartitionMap(FSMProxy): msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition) raise PartitionError(msg) # Actually unmap the partitions - log_check_call(['kpartx', '-d', volume.device_path]) + log_check_call(['kpartx', '-ds', volume.device_path]) # Call unmap on all partitions for partition in self.partitions: partition.unmap() diff --git a/bootstrapvz/providers/gce/__init__.py b/bootstrapvz/providers/gce/__init__.py new file mode 100644 index 0000000..fc7f322 --- /dev/null +++ b/bootstrapvz/providers/gce/__init__.py @@ -0,0 +1,82 @@ +import tasks.apt +import tasks.boot +import tasks.configuration +import tasks.image +import tasks.host +import tasks.packages +from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tasks import partitioning +from bootstrapvz.common.tasks import filesystem +from bootstrapvz.common.tasks import security +from bootstrapvz.common.tasks import network +from bootstrapvz.common.tasks import initd +from bootstrapvz.common.tasks import workspace + + +def initialize(): + pass + + +def validate_manifest(data, validator, error): + import os.path + schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json')) + validator(data, schema_path) + + +def resolve_tasks(tasklist, manifest): + import bootstrapvz.common.task_groups + tasklist.update(bootstrapvz.common.task_groups.base_set) + tasklist.update(bootstrapvz.common.task_groups.volume_set) + tasklist.update(bootstrapvz.common.task_groups.mounting_set) + tasklist.update(bootstrapvz.common.task_groups.get_apt_set(manifest)) + tasklist.update(bootstrapvz.common.task_groups.locale_set) + + tasklist.update(bootstrapvz.common.task_groups.bootloader_set.get(manifest.system['bootloader'])) + + if manifest.volume['partitions']['type'] != 'none': + tasklist.update(bootstrapvz.common.task_groups.partitioning_set) + + tasklist.update([bootstrapvz.plugins.cloud_init.tasks.AddBackports, + loopback.Create, + tasks.apt.SetPackageRepositories, + tasks.apt.ImportGoogleKey, + tasks.packages.DefaultPackages, + tasks.packages.GooglePackages, + tasks.packages.InstallGSUtil, + + tasks.configuration.GatherReleaseInformation, + + security.EnableShadowConfig, + network.RemoveDNSInfo, + network.RemoveHostname, + network.ConfigureNetworkIF, + tasks.host.DisableIPv6, + tasks.host.SetHostname, + tasks.boot.ConfigureGrub, + initd.AddSSHKeyGeneration, + initd.InstallInitScripts, + tasks.apt.CleanGoogleRepositoriesAndKeys, + + loopback.MoveImage, + tasks.image.CreateTarball, + ]) + + if 'gcs_destination' in manifest.image: + tasklist.add(tasks.image.UploadImage) + if 'gce_project' in manifest.image: + tasklist.add(tasks.image.RegisterImage) + + tasklist.update(bootstrapvz.common.task_groups.get_fs_specific_set(manifest.volume['partitions'])) + + if 'boot' in manifest.volume['partitions']: + tasklist.update(bootstrapvz.common.task_groups.boot_partition_set) + + +def resolve_rollback_tasks(tasklist, manifest, counter_task): + counter_task(loopback.Create, volume.Delete) + counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir) + counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions) + counter_task(filesystem.MountRoot, filesystem.UnmountRoot) + counter_task(volume.Attach, volume.Detach) + counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace) diff --git a/bootstrapvz/providers/gce/manifest-schema.json b/bootstrapvz/providers/gce/manifest-schema.json new file mode 100644 index 0000000..4e1d02d --- /dev/null +++ b/bootstrapvz/providers/gce/manifest-schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "GCE manifest", + "type": "object", + "properties": { + "image": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "gcs_destination": { + "type": "string" + }, + "gce_project": { + "type": "string" + } + } + }, + "system": { + "type": "object", + "properties": { + "bootloader": { + "type": "string", + "enum": ["grub", "extlinux"] + } + } + }, + "volume": { + "type": "object", + "properties": { + "partitions": { + "type": "object", + "properties": { + "type": { "enum": ["msdos"] } + } + } + }, + "required": ["partitions"] + } + } +} + diff --git a/bootstrapvz/providers/gce/tasks/__init__.py b/bootstrapvz/providers/gce/tasks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrapvz/providers/gce/tasks/apt.py b/bootstrapvz/providers/gce/tasks/apt.py new file mode 100644 index 0000000..fb6b38f --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/apt.py @@ -0,0 +1,56 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +from bootstrapvz.common.tools import log_check_call +import os + + +class SetPackageRepositories(Task): + description = 'Adding apt sources' + phase = phases.preparation + successors = [apt.AddManifestSources] + + @classmethod + def run(cls, info): + sections = 'main' + if 'sections' in info.manifest.system: + sections = ' '.join(info.manifest.system['sections']) + info.source_lists.add('main', 'deb http://http.debian.net/debian {system.release} ' + sections) + info.source_lists.add('main', 'deb-src http://http.debian.net/debian {system.release} ' + sections) + info.source_lists.add('backports', 'deb http://http.debian.net/debian {system.release}-backports ' + sections) + info.source_lists.add('backports', 'deb-src http://http.debian.net/debian {system.release}-backports ' + sections) + info.source_lists.add('goog', 'deb http://goog-repo.appspot.com/debian pigeon main') + + +class ImportGoogleKey(Task): + description = 'Adding Google key' + phase = phases.package_installation + predecessors = [apt.InstallTrustedKeys] + successors = [apt.WriteSources] + + @classmethod + def run(cls, info): + key_file = os.path.join(info.root, 'google.gpg.key') + log_check_call(['wget', 'https://goog-repo.appspot.com/debian/key/public.gpg.key', '-O', key_file]) + log_check_call(['chroot', info.root, 'apt-key', 'add', 'google.gpg.key']) + os.remove(key_file) + + +class CleanGoogleRepositoriesAndKeys(Task): + description = 'Removing Google key and apt source files' + phase = phases.system_cleaning + successors = [apt.AptClean] + + @classmethod + def run(cls, info): + keys = log_check_call(['chroot', info.root, 'apt-key', + 'adv', '--with-colons', '--list-keys']) + # protect against first lines with debug information, + # not apt-key output + key_id = [key.split(':')[4] for key in keys + if len(key.split(':')) == 13 and + key.split(':')[9].find('@google.com') > 0] + log_check_call(['chroot', info.root, 'apt-key', 'del', key_id[0]]) + apt_file = os.path.join(info.root, 'etc/apt/sources.list.d/goog.list') + os.remove(apt_file) + log_check_call(['chroot', info.root, 'apt-get', 'update']) diff --git a/bootstrapvz/providers/gce/tasks/boot.py b/bootstrapvz/providers/gce/tasks/boot.py new file mode 100644 index 0000000..1100210 --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/boot.py @@ -0,0 +1,17 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import boot +import os.path + + +class ConfigureGrub(Task): + description = 'Change grub configuration to allow for ttyS0 output' + phase = phases.system_modification + successors = [boot.InstallGrub] + + @classmethod + def run(cls, info): + from bootstrapvz.common.tools import sed_i + grub_config = os.path.join(info.root, 'etc/default/grub') + sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1console=ttyS0,38400n8"') + sed_i(grub_config, r'^.*(GRUB_TIMEOUT=).*$', r'GRUB_TIMEOUT=0') diff --git a/bootstrapvz/providers/gce/tasks/configuration.py b/bootstrapvz/providers/gce/tasks/configuration.py new file mode 100644 index 0000000..fbb92f5 --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/configuration.py @@ -0,0 +1,17 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tools import log_check_call + + +class GatherReleaseInformation(Task): + description = 'Gathering release information about created image' + phase = phases.system_modification + + @classmethod + def run(cls, info): + lsb_distribution = log_check_call(['chroot', info.root, 'lsb_release', '-i', '-s']) + lsb_description = log_check_call(['chroot', info.root, 'lsb_release', '-d', '-s']) + lsb_release = log_check_call(['chroot', info.root, 'lsb_release', '-r', '-s']) + info._gce['lsb_distribution'] = lsb_distribution[0] + info._gce['lsb_description'] = lsb_description[0] + info._gce['lsb_release'] = lsb_release[0] diff --git a/bootstrapvz/providers/gce/tasks/host.py b/bootstrapvz/providers/gce/tasks/host.py new file mode 100644 index 0000000..bd61878 --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/host.py @@ -0,0 +1,28 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import network +from bootstrapvz.common.tools import log_check_call +import os.path + + +class DisableIPv6(Task): + description = "Disabling IPv6 support" + phase = phases.system_modification + predecessors = [network.ConfigureNetworkIF] + + @classmethod + def run(cls, info): + network_configuration_path = os.path.join(info.root, 'etc/sysctl.d/70-disable-ipv6.conf') + with open(network_configuration_path, 'w') as config_file: + print >>config_file, "net.ipv6.conf.all.disable_ipv6 = 1" + + +class SetHostname(Task): + description = "Setting hostname" + phase = phases.system_modification + + @classmethod + def run(cls, info): + log_check_call(['chroot', info.root, 'ln', '-s', + '/usr/share/google/set-hostname', + '/etc/dhcp/dhclient-exit-hooks.d/set-hostname']) diff --git a/bootstrapvz/providers/gce/tasks/image.py b/bootstrapvz/providers/gce/tasks/image.py new file mode 100644 index 0000000..d1967e4 --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/image.py @@ -0,0 +1,61 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import loopback +from bootstrapvz.common.tools import log_check_call +import os.path + + +class CreateTarball(Task): + description = 'Creating tarball with image' + phase = phases.image_registration + predecessors = [loopback.MoveImage] + + @classmethod + def run(cls, info): + import datetime + image_name = info.manifest.image['name'].format(**info.manifest_vars) + filename = '{image_name}.{ext}'.format(image_name=image_name, ext=info.volume.extension) + today = datetime.datetime.today() + name_suffix = today.strftime('%Y%m%d') + image_name_format = '{lsb_distribution}-{lsb_release}-{release}-v{name_suffix}' + image_name = image_name_format.format(lsb_distribution=info._gce['lsb_distribution'], + lsb_release=info._gce['lsb_release'], + release=info.manifest.system['release'], + name_suffix=name_suffix) + # ensure that we do not use disallowed characters in image name + image_name = image_name.lower() + image_name = image_name.replace(".", "-") + info._gce['image_name'] = image_name + tarball_name = '{image_name}.tar.gz'.format(image_name=image_name) + tarball_path = os.path.join(info.manifest.bootstrapper['workspace'], tarball_name) + info._gce['tarball_name'] = tarball_name + info._gce['tarball_path'] = tarball_path + log_check_call(['tar', '--sparse', '-C', info.manifest.bootstrapper['workspace'], + '-caf', tarball_path, filename]) + + +class UploadImage(Task): + description = 'Uploading image to GSE' + phase = phases.image_registration + predecessors = [CreateTarball] + + @classmethod + def run(cls, info): + log_check_call(['gsutil', 'cp', info._gce['tarball_path'], + info.manifest.image['gcs_destination'] + info._gce['tarball_name']]) + + +class RegisterImage(Task): + description = 'Registering image with GCE' + phase = phases.image_registration + predecessors = [UploadImage] + + @classmethod + def run(cls, info): + image_description = info._gce['lsb_description'] + if 'description' in info.manifest.image: + image_description = info.manifest.image['description'] + log_check_call(['gcutil', '--project={}'.format(info.manifest.image['gce_project']), + 'addimage', info._gce['image_name'], + info.manifest.image['gcs_destination'] + info._gce['tarball_name'], + '--description={}'.format(image_description)]) diff --git a/bootstrapvz/providers/gce/tasks/packages.py b/bootstrapvz/providers/gce/tasks/packages.py new file mode 100644 index 0000000..1667d35 --- /dev/null +++ b/bootstrapvz/providers/gce/tasks/packages.py @@ -0,0 +1,56 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt +from bootstrapvz.common.tools import log_check_call +import os +import os.path + + +class DefaultPackages(Task): + description = 'Adding image packages required for GCE' + phase = phases.preparation + predecessors = [apt.AddDefaultSources] + + @classmethod + def run(cls, info): + info.packages.add('python') + info.packages.add('sudo') + info.packages.add('ntp') + info.packages.add('lsb-release') + info.packages.add('acpi-support-base') + info.packages.add('openssh-client') + info.packages.add('openssh-server') + info.packages.add('dhcpd') + + kernel_packages_path = os.path.join(os.path.dirname(__file__), '../../ec2/tasks/packages-kernels.json') + from bootstrapvz.common.tools import config_get + kernel_package = config_get(kernel_packages_path, [info.release_codename, + info.manifest.system['architecture']]) + info.packages.add(kernel_package) + + +class GooglePackages(Task): + description = 'Adding image packages required for GCE from Google repositories' + phase = phases.preparation + predecessors = [DefaultPackages] + + @classmethod + def run(cls, info): + info.packages.add('google-compute-daemon') + info.packages.add('google-startup-scripts') + info.packages.add('python-gcimagebundle') + info.packages.add('gcutil') + + +class InstallGSUtil(Task): + description = 'Install gsutil, not yet packaged' + phase = phases.package_installation + + @classmethod + def run(cls, info): + log_check_call(['wget', 'http://storage.googleapis.com/pub/gsutil.tar.gz']) + gsutil_directory = os.path.join(info.root, 'usr/local/share/google') + gsutil_binary = os.path.join(os.path.join(info.root, 'usr/local/bin'), 'gsutil') + os.makedirs(gsutil_directory) + log_check_call(['tar', 'xaf', 'gsutil.tar.gz', '-C', gsutil_directory]) + log_check_call(['ln', '-s', '../share/google/gsutil/gsutil', gsutil_binary]) diff --git a/manifests/gce.manifest.json b/manifests/gce.manifest.json new file mode 100644 index 0000000..2c6fdad --- /dev/null +++ b/manifests/gce.manifest.json @@ -0,0 +1,46 @@ +{ + "provider": "gce", + "bootstrapper": { + "workspace": "/target" + }, + "image": { + "name": "disk", + "description": "Debian {system.release} {system.architecture}" + }, + "system": { + "release": "wheezy", + "sections": ["main", "contrib", "non-free"], + "architecture": "amd64", + "bootloader": "grub", + "timezone": "UTC", + "locale": "en_US", + "charmap": "UTF-8" + }, + "packages": { + "mirror": "http://gce_debian_mirror.storage.googleapis.com/", + "preferences": { + "backport-kernel": [ + { + "package": "linux-image-* initramfs-tools", + "pin": "release n=wheezy-backports", + "pin-priority": 500 + } + ] + } + }, + "plugins": { + "ntp": { + "servers": ["metadata.google.internal"] + } + }, + "volume": { + "backing": "raw", + "partitions": { + "type": "msdos", + "root": { + "size": "10GiB", + "filesystem": "ext4" + } + } + } +}