Add Google Cloud Engine (GCE) provider:

* force kpartx to use synchronous mode, removing sleep(10)
 * get image configuration, use it during tarball creation and registration
 * add (non-working, path problems) image registration
 * add cleaning of image from Google keys and repositories
 * add NTP server address in manifest
 * add preference for backport kernels in manifest
 * disable IPv6
 * correctly set host name

CHANGELOG.md

@@ -0,0 +1,4 @@
+2014-05-02:
+  Tomasz Rybak:
+   * Added Google Compute Engine Provider
+

bootstrapvz/base/fs/partitionmaps/abstract.py

@@ -78,7 +78,7 @@ class AbstractPartitionMap(FSMProxy):
 			                    '(?P<start_blk>\d) (?P<num_blks>\d+) '
 			                    '{device_path} (?P<blk_offset>\d+)$'
 			                    .format(device_path=volume.device_path))
-			log_check_call(['kpartx', '-a', volume.device_path])
+			log_check_call(['kpartx', '-as', volume.device_path])
 			import os.path
 			# Run through the kpartx output and map the paths to the partitions
 			for mapping in mappings:
@@ -99,7 +99,7 @@ class AbstractPartitionMap(FSMProxy):
 			for partition in self.partitions:
 				if not partition.fsm.can('unmap'):
 					partition.unmap()
-			log_check_call(['kpartx', '-d', volume.device_path])
+			log_check_call(['kpartx', '-ds', volume.device_path])
 			raise e
 
 	def unmap(self, volume):
@@ -122,7 +122,7 @@ class AbstractPartitionMap(FSMProxy):
 				msg = 'The partition {partition} prevents the unmap procedure'.format(partition=partition)
 				raise PartitionError(msg)
 		# Actually unmap the partitions
-		log_check_call(['kpartx', '-d', volume.device_path])
+		log_check_call(['kpartx', '-ds', volume.device_path])
 		# Call unmap on all partitions
 		for partition in self.partitions:
 			partition.unmap()

bootstrapvz/providers/gce/__init__.py

@@ -0,0 +1,82 @@
+import tasks.apt
+import tasks.boot
+import tasks.configuration
+import tasks.image
+import tasks.host
+import tasks.packages
+from bootstrapvz.common.tasks import volume
+from bootstrapvz.common.tasks import loopback
+from bootstrapvz.common.tasks import partitioning
+from bootstrapvz.common.tasks import filesystem
+from bootstrapvz.common.tasks import security
+from bootstrapvz.common.tasks import network
+from bootstrapvz.common.tasks import initd
+from bootstrapvz.common.tasks import workspace
+
+
+def initialize():
+	pass
+
+
+def validate_manifest(data, validator, error):
+	import os.path
+	schema_path = os.path.normpath(os.path.join(os.path.dirname(__file__), 'manifest-schema.json'))
+	validator(data, schema_path)
+
+
+def resolve_tasks(tasklist, manifest):
+	import bootstrapvz.common.task_groups
+	tasklist.update(bootstrapvz.common.task_groups.base_set)
+	tasklist.update(bootstrapvz.common.task_groups.volume_set)
+	tasklist.update(bootstrapvz.common.task_groups.mounting_set)
+	tasklist.update(bootstrapvz.common.task_groups.get_apt_set(manifest))
+	tasklist.update(bootstrapvz.common.task_groups.locale_set)
+
+	tasklist.update(bootstrapvz.common.task_groups.bootloader_set.get(manifest.system['bootloader']))
+
+	if manifest.volume['partitions']['type'] != 'none':
+		tasklist.update(bootstrapvz.common.task_groups.partitioning_set)
+
+	tasklist.update([bootstrapvz.plugins.cloud_init.tasks.AddBackports,
+	                 loopback.Create,
+	                 tasks.apt.SetPackageRepositories,
+	                 tasks.apt.ImportGoogleKey,
+	                 tasks.packages.DefaultPackages,
+	                 tasks.packages.GooglePackages,
+	                 tasks.packages.InstallGSUtil,
+
+	                 tasks.configuration.GatherReleaseInformation,
+
+	                 security.EnableShadowConfig,
+	                 network.RemoveDNSInfo,
+	                 network.RemoveHostname,
+	                 network.ConfigureNetworkIF,
+	                 tasks.host.DisableIPv6,
+	                 tasks.host.SetHostname,
+	                 tasks.boot.ConfigureGrub,
+	                 initd.AddSSHKeyGeneration,
+	                 initd.InstallInitScripts,
+	                 tasks.apt.CleanGoogleRepositoriesAndKeys,
+
+	                 loopback.MoveImage,
+	                 tasks.image.CreateTarball,
+	                 ])
+
+	if 'gcs_destination' in manifest.image:
+		tasklist.add(tasks.image.UploadImage)
+		if 'gce_project' in manifest.image:
+			tasklist.add(tasks.image.RegisterImage)
+
+	tasklist.update(bootstrapvz.common.task_groups.get_fs_specific_set(manifest.volume['partitions']))
+
+	if 'boot' in manifest.volume['partitions']:
+		tasklist.update(bootstrapvz.common.task_groups.boot_partition_set)
+
+
+def resolve_rollback_tasks(tasklist, manifest, counter_task):
+	counter_task(loopback.Create, volume.Delete)
+	counter_task(filesystem.CreateMountDir, filesystem.DeleteMountDir)
+	counter_task(partitioning.MapPartitions, partitioning.UnmapPartitions)
+	counter_task(filesystem.MountRoot, filesystem.UnmountRoot)
+	counter_task(volume.Attach, volume.Detach)
+	counter_task(workspace.CreateWorkspace, workspace.DeleteWorkspace)

bootstrapvz/providers/gce/manifest-schema.json

@@ -0,0 +1,43 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "GCE manifest",
+	"type": "object",
+	"properties": {
+		"image": {
+			"type": "object",
+				"properties": {
+					"description": {
+						"type": "string"
+					},
+					"gcs_destination": {
+						"type": "string"
+					},
+					"gce_project": {
+						"type": "string"
+					}
+			}
+		},
+		"system": {
+			"type": "object",
+			"properties": {
+				"bootloader": {
+					"type": "string",
+					"enum": ["grub", "extlinux"]
+				}
+			}
+		},
+		"volume": {
+			"type": "object",
+			"properties": {
+				"partitions": {
+					"type": "object",
+					"properties": {
+						"type": { "enum": ["msdos"] }
+					}
+				}
+			},
+			"required": ["partitions"]
+		}
+	}
+}
+

bootstrapvz/providers/gce/tasks/apt.py

@@ -0,0 +1,56 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tools import log_check_call
+import os
+
+
+class SetPackageRepositories(Task):
+	description = 'Adding apt sources'
+	phase = phases.preparation
+	successors = [apt.AddManifestSources]
+
+	@classmethod
+	def run(cls, info):
+		sections = 'main'
+		if 'sections' in info.manifest.system:
+			sections = ' '.join(info.manifest.system['sections'])
+		info.source_lists.add('main', 'deb     http://http.debian.net/debian {system.release} ' + sections)
+		info.source_lists.add('main', 'deb-src http://http.debian.net/debian {system.release} ' + sections)
+		info.source_lists.add('backports', 'deb     http://http.debian.net/debian {system.release}-backports ' + sections)
+		info.source_lists.add('backports', 'deb-src http://http.debian.net/debian {system.release}-backports ' + sections)
+		info.source_lists.add('goog', 'deb http://goog-repo.appspot.com/debian pigeon main')
+
+
+class ImportGoogleKey(Task):
+	description = 'Adding Google key'
+	phase = phases.package_installation
+	predecessors = [apt.InstallTrustedKeys]
+	successors = [apt.WriteSources]
+
+	@classmethod
+	def run(cls, info):
+		key_file = os.path.join(info.root, 'google.gpg.key')
+		log_check_call(['wget', 'https://goog-repo.appspot.com/debian/key/public.gpg.key', '-O', key_file])
+		log_check_call(['chroot', info.root, 'apt-key', 'add', 'google.gpg.key'])
+		os.remove(key_file)
+
+
+class CleanGoogleRepositoriesAndKeys(Task):
+	description = 'Removing Google key and apt source files'
+	phase = phases.system_cleaning
+	successors = [apt.AptClean]
+
+	@classmethod
+	def run(cls, info):
+		keys = log_check_call(['chroot', info.root, 'apt-key',
+		                       'adv', '--with-colons', '--list-keys'])
+		# protect against first lines with debug information,
+		# not apt-key output
+		key_id = [key.split(':')[4] for key in keys
+		          if len(key.split(':')) == 13 and
+		          key.split(':')[9].find('@google.com') > 0]
+		log_check_call(['chroot', info.root, 'apt-key', 'del', key_id[0]])
+		apt_file = os.path.join(info.root, 'etc/apt/sources.list.d/goog.list')
+		os.remove(apt_file)
+		log_check_call(['chroot', info.root, 'apt-get', 'update'])

bootstrapvz/providers/gce/tasks/boot.py

@@ -0,0 +1,17 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import boot
+import os.path
+
+
+class ConfigureGrub(Task):
+	description = 'Change grub configuration to allow for ttyS0 output'
+	phase = phases.system_modification
+	successors = [boot.InstallGrub]
+
+	@classmethod
+	def run(cls, info):
+		from bootstrapvz.common.tools import sed_i
+		grub_config = os.path.join(info.root, 'etc/default/grub')
+		sed_i(grub_config, r'^(GRUB_CMDLINE_LINUX*=".*)"\s*$', r'\1console=ttyS0,38400n8"')
+		sed_i(grub_config, r'^.*(GRUB_TIMEOUT=).*$', r'GRUB_TIMEOUT=0')

bootstrapvz/providers/gce/tasks/configuration.py

@@ -0,0 +1,17 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tools import log_check_call
+
+
+class GatherReleaseInformation(Task):
+	description = 'Gathering release information about created image'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		lsb_distribution = log_check_call(['chroot', info.root, 'lsb_release', '-i', '-s'])
+		lsb_description = log_check_call(['chroot', info.root, 'lsb_release', '-d', '-s'])
+		lsb_release = log_check_call(['chroot', info.root, 'lsb_release', '-r', '-s'])
+		info._gce['lsb_distribution'] = lsb_distribution[0]
+		info._gce['lsb_description'] = lsb_description[0]
+		info._gce['lsb_release'] = lsb_release[0]

bootstrapvz/providers/gce/tasks/host.py

@@ -0,0 +1,28 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import network
+from bootstrapvz.common.tools import log_check_call
+import os.path
+
+
+class DisableIPv6(Task):
+	description = "Disabling IPv6 support"
+	phase = phases.system_modification
+	predecessors = [network.ConfigureNetworkIF]
+
+	@classmethod
+	def run(cls, info):
+		network_configuration_path = os.path.join(info.root, 'etc/sysctl.d/70-disable-ipv6.conf')
+		with open(network_configuration_path, 'w') as config_file:
+			print >>config_file, "net.ipv6.conf.all.disable_ipv6 = 1"
+
+
+class SetHostname(Task):
+	description = "Setting hostname"
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		log_check_call(['chroot', info.root, 'ln', '-s',
+		                '/usr/share/google/set-hostname',
+		                '/etc/dhcp/dhclient-exit-hooks.d/set-hostname'])

bootstrapvz/providers/gce/tasks/image.py

@@ -0,0 +1,61 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import loopback
+from bootstrapvz.common.tools import log_check_call
+import os.path
+
+
+class CreateTarball(Task):
+	description = 'Creating tarball with image'
+	phase = phases.image_registration
+	predecessors = [loopback.MoveImage]
+
+	@classmethod
+	def run(cls, info):
+		import datetime
+		image_name = info.manifest.image['name'].format(**info.manifest_vars)
+		filename = '{image_name}.{ext}'.format(image_name=image_name, ext=info.volume.extension)
+		today = datetime.datetime.today()
+		name_suffix = today.strftime('%Y%m%d')
+		image_name_format = '{lsb_distribution}-{lsb_release}-{release}-v{name_suffix}'
+		image_name = image_name_format.format(lsb_distribution=info._gce['lsb_distribution'],
+		                                      lsb_release=info._gce['lsb_release'],
+		                                      release=info.manifest.system['release'],
+		                                      name_suffix=name_suffix)
+		# ensure that we do not use disallowed characters in image name
+		image_name = image_name.lower()
+		image_name = image_name.replace(".", "-")
+		info._gce['image_name'] = image_name
+		tarball_name = '{image_name}.tar.gz'.format(image_name=image_name)
+		tarball_path = os.path.join(info.manifest.bootstrapper['workspace'], tarball_name)
+		info._gce['tarball_name'] = tarball_name
+		info._gce['tarball_path'] = tarball_path
+		log_check_call(['tar', '--sparse', '-C', info.manifest.bootstrapper['workspace'],
+		                '-caf', tarball_path, filename])
+
+
+class UploadImage(Task):
+	description = 'Uploading image to GSE'
+	phase = phases.image_registration
+	predecessors = [CreateTarball]
+
+	@classmethod
+	def run(cls, info):
+		log_check_call(['gsutil', 'cp', info._gce['tarball_path'],
+		                info.manifest.image['gcs_destination'] + info._gce['tarball_name']])
+
+
+class RegisterImage(Task):
+	description = 'Registering image with GCE'
+	phase = phases.image_registration
+	predecessors = [UploadImage]
+
+	@classmethod
+	def run(cls, info):
+		image_description = info._gce['lsb_description']
+		if 'description' in info.manifest.image:
+			image_description = info.manifest.image['description']
+		log_check_call(['gcutil', '--project={}'.format(info.manifest.image['gce_project']),
+		                'addimage', info._gce['image_name'],
+		                info.manifest.image['gcs_destination'] + info._gce['tarball_name'],
+		                '--description={}'.format(image_description)])

bootstrapvz/providers/gce/tasks/packages.py

@@ -0,0 +1,56 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import apt
+from bootstrapvz.common.tools import log_check_call
+import os
+import os.path
+
+
+class DefaultPackages(Task):
+	description = 'Adding image packages required for GCE'
+	phase = phases.preparation
+	predecessors = [apt.AddDefaultSources]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('python')
+		info.packages.add('sudo')
+		info.packages.add('ntp')
+		info.packages.add('lsb-release')
+		info.packages.add('acpi-support-base')
+		info.packages.add('openssh-client')
+		info.packages.add('openssh-server')
+		info.packages.add('dhcpd')
+
+		kernel_packages_path = os.path.join(os.path.dirname(__file__), '../../ec2/tasks/packages-kernels.json')
+		from bootstrapvz.common.tools import config_get
+		kernel_package = config_get(kernel_packages_path, [info.release_codename,
+		                                                   info.manifest.system['architecture']])
+		info.packages.add(kernel_package)
+
+
+class GooglePackages(Task):
+	description = 'Adding image packages required for GCE from Google repositories'
+	phase = phases.preparation
+	predecessors = [DefaultPackages]
+
+	@classmethod
+	def run(cls, info):
+		info.packages.add('google-compute-daemon')
+		info.packages.add('google-startup-scripts')
+		info.packages.add('python-gcimagebundle')
+		info.packages.add('gcutil')
+
+
+class InstallGSUtil(Task):
+	description = 'Install gsutil, not yet packaged'
+	phase = phases.package_installation
+
+	@classmethod
+	def run(cls, info):
+		log_check_call(['wget', 'http://storage.googleapis.com/pub/gsutil.tar.gz'])
+		gsutil_directory = os.path.join(info.root, 'usr/local/share/google')
+		gsutil_binary = os.path.join(os.path.join(info.root, 'usr/local/bin'), 'gsutil')
+		os.makedirs(gsutil_directory)
+		log_check_call(['tar', 'xaf', 'gsutil.tar.gz', '-C', gsutil_directory])
+		log_check_call(['ln', '-s', '../share/google/gsutil/gsutil', gsutil_binary])

manifests/gce.manifest.json

@@ -0,0 +1,46 @@
+{
+	"provider": "gce",
+	"bootstrapper": {
+		"workspace":      "/target"
+	},
+	"image": {
+		"name":        "disk",
+		"description": "Debian {system.release} {system.architecture}"
+	},
+	"system": {
+		"release":      "wheezy",
+		"sections":     ["main", "contrib", "non-free"],
+		"architecture": "amd64",
+		"bootloader":   "grub",
+		"timezone":     "UTC",
+		"locale":       "en_US",
+		"charmap":      "UTF-8"
+	},
+	"packages": {
+		"mirror": "http://gce_debian_mirror.storage.googleapis.com/",
+		"preferences": {
+			"backport-kernel": [
+			{
+				"package": "linux-image-* initramfs-tools",
+				"pin": "release n=wheezy-backports",
+				"pin-priority": 500
+			}
+			]
+		}
+	},
+	"plugins": {
+		"ntp": {
+			"servers": ["metadata.google.internal"]
+		}
+	},
+	"volume": {
+		"backing": "raw",
+		"partitions": {
+			"type": "msdos",
+			"root": {
+				"size":       "10GiB",
+				"filesystem": "ext4"
+			}
+		}
+	}
+}