Installing Cloud SDK which includes gcutil and gsutil. We still don't

have a package for it so laying out the tarball for now.

Change-Id: If66f0f1c074e6077e1ca57375cac9c4832bbd7fc

bootstrapvz/plugins/google_cloud_sdk/__init__.py

@@ -0,0 +1,6 @@
+import tasks
+
+
+def resolve_tasks(taskset, manifest):
+	taskset.add(tasks.InstallCloudSDK)
+	taskset.add(tasks.RemoveCloudSDKTarball)

bootstrapvz/plugins/google_cloud_sdk/tasks.py

@@ -0,0 +1,74 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import workspace
+from bootstrapvz.common.tools import log_check_call
+import os
+
+
+class InstallCloudSDK(Task):
+	description = 'Install Cloud SDK, not yet packaged'
+	phase = phases.system_modification
+
+	@classmethod
+	def run(cls, info):
+		import contextlib
+		import re
+		import urllib
+		import urlparse
+
+		# The current download URL needs to be determined dynamically via a sha1sum file. Here's the
+		# necessary logic.
+
+		cloudsdk_download_site = 'https://dl.google.com/dl/cloudsdk/release/'
+		cloudsdk_filelist_url = urlparse.urljoin(cloudsdk_download_site, 'sha1.txt')
+		cloudsdk_pathname_regexp = r'^packages/google-cloud-sdk-coretools-linux-[0-9]+\.tar\.gz$'
+		cloudsdk_filename = ''  # This is set in the 'with' block below.
+
+		with contextlib.closing(urllib.urlopen(cloudsdk_filelist_url)) as cloudsdk_filelist:
+			# cloudsdk_filelist is in sha1sum format, so <hash><whitespace><pathname>
+			# pathname is a suffix relative to cloudsdk_download_site
+			#
+			# Retrieve the pathname which matches cloudsdk_pathname_regexp. It's currently safe to
+			# assume that only one pathname will match.
+			for cloudsdk_filelist_line in cloudsdk_filelist:
+				_, pathname = cloudsdk_filelist_line.split()
+				if re.match(cloudsdk_pathname_regexp, pathname):
+					# Don't use os.path.basename since we're actually parsing a URL
+					# suffix, not a path. Same probable result, but wrong semantics.
+					#
+					# The format of pathname is already known to match
+					# cloudsdk_pathname_regexp, so this is safe.
+					_, cloudsdk_filename = pathname.rsplit('/', 1)
+					break
+
+		cloudsdk_download_dest = os.path.join(info.workspace, cloudsdk_filename)
+
+		cloudsdk_url = urlparse.urljoin(cloudsdk_download_site, pathname)
+
+		urllib.urlretrieve(cloudsdk_url, cloudsdk_download_dest)
+
+		# Make a "mental note" of which file to remove in the system cleaning phase.
+		info._google_cloud_sdk['tarball_pathname'] = cloudsdk_download_dest
+
+		cloudsdk_directory = os.path.join(info.root, 'usr/local/share/google')
+		os.makedirs(cloudsdk_directory)
+		log_check_call(['file', cloudsdk_download_dest, cloudsdk_directory])
+		log_check_call(['tar', 'xaf', cloudsdk_download_dest, '-C', cloudsdk_directory])
+
+		# We need to symlink certain programs from the Cloud SDK bin directory into /usr/local/bin.
+		# Keep a list and do it in a unified way. Naturally this will go away with proper packaging.
+		gcloud_programs = ['bq', 'gsutil', 'gcutil', 'gcloud', 'git-credential-gcloud.sh']
+		for prog in gcloud_programs:
+			src = os.path.join('..', 'share', 'google', 'google-cloud-sdk', 'bin', prog)
+			dest = os.path.join(info.root, 'usr', 'local', 'bin', prog)
+			os.symlink(src, dest)
+
+
+class RemoveCloudSDKTarball(Task):
+	description = 'Remove tarball for Cloud SDK'
+	phase = phases.system_cleaning
+	successors = [workspace.DeleteWorkspace]
+
+	@classmethod
+	def run(cls, info):
+		os.remove(info._google_cloud_sdk['tarball_pathname'])

bootstrapvz/providers/gce/__init__.py

@@ -32,7 +32,6 @@ def resolve_tasks(taskset, manifest):
 	                tasks.apt.ImportGoogleKey,
 	                tasks.packages.DefaultPackages,
 	                tasks.packages.GooglePackages,
-	                tasks.packages.InstallGSUtil,
 
 	                tasks.configuration.GatherReleaseInformation,
 

bootstrapvz/providers/gce/tasks/packages.py

@@ -1,7 +1,6 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
 from bootstrapvz.common.tasks import apt
-from bootstrapvz.common.tools import log_check_call
 import os
 
 
@@ -39,21 +38,3 @@ class GooglePackages(Task):
 		info.packages.add('google-compute-daemon')
 		info.packages.add('google-startup-scripts')
 		info.packages.add('python-gcimagebundle')
-		info.packages.add('gcutil')
-
-
-class InstallGSUtil(Task):
-	description = 'Install gsutil, not yet packaged'
-	phase = phases.package_installation
-
-	@classmethod
-	def run(cls, info):
-		gsutil_tarball = os.path.join(info.manifest.bootstrapper['workspace'], 'gsutil.tar.gz')
-		log_check_call(['wget', '--output-document', gsutil_tarball,
-		                'http://storage.googleapis.com/pub/gsutil.tar.gz'])
-		gsutil_directory = os.path.join(info.root, 'usr/local/share/google')
-		gsutil_binary = os.path.join(os.path.join(info.root, 'usr/local/bin'), 'gsutil')
-		os.makedirs(gsutil_directory)
-		log_check_call(['tar', 'xaf', gsutil_tarball, '-C', gsutil_directory])
-		os.remove(gsutil_tarball)
-		log_check_call(['ln', '-s', '../share/google/gsutil/gsutil', gsutil_binary])

manifests/gce-backports.manifest.yml

@@ -36,6 +36,7 @@ packages:
       pin: release n=wheezy-backports
       pin-priority: 500
 plugins:
+  google_cloud_sdk: {}
   ntp:
     servers:
     - metadata.google.internal

manifests/gce.manifest.yml

@@ -27,6 +27,7 @@ packages:
   - contrib
   - non-free
 plugins:
+  google_cloud_sdk: {}
   ntp:
     servers:
     - metadata.google.internal