Merge pull request #360 from zmarano/master

Create a plugin for expand_root to replace the internal common task.

bootstrapvz/plugins/expand_root/README.rst

@@ -0,0 +1,11 @@
+Expand Root
+-----------
+
+This plugin adds support to expand the root partition and filesystem dynamically on boot. It adds a shell script to call growpart and the proper filesystem expansion tool for a given device, partition, and filesystem. The growpart script is part of the cloud-guest-utils package in stretch and jessie-backports. The version of this script in jessie is broken in several ways and so this plugin installs the version from jessie-backports which works correctly. This plugin should not be used in conjunction with common.tasks.initd.AddExpandRoot and common.tasks.initd.AdjustExpandRootScript. It is meant to replace the existing internal common version of expand-root.
+
+Settings
+--------
+
+-  ``filesystem_type``: The type of filesystem to grow, one of ext2, ext3, ext4, of xfs.
+-  ``root_device``: The root device we are growing, /dev/sda as an example.
+-  ``root_partition``: The root partition ID we are growing, 1 (which becomes /dev/sda1). This is specified so you could grow a different partition on the root_device if you have a multi partition setup and because growpart takes the partition number as a separate argument.

bootstrapvz/plugins/expand_root/__init__.py

@@ -0,0 +1,11 @@
+import tasks
+from bootstrapvz.common.tools import rel_path
+
+
+def validate_manifest(data, validator, error):
+    validator(data, rel_path(__file__, 'manifest-schema.yml'))
+
+
+def resolve_tasks(taskset, manifest):
+    taskset.add(tasks.InstallGrowpart)
+    taskset.add(tasks.InstallExpandRootScripts)

bootstrapvz/plugins/expand_root/assets/expand-root.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Expand the root partition and filesystem on boot
+After=local-fs.target
+Wants=local-fs.target
+
+[Service]
+ExecStart=/usr/bin/expand-root.sh DEVICE PARTITION FILESYSTEM
+Type=oneshot
+
+[Install]
+WantedBy=multi-user.target

bootstrapvz/plugins/expand_root/assets/expand-root.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# Expands a partition and filesystem using growpart and an appropriate
+# filesystem tool for live filesystem expansion. Takes three arguments:
+# DEVICE, such as "/dev/sda"
+# PARTITION, such as "1"
+# FILESYSTEM, such as "ext4"
+
+DEVICE="${1}"
+PARTITION="${2}"
+FILESYSTEM="${3}"
+
+if [[ -z "${DEVICE}" || -z "${PARTITION}" || -z "${FILESYSTEM}" ]]; then
+  echo "Requires: $0 DEVICE PARTITION FILESYSTEM"
+  exit 1
+fi
+
+# Grow partition using growpart
+if [[ -x /usr/bin/growpart ]]; then
+  echo "Growing partition ${DEVICE}${PARTITION}"
+  /usr/bin/growpart "${DEVICE}" "${PARTITION}"
+else
+  echo "/usr/bin/growpart was not found"
+  exit 1
+fi
+
+echo "Resizing ${FILESYSTEM} filesystem on ${DEVICE}${PARTITION}"
+case "${FILESYSTEM}" in
+  xfs)  xfs_growfs / ;;
+  ext2) resize2fs "${DEVICE}${PARTITION}" ;;
+  ext3) resize2fs "${DEVICE}${PARTITION}" ;;
+  ext4) resize2fs "${DEVICE}${PARTITION}" ;;
+  *) echo "Unsupported filesystem,  unable to expand size." ;;
+esac

bootstrapvz/plugins/expand_root/manifest-schema.yml

@@ -0,0 +1,24 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+title: Expand root plugin manifest
+type: object
+properties:
+  plugins:
+    type: object
+    properties:
+      expand_root:
+        type: object
+        properties:
+          filesystem_type:
+            enum:
+            - ext2
+            - ext3
+            - ext4
+            - xfs
+          root_device: {type: string}
+          root_partition: {type: integer}
+        required:
+        - filesystem_type
+        - root_device
+        - root_partition
+        additionalProperties: false

bootstrapvz/plugins/expand_root/tasks.py

@@ -0,0 +1,58 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import initd
+from bootstrapvz.common.tasks import packages
+from bootstrapvz.common.tools import log_check_call
+from bootstrapvz.common.tools import rel_path
+from bootstrapvz.common.tools import sed_i
+import os
+import shutil
+
+ASSETS_DIR = rel_path(__file__, 'assets')
+
+
+class InstallGrowpart(Task):
+    description = 'Adding necessary packages for growpart.'
+    phase = phases.preparation
+    successors = [packages.AddManifestPackages]
+
+    @classmethod
+    def run(cls, info):
+        # Use the cloud-guest-utils package from jessie-backports which has
+        # several significant bug fixes from the mainline growpart script.
+        target = None
+        from bootstrapvz.common.releases import jessie
+        if info.manifest.release == jessie:
+            target = '{system.release}-backports'
+        info.packages.add('cloud-guest-utils', target)
+
+
+class InstallExpandRootScripts(Task):
+    description = 'Installing scripts for expand-root.'
+    phase = phases.system_modification
+    successors = [initd.InstallInitScripts]
+
+    @classmethod
+    def run(cls, info):
+        expand_root_script = os.path.join(ASSETS_DIR, 'expand-root.sh')
+        expand_root_service = os.path.join(ASSETS_DIR, 'expand-root.service')
+
+        expand_root_script_dest = os.path.join(info.root, 'usr/bin/expand-root.sh')
+        expand_root_service_dest = os.path.join(info.root, 'lib/systemd/system/expand-root.service')
+
+        filesystem_type = info.manifest.plugins['expand_root'].get('filesystem_type')
+        root_device = info.manifest.plugins['expand_root'].get('root_device')
+        root_partition = info.manifest.plugins['expand_root'].get('root_partition')
+
+        # Copy files over
+        shutil.copy(expand_root_script, expand_root_script_dest)
+        os.chmod(expand_root_script_dest, 0750)
+        shutil.copy(expand_root_service, expand_root_service_dest)
+
+        # Expand out options into expand-root.sh script.
+        opts = '%s %s %s' % (root_device, root_partition, filesystem_type)
+        sed_i(expand_root_service_dest, r'^ExecStart=/usr/bin/expand-root.sh.*$',
+              'ExecStart=/usr/bin/expand-root.sh %s' % opts)
+
+        # Enable systemd service
+        log_check_call(['chroot', info.root,  'systemctl', 'enable', 'expand-root.service'])

bootstrapvz/providers/gce/__init__.py

@@ -3,7 +3,6 @@ import tasks.apt
 import tasks.boot
 import tasks.configuration
 import tasks.image
-import tasks.initd
 import tasks.host
 import tasks.packages
 from bootstrapvz.common.tasks import apt, boot, image, loopback, initd
@@ -16,7 +15,6 @@ def validate_manifest(data, validator, error):
 
 
 def resolve_tasks(taskset, manifest):
-    from bootstrapvz.common.releases import stretch
     taskset.update(task_groups.get_standard_groups(manifest))
     taskset.update([apt.AddBackports,
                     apt.AddDefaultSources,
@@ -27,9 +25,6 @@ def resolve_tasks(taskset, manifest):
                     tasks.host.DisableIPv6,
                     tasks.boot.ConfigureGrub,
                     initd.InstallInitScripts,
-                    initd.AddExpandRoot,
-                    initd.AdjustExpandRootScript,
-                    tasks.initd.AdjustExpandRootDev,
                     boot.BlackListModules,
                     boot.UpdateInitramfs,
                     ssh.AddSSHKeyGeneration,
@@ -42,13 +37,6 @@ def resolve_tasks(taskset, manifest):
                     ])
     taskset.discard(grub.SetGrubConsolOutputDeviceToSerial)
 
-    # Temporary fix for Stretch
-    if manifest.release == stretch:
-        taskset.discard(initd.InstallInitScripts)
-        taskset.discard(initd.AddExpandRoot)
-        taskset.discard(initd.AdjustExpandRootScript)
-        taskset.discard(tasks.initd.AdjustExpandRootDev)
-
     if 'gcs_destination' in manifest.provider:
         taskset.add(tasks.image.UploadImage)
         if 'gce_project' in manifest.provider:

bootstrapvz/providers/gce/tasks/initd.py

@@ -1,16 +0,0 @@
-from bootstrapvz.base import Task
-from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import initd
-import os.path
-
-
-class AdjustExpandRootDev(Task):
-    description = 'Adjusting the expand-root device'
-    phase = phases.system_modification
-    predecessors = [initd.AddExpandRoot, initd.AdjustExpandRootScript]
-
-    @classmethod
-    def run(cls, info):
-        from bootstrapvz.common.tools import sed_i
-        script = os.path.join(info.root, 'etc/init.d/expand-root')
-        sed_i(script, '/dev/loop0', '/dev/sda')

manifests/official/gce/jessie.yml

@@ -26,8 +26,6 @@ packages:
       - deb http://packages.cloud.google.com/apt cloud-sdk-{system.release} main
       - deb http://packages.cloud.google.com/apt google-cloud-compute-{system.release} main
   install:
-    - cloud-initramfs-growroot
-    - cloud-utils
     - google-cloud-sdk
     - google-compute-engine-{system.release}
     - google-compute-engine-init-{system.release}
@@ -41,12 +39,11 @@ packages:
     - package: python-crcmod
       pin: release n=jessie-backports
       pin-priority: 500
-    # cloud-utils in backports has a fixed version of growpart to allow root disk expansion on disks > 2TB.
-    backport-cloud-utils:
-    - package: cloud-utils
-      pin: release n=jessie-backports
-      pin-priority: 500
 plugins:
+  expand_root:
+    filesystem_type: ext4
+    root_device: /dev/sda
+    root_partition: 1
   google_cloud_repo:
     cleanup_bootstrap_key: true
     enable_keyring_repo: true

manifests/official/gce/stretch.yml

@@ -26,16 +26,20 @@ packages:
       - deb http://packages.cloud.google.com/apt cloud-sdk-{system.release} main
       - deb http://packages.cloud.google.com/apt google-cloud-compute-{system.release} main
   install:
-    - cloud-initramfs-growroot
-    - cloud-utils
     - google-cloud-sdk
     - google-compute-engine-{system.release}
     - google-compute-engine-init-{system.release}
     - google-config-{system.release}
+    - man
+    - net-tools
     - python-crcmod
     - screen
     - vim
 plugins:
+  expand_root:
+    filesystem_type: ext4
+    root_device: /dev/sda
+    root_partition: 1
   google_cloud_repo:
     cleanup_bootstrap_key: true
     enable_keyring_repo: true