Merge pull request #387 from liori/minimize-size-qemu-repacking

minimize_size: New `repack` option for qemu-img-based repacking

bootstrapvz/plugins/minimize_size/README.rst

@@ -15,17 +15,21 @@ footprint:
    deallocate unused sectors on the volume. On an unpartitioned volume
    this will be done for the entire volume, while it will only happen on
    the root partition for partitioned volumes.
--  Use
+-  Shrink the real volume size. Supported tools are:
-   `vmware-vdiskmanager <https://www.vmware.com/support/ws45/doc/disks_vdiskmanager_eg_ws.html>`__
+
-   to shrink the real volume size (only applicable when using vmdk
+   -  `vmware-vdiskmanager <https://www.vmware.com/support/ws45/doc/disks_vdiskmanager_eg_ws.html>`__
-   backing). The tool is part of the `VMWare
+      (only applicable when using vmdk backing). The tool is part of the
-   Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
+      `VMWare Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
-   package.
+      package.
+   -  `qemu-img` (only applicaple when using vmdk or vdi backing). This
+      tool is part of the `QEMU emulator <https://www.qemu.org/>`__.
+
 -  Tell apt to only download specific language files. See the
    `apt.conf manpage <http://manpages.debian.org/cgi-bin/man.cgi?query=apt.conf>`__
    for more details ("Languages" in the "Acquire group" section).
 -  Configure debootstrap and dpkg to filter out specific paths when installing packages
 
+
 Settings
 ~~~~~~~~
 
@@ -35,8 +39,13 @@ Settings
    Default: false
    ``optional``
 -  ``shrink``: Whether the volume should be shrunk. This setting works
-   best in conjunction with the zerofree tool.
+   best in conjunction with the zerofree tool. Valid values:
-   Valid values: true, false
+
+   -  false: Do not shrink.
+   -  ``vmware-vdiskmanager`` or true: Shrink using the `vmware-vdiskmanager`
+      utility.
+   -  ``qemu-img``: Shrink using the `qemu-img` utility.
+
    Default: false
    ``optional``
 -  ``apt``: Apt specific configurations. ``optional``

bootstrapvz/plugins/minimize_size/__init__.py

@@ -5,12 +5,29 @@ import tasks.dpkg
 from bootstrapvz.common.tasks import locale
 
 
+def get_shrink_type(plugins):
+    """Gets the type of shrinking process requested by the user, taking into account backward compatibility
+    values
+
+    :param dict plugins: the part of the manifest related to plugins
+    :return: None (if none selected), "vmware-vdiskmanager" or "qemu-img" (tool to be used)"""
+    shrink_type = plugins['minimize_size'].get('shrink')
+    if shrink_type is True:
+        shrink_type = 'vmware-vdiskmanager'
+    elif shrink_type is False:
+        shrink_type = None
+    return shrink_type
+
+
 def validate_manifest(data, validator, error):
     from bootstrapvz.common.tools import rel_path
     validator(data, rel_path(__file__, 'manifest-schema.yml'))
 
-    if data['plugins']['minimize_size'].get('shrink', False) and data['volume']['backing'] != 'vmdk':
+    shrink_type = get_shrink_type(data['plugins'])
-        error('Can only shrink vmdk images', ['plugins', 'minimize_size', 'shrink'])
+    if shrink_type == 'vmware-vdiskmanager' and data['volume']['backing'] != 'vmdk':
+        error('Can only shrink vmdk images with vmware-vdiskmanager', ['plugins', 'minimize_size', 'shrink'])
+    if shrink_type == 'qemu-img' and data['volume']['backing'] not in ('vmdk', 'vdi'):
+        error('Can only shrink vmdk and vdi images with qemu-img', ['plugins', 'minimize_size', 'shrink'])
 
 
 def resolve_tasks(taskset, manifest):
@@ -18,11 +35,14 @@ def resolve_tasks(taskset, manifest):
                     tasks.mounts.RemoveFolderMounts,
                     ])
     if manifest.plugins['minimize_size'].get('zerofree', False):
-        taskset.add(tasks.shrink.AddRequiredCommands)
+        taskset.add(tasks.shrink.AddRequiredZeroFreeCommand)
         taskset.add(tasks.shrink.Zerofree)
-    if manifest.plugins['minimize_size'].get('shrink', False):
+    if get_shrink_type(manifest.plugins) == 'vmware-vdiskmanager':
-        taskset.add(tasks.shrink.AddRequiredCommands)
+        taskset.add(tasks.shrink.AddRequiredVDiskManagerCommand)
-        taskset.add(tasks.shrink.ShrinkVolume)
+        taskset.add(tasks.shrink.ShrinkVolumeWithVDiskManager)
+    if get_shrink_type(manifest.plugins) == 'qemu-img':
+        taskset.add(tasks.shrink.AddRequiredQemuImgCommand)
+        taskset.add(tasks.shrink.ShrinkVolumeWithQemuImg)
     if 'apt' in manifest.plugins['minimize_size']:
         apt = manifest.plugins['minimize_size']['apt']
         if apt.get('autoclean', False):

bootstrapvz/plugins/minimize_size/manifest-schema.yml

@@ -6,7 +6,9 @@ properties:
       minimize_size:
         properties:
           shrink:
-            type: boolean
+            anyOf:
+              - type: boolean
+              - enum: [vmware-vdiskmanager, qemu-img]
           zerofree:
             type: boolean
           apt:

bootstrapvz/plugins/minimize_size/tasks/shrink.py

@@ -8,18 +8,35 @@ from bootstrapvz.common.tools import log_check_call
 import os
 
 
-class AddRequiredCommands(Task):
+class AddRequiredZeroFreeCommand(Task):
-    description = 'Adding commands required for reducing volume size'
+    description = 'Adding command required for zero-ing volume'
     phase = phases.validation
     successors = [host.CheckExternalCommands]
 
     @classmethod
     def run(cls, info):
-        if info.manifest.plugins['minimize_size'].get('zerofree', False):
+        info.host_dependencies['zerofree'] = 'zerofree'
-            info.host_dependencies['zerofree'] = 'zerofree'
+
-        if info.manifest.plugins['minimize_size'].get('shrink', False):
+
-            link = 'https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0'
+class AddRequiredVDiskManagerCommand(Task):
-            info.host_dependencies['vmware-vdiskmanager'] = link
+    description = 'Adding vmware-vdiskmanager command required for reducing volume size'
+    phase = phases.validation
+    successors = [host.CheckExternalCommands]
+
+    @classmethod
+    def run(cls, info):
+        link = 'https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0'
+        info.host_dependencies['vmware-vdiskmanager'] = link
+
+
+class AddRequiredQemuImgCommand(Task):
+    description = 'Adding qemu-img command required for reducing volume size'
+    phase = phases.validation
+    successors = [host.CheckExternalCommands]
+
+    @classmethod
+    def run(cls, info):
+        info.host_dependencies['qemu-img'] = 'qemu-img'
 
 
 class Zerofree(Task):
@@ -33,8 +50,8 @@ class Zerofree(Task):
         log_check_call(['zerofree', info.volume.partition_map.root.device_path])
 
 
-class ShrinkVolume(Task):
+class ShrinkVolumeWithVDiskManager(Task):
-    description = 'Shrinking the volume'
+    description = 'Shrinking the volume with vmware-vdiskmanager'
     phase = phases.volume_unmounting
     predecessors = [volume.Detach]
 
@@ -43,3 +60,16 @@ class ShrinkVolume(Task):
         perm = os.stat(info.volume.image_path).st_mode & 0777
         log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path])
         os.chmod(info.volume.image_path, perm)
+
+
+class ShrinkVolumeWithQemuImg(Task):
+    description = 'Shrinking the volume with qemu-img'
+    phase = phases.volume_unmounting
+    predecessors = [volume.Detach]
+
+    @classmethod
+    def run(cls, info):
+        tmp_name = os.path.join(info.workspace, 'shrinked.' + info.volume.extension)
+        log_check_call(
+            ['qemu-img', 'convert', '-O', info.volume.extension, info.volume.image_path, tmp_name])
+        os.rename(tmp_name, info.volume.image_path)