Implement some apt minimization stuff from docker

See here for more: 1d775a54cc/contrib/mkimage/debootstrap

bootstrapvz/plugins/minimize_size/README.rst

@@ -6,8 +6,8 @@ virtual volumes are much smaller than their reported size until any data
 is written to them. During the bootstrapping process temporary data like
 the aptitude cache is written to the volume only to be removed again.
 
-The minimize size plugin employs three different strategies to keep a
+The minimize size plugin employs various strategies to keep a low volume
-low volume footprint:
+footprint:
 
 -  Mount folders from the host into key locations of the image volume to
    avoid any unneccesary disk writes.
@@ -21,6 +21,9 @@ low volume footprint:
    backing). The tool is part of the `VMWare
    Workstation <https://my.vmware.com/web/vmware/info/slug/desktop_end_user_computing/vmware_workstation/10_0>`__
    package.
+-  Tell apt to only download specific language files. See the
+   `apt.conf manpage <http://manpages.debian.org/cgi-bin/man.cgi?query=apt.conf>`__
+   for more details ("Languages" in the "Acquire group" section).
 
 Settings
 ~~~~~~~~
@@ -35,3 +38,6 @@ Settings
    Valid values: true, false
    Default: false
    ``optional``
+-  ``apt_languages``: List of languages apt should download. Use ``[none]`` to
+   not download any languages at all.
+   ``optional``

bootstrapvz/plugins/minimize_size/__init__.py

@@ -19,6 +19,16 @@ def resolve_tasks(taskset, manifest):
 	if manifest.plugins['minimize_size'].get('shrink', False):
 		taskset.add(tasks.AddRequiredCommands)
 		taskset.add(tasks.ShrinkVolume)
+	if 'apt' in manifest.plugins['minimize_size']:
+		apt = manifest.plugins['minimize_size']['apt']
+		if apt.get('autoclean', False):
+			taskset.add(tasks.AutomateAptClean)
+		if 'languages' in apt:
+			taskset.add(tasks.FilterTranslationFiles)
+		if apt.get('gzip_indexes', False):
+			taskset.add(tasks.AptGzipIndexes)
+		if apt.get('autoremove_suggests', False):
+			taskset.add(tasks.AptAutoremoveSuggests)
 
 
 def resolve_rollback_tasks(taskset, manifest, completed, counter_task):

bootstrapvz/plugins/minimize_size/assets/apt-autoremove-suggests

@@ -0,0 +1,13 @@
+# Since Docker users are looking for the smallest possible final images, the
+# following emerges as a very common pattern:
+#   RUN apt-get update \
+#       && apt-get install -y <packages> \
+#       && <do some compilation work> \
+#       && apt-get purge -y --auto-remove <packages>
+# By default, APT will actually _keep_ packages installed via Recommends or
+# Depends if another package Suggests them, even and including if the package
+# that originally caused them to be installed is removed.  Setting this to
+# "false" ensures that APT is appropriately aggressive about removing the
+# packages it added.
+# https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant
+Apt::AutoRemove::SuggestsImportant "false";

bootstrapvz/plugins/minimize_size/assets/apt-clean

@@ -0,0 +1,16 @@
+# Since for most Docker users, package installs happen in "docker build" steps,
+# they essentially become individual layers due to the way Docker handles
+# layering, especially using CoW filesystems.  What this means for us is that
+# the caches that APT keeps end up just wasting space in those layers, making
+# our layers unnecessarily large (especially since we'll normally never use
+# these caches again and will instead just "docker build" again and make a brand
+# new image).
+# Ideally, these would just be invoking "apt-get clean", but in our testing,
+# that ended up being cyclic and we got stuck on APT's lock, so we get this fun
+# creation that's essentially just "apt-get clean".
+DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };
+Dir::Cache::pkgcache "";
+Dir::Cache::srcpkgcache "";
+# Note that we do realize this isn't the ideal way to do this, and are always
+# open to better suggestions (https://github.com/docker/docker/issues).

bootstrapvz/plugins/minimize_size/assets/apt-gzip-indexes

@@ -0,0 +1,9 @@
+# Since Docker users using "RUN apt-get update && apt-get install -y ..." in
+# their Dockerfiles don't go delete the lists files afterwards, we want them to
+# be as small as possible on-disk, so we explicitly request "gz" versions and
+# tell Apt to keep them gzipped on-disk.
+# For comparison, an "apt-get update" layer without this on a pristine
+# "debian:wheezy" base image was "29.88 MB", where with this it was only
+# "8.273 MB".
+Acquire::GzipIndexes "true";
+Acquire::CompressionTypes::Order:: "gz";

bootstrapvz/plugins/minimize_size/assets/apt-languages

@@ -0,0 +1,4 @@
+# In Docker, we don't often need the "Translations" files, so we're just wasting
+# time and space by downloading them, and this inhibits that.  For users that do
+# need them, it's a simple matter to delete this file and "apt-get update". :)
+Acquire::Languages { ACQUIRE_LANGUAGES_FILTER };

bootstrapvz/plugins/minimize_size/manifest-schema.yml

@@ -9,6 +9,25 @@ properties:
             type: boolean
           zerofree:
             type: boolean
+          apt:
+            type: object
+            properties:
+              autoclean:
+                type: boolean
+              languages:
+                type: array
+                minItems: 1
+                items:
+                  type: string
+              gzip_indexes:
+                type: boolean
+              autoremove_suggests:
+                type: boolean
+              locales:
+                type: array
+                minItems: 1
+                items:
+                  type: string
         type: object
         additionalProperties: false
     type: object

bootstrapvz/plugins/minimize_size/tasks.py

@@ -6,8 +6,12 @@ from bootstrapvz.common.tasks import filesystem
 from bootstrapvz.common.tasks import host
 from bootstrapvz.common.tasks import partitioning
 from bootstrapvz.common.tasks import volume
+from bootstrapvz.common.tools import sed_i
+from bootstrapvz.common.tools import log_check_call
 import os
+import shutil
 
+assets = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets'))
 folders = ['tmp', 'var/lib/apt/lists']
 
 
@@ -69,7 +73,6 @@ class Zerofree(Task):
 
 	@classmethod
 	def run(cls, info):
-		from bootstrapvz.common.tools import log_check_call
 		log_check_call(['zerofree', info.volume.partition_map.root.device_path])
 
 
@@ -84,3 +87,58 @@ class ShrinkVolume(Task):
 		perm = os.stat(info.volume.image_path).st_mode & 0777
 		log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path])
 		os.chmod(info.volume.image_path, perm)
+
+
+class AutomateAptClean(Task):
+	description = 'Configuring apt to always clean everything out when it\'s done'
+	phase = phases.package_installation
+	successors = [apt.AptUpdate]
+	# Snatched from:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+
+	@classmethod
+	def run(cls, info):
+		shutil.copy(os.path.join(assets, 'apt-clean'),
+		            os.path.join(info.root, 'etc/apt/apt.conf.d/90clean'))
+
+
+class FilterTranslationFiles(Task):
+	description = 'Configuring apt to only download and use specific translation files'
+	phase = phases.package_installation
+	successors = [apt.AptUpdate]
+	# Snatched from:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+
+	@classmethod
+	def run(cls, info):
+		langs = info.manifest.plugins['minimize_size']['apt']['languages']
+		config = '; '.join(map(lambda l: '"' + l + '"', langs))
+		config_path = os.path.join(info.root, 'etc/apt/apt.conf.d/20languages')
+		shutil.copy(os.path.join(assets, 'apt-languages'), config_path)
+		sed_i(config_path, r'ACQUIRE_LANGUAGES_FILTER', config)
+
+
+class AptGzipIndexes(Task):
+	description = 'Configuring apt to always gzip lists files'
+	phase = phases.package_installation
+	successors = [apt.AptUpdate]
+	# Snatched from:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+
+	@classmethod
+	def run(cls, info):
+		shutil.copy(os.path.join(assets, 'apt-gzip-indexes'),
+		            os.path.join(info.root, 'etc/apt/apt.conf.d/20gzip-indexes'))
+
+
+class AptAutoremoveSuggests(Task):
+	description = 'Configuring apt to not consider suggested important'
+	phase = phases.package_installation
+	successors = [apt.AptUpdate]
+	# Snatched from:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+
+	@classmethod
+	def run(cls, info):
+		shutil.copy(os.path.join(assets, 'apt-autoremove-suggests'),
+		            os.path.join(info.root, 'etc/apt/apt.conf.d/20autoremove-suggests'))

manifests/examples/docker/jessie.yml

@@ -24,3 +24,10 @@ volume:
     root:
       filesystem: ext4
       size: 1GiB
+plugins:
+  minimize_size:
+    apt:
+      autoclean: true
+      languages: [none]
+      gzip_indexes: true
+      autoremove_suggests: true