Implement locale filter in minimize_size plugin

This filter is rather aggressive, since it also hooks into the
bootstrapping process itself to prevent debootstrap from unpacking
specific locale files

bootstrapvz/plugins/minimize_size/__init__.py

@@ -29,7 +29,13 @@ def resolve_tasks(taskset, manifest):
 			taskset.add(tasks.AptGzipIndexes)
 		if apt.get('autoremove_suggests', False):
 			taskset.add(tasks.AptAutoremoveSuggests)
+		if 'locales' in apt:
+			taskset.update([tasks.CreateBootstrapFilterScripts,
+			                tasks.DeleteBootstrapFilterScripts,
+			                tasks.FilterLocales,
+			                ])
 
 
 def resolve_rollback_tasks(taskset, manifest, completed, counter_task):
 	counter_task(taskset, tasks.AddFolderMounts, tasks.RemoveFolderMounts)
+	counter_task(taskset, tasks.CreateBootstrapFilterScripts, tasks.DeleteBootstrapFilterScripts)

bootstrapvz/plugins/minimize_size/assets/bootstrap-files-filter.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+grep 'EXCLUDE_PATTERN' | grep --invert-match --fixed-strings 'INCLUDE_PATHS'

bootstrapvz/plugins/minimize_size/assets/bootstrap-script.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# This script does not override anything defined in /usr/share/debootstrap/scripts
+# Instead we use it to redefine extract_dpkg_deb_data(), so that we may exclude
+# certain files during bootstrapping.
+
+extract_dpkg_deb_data () {
+  local pkg="$1"
+  local excludes_file="/tmp/bootstrap-vz-excludes-$$"
+  # List all files in $pkg and run them through the filter (avoid exit status >0 if no matches are found)
+  dpkg-deb --fsys-tarfile "$pkg" | tar -t | BOOTSTRAP_FILES_FILTER_PATH > "$excludes_file" || true
+  dpkg-deb --fsys-tarfile "$pkg" | tar --exclude-from "$excludes_file" -xf -
+  rm "$excludes_file"
+}
+
+# Direct copypasta from the debootstrap script where it determines
+# which script to run. We do exactly the same but leave out the
+# if [ "$4" != "" ] part so that we can source the script that
+# should've been sourced in this scripts place.
+
+SCRIPT="$DEBOOTSTRAP_DIR/scripts/$SUITE"
+if [ -n "$VARIANT" ] && [ -e "${SCRIPT}.${VARIANT}" ]; then
+  SCRIPT="${SCRIPT}.${VARIANT}"
+fi
+
+. $SCRIPT

bootstrapvz/plugins/minimize_size/tasks.py

@@ -6,6 +6,7 @@ from bootstrapvz.common.tasks import filesystem
 from bootstrapvz.common.tasks import host
 from bootstrapvz.common.tasks import partitioning
 from bootstrapvz.common.tasks import volume
+from bootstrapvz.common.tasks import workspace
 from bootstrapvz.common.tools import sed_i
 from bootstrapvz.common.tools import log_check_call
 import os
@@ -83,12 +84,100 @@ class ShrinkVolume(Task):
 
 	@classmethod
 	def run(cls, info):
-		from bootstrapvz.common.tools import log_check_call
 		perm = os.stat(info.volume.image_path).st_mode & 0777
 		log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path])
 		os.chmod(info.volume.image_path, perm)
 
 
+class CreateBootstrapFilterScripts(Task):
+	description = 'Creating the bootstrapping locales filter script'
+	phase = phases.os_installation
+	successors = [bootstrap.Bootstrap]
+	# Inspired by:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+
+	@classmethod
+	def run(cls, info):
+		if info.bootstrap_script is not None:
+			from bootstrapvz.common.exceptions import TaskError
+			raise TaskError('info.bootstrap_script seems to already be set '
+			                'and is conflicting with this task')
+
+		bootstrap_script = os.path.join(info.workspace, 'bootstrap_script.sh')
+		filter_script = os.path.join(info.workspace, 'bootstrap_files_filter.sh')
+
+		shutil.copy(os.path.join(assets, 'bootstrap-script.sh'), bootstrap_script)
+		shutil.copy(os.path.join(assets, 'bootstrap-files-filter.sh'), filter_script)
+
+		sed_i(bootstrap_script, r'BOOTSTRAP_FILES_FILTER_PATH', filter_script)
+		sed_i(filter_script, r'EXCLUDE_PATTERN', "./usr/share/locale/.\+\|./usr/share/man/.\+")
+
+		keep_files = ['./usr/share/locale/locale.alias',
+		              './usr/share/man/man1',
+		              './usr/share/man/man2',
+		              './usr/share/man/man3',
+		              './usr/share/man/man4',
+		              './usr/share/man/man5',
+		              './usr/share/man/man6',
+		              './usr/share/man/man7',
+		              './usr/share/man/man8',
+		              './usr/share/man/man9',
+		              ]
+		locales = info.manifest.plugins['minimize_size']['apt']['locales']
+		keep_files.extend(map(lambda l: './usr/share/locale/' + l + '/', locales))
+		keep_files.extend(map(lambda l: './usr/share/man/' + l + '/', locales))
+
+		sed_i(filter_script, r'INCLUDE_PATHS', "\n".join(keep_files))
+		os.chmod(filter_script, 0755)
+
+		info.bootstrap_script = bootstrap_script
+		info._minimize_size['filter_script'] = filter_script
+
+
+class DeleteBootstrapFilterScripts(Task):
+	description = 'Deleting the bootstrapping locales filter script'
+	phase = phases.cleaning
+	successors = [workspace.DeleteWorkspace]
+
+	@classmethod
+	def run(cls, info):
+		os.remove(info._minimize_size['filter_script'])
+		del info._minimize_size['filter_script']
+		os.remove(info.bootstrap_script)
+
+
+class FilterLocales(Task):
+	description = 'Configuring dpkg to only include specific locales/manpages when installing packages'
+	phase = phases.os_installation
+	successors = [bootstrap.Bootstrap]
+	# Snatched from:
+	# https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
+	# and
+	# https://raphaelhertzog.com/2010/11/15/save-disk-space-by-excluding-useless-files-with-dpkg/
+
+	@classmethod
+	def run(cls, info):
+		# This is before we start bootstrapping, so we create dpkg.cfg.d manually
+		os.makedirs(os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d'))
+
+		locale_lines = ['path-exclude=/usr/share/locale/*',
+		                'path-include=/usr/share/locale/locale.alias']
+		manpages_lines = ['path-exclude=/usr/share/man/*',
+		                  'path-include=/usr/share/man/man[1-9]']
+
+		locales = info.manifest.plugins['minimize_size']['apt']['locales']
+		locale_lines.extend(map(lambda l: 'path-include=/usr/share/locale/' + l + '/*', locales))
+		manpages_lines.extend(map(lambda l: 'path-include=/usr/share/man/' + l + '/*', locales))
+
+		locales_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d/10filter-locales')
+		manpages_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d/10filter-manpages')
+
+		with open(locales_path, 'w') as locale_filter:
+			locale_filter.write('\n'.join(locale_lines) + '\n')
+		with open(manpages_path, 'w') as manpages_filter:
+			manpages_filter.write('\n'.join(manpages_lines) + '\n')
+
+
 class AutomateAptClean(Task):
 	description = 'Configuring apt to always clean everything out when it\'s done'
 	phase = phases.package_installation

manifests/examples/docker/jessie.yml

@@ -31,3 +31,6 @@ plugins:
       languages: [none]
       gzip_indexes: true
       autoremove_suggests: true
+      locales:
+        - en
+        - en_US