From 6ae859f8861f66d603f2a4f0c51a97e00854591d Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Fri, 11 Dec 2015 00:50:26 +0100 Subject: [PATCH] Implement locale filter in minimize_size plugin This filter is rather aggressive, since it also hooks into the bootstrapping process itself to prevent debootstrap from unpacking specific locale files --- bootstrapvz/plugins/minimize_size/__init__.py | 6 ++ .../assets/bootstrap-files-filter.sh | 2 + .../minimize_size/assets/bootstrap-script.sh | 26 ++++++ bootstrapvz/plugins/minimize_size/tasks.py | 91 ++++++++++++++++++- manifests/examples/docker/jessie.yml | 3 + 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 bootstrapvz/plugins/minimize_size/assets/bootstrap-files-filter.sh create mode 100644 bootstrapvz/plugins/minimize_size/assets/bootstrap-script.sh diff --git a/bootstrapvz/plugins/minimize_size/__init__.py b/bootstrapvz/plugins/minimize_size/__init__.py index 1f9daac..a155470 100644 --- a/bootstrapvz/plugins/minimize_size/__init__.py +++ b/bootstrapvz/plugins/minimize_size/__init__.py @@ -29,7 +29,13 @@ def resolve_tasks(taskset, manifest): taskset.add(tasks.AptGzipIndexes) if apt.get('autoremove_suggests', False): taskset.add(tasks.AptAutoremoveSuggests) + if 'locales' in apt: + taskset.update([tasks.CreateBootstrapFilterScripts, + tasks.DeleteBootstrapFilterScripts, + tasks.FilterLocales, + ]) def resolve_rollback_tasks(taskset, manifest, completed, counter_task): counter_task(taskset, tasks.AddFolderMounts, tasks.RemoveFolderMounts) + counter_task(taskset, tasks.CreateBootstrapFilterScripts, tasks.DeleteBootstrapFilterScripts) diff --git a/bootstrapvz/plugins/minimize_size/assets/bootstrap-files-filter.sh b/bootstrapvz/plugins/minimize_size/assets/bootstrap-files-filter.sh new file mode 100644 index 0000000..886187a --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/bootstrap-files-filter.sh @@ -0,0 +1,2 @@ +#!/bin/sh +grep 'EXCLUDE_PATTERN' | grep --invert-match --fixed-strings 'INCLUDE_PATHS' diff --git a/bootstrapvz/plugins/minimize_size/assets/bootstrap-script.sh b/bootstrapvz/plugins/minimize_size/assets/bootstrap-script.sh new file mode 100644 index 0000000..4f5a2be --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/bootstrap-script.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# This script does not override anything defined in /usr/share/debootstrap/scripts +# Instead we use it to redefine extract_dpkg_deb_data(), so that we may exclude +# certain files during bootstrapping. + +extract_dpkg_deb_data () { + local pkg="$1" + local excludes_file="/tmp/bootstrap-vz-excludes-$$" + # List all files in $pkg and run them through the filter (avoid exit status >0 if no matches are found) + dpkg-deb --fsys-tarfile "$pkg" | tar -t | BOOTSTRAP_FILES_FILTER_PATH > "$excludes_file" || true + dpkg-deb --fsys-tarfile "$pkg" | tar --exclude-from "$excludes_file" -xf - + rm "$excludes_file" +} + +# Direct copypasta from the debootstrap script where it determines +# which script to run. We do exactly the same but leave out the +# if [ "$4" != "" ] part so that we can source the script that +# should've been sourced in this scripts place. + +SCRIPT="$DEBOOTSTRAP_DIR/scripts/$SUITE" +if [ -n "$VARIANT" ] && [ -e "${SCRIPT}.${VARIANT}" ]; then + SCRIPT="${SCRIPT}.${VARIANT}" +fi + +. $SCRIPT diff --git a/bootstrapvz/plugins/minimize_size/tasks.py b/bootstrapvz/plugins/minimize_size/tasks.py index 5a2470f..f885089 100644 --- a/bootstrapvz/plugins/minimize_size/tasks.py +++ b/bootstrapvz/plugins/minimize_size/tasks.py @@ -6,6 +6,7 @@ from bootstrapvz.common.tasks import filesystem from bootstrapvz.common.tasks import host from bootstrapvz.common.tasks import partitioning from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tasks import workspace from bootstrapvz.common.tools import sed_i from bootstrapvz.common.tools import log_check_call import os @@ -83,12 +84,100 @@ class ShrinkVolume(Task): @classmethod def run(cls, info): - from bootstrapvz.common.tools import log_check_call perm = os.stat(info.volume.image_path).st_mode & 0777 log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path]) os.chmod(info.volume.image_path, perm) +class CreateBootstrapFilterScripts(Task): + description = 'Creating the bootstrapping locales filter script' + phase = phases.os_installation + successors = [bootstrap.Bootstrap] + # Inspired by: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + + @classmethod + def run(cls, info): + if info.bootstrap_script is not None: + from bootstrapvz.common.exceptions import TaskError + raise TaskError('info.bootstrap_script seems to already be set ' + 'and is conflicting with this task') + + bootstrap_script = os.path.join(info.workspace, 'bootstrap_script.sh') + filter_script = os.path.join(info.workspace, 'bootstrap_files_filter.sh') + + shutil.copy(os.path.join(assets, 'bootstrap-script.sh'), bootstrap_script) + shutil.copy(os.path.join(assets, 'bootstrap-files-filter.sh'), filter_script) + + sed_i(bootstrap_script, r'BOOTSTRAP_FILES_FILTER_PATH', filter_script) + sed_i(filter_script, r'EXCLUDE_PATTERN', "./usr/share/locale/.\+\|./usr/share/man/.\+") + + keep_files = ['./usr/share/locale/locale.alias', + './usr/share/man/man1', + './usr/share/man/man2', + './usr/share/man/man3', + './usr/share/man/man4', + './usr/share/man/man5', + './usr/share/man/man6', + './usr/share/man/man7', + './usr/share/man/man8', + './usr/share/man/man9', + ] + locales = info.manifest.plugins['minimize_size']['apt']['locales'] + keep_files.extend(map(lambda l: './usr/share/locale/' + l + '/', locales)) + keep_files.extend(map(lambda l: './usr/share/man/' + l + '/', locales)) + + sed_i(filter_script, r'INCLUDE_PATHS', "\n".join(keep_files)) + os.chmod(filter_script, 0755) + + info.bootstrap_script = bootstrap_script + info._minimize_size['filter_script'] = filter_script + + +class DeleteBootstrapFilterScripts(Task): + description = 'Deleting the bootstrapping locales filter script' + phase = phases.cleaning + successors = [workspace.DeleteWorkspace] + + @classmethod + def run(cls, info): + os.remove(info._minimize_size['filter_script']) + del info._minimize_size['filter_script'] + os.remove(info.bootstrap_script) + + +class FilterLocales(Task): + description = 'Configuring dpkg to only include specific locales/manpages when installing packages' + phase = phases.os_installation + successors = [bootstrap.Bootstrap] + # Snatched from: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + # and + # https://raphaelhertzog.com/2010/11/15/save-disk-space-by-excluding-useless-files-with-dpkg/ + + @classmethod + def run(cls, info): + # This is before we start bootstrapping, so we create dpkg.cfg.d manually + os.makedirs(os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d')) + + locale_lines = ['path-exclude=/usr/share/locale/*', + 'path-include=/usr/share/locale/locale.alias'] + manpages_lines = ['path-exclude=/usr/share/man/*', + 'path-include=/usr/share/man/man[1-9]'] + + locales = info.manifest.plugins['minimize_size']['apt']['locales'] + locale_lines.extend(map(lambda l: 'path-include=/usr/share/locale/' + l + '/*', locales)) + manpages_lines.extend(map(lambda l: 'path-include=/usr/share/man/' + l + '/*', locales)) + + locales_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d/10filter-locales') + manpages_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d/10filter-manpages') + + with open(locales_path, 'w') as locale_filter: + locale_filter.write('\n'.join(locale_lines) + '\n') + with open(manpages_path, 'w') as manpages_filter: + manpages_filter.write('\n'.join(manpages_lines) + '\n') + + class AutomateAptClean(Task): description = 'Configuring apt to always clean everything out when it\'s done' phase = phases.package_installation diff --git a/manifests/examples/docker/jessie.yml b/manifests/examples/docker/jessie.yml index 4e118c6..67ed959 100644 --- a/manifests/examples/docker/jessie.yml +++ b/manifests/examples/docker/jessie.yml @@ -31,3 +31,6 @@ plugins: languages: [none] gzip_indexes: true autoremove_suggests: true + locales: + - en + - en_US