diff --git a/bootstrapvz/plugins/minimize_size/README.rst b/bootstrapvz/plugins/minimize_size/README.rst index bb7860c..b45ddee 100644 --- a/bootstrapvz/plugins/minimize_size/README.rst +++ b/bootstrapvz/plugins/minimize_size/README.rst @@ -6,8 +6,8 @@ virtual volumes are much smaller than their reported size until any data is written to them. During the bootstrapping process temporary data like the aptitude cache is written to the volume only to be removed again. -The minimize size plugin employs three different strategies to keep a -low volume footprint: +The minimize size plugin employs various strategies to keep a low volume +footprint: - Mount folders from the host into key locations of the image volume to avoid any unneccesary disk writes. @@ -21,6 +21,9 @@ low volume footprint: backing). The tool is part of the `VMWare Workstation `__ package. +- Tell apt to only download specific language files. See the + `apt.conf manpage `__ + for more details ("Languages" in the "Acquire group" section). Settings ~~~~~~~~ @@ -35,3 +38,6 @@ Settings Valid values: true, false Default: false ``optional`` +- ``apt_languages``: List of languages apt should download. Use ``[none]`` to + not download any languages at all. + ``optional`` diff --git a/bootstrapvz/plugins/minimize_size/__init__.py b/bootstrapvz/plugins/minimize_size/__init__.py index 107da6e..1f9daac 100644 --- a/bootstrapvz/plugins/minimize_size/__init__.py +++ b/bootstrapvz/plugins/minimize_size/__init__.py @@ -19,6 +19,16 @@ def resolve_tasks(taskset, manifest): if manifest.plugins['minimize_size'].get('shrink', False): taskset.add(tasks.AddRequiredCommands) taskset.add(tasks.ShrinkVolume) + if 'apt' in manifest.plugins['minimize_size']: + apt = manifest.plugins['minimize_size']['apt'] + if apt.get('autoclean', False): + taskset.add(tasks.AutomateAptClean) + if 'languages' in apt: + taskset.add(tasks.FilterTranslationFiles) + if apt.get('gzip_indexes', False): + taskset.add(tasks.AptGzipIndexes) + if apt.get('autoremove_suggests', False): + taskset.add(tasks.AptAutoremoveSuggests) def resolve_rollback_tasks(taskset, manifest, completed, counter_task): diff --git a/bootstrapvz/plugins/minimize_size/assets/apt-autoremove-suggests b/bootstrapvz/plugins/minimize_size/assets/apt-autoremove-suggests new file mode 100644 index 0000000..5e47797 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/apt-autoremove-suggests @@ -0,0 +1,13 @@ +# Since Docker users are looking for the smallest possible final images, the +# following emerges as a very common pattern: +# RUN apt-get update \ +# && apt-get install -y \ +# && \ +# && apt-get purge -y --auto-remove +# By default, APT will actually _keep_ packages installed via Recommends or +# Depends if another package Suggests them, even and including if the package +# that originally caused them to be installed is removed. Setting this to +# "false" ensures that APT is appropriately aggressive about removing the +# packages it added. +# https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant +Apt::AutoRemove::SuggestsImportant "false"; diff --git a/bootstrapvz/plugins/minimize_size/assets/apt-clean b/bootstrapvz/plugins/minimize_size/assets/apt-clean new file mode 100644 index 0000000..f9348ed --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/apt-clean @@ -0,0 +1,16 @@ +# Since for most Docker users, package installs happen in "docker build" steps, +# they essentially become individual layers due to the way Docker handles +# layering, especially using CoW filesystems. What this means for us is that +# the caches that APT keeps end up just wasting space in those layers, making +# our layers unnecessarily large (especially since we'll normally never use +# these caches again and will instead just "docker build" again and make a brand +# new image). +# Ideally, these would just be invoking "apt-get clean", but in our testing, +# that ended up being cyclic and we got stuck on APT's lock, so we get this fun +# creation that's essentially just "apt-get clean". +DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; }; +APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; }; +Dir::Cache::pkgcache ""; +Dir::Cache::srcpkgcache ""; +# Note that we do realize this isn't the ideal way to do this, and are always +# open to better suggestions (https://github.com/docker/docker/issues). diff --git a/bootstrapvz/plugins/minimize_size/assets/apt-gzip-indexes b/bootstrapvz/plugins/minimize_size/assets/apt-gzip-indexes new file mode 100644 index 0000000..cdb6621 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/apt-gzip-indexes @@ -0,0 +1,9 @@ +# Since Docker users using "RUN apt-get update && apt-get install -y ..." in +# their Dockerfiles don't go delete the lists files afterwards, we want them to +# be as small as possible on-disk, so we explicitly request "gz" versions and +# tell Apt to keep them gzipped on-disk. +# For comparison, an "apt-get update" layer without this on a pristine +# "debian:wheezy" base image was "29.88 MB", where with this it was only +# "8.273 MB". +Acquire::GzipIndexes "true"; +Acquire::CompressionTypes::Order:: "gz"; diff --git a/bootstrapvz/plugins/minimize_size/assets/apt-languages b/bootstrapvz/plugins/minimize_size/assets/apt-languages new file mode 100644 index 0000000..2d55968 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/assets/apt-languages @@ -0,0 +1,4 @@ +# In Docker, we don't often need the "Translations" files, so we're just wasting +# time and space by downloading them, and this inhibits that. For users that do +# need them, it's a simple matter to delete this file and "apt-get update". :) +Acquire::Languages { ACQUIRE_LANGUAGES_FILTER }; diff --git a/bootstrapvz/plugins/minimize_size/manifest-schema.yml b/bootstrapvz/plugins/minimize_size/manifest-schema.yml index 5a36be8..268791e 100644 --- a/bootstrapvz/plugins/minimize_size/manifest-schema.yml +++ b/bootstrapvz/plugins/minimize_size/manifest-schema.yml @@ -9,6 +9,25 @@ properties: type: boolean zerofree: type: boolean + apt: + type: object + properties: + autoclean: + type: boolean + languages: + type: array + minItems: 1 + items: + type: string + gzip_indexes: + type: boolean + autoremove_suggests: + type: boolean + locales: + type: array + minItems: 1 + items: + type: string type: object additionalProperties: false type: object diff --git a/bootstrapvz/plugins/minimize_size/tasks.py b/bootstrapvz/plugins/minimize_size/tasks.py index 5cba758..5a2470f 100644 --- a/bootstrapvz/plugins/minimize_size/tasks.py +++ b/bootstrapvz/plugins/minimize_size/tasks.py @@ -6,8 +6,12 @@ from bootstrapvz.common.tasks import filesystem from bootstrapvz.common.tasks import host from bootstrapvz.common.tasks import partitioning from bootstrapvz.common.tasks import volume +from bootstrapvz.common.tools import sed_i +from bootstrapvz.common.tools import log_check_call import os +import shutil +assets = os.path.normpath(os.path.join(os.path.dirname(__file__), 'assets')) folders = ['tmp', 'var/lib/apt/lists'] @@ -69,7 +73,6 @@ class Zerofree(Task): @classmethod def run(cls, info): - from bootstrapvz.common.tools import log_check_call log_check_call(['zerofree', info.volume.partition_map.root.device_path]) @@ -84,3 +87,58 @@ class ShrinkVolume(Task): perm = os.stat(info.volume.image_path).st_mode & 0777 log_check_call(['/usr/bin/vmware-vdiskmanager', '-k', info.volume.image_path]) os.chmod(info.volume.image_path, perm) + + +class AutomateAptClean(Task): + description = 'Configuring apt to always clean everything out when it\'s done' + phase = phases.package_installation + successors = [apt.AptUpdate] + # Snatched from: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + + @classmethod + def run(cls, info): + shutil.copy(os.path.join(assets, 'apt-clean'), + os.path.join(info.root, 'etc/apt/apt.conf.d/90clean')) + + +class FilterTranslationFiles(Task): + description = 'Configuring apt to only download and use specific translation files' + phase = phases.package_installation + successors = [apt.AptUpdate] + # Snatched from: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + + @classmethod + def run(cls, info): + langs = info.manifest.plugins['minimize_size']['apt']['languages'] + config = '; '.join(map(lambda l: '"' + l + '"', langs)) + config_path = os.path.join(info.root, 'etc/apt/apt.conf.d/20languages') + shutil.copy(os.path.join(assets, 'apt-languages'), config_path) + sed_i(config_path, r'ACQUIRE_LANGUAGES_FILTER', config) + + +class AptGzipIndexes(Task): + description = 'Configuring apt to always gzip lists files' + phase = phases.package_installation + successors = [apt.AptUpdate] + # Snatched from: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + + @classmethod + def run(cls, info): + shutil.copy(os.path.join(assets, 'apt-gzip-indexes'), + os.path.join(info.root, 'etc/apt/apt.conf.d/20gzip-indexes')) + + +class AptAutoremoveSuggests(Task): + description = 'Configuring apt to not consider suggested important' + phase = phases.package_installation + successors = [apt.AptUpdate] + # Snatched from: + # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap + + @classmethod + def run(cls, info): + shutil.copy(os.path.join(assets, 'apt-autoremove-suggests'), + os.path.join(info.root, 'etc/apt/apt.conf.d/20autoremove-suggests')) diff --git a/manifests/examples/docker/jessie.yml b/manifests/examples/docker/jessie.yml index 4eee878..4e118c6 100644 --- a/manifests/examples/docker/jessie.yml +++ b/manifests/examples/docker/jessie.yml @@ -24,3 +24,10 @@ volume: root: filesystem: ext4 size: 1GiB +plugins: + minimize_size: + apt: + autoclean: true + languages: [none] + gzip_indexes: true + autoremove_suggests: true