From c19eb61bbf7ecfaa1239222e0f889fcddf6d05c0 Mon Sep 17 00:00:00 2001 From: Carlos Meza Date: Sat, 10 Feb 2018 10:09:04 +0000 Subject: [PATCH] add docker specific settings --- bootstrapvz/common/tasks/dpkg.py | 16 ++++++ bootstrapvz/plugins/minimize_size/__init__.py | 12 ++--- .../plugins/minimize_size/tasks/dpkg.py | 17 ++----- bootstrapvz/providers/docker/README.rst | 14 ++++- bootstrapvz/providers/docker/__init__.py | 9 +++- .../docker/assets/docker-apt-speedup | 9 ++++ .../providers/docker/tasks/__init__.py | 3 ++ .../providers/docker/tasks/settings.py | 51 +++++++++++++++++++ .../examples/docker/stretch-minimized.yml | 47 +++++++++++++++++ manifests/examples/docker/stretch.yml | 42 +++++++++++++++ 10 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 bootstrapvz/common/tasks/dpkg.py create mode 100644 bootstrapvz/providers/docker/assets/docker-apt-speedup create mode 100644 bootstrapvz/providers/docker/tasks/settings.py create mode 100644 manifests/examples/docker/stretch-minimized.yml create mode 100644 manifests/examples/docker/stretch.yml diff --git a/bootstrapvz/common/tasks/dpkg.py b/bootstrapvz/common/tasks/dpkg.py new file mode 100644 index 0000000..0777a59 --- /dev/null +++ b/bootstrapvz/common/tasks/dpkg.py @@ -0,0 +1,16 @@ +from bootstrapvz.base import Task +from bootstrapvz.common.tasks import bootstrap +from .. import phases +import os.path + + +class CreateDpkgCfg(Task): + description = 'Creating /etc/dpkg/dpkg.cfg.d before bootstrapping' + phase = phases.os_installation + successors = [bootstrap.Bootstrap] + + @classmethod + def run(cls, info): + dpkgcfg_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d') + if not os.path.exists(dpkgcfg_path): + os.makedirs(dpkgcfg_path) diff --git a/bootstrapvz/plugins/minimize_size/__init__.py b/bootstrapvz/plugins/minimize_size/__init__.py index 10420ae..9bc68a5 100644 --- a/bootstrapvz/plugins/minimize_size/__init__.py +++ b/bootstrapvz/plugins/minimize_size/__init__.py @@ -2,7 +2,7 @@ import tasks.mounts import tasks.shrink import tasks.apt import tasks.dpkg -from bootstrapvz.common.tasks import locale +from bootstrapvz.common.tasks import dpkg, locale def get_shrink_type(plugins): @@ -54,20 +54,20 @@ def resolve_tasks(taskset, manifest): if apt.get('autoremove_suggests', False): taskset.add(tasks.apt.AptAutoremoveSuggests) if 'dpkg' in manifest.plugins['minimize_size']: - filter_tasks = [tasks.dpkg.CreateDpkgCfg, + filter_tasks = [dpkg.CreateDpkgCfg, tasks.dpkg.InitializeBootstrapFilterList, tasks.dpkg.CreateBootstrapFilterScripts, tasks.dpkg.DeleteBootstrapFilterScripts, ] - dpkg = manifest.plugins['minimize_size']['dpkg'] - if 'locales' in dpkg: + msdpkg = manifest.plugins['minimize_size']['dpkg'] + if 'locales' in msdpkg: taskset.update(filter_tasks) taskset.add(tasks.dpkg.FilterLocales) # If no locales are selected, we don't need the locale package - if len(dpkg['locales']) == 0: + if msdpkg['locales']: taskset.discard(locale.LocaleBootstrapPackage) taskset.discard(locale.GenerateLocale) - if dpkg.get('exclude_docs', False): + if msdpkg.get('exclude_docs', False): taskset.update(filter_tasks) taskset.add(tasks.dpkg.ExcludeDocs) diff --git a/bootstrapvz/plugins/minimize_size/tasks/dpkg.py b/bootstrapvz/plugins/minimize_size/tasks/dpkg.py index c39f0c3..73d2aa6 100644 --- a/bootstrapvz/plugins/minimize_size/tasks/dpkg.py +++ b/bootstrapvz/plugins/minimize_size/tasks/dpkg.py @@ -1,23 +1,12 @@ from bootstrapvz.base import Task from bootstrapvz.common import phases -from bootstrapvz.common.tasks import bootstrap -from bootstrapvz.common.tasks import workspace +from bootstrapvz.common.tasks import bootstrap, dpkg, workspace from bootstrapvz.common.tools import sed_i import os import shutil from . import assets -class CreateDpkgCfg(Task): - description = 'Creating /etc/dpkg/dpkg.cfg.d before bootstrapping' - phase = phases.os_installation - successors = [bootstrap.Bootstrap] - - @classmethod - def run(cls, info): - os.makedirs(os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d')) - - class InitializeBootstrapFilterList(Task): description = 'Initializing the bootstrapping filter list' phase = phases.preparation @@ -68,7 +57,7 @@ class CreateBootstrapFilterScripts(Task): class FilterLocales(Task): description = 'Configuring dpkg and debootstrap to only include specific locales/manpages when installing packages' phase = phases.os_installation - predecessors = [CreateDpkgCfg] + predecessors = [dpkg.CreateDpkgCfg] successors = [CreateBootstrapFilterScripts] # Snatched from: # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap @@ -122,7 +111,7 @@ class FilterLocales(Task): class ExcludeDocs(Task): description = 'Configuring dpkg and debootstrap to not install additional documentation for packages' phase = phases.os_installation - predecessors = [CreateDpkgCfg] + predecessors = [dpkg.CreateDpkgCfg] successors = [CreateBootstrapFilterScripts] @classmethod diff --git a/bootstrapvz/providers/docker/README.rst b/bootstrapvz/providers/docker/README.rst index c907bb5..8da39cd 100644 --- a/bootstrapvz/providers/docker/README.rst +++ b/bootstrapvz/providers/docker/README.rst @@ -5,8 +5,11 @@ The `Docker `__ provider creates a docker image from scratch, creates a Dockerfile for it and imports the image to a repo specified in the manifest. -In order to reduce the size of the image, it is highly recommend -to make use of the `minimize_size <../../plugins/minimize_size>`__ plugin. +In order to mimick `Moby's bootstrap `__ +the `minimize_size <../../plugins/minimize_size>`__ plugin is required. + +The image can also be shrunk even futher using the +`minimize_size <../../plugins/minimize_size>`__ plugin. With optimal settings a 64-bit jessie image can be whittled down to 81.95 MB (built on Dec 13th 2015 with ``manifests/examples/docker/jessie-minimized.yml``). @@ -54,3 +57,10 @@ Example: labels: name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d} description: Debian {system.release} {system.architecture} + plugins: + minimize_size: + apt: + autoclean: true + languages: [none] + gzip_indexes: true + autoremove_suggests: true diff --git a/bootstrapvz/providers/docker/__init__.py b/bootstrapvz/providers/docker/__init__.py index 05a16a0..88eac94 100644 --- a/bootstrapvz/providers/docker/__init__.py +++ b/bootstrapvz/providers/docker/__init__.py @@ -1,8 +1,9 @@ from bootstrapvz.common import task_groups -from bootstrapvz.common.tasks import apt, folder, filesystem +from bootstrapvz.common.tasks import apt, dpkg, folder, filesystem from bootstrapvz.common.tools import rel_path import tasks.commands import tasks.image +import tasks.settings def validate_manifest(data, validator, error): @@ -12,7 +13,8 @@ def validate_manifest(data, validator, error): def resolve_tasks(taskset, manifest): taskset.update(task_groups.get_base_group(manifest)) - taskset.update([folder.Create, + taskset.update([dpkg.CreateDpkgCfg, + folder.Create, filesystem.CopyMountTable, filesystem.RemoveMountTable, folder.Delete, @@ -29,6 +31,9 @@ def resolve_tasks(taskset, manifest): taskset.update([tasks.commands.AddRequiredCommands, tasks.image.CreateDockerfileEntry, tasks.image.CreateImage, + tasks.settings.DpkgUnsafeIo, + tasks.settings.AutoRemoveKernel, + tasks.settings.SystemdContainer ]) if 'labels' in manifest.provider: taskset.add(tasks.image.PopulateLabels) diff --git a/bootstrapvz/providers/docker/assets/docker-apt-speedup b/bootstrapvz/providers/docker/assets/docker-apt-speedup new file mode 100644 index 0000000..2b0a7fb --- /dev/null +++ b/bootstrapvz/providers/docker/assets/docker-apt-speedup @@ -0,0 +1,9 @@ +# This file was created by bootstrap-vz. +# See https://github.com/andsens/bootstrap-vz/blob/master/LICENSE for +# legal notices and disclaimers. + +# For most Docker users, package installs happen during "docker build", which +# doesn't survive power loss and gets restarted clean afterwards anyhow, so +# this minor tweak gives us a nice speedup (much nicer on spinning disks, +# obviously). +force-unsafe-io diff --git a/bootstrapvz/providers/docker/tasks/__init__.py b/bootstrapvz/providers/docker/tasks/__init__.py index e69de29..9026602 100644 --- a/bootstrapvz/providers/docker/tasks/__init__.py +++ b/bootstrapvz/providers/docker/tasks/__init__.py @@ -0,0 +1,3 @@ +from bootstrapvz.common.tools import rel_path + +assets = rel_path(__file__, '../assets') diff --git a/bootstrapvz/providers/docker/tasks/settings.py b/bootstrapvz/providers/docker/tasks/settings.py new file mode 100644 index 0000000..42314df --- /dev/null +++ b/bootstrapvz/providers/docker/tasks/settings.py @@ -0,0 +1,51 @@ +from bootstrapvz.base import Task +from bootstrapvz.common import phases +from bootstrapvz.common.tasks import apt, bootstrap, dpkg +from . import assets +import os +import shutil + + +class DpkgUnsafeIo(Task): + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L94 + description = 'Force dpkg not to call sync()' + phase = phases.os_installation + predecessors = [dpkg.CreateDpkgCfg] + successors = [bootstrap.Bootstrap] + + @classmethod + def run(cls, info): + dpkgcfg_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d') + + shutil.copy( + os.path.join(assets, 'docker-apt-speedup'), + os.path.join(info.root, dpkgcfg_path, 'docker-apt-speedup')) + + +class AutoRemoveKernel(Task): + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L87 + description = 'Do not prevent autoremove of current kernel' + phase = phases.package_installation + successors = [apt.AptUpdate] + + @classmethod + def run(cls, info): + # this file is one APT creates to make sure we don't "autoremove" our currently + # in-use kernel, which doesn't really apply to debootstraps/Docker images that + # don't even have kernels installed + autoremovekernels = os.path.join(info.root, 'etc/apt/apt.conf.d/01autoremove-kernels') + if os.path.isfile(autoremovekernels): + os.remove(autoremovekernels) + + +class SystemdContainer(Task): + # https://github.com/systemd/systemd/blob/aa0c34279ee40bce2f9681b496922dedbadfca19/src/basic/virt.c#L434 + description = 'Make systemd-detect-virt return "docker"' + phase = phases.package_installation + successors = [apt.AptUpdate] + + @classmethod + def run(cls, info): + os.makedirs(os.path.join(info.root, 'run/systemd')) + with open(os.path.join(info.root, 'run/systemd/container'), 'w') as systemd: + systemd.write('docker') diff --git a/manifests/examples/docker/stretch-minimized.yml b/manifests/examples/docker/stretch-minimized.yml new file mode 100644 index 0000000..3126368 --- /dev/null +++ b/manifests/examples/docker/stretch-minimized.yml @@ -0,0 +1,47 @@ +--- +name: debian-min-{system.release}-{system.architecture}:latest +provider: + name: docker + labels: + name: debian-min-{system.release}-{system.architecture}-{%Y}.{%m}.{%d} + summary: Debian minimized {system.release} {system.architecture} + description: >- + Minimized version of Debian {system.release} + without any manpages, additional documentation. + Built on: {%Y}.{%m}.{%d} + distribution-scope: public + dockerfile: + - CMD /bin/bash +bootstrapper: + workspace: /target + variant: minbase +system: + release: stretch + architecture: amd64 + bootloader: none + charmap: UTF-8 + locale: en_US + timezone: UTC +volume: + backing: folder + partitions: + type: none + root: + filesystem: ext4 + size: 1GiB +plugins: + minimize_size: + # This plugin is used to help implement some of the modifications from Moby's debootstrap + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap + apt: + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L107 + autoclean: true + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L132 + languages: [none] + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L142 + gzip_indexes: true + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L157 + autoremove_suggests: true + dpkg: # dpkg is required for the Docker provider + locales: [] + exclude_docs: true diff --git a/manifests/examples/docker/stretch.yml b/manifests/examples/docker/stretch.yml new file mode 100644 index 0000000..335607e --- /dev/null +++ b/manifests/examples/docker/stretch.yml @@ -0,0 +1,42 @@ +--- +name: debian-{system.release}-{system.architecture}:latest +provider: + name: docker + labels: + name: debian-{system.release}-{system.architecture}-{%Y}.{%m}.{%d} + summary: Debian {system.release} {system.architecture} + description: >- + Debian {system.release} {system.architecture}, built on: {%Y}.{%m}.{%d} + distribution-scope: public + dockerfile: + - CMD /bin/bash +bootstrapper: + workspace: /target + variant: minbase +system: + release: stretch + architecture: amd64 + bootloader: none + charmap: UTF-8 + locale: en_US + timezone: UTC +volume: + backing: folder + partitions: + type: none + root: + filesystem: ext4 + size: 1GiB +plugins: + minimize_size: + # This plugin is used to help implement some of the modifications from Moby's debootstrap + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap + apt: + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L107 + autoclean: true + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L132 + languages: [none] + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L142 + gzip_indexes: true + # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L157 + autoremove_suggests: true