Merge pull request #458 from CMeza99/mobify

Add Docker specific settings

bootstrapvz/common/tasks/dpkg.py

@@ -0,0 +1,16 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common.tasks import bootstrap
+from .. import phases
+import os.path
+
+
+class CreateDpkgCfg(Task):
+    description = 'Creating /etc/dpkg/dpkg.cfg.d before bootstrapping'
+    phase = phases.os_installation
+    successors = [bootstrap.Bootstrap]
+
+    @classmethod
+    def run(cls, info):
+        dpkgcfg_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d')
+        if not os.path.exists(dpkgcfg_path):
+            os.makedirs(dpkgcfg_path)

bootstrapvz/plugins/minimize_size/__init__.py

@@ -2,7 +2,7 @@ import tasks.mounts
 import tasks.shrink
 import tasks.apt
 import tasks.dpkg
-from bootstrapvz.common.tasks import locale
+from bootstrapvz.common.tasks import dpkg, locale
 
 
 def get_shrink_type(plugins):
@@ -56,20 +56,20 @@ def resolve_tasks(taskset, manifest):
         if apt.get('autoremove_suggests', False):
             taskset.add(tasks.apt.AptAutoremoveSuggests)
     if 'dpkg' in manifest.plugins['minimize_size']:
-        filter_tasks = [tasks.dpkg.CreateDpkgCfg,
+        filter_tasks = [dpkg.CreateDpkgCfg,
                         tasks.dpkg.InitializeBootstrapFilterList,
                         tasks.dpkg.CreateBootstrapFilterScripts,
                         tasks.dpkg.DeleteBootstrapFilterScripts,
                         ]
-        dpkg = manifest.plugins['minimize_size']['dpkg']
+        msdpkg = manifest.plugins['minimize_size']['dpkg']
-        if 'locales' in dpkg:
+        if 'locales' in msdpkg:
             taskset.update(filter_tasks)
             taskset.add(tasks.dpkg.FilterLocales)
             # If no locales are selected, we don't need the locale package
-            if len(dpkg['locales']) == 0:
+            if msdpkg['locales']:
                 taskset.discard(locale.LocaleBootstrapPackage)
                 taskset.discard(locale.GenerateLocale)
-        if dpkg.get('exclude_docs', False):
+        if msdpkg.get('exclude_docs', False):
             taskset.update(filter_tasks)
             taskset.add(tasks.dpkg.ExcludeDocs)
 

bootstrapvz/plugins/minimize_size/tasks/dpkg.py

@@ -1,23 +1,12 @@
 from bootstrapvz.base import Task
 from bootstrapvz.common import phases
-from bootstrapvz.common.tasks import bootstrap
+from bootstrapvz.common.tasks import bootstrap, dpkg, workspace
-from bootstrapvz.common.tasks import workspace
 from bootstrapvz.common.tools import sed_i
 import os
 import shutil
 from . import assets
 
 
-class CreateDpkgCfg(Task):
-    description = 'Creating /etc/dpkg/dpkg.cfg.d before bootstrapping'
-    phase = phases.os_installation
-    successors = [bootstrap.Bootstrap]
-
-    @classmethod
-    def run(cls, info):
-        os.makedirs(os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d'))
-
-
 class InitializeBootstrapFilterList(Task):
     description = 'Initializing the bootstrapping filter list'
     phase = phases.preparation
@@ -68,7 +57,7 @@ class CreateBootstrapFilterScripts(Task):
 class FilterLocales(Task):
     description = 'Configuring dpkg and debootstrap to only include specific locales/manpages when installing packages'
     phase = phases.os_installation
-    predecessors = [CreateDpkgCfg]
+    predecessors = [dpkg.CreateDpkgCfg]
     successors = [CreateBootstrapFilterScripts]
     # Snatched from:
     # https://github.com/docker/docker/blob/1d775a54cc67e27f755c7338c3ee938498e845d7/contrib/mkimage/debootstrap
@@ -122,7 +111,7 @@ class FilterLocales(Task):
 class ExcludeDocs(Task):
     description = 'Configuring dpkg and debootstrap to not install additional documentation for packages'
     phase = phases.os_installation
-    predecessors = [CreateDpkgCfg]
+    predecessors = [dpkg.CreateDpkgCfg]
     successors = [CreateBootstrapFilterScripts]
 
     @classmethod

bootstrapvz/providers/docker/README.rst

@@ -5,8 +5,11 @@ The `Docker <https://www.docker.com/>`__ provider creates a docker image
 from scratch, creates a Dockerfile for it and imports the image to a repo
 specified in the manifest.
 
-In order to reduce the size of the image, it is highly recommend
+In order to mimick `Moby's bootstrap <https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap>`__
-to make use of the `minimize_size <../../plugins/minimize_size>`__ plugin.
+the `minimize_size <../../plugins/minimize_size>`__ plugin is required.
+
+The image can also be shrunk even futher using the 
+`minimize_size <../../plugins/minimize_size>`__ plugin.
 With optimal settings a 64-bit jessie image can be whittled down to 81.95 MB
 (built on Dec 13th 2015 with ``manifests/examples/docker/jessie-minimized.yml``).
 
@@ -54,3 +57,10 @@ Example:
       labels:
         name: debian-{system.release}-{system.architecture}-{%y}{%m}{%d}
         description: Debian {system.release} {system.architecture}
+    plugins:
+      minimize_size:
+        apt:
+          autoclean: true
+          languages: [none]
+          gzip_indexes: true
+          autoremove_suggests: true

bootstrapvz/providers/docker/__init__.py

@@ -1,8 +1,9 @@
 from bootstrapvz.common import task_groups
-from bootstrapvz.common.tasks import apt, folder, filesystem
+from bootstrapvz.common.tasks import apt, dpkg, folder, filesystem
 from bootstrapvz.common.tools import rel_path
 import tasks.commands
 import tasks.image
+import tasks.settings
 
 
 def validate_manifest(data, validator, error):
@@ -12,7 +13,8 @@ def validate_manifest(data, validator, error):
 
 def resolve_tasks(taskset, manifest):
     taskset.update(task_groups.get_base_group(manifest))
-    taskset.update([folder.Create,
+    taskset.update([dpkg.CreateDpkgCfg,
+                    folder.Create,
                     filesystem.CopyMountTable,
                     filesystem.RemoveMountTable,
                     folder.Delete,
@@ -29,6 +31,9 @@ def resolve_tasks(taskset, manifest):
     taskset.update([tasks.commands.AddRequiredCommands,
                     tasks.image.CreateDockerfileEntry,
                     tasks.image.CreateImage,
+                    tasks.settings.DpkgUnsafeIo,
+                    tasks.settings.AutoRemoveKernel,
+                    tasks.settings.SystemdContainer
                     ])
     if 'labels' in manifest.provider:
         taskset.add(tasks.image.PopulateLabels)

bootstrapvz/providers/docker/assets/docker-apt-speedup

@@ -0,0 +1,9 @@
+# This file was created by bootstrap-vz.
+# See https://github.com/andsens/bootstrap-vz/blob/master/LICENSE for
+# legal notices and disclaimers.
+
+# For most Docker users, package installs happen during "docker build", which
+# doesn't survive power loss and gets restarted clean afterwards anyhow, so
+# this minor tweak gives us a nice speedup (much nicer on spinning disks,
+# obviously).
+force-unsafe-io

bootstrapvz/providers/docker/tasks/__init__.py

@@ -0,0 +1,3 @@
+from bootstrapvz.common.tools import rel_path
+
+assets = rel_path(__file__, '../assets')

bootstrapvz/providers/docker/tasks/settings.py

@@ -0,0 +1,51 @@
+from bootstrapvz.base import Task
+from bootstrapvz.common import phases
+from bootstrapvz.common.tasks import apt, bootstrap, dpkg
+from . import assets
+import os
+import shutil
+
+
+class DpkgUnsafeIo(Task):
+    # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L94
+    description = 'Force dpkg not to call sync()'
+    phase = phases.os_installation
+    predecessors = [dpkg.CreateDpkgCfg]
+    successors = [bootstrap.Bootstrap]
+
+    @classmethod
+    def run(cls, info):
+        dpkgcfg_path = os.path.join(info.root, 'etc/dpkg/dpkg.cfg.d')
+
+        shutil.copy(
+            os.path.join(assets, 'docker-apt-speedup'),
+            os.path.join(info.root, dpkgcfg_path, 'docker-apt-speedup'))
+
+
+class AutoRemoveKernel(Task):
+    # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L87
+    description = 'Do not prevent autoremove of current kernel'
+    phase = phases.package_installation
+    successors = [apt.AptUpdate]
+
+    @classmethod
+    def run(cls, info):
+        # this file is one APT creates to make sure we don't "autoremove" our currently
+        # in-use kernel, which doesn't really apply to debootstraps/Docker images that
+        # don't even have kernels installed
+        autoremovekernels = os.path.join(info.root, 'etc/apt/apt.conf.d/01autoremove-kernels')
+        if os.path.isfile(autoremovekernels):
+            os.remove(autoremovekernels)
+
+
+class SystemdContainer(Task):
+    # https://github.com/systemd/systemd/blob/aa0c34279ee40bce2f9681b496922dedbadfca19/src/basic/virt.c#L434
+    description = 'Make systemd-detect-virt return "docker"'
+    phase = phases.package_installation
+    successors = [apt.AptUpdate]
+
+    @classmethod
+    def run(cls, info):
+        os.makedirs(os.path.join(info.root, 'run/systemd'))
+        with open(os.path.join(info.root, 'run/systemd/container'), 'w') as systemd:
+            systemd.write('docker')

manifests/examples/docker/stretch-minimized.yml

@@ -0,0 +1,47 @@
+---
+name: debian-min-{system.release}-{system.architecture}:latest
+provider:
+  name: docker
+  labels:
+    name: debian-min-{system.release}-{system.architecture}-{%Y}.{%m}.{%d}
+    summary: Debian minimized {system.release} {system.architecture}
+    description: >-
+      Minimized version of Debian {system.release}
+      without any manpages, additional documentation.
+      Built on: {%Y}.{%m}.{%d}
+    distribution-scope: public
+  dockerfile:
+    - CMD /bin/bash
+bootstrapper:
+  workspace: /target
+  variant: minbase
+system:
+  release: stretch
+  architecture: amd64
+  bootloader: none
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: folder
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 1GiB
+plugins:
+  minimize_size:
+    # This plugin is used to help implement some of the modifications from Moby's debootstrap
+    # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap
+    apt:
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L107
+      autoclean: true
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L132
+      languages: [none]
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L142
+      gzip_indexes: true
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L157
+      autoremove_suggests: true
+    dpkg:  # dpkg is required for the Docker provider
+      locales: []
+      exclude_docs: true

manifests/examples/docker/stretch.yml

@@ -0,0 +1,42 @@
+---
+name: debian-{system.release}-{system.architecture}:latest
+provider:
+  name: docker
+  labels:
+    name: debian-{system.release}-{system.architecture}-{%Y}.{%m}.{%d}
+    summary: Debian {system.release} {system.architecture}
+    description: >-
+      Debian {system.release} {system.architecture}, built on: {%Y}.{%m}.{%d}
+    distribution-scope: public
+  dockerfile:
+    - CMD /bin/bash
+bootstrapper:
+  workspace: /target
+  variant: minbase
+system:
+  release: stretch
+  architecture: amd64
+  bootloader: none
+  charmap: UTF-8
+  locale: en_US
+  timezone: UTC
+volume:
+  backing: folder
+  partitions:
+    type: none
+    root:
+      filesystem: ext4
+      size: 1GiB
+plugins:
+  minimize_size:
+    # This plugin is used to help implement some of the modifications from Moby's debootstrap
+    # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap
+    apt:
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L107
+      autoclean: true
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L132
+      languages: [none]
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L142
+      gzip_indexes: true
+      # https://github.com/moby/moby/blob/e2e5d4bc9da5ba17bb2822909611f8300fdd80f0/contrib/mkimage/debootstrap#L157
+      autoremove_suggests: true