From d3f306a6d9f5702a7e8aa4fe58c06bd03e00ecbe Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 14:12:00 +0200 Subject: [PATCH 01/19] Build docs through tox --- docs/Makefile | 177 -------------------------------------------------- tox.ini | 7 ++ 2 files changed, 7 insertions(+), 177 deletions(-) delete mode 100644 docs/Makefile diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 66d46ad..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bootstrap-vz.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bootstrap-vz.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/bootstrap-vz" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bootstrap-vz" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/tox.ini b/tox.ini index b4a76b4..d147ed8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,3 +14,10 @@ deps = nose nose-cov commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive + +[testenv:docs] +changedir = docs +deps = + sphinx +commands = + sphinx-build -b html -d _build/html/doctrees . _build/html \ No newline at end of file From 6f87287c31df9b04a0c9aab7c3a5cc1095a08a92 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 14:13:55 +0200 Subject: [PATCH 02/19] Also build task graph when building docs --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index d147ed8..d428d52 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,8 @@ deps = commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive [testenv:docs] -changedir = docs deps = sphinx commands = - sphinx-build -b html -d _build/html/doctrees . _build/html \ No newline at end of file + sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html + ./taskoverview.py --output docs/_static/graph.json From 0276b7091042e7478b783d12a3ed43a5a3936572 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 14:35:39 +0200 Subject: [PATCH 03/19] Fix taskoverview graph --- docs/_static/graph.json | 2 +- docs/_static/taskoverview.coffee | 6 +++--- tox.ini | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/_static/graph.json b/docs/_static/graph.json index 3caaa2b..3485725 100644 --- a/docs/_static/graph.json +++ b/docs/_static/graph.json @@ -1 +1 @@ -{"phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}], "modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "nodes": [{"phase": 0, "name": "AddDefaultSources", "module": 0}, {"phase": 0, "name": "AddManifestPreferences", "module": 0}, {"phase": 0, "name": "AddManifestSources", "module": 0}, {"phase": 7, "name": "AptClean", "module": 0}, {"phase": 5, "name": "AptUpdate", "module": 0}, {"phase": 5, "name": "AptUpgrade", "module": 0}, {"phase": 5, "name": "DisableDaemonAutostart", "module": 0}, {"phase": 7, "name": "EnableDaemonAutostart", "module": 0}, {"phase": 5, "name": "InstallTrustedKeys", "module": 0}, {"phase": 7, "name": "PurgeUnusedPackages", "module": 0}, {"phase": 5, "name": "WritePreferences", "module": 0}, {"phase": 5, "name": "WriteSources", "module": 0}, {"phase": 0, "name": "AddExtlinuxPackage", "module": 1}, {"phase": 0, "name": "AddGrubPackage", "module": 1}, {"phase": 6, "name": "BlackListModules", "module": 1}, {"phase": 6, "name": "ConfigureGrub", "module": 1}, {"phase": 6, "name": "DisableGetTTYs", "module": 1}, {"phase": 6, "name": "InstallExtLinux", "module": 1}, {"phase": 6, "name": "InstallGrub", "module": 1}, {"phase": 0, "name": "AddRequiredCommands", "module": 2}, {"phase": 4, "name": "Bootstrap", "module": 2}, {"phase": 0, "name": "ExcludePackagesInBootstrap", "module": 2}, {"phase": 0, "name": "IncludePackagesInBootstrap", "module": 2}, {"phase": 4, "name": "MakeTarball", "module": 2}, {"phase": 7, "name": "CleanTMP", "module": 3}, {"phase": 7, "name": "ClearMOTD", "module": 3}, {"phase": 10, "name": "TriggerRollback", "module": 4}, {"phase": 0, "name": "AddRequiredCommands", "module": 5}, {"phase": 0, "name": "AddXFSProgs", "module": 5}, {"phase": 3, "name": "CreateBootMountDir", "module": 5}, {"phase": 3, "name": "CreateMountDir", "module": 5}, {"phase": 8, "name": "DeleteMountDir", "module": 5}, {"phase": 6, "name": "FStab", "module": 5}, {"phase": 2, "name": "Format", "module": 5}, {"phase": 3, "name": "MountBoot", "module": 5}, {"phase": 3, "name": "MountRoot", "module": 5}, {"phase": 4, "name": "MountSpecials", "module": 5}, {"phase": 2, "name": "TuneVolumeFS", "module": 5}, {"phase": 8, "name": "UnmountRoot", "module": 5}, {"phase": 0, "name": "CheckExternalCommands", "module": 6}, {"phase": 6, "name": "AddExpandRoot", "module": 7}, {"phase": 6, "name": "AdjustExpandRootScript", "module": 7}, {"phase": 6, "name": "InstallInitScripts", "module": 7}, {"phase": 6, "name": "RemoveHWClock", "module": 7}, {"phase": 5, "name": "GenerateLocale", "module": 8}, {"phase": 0, "name": "LocaleBootstrapPackage", "module": 8}, {"phase": 6, "name": "SetTimezone", "module": 8}, {"phase": 0, "name": "AddRequiredCommands", "module": 9}, {"phase": 1, "name": "Create", "module": 9}, {"phase": 9, "name": "MoveImage", "module": 9}, {"phase": 6, "name": "ConfigureNetworkIF", "module": 10}, {"phase": 6, "name": "RemoveDNSInfo", "module": 10}, {"phase": 6, "name": "RemoveHostname", "module": 10}, {"phase": 6, "name": "SetHostname", "module": 10}, {"phase": 0, "name": "AddManifestPackages", "module": 11}, {"phase": 5, "name": "AddTaskselStandardPackages", "module": 11}, {"phase": 5, "name": "InstallPackages", "module": 11}, {"phase": 0, "name": "AddRequiredCommands", "module": 12}, {"phase": 2, "name": "MapPartitions", "module": 12}, {"phase": 2, "name": "PartitionVolume", "module": 12}, {"phase": 8, "name": "UnmapPartitions", "module": 12}, {"phase": 6, "name": "EnableShadowConfig", "module": 13}, {"phase": 0, "name": "AddOpenSSHPackage", "module": 14}, {"phase": 6, "name": "AddSSHKeyGeneration", "module": 14}, {"phase": 6, "name": "DisableSSHDNSLookup", "module": 14}, {"phase": 6, "name": "DisableSSHPasswordAuthentication", "module": 14}, {"phase": 7, "name": "ShredHostkeys", "module": 14}, {"phase": 1, "name": "Attach", "module": 15}, {"phase": 10, "name": "Delete", "module": 15}, {"phase": 8, "name": "Detach", "module": 15}, {"phase": 0, "name": "CreateWorkspace", "module": 16}, {"phase": 10, "name": "DeleteWorkspace", "module": 16}, {"phase": 0, "name": "AddSudoPackage", "module": 17}, {"phase": 6, "name": "AdminUserCredentials", "module": 17}, {"phase": 6, "name": "CreateAdminUser", "module": 17}, {"phase": 6, "name": "DisableRootLogin", "module": 17}, {"phase": 6, "name": "PasswordlessSudo", "module": 17}, {"phase": 7, "name": "RemoveAptProxy", "module": 18}, {"phase": 5, "name": "SetAptProxy", "module": 18}, {"phase": 0, "name": "AddPackages", "module": 19}, {"phase": 0, "name": "CheckAssetsPath", "module": 19}, {"phase": 6, "name": "CopyChefAssets", "module": 19}, {"phase": 0, "name": "AddBackports", "module": 20}, {"phase": 0, "name": "AddCloudInitPackages", "module": 20}, {"phase": 6, "name": "DisableModules", "module": 20}, {"phase": 5, "name": "SetMetadataSource", "module": 20}, {"phase": 6, "name": "SetUsername", "module": 20}, {"phase": 6, "name": "ImageExecuteCommand", "module": 21}, {"phase": 4, "name": "AddFolderMounts", "module": 22}, {"phase": 0, "name": "AddRequiredCommands", "module": 22}, {"phase": 7, "name": "RemoveFolderMounts", "module": 22}, {"phase": 8, "name": "ShrinkVolume", "module": 22}, {"phase": 8, "name": "Zerofree", "module": 22}, {"phase": 5, "name": "AddNtpPackage", "module": 23}, {"phase": 6, "name": "SetNtpServers", "module": 23}, {"phase": 0, "name": "AddBackports", "module": 24}, {"phase": 0, "name": "AddONEContextPackage", "module": 24}, {"phase": 5, "name": "CopyImage", "module": 25}, {"phase": 1, "name": "CreateFromImage", "module": 25}, {"phase": 1, "name": "CreateFromSnapshot", "module": 25}, {"phase": 5, "name": "Snapshot", "module": 25}, {"phase": 0, "name": "AddPackages", "module": 26}, {"phase": 6, "name": "ApplyPuppetManifest", "module": 26}, {"phase": 0, "name": "CheckAssetsPath", "module": 26}, {"phase": 0, "name": "CheckManifestPath", "module": 26}, {"phase": 6, "name": "CopyPuppetAssets", "module": 26}, {"phase": 6, "name": "EnableAgent", "module": 26}, {"phase": 6, "name": "SetRootPassword", "module": 27}, {"phase": 5, "name": "BootstrapSaltMinion", "module": 28}, {"phase": 5, "name": "InstallSaltDependencies", "module": 28}, {"phase": 6, "name": "SetSaltGrains", "module": 28}, {"phase": 0, "name": "AddUnattendedUpgradesPackage", "module": 29}, {"phase": 6, "name": "EnablePeriodicUpgrades", "module": 29}, {"phase": 6, "name": "AddInsecurePublicKey", "module": 30}, {"phase": 0, "name": "AddPackages", "module": 30}, {"phase": 0, "name": "CheckBoxPath", "module": 30}, {"phase": 0, "name": "CreateVagrantBoxDir", "module": 30}, {"phase": 6, "name": "CreateVagrantUser", "module": 30}, {"phase": 9, "name": "PackageBox", "module": 30}, {"phase": 6, "name": "PasswordlessSudo", "module": 30}, {"phase": 10, "name": "RemoveVagrantBoxDir", "module": 30}, {"phase": 6, "name": "SetRootPassword", "module": 30}, {"phase": 6, "name": "ConfigureGrub", "module": 31}, {"phase": 9, "name": "ConvertToVhd", "module": 32}, {"phase": 0, "name": "DefaultPackages", "module": 33}, {"phase": 5, "name": "Waagent", "module": 33}, {"phase": 0, "name": "AMIName", "module": 34}, {"phase": 9, "name": "BundleImage", "module": 34}, {"phase": 9, "name": "RegisterAMI", "module": 34}, {"phase": 10, "name": "RemoveBundle", "module": 34}, {"phase": 9, "name": "UploadImage", "module": 34}, {"phase": 6, "name": "ConfigurePVGrub", "module": 35}, {"phase": 0, "name": "Connect", "module": 36}, {"phase": 0, "name": "GetCredentials", "module": 36}, {"phase": 1, "name": "Attach", "module": 37}, {"phase": 1, "name": "Create", "module": 37}, {"phase": 9, "name": "Snapshot", "module": 37}, {"phase": 6, "name": "S3FStab", "module": 38}, {"phase": 0, "name": "AddExternalCommands", "module": 39}, {"phase": 0, "name": "GetInstanceMetadata", "module": 39}, {"phase": 0, "name": "SetRegion", "module": 39}, {"phase": 6, "name": "AddEC2InitScripts", "module": 40}, {"phase": 0, "name": "AddBuildEssentialPackage", "module": 41}, {"phase": 6, "name": "EnableDHCPCDDNS", "module": 41}, {"phase": 5, "name": "InstallEnhancedNetworking", "module": 41}, {"phase": 0, "name": "DefaultPackages", "module": 42}, {"phase": 7, "name": "CleanGoogleRepositoriesAndKeys", "module": 43}, {"phase": 5, "name": "ImportGoogleKey", "module": 43}, {"phase": 0, "name": "SetPackageRepositories", "module": 43}, {"phase": 6, "name": "ConfigureGrub", "module": 44}, {"phase": 6, "name": "GatherReleaseInformation", "module": 45}, {"phase": 6, "name": "DisableIPv6", "module": 46}, {"phase": 6, "name": "SetHostname", "module": 46}, {"phase": 9, "name": "CreateTarball", "module": 47}, {"phase": 9, "name": "RegisterImage", "module": 47}, {"phase": 9, "name": "UploadImage", "module": 47}, {"phase": 0, "name": "DefaultPackages", "module": 48}, {"phase": 0, "name": "GooglePackages", "module": 48}, {"phase": 5, "name": "InstallGSUtil", "module": 48}, {"phase": 0, "name": "DefaultPackages", "module": 49}, {"phase": 6, "name": "VirtIO", "module": 50}, {"phase": 5, "name": "AddGuestAdditionsPackages", "module": 51}, {"phase": 0, "name": "CheckGuestAdditionsPath", "module": 51}, {"phase": 5, "name": "InstallGuestAdditions", "module": 51}, {"phase": 0, "name": "DefaultPackages", "module": 52}], "links": [{"source": 19, "target": 39, "definer": 19}, {"source": 21, "target": 20, "definer": 21}, {"source": 22, "target": 20, "definer": 22}, {"source": 27, "target": 39, "definer": 27}, {"source": 38, "target": 69, "definer": 38}, {"source": 40, "target": 42, "definer": 40}, {"source": 43, "target": 42, "definer": 43}, {"source": 47, "target": 39, "definer": 47}, {"source": 48, "target": 67, "definer": 48}, {"source": 55, "target": 56, "definer": 55}, {"source": 57, "target": 39, "definer": 57}, {"source": 58, "target": 33, "definer": 58}, {"source": 60, "target": 69, "definer": 60}, {"source": 63, "target": 42, "definer": 63}, {"source": 68, "target": 71, "definer": 68}, {"source": 78, "target": 4, "definer": 78}, {"source": 85, "target": 4, "definer": 85}, {"source": 89, "target": 39, "definer": 89}, {"source": 90, "target": 3, "definer": 90}, {"source": 92, "target": 60, "definer": 92}, {"source": 92, "target": 69, "definer": 92}, {"source": 93, "target": 56, "definer": 93}, {"source": 98, "target": 67, "definer": 98}, {"source": 99, "target": 134, "definer": 99}, {"source": 102, "target": 52, "definer": 102}, {"source": 102, "target": 51, "definer": 102}, {"source": 120, "target": 71, "definer": 120}, {"source": 122, "target": 18, "definer": 122}, {"source": 129, "target": 71, "definer": 129}, {"source": 138, "target": 39, "definer": 138}, {"source": 141, "target": 42, "definer": 141}, {"source": 146, "target": 3, "definer": 146}, {"source": 147, "target": 11, "definer": 147}, {"source": 148, "target": 2, "definer": 148}, {"source": 149, "target": 18, "definer": 149}, {"source": 161, "target": 56, "definer": 161}, {"source": 2, "target": 0, "definer": 0}, {"source": 44, "target": 4, "definer": 4}, {"source": 11, "target": 4, "definer": 4}, {"source": 4, "target": 5, "definer": 5}, {"source": 6, "target": 5, "definer": 5}, {"source": 11, "target": 10, "definer": 10}, {"source": 8, "target": 11, "definer": 11}, {"source": 0, "target": 12, "definer": 12}, {"source": 0, "target": 13, "definer": 13}, {"source": 32, "target": 15, "definer": 15}, {"source": 32, "target": 17, "definer": 17}, {"source": 32, "target": 18, "definer": 18}, {"source": 23, "target": 20, "definer": 20}, {"source": 0, "target": 28, "definer": 28}, {"source": 35, "target": 29, "definer": 29}, {"source": 38, "target": 31, "definer": 31}, {"source": 29, "target": 34, "definer": 34}, {"source": 30, "target": 35, "definer": 35}, {"source": 20, "target": 36, "definer": 36}, {"source": 33, "target": 37, "definer": 37}, {"source": 42, "target": 41, "definer": 41}, {"source": 0, "target": 54, "definer": 54}, {"source": 4, "target": 55, "definer": 55}, {"source": 5, "target": 56, "definer": 56}, {"source": 59, "target": 58, "definer": 58}, {"source": 38, "target": 60, "definer": 60}, {"source": 0, "target": 62, "definer": 62}, {"source": 0, "target": 72, "definer": 72}, {"source": 42, "target": 73, "definer": 73}, {"source": 0, "target": 79, "definer": 79}, {"source": 0, "target": 83, "definer": 83}, {"source": 82, "target": 83, "definer": 83}, {"source": 44, "target": 85, "definer": 85}, {"source": 20, "target": 88, "definer": 88}, {"source": 69, "target": 91, "definer": 91}, {"source": 38, "target": 92, "definer": 92}, {"source": 0, "target": 96, "definer": 96}, {"source": 95, "target": 96, "definer": 96}, {"source": 56, "target": 97, "definer": 97}, {"source": 163, "target": 97, "definer": 97}, {"source": 56, "target": 100, "definer": 100}, {"source": 163, "target": 100, "definer": 100}, {"source": 0, "target": 101, "definer": 101}, {"source": 105, "target": 102, "definer": 102}, {"source": 56, "target": 108, "definer": 108}, {"source": 0, "target": 109, "definer": 109}, {"source": 0, "target": 111, "definer": 111}, {"source": 117, "target": 113, "definer": 113}, {"source": 0, "target": 114, "definer": 114}, {"source": 70, "target": 116, "definer": 116}, {"source": 115, "target": 116, "definer": 116}, {"source": 0, "target": 124, "definer": 124}, {"source": 56, "target": 125, "definer": 125}, {"source": 132, "target": 126, "definer": 126}, {"source": 136, "target": 128, "definer": 128}, {"source": 130, "target": 128, "definer": 128}, {"source": 127, "target": 130, "definer": 130}, {"source": 133, "target": 132, "definer": 132}, {"source": 139, "target": 132, "definer": 132}, {"source": 140, "target": 132, "definer": 132}, {"source": 135, "target": 134, "definer": 134}, {"source": 0, "target": 142, "definer": 142}, {"source": 0, "target": 145, "definer": 145}, {"source": 8, "target": 147, "definer": 147}, {"source": 50, "target": 151, "definer": 151}, {"source": 49, "target": 153, "definer": 153}, {"source": 155, "target": 154, "definer": 154}, {"source": 153, "target": 155, "definer": 155}, {"source": 0, "target": 156, "definer": 156}, {"source": 156, "target": 157, "definer": 157}, {"source": 0, "target": 159, "definer": 159}, {"source": 56, "target": 163, "definer": 163}, {"source": 0, "target": 164, "definer": 164}]} \ No newline at end of file +{"modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.extlinux"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.grub"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.kernel"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.docker_daemon.tasks"}, {"name": "bootstrapvz.plugins.file_copy.tasks"}, {"name": "bootstrapvz.plugins.google_cloud_sdk.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.pip_install.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.initd"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "links": [{"target": 46, "definer": 15, "source": 15}, {"target": 53, "definer": 27, "source": 27}, {"target": 46, "definer": 28, "source": 28}, {"target": 41, "definer": 39, "source": 39}, {"target": 79, "definer": 41, "source": 41}, {"target": 53, "definer": 45, "source": 45}, {"target": 49, "definer": 47, "source": 47}, {"target": 49, "definer": 50, "source": 50}, {"target": 66, "definer": 51, "source": 51}, {"target": 46, "definer": 57, "source": 57}, {"target": 77, "definer": 58, "source": 58}, {"target": 66, "definer": 65, "source": 65}, {"target": 46, "definer": 67, "source": 67}, {"target": 35, "definer": 68, "source": 68}, {"target": 79, "definer": 70, "source": 70}, {"target": 49, "definer": 73, "source": 73}, {"target": 81, "definer": 78, "source": 78}, {"target": 5, "definer": 89, "source": 89}, {"target": 5, "definer": 95, "source": 95}, {"target": 49, "definer": 99, "source": 99}, {"target": 44, "definer": 100, "source": 100}, {"target": 45, "definer": 100, "source": 100}, {"target": 46, "definer": 109, "source": 109}, {"target": 4, "definer": 110, "source": 110}, {"target": 70, "definer": 112, "source": 112}, {"target": 79, "definer": 112, "source": 112}, {"target": 66, "definer": 113, "source": 113}, {"target": 77, "definer": 119, "source": 119}, {"target": 155, "definer": 120, "source": 120}, {"target": 81, "definer": 141, "source": 141}, {"target": 44, "definer": 143, "source": 143}, {"target": 45, "definer": 143, "source": 143}, {"target": 81, "definer": 150, "source": 150}, {"target": 46, "definer": 159, "source": 159}, {"target": 49, "definer": 162, "source": 162}, {"target": 53, "definer": 165, "source": 165}, {"target": 4, "definer": 167, "source": 167}, {"target": 61, "definer": 167, "source": 167}, {"target": 12, "definer": 168, "source": 168}, {"target": 44, "definer": 170, "source": 170}, {"target": 45, "definer": 170, "source": 170}, {"target": 66, "definer": 183, "source": 183}, {"target": 0, "definer": 0, "source": 1}, {"target": 1, "definer": 1, "source": 3}, {"target": 5, "definer": 5, "source": 54}, {"target": 5, "definer": 5, "source": 12}, {"target": 5, "definer": 5, "source": 11}, {"target": 6, "definer": 6, "source": 5}, {"target": 6, "definer": 6, "source": 7}, {"target": 11, "definer": 11, "source": 12}, {"target": 12, "definer": 12, "source": 9}, {"target": 16, "definer": 16, "source": 19}, {"target": 23, "definer": 23, "source": 1}, {"target": 26, "definer": 26, "source": 34}, {"target": 26, "definer": 26, "source": 24}, {"target": 27, "definer": 27, "source": 34}, {"target": 27, "definer": 27, "source": 25}, {"target": 29, "definer": 29, "source": 1}, {"target": 30, "definer": 30, "source": 38}, {"target": 31, "definer": 31, "source": 37}, {"target": 33, "definer": 33, "source": 41}, {"target": 36, "definer": 36, "source": 31}, {"target": 37, "definer": 37, "source": 32}, {"target": 38, "definer": 38, "source": 16}, {"target": 40, "definer": 40, "source": 35}, {"target": 42, "definer": 42, "source": 1}, {"target": 43, "definer": 43, "source": 34}, {"target": 44, "definer": 44, "source": 34}, {"target": 45, "definer": 45, "source": 34}, {"target": 48, "definer": 48, "source": 49}, {"target": 52, "definer": 52, "source": 66}, {"target": 64, "definer": 64, "source": 1}, {"target": 65, "definer": 65, "source": 5}, {"target": 66, "definer": 66, "source": 6}, {"target": 68, "definer": 68, "source": 69}, {"target": 70, "definer": 70, "source": 41}, {"target": 72, "definer": 72, "source": 1}, {"target": 82, "definer": 82, "source": 1}, {"target": 83, "definer": 83, "source": 49}, {"target": 90, "definer": 90, "source": 1}, {"target": 93, "definer": 93, "source": 1}, {"target": 93, "definer": 93, "source": 0}, {"target": 95, "definer": 95, "source": 54}, {"target": 100, "definer": 100, "source": 43}, {"target": 100, "definer": 100, "source": 170}, {"target": 101, "definer": 101, "source": 97}, {"target": 102, "definer": 102, "source": 103}, {"target": 108, "definer": 108, "source": 16}, {"target": 111, "definer": 111, "source": 79}, {"target": 112, "definer": 112, "source": 41}, {"target": 115, "definer": 115, "source": 1}, {"target": 115, "definer": 115, "source": 0}, {"target": 116, "definer": 116, "source": 1}, {"target": 118, "definer": 118, "source": 66}, {"target": 118, "definer": 118, "source": 185}, {"target": 121, "definer": 121, "source": 66}, {"target": 121, "definer": 121, "source": 185}, {"target": 122, "definer": 122, "source": 1}, {"target": 123, "definer": 123, "source": 126}, {"target": 129, "definer": 129, "source": 66}, {"target": 130, "definer": 130, "source": 1}, {"target": 132, "definer": 132, "source": 1}, {"target": 134, "definer": 134, "source": 138}, {"target": 135, "definer": 135, "source": 1}, {"target": 137, "definer": 137, "source": 80}, {"target": 137, "definer": 137, "source": 136}, {"target": 145, "definer": 145, "source": 1}, {"target": 146, "definer": 146, "source": 66}, {"target": 147, "definer": 147, "source": 153}, {"target": 149, "definer": 149, "source": 157}, {"target": 149, "definer": 149, "source": 151}, {"target": 151, "definer": 151, "source": 148}, {"target": 153, "definer": 153, "source": 154}, {"target": 153, "definer": 153, "source": 160}, {"target": 153, "definer": 153, "source": 161}, {"target": 155, "definer": 155, "source": 156}, {"target": 163, "definer": 163, "source": 1}, {"target": 166, "definer": 166, "source": 1}, {"target": 168, "definer": 168, "source": 9}, {"target": 169, "definer": 169, "source": 3}, {"target": 169, "definer": 169, "source": 0}, {"target": 172, "definer": 172, "source": 60}, {"target": 174, "definer": 174, "source": 59}, {"target": 175, "definer": 175, "source": 176}, {"target": 176, "definer": 176, "source": 174}, {"target": 177, "definer": 177, "source": 47}, {"target": 177, "definer": 177, "source": 48}, {"target": 178, "definer": 178, "source": 1}, {"target": 179, "definer": 179, "source": 178}, {"target": 180, "definer": 180, "source": 1}, {"target": 180, "definer": 180, "source": 0}, {"target": 180, "definer": 180, "source": 178}, {"target": 181, "definer": 181, "source": 1}, {"target": 185, "definer": 185, "source": 66}, {"target": 186, "definer": 186, "source": 1}], "nodes": [{"name": "AddBackports", "module": 0, "phase": 0}, {"name": "AddDefaultSources", "module": 0, "phase": 0}, {"name": "AddManifestPreferences", "module": 0, "phase": 0}, {"name": "AddManifestSources", "module": 0, "phase": 0}, {"name": "AptClean", "module": 0, "phase": 7}, {"name": "AptUpdate", "module": 0, "phase": 5}, {"name": "AptUpgrade", "module": 0, "phase": 5}, {"name": "DisableDaemonAutostart", "module": 0, "phase": 5}, {"name": "EnableDaemonAutostart", "module": 0, "phase": 7}, {"name": "InstallTrustedKeys", "module": 0, "phase": 5}, {"name": "PurgeUnusedPackages", "module": 0, "phase": 7}, {"name": "WritePreferences", "module": 0, "phase": 5}, {"name": "WriteSources", "module": 0, "phase": 5}, {"name": "BlackListModules", "module": 1, "phase": 6}, {"name": "DisableGetTTYs", "module": 1, "phase": 6}, {"name": "AddRequiredCommands", "module": 2, "phase": 0}, {"name": "Bootstrap", "module": 2, "phase": 4}, {"name": "ExcludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "IncludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "MakeTarball", "module": 2, "phase": 4}, {"name": "CleanTMP", "module": 3, "phase": 7}, {"name": "ClearMOTD", "module": 3, "phase": 7}, {"name": "TriggerRollback", "module": 4, "phase": 10}, {"name": "AddExtlinuxPackage", "module": 5, "phase": 0}, {"name": "ConfigureExtlinux", "module": 5, "phase": 6}, {"name": "ConfigureExtlinuxJessie", "module": 5, "phase": 6}, {"name": "InstallExtlinux", "module": 5, "phase": 6}, {"name": "InstallExtlinuxJessie", "module": 5, "phase": 6}, {"name": "AddRequiredCommands", "module": 6, "phase": 0}, {"name": "AddXFSProgs", "module": 6, "phase": 0}, {"name": "CopyMountTable", "module": 6, "phase": 4}, {"name": "CreateBootMountDir", "module": 6, "phase": 3}, {"name": "CreateMountDir", "module": 6, "phase": 3}, {"name": "DeleteMountDir", "module": 6, "phase": 8}, {"name": "FStab", "module": 6, "phase": 6}, {"name": "Format", "module": 6, "phase": 2}, {"name": "MountBoot", "module": 6, "phase": 3}, {"name": "MountRoot", "module": 6, "phase": 3}, {"name": "MountSpecials", "module": 6, "phase": 4}, {"name": "RemoveMountTable", "module": 6, "phase": 8}, {"name": "TuneVolumeFS", "module": 6, "phase": 2}, {"name": "UnmountRoot", "module": 6, "phase": 8}, {"name": "AddGrubPackage", "module": 7, "phase": 0}, {"name": "ConfigureGrub", "module": 7, "phase": 6}, {"name": "InstallGrub_1_99", "module": 7, "phase": 6}, {"name": "InstallGrub_2", "module": 7, "phase": 6}, {"name": "CheckExternalCommands", "module": 8, "phase": 0}, {"name": "AddExpandRoot", "module": 9, "phase": 6}, {"name": "AdjustExpandRootScript", "module": 9, "phase": 6}, {"name": "InstallInitScripts", "module": 9, "phase": 6}, {"name": "RemoveHWClock", "module": 9, "phase": 6}, {"name": "AddDKMSPackages", "module": 10, "phase": 5}, {"name": "DetermineKernelVersion", "module": 10, "phase": 5}, {"name": "UpdateInitramfs", "module": 10, "phase": 6}, {"name": "GenerateLocale", "module": 11, "phase": 5}, {"name": "LocaleBootstrapPackage", "module": 11, "phase": 0}, {"name": "SetTimezone", "module": 11, "phase": 6}, {"name": "AddRequiredCommands", "module": 12, "phase": 0}, {"name": "Create", "module": 12, "phase": 1}, {"name": "MoveImage", "module": 12, "phase": 9}, {"name": "ConfigureNetworkIF", "module": 13, "phase": 6}, {"name": "RemoveDNSInfo", "module": 13, "phase": 7}, {"name": "RemoveHostname", "module": 13, "phase": 7}, {"name": "SetHostname", "module": 13, "phase": 6}, {"name": "AddManifestPackages", "module": 14, "phase": 0}, {"name": "AddTaskselStandardPackages", "module": 14, "phase": 5}, {"name": "InstallPackages", "module": 14, "phase": 5}, {"name": "AddRequiredCommands", "module": 15, "phase": 0}, {"name": "MapPartitions", "module": 15, "phase": 2}, {"name": "PartitionVolume", "module": 15, "phase": 2}, {"name": "UnmapPartitions", "module": 15, "phase": 8}, {"name": "EnableShadowConfig", "module": 16, "phase": 6}, {"name": "AddOpenSSHPackage", "module": 17, "phase": 0}, {"name": "AddSSHKeyGeneration", "module": 17, "phase": 6}, {"name": "DisableSSHDNSLookup", "module": 17, "phase": 6}, {"name": "DisableSSHPasswordAuthentication", "module": 17, "phase": 6}, {"name": "ShredHostkeys", "module": 17, "phase": 7}, {"name": "Attach", "module": 18, "phase": 1}, {"name": "Delete", "module": 18, "phase": 10}, {"name": "Detach", "module": 18, "phase": 8}, {"name": "CreateWorkspace", "module": 19, "phase": 0}, {"name": "DeleteWorkspace", "module": 19, "phase": 10}, {"name": "AddSudoPackage", "module": 20, "phase": 0}, {"name": "AdminUserCredentials", "module": 20, "phase": 6}, {"name": "CreateAdminUser", "module": 20, "phase": 6}, {"name": "DisableRootLogin", "module": 20, "phase": 6}, {"name": "PasswordlessSudo", "module": 20, "phase": 6}, {"name": "CheckAptProxy", "module": 21, "phase": 0}, {"name": "RemoveAptProxy", "module": 21, "phase": 7}, {"name": "SetAptProxy", "module": 21, "phase": 5}, {"name": "AddPackages", "module": 22, "phase": 0}, {"name": "CheckAssetsPath", "module": 22, "phase": 0}, {"name": "CopyChefAssets", "module": 22, "phase": 6}, {"name": "AddCloudInitPackages", "module": 23, "phase": 0}, {"name": "DisableModules", "module": 23, "phase": 6}, {"name": "SetMetadataSource", "module": 23, "phase": 5}, {"name": "SetUsername", "module": 23, "phase": 6}, {"name": "AddDockerBinary", "module": 24, "phase": 6}, {"name": "AddDockerDeps", "module": 24, "phase": 5}, {"name": "AddDockerInit", "module": 24, "phase": 6}, {"name": "EnableMemoryCgroup", "module": 24, "phase": 6}, {"name": "PullDockerImages", "module": 24, "phase": 6}, {"name": "FileCopyCommand", "module": 25, "phase": 6}, {"name": "MkdirCommand", "module": 25, "phase": 6}, {"name": "ValidateSourcePaths", "module": 25, "phase": 0}, {"name": "InstallCloudSDK", "module": 26, "phase": 6}, {"name": "RemoveCloudSDKTarball", "module": 26, "phase": 7}, {"name": "ImageExecuteCommand", "module": 27, "phase": 6}, {"name": "AddFolderMounts", "module": 28, "phase": 4}, {"name": "AddRequiredCommands", "module": 28, "phase": 0}, {"name": "RemoveFolderMounts", "module": 28, "phase": 7}, {"name": "ShrinkVolume", "module": 28, "phase": 8}, {"name": "Zerofree", "module": 28, "phase": 8}, {"name": "AddNtpPackage", "module": 29, "phase": 5}, {"name": "SetNtpServers", "module": 29, "phase": 6}, {"name": "AddONEContextPackage", "module": 30, "phase": 0}, {"name": "AddPipPackage", "module": 31, "phase": 0}, {"name": "PipInstallCommand", "module": 31, "phase": 6}, {"name": "CopyImage", "module": 32, "phase": 5}, {"name": "CreateFromImage", "module": 32, "phase": 1}, {"name": "CreateFromSnapshot", "module": 32, "phase": 1}, {"name": "Snapshot", "module": 32, "phase": 5}, {"name": "AddPackages", "module": 33, "phase": 0}, {"name": "ApplyPuppetManifest", "module": 33, "phase": 6}, {"name": "CheckAssetsPath", "module": 33, "phase": 0}, {"name": "CheckManifestPath", "module": 33, "phase": 0}, {"name": "CopyPuppetAssets", "module": 33, "phase": 6}, {"name": "EnableAgent", "module": 33, "phase": 6}, {"name": "SetRootPassword", "module": 34, "phase": 6}, {"name": "BootstrapSaltMinion", "module": 35, "phase": 5}, {"name": "InstallSaltDependencies", "module": 35, "phase": 0}, {"name": "SetSaltGrains", "module": 35, "phase": 6}, {"name": "AddUnattendedUpgradesPackage", "module": 36, "phase": 0}, {"name": "EnablePeriodicUpgrades", "module": 36, "phase": 6}, {"name": "AddInsecurePublicKey", "module": 37, "phase": 6}, {"name": "AddPackages", "module": 37, "phase": 0}, {"name": "CheckBoxPath", "module": 37, "phase": 0}, {"name": "CreateVagrantBoxDir", "module": 37, "phase": 0}, {"name": "CreateVagrantUser", "module": 37, "phase": 6}, {"name": "PackageBox", "module": 37, "phase": 9}, {"name": "PasswordlessSudo", "module": 37, "phase": 6}, {"name": "RemoveVagrantBoxDir", "module": 37, "phase": 10}, {"name": "SetRootPassword", "module": 37, "phase": 6}, {"name": "ConfigureGrub", "module": 38, "phase": 6}, {"name": "ConvertToVhd", "module": 39, "phase": 9}, {"name": "DefaultPackages", "module": 40, "phase": 0}, {"name": "Waagent", "module": 40, "phase": 5}, {"name": "AMIName", "module": 41, "phase": 0}, {"name": "BundleImage", "module": 41, "phase": 9}, {"name": "RegisterAMI", "module": 41, "phase": 9}, {"name": "RemoveBundle", "module": 41, "phase": 10}, {"name": "UploadImage", "module": 41, "phase": 9}, {"name": "ConfigurePVGrub", "module": 42, "phase": 6}, {"name": "Connect", "module": 43, "phase": 0}, {"name": "GetCredentials", "module": 43, "phase": 0}, {"name": "Attach", "module": 44, "phase": 1}, {"name": "Create", "module": 44, "phase": 1}, {"name": "Snapshot", "module": 44, "phase": 9}, {"name": "S3FStab", "module": 45, "phase": 6}, {"name": "AddExternalCommands", "module": 46, "phase": 0}, {"name": "GetInstanceMetadata", "module": 46, "phase": 0}, {"name": "SetRegion", "module": 46, "phase": 0}, {"name": "AddEC2InitScripts", "module": 47, "phase": 6}, {"name": "AddBuildEssentialPackage", "module": 48, "phase": 0}, {"name": "EnableDHCPCDDNS", "module": 48, "phase": 6}, {"name": "InstallEnhancedNetworking", "module": 48, "phase": 6}, {"name": "DefaultPackages", "module": 49, "phase": 0}, {"name": "CleanGoogleRepositoriesAndKeys", "module": 50, "phase": 7}, {"name": "ImportGoogleKey", "module": 50, "phase": 5}, {"name": "SetPackageRepositories", "module": 50, "phase": 0}, {"name": "ConfigureGrub", "module": 51, "phase": 6}, {"name": "GatherReleaseInformation", "module": 52, "phase": 6}, {"name": "DisableIPv6", "module": 53, "phase": 6}, {"name": "InstallHostnameHook", "module": 53, "phase": 6}, {"name": "CreateTarball", "module": 54, "phase": 9}, {"name": "RegisterImage", "module": 54, "phase": 9}, {"name": "UploadImage", "module": 54, "phase": 9}, {"name": "AdjustExpandRootDev", "module": 55, "phase": 6}, {"name": "DefaultPackages", "module": 56, "phase": 0}, {"name": "GooglePackages", "module": 56, "phase": 0}, {"name": "ReleasePackages", "module": 56, "phase": 0}, {"name": "DefaultPackages", "module": 57, "phase": 0}, {"name": "VirtIO", "module": 58, "phase": 6}, {"name": "AddGuestAdditionsPackages", "module": 59, "phase": 5}, {"name": "CheckGuestAdditionsPath", "module": 59, "phase": 0}, {"name": "InstallGuestAdditions", "module": 59, "phase": 5}, {"name": "DefaultPackages", "module": 60, "phase": 0}], "phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}]} \ No newline at end of file diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee index d9d3b94..4be9fe3 100644 --- a/docs/_static/taskoverview.coffee +++ b/docs/_static/taskoverview.coffee @@ -1,10 +1,10 @@ class window.TaskOverview viewBoxHeight = 800 - viewBoxWidth = 200 + viewBoxWidth = 800 margins = - top: 100 + top: 200 left: 50 - bottom: 100 + bottom: 200 right: 50 gravity = lateral: .1 diff --git a/tox.ini b/tox.ini index d428d52..0d736c1 100644 --- a/tox.ini +++ b/tox.ini @@ -19,5 +19,5 @@ commands = nosetests -v tests/integration --with-coverage --cover-package=bootst deps = sphinx commands = - sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html ./taskoverview.py --output docs/_static/graph.json + sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html From be5590f411d197a65d6a3cfb89648c0cd42bfe7b Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 15:06:41 +0200 Subject: [PATCH 04/19] Add highlighting of tasks in same module in taskoverview and a docs-serve tox target --- docs/_static/taskoverview.coffee | 14 +++++++++++--- docs/_static/taskoverview.less | 5 +++++ tox.ini | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee index 4be9fe3..ccae87d 100644 --- a/docs/_static/taskoverview.coffee +++ b/docs/_static/taskoverview.coffee @@ -118,15 +118,23 @@ class window.TaskOverview .selectAll('line').data(layout.links()).enter() .append('line').attr('marker-end', 'url(#right-arrowhead)') + mouseOver = (d) -> + labels.classed 'hover', (l) -> d is l + nodes.classed 'highlight', (n) -> d.module is n.module + + mouseOut = (d) -> + labels.classed 'hover', no + nodes.classed 'highlight', no + nodes = @svg.append('g').attr('class', 'nodes') .selectAll('g.partition').data(groups).enter() .append('g').attr('class', 'partition') .selectAll('circle').data((d) -> d.values).enter() .append('circle').attr('r', (d) -> d.radius) - .style('fill', (d, i) -> nodeColors(d[nodeColorKey])) + .style('fill', (d) -> nodeColors(d[nodeColorKey])) .call(layout.drag) - .on('mouseover', (d) -> (labels.filter (l) -> d is l).classed 'hover', true) - .on('mouseout', (d) -> (labels.filter (l) -> d is l).classed 'hover', false) + .on('mouseover', mouseOver) + .on('mouseout', mouseOut) labels = @svg.append('g').attr('class', 'node-labels') .selectAll('g.partition').data(groups).enter() diff --git a/docs/_static/taskoverview.less b/docs/_static/taskoverview.less index 2903816..8c1e67c 100644 --- a/docs/_static/taskoverview.less +++ b/docs/_static/taskoverview.less @@ -6,6 +6,11 @@ } g.nodes circle { stroke: #000000; + &.highlight { + stroke: #555599; + stroke-width: 2.5px; + fill: #EEAAAA !important; + } opacity: .9; stroke-width: 1.5px; } diff --git a/tox.ini b/tox.ini index 0d736c1..17db141 100644 --- a/tox.ini +++ b/tox.ini @@ -21,3 +21,7 @@ deps = commands = ./taskoverview.py --output docs/_static/graph.json sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html + +[testenv:docs-serve] +changedir = docs/_build/html +commands = python -m SimpleHTTPServer 8080 From d06fbb4a2d9278328cfa0693326e33828f5c5c26 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 15:11:50 +0200 Subject: [PATCH 05/19] Turn on nazi option for sphinx and fix warnings --- bootstrapvz/base/fs/volume.py | 1 + docs/base/pkg.rst | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bootstrapvz/base/fs/volume.py b/bootstrapvz/base/fs/volume.py index 0a17f61..043094a 100644 --- a/bootstrapvz/base/fs/volume.py +++ b/bootstrapvz/base/fs/volume.py @@ -70,6 +70,7 @@ class Volume(FSMProxy): rather than a loopback device or a network block device. :param _e_obj e: Event object containing arguments to create() + Keyword arguments to link_dm_node() are: :param int logical_start_sector: The sector the volume should start at in the new volume diff --git a/docs/base/pkg.rst b/docs/base/pkg.rst index 1a8db8b..4625358 100644 --- a/docs/base/pkg.rst +++ b/docs/base/pkg.rst @@ -16,7 +16,7 @@ Sources list :private-members: Preferences list ------------- +---------------- .. automodule:: bootstrapvz.base.pkg.preferenceslist :members: :private-members: diff --git a/tox.ini b/tox.ini index 17db141..e229a8d 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ deps = sphinx commands = ./taskoverview.py --output docs/_static/graph.json - sphinx-build -b html -d docs/_build/html/doctrees docs docs/_build/html + sphinx-build -W -b html -d docs/_build/html/doctrees docs docs/_build/html [testenv:docs-serve] changedir = docs/_build/html From 73972e1b60e8c41a4a09c02d7572c1bb53552bc7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 15:16:18 +0200 Subject: [PATCH 06/19] Move taskoverview exec into docs/ --- taskoverview.py => docs/taskoverview.py | 4 ++++ tox.ini | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) rename taskoverview.py => docs/taskoverview.py (95%) diff --git a/taskoverview.py b/docs/taskoverview.py similarity index 95% rename from taskoverview.py rename to docs/taskoverview.py index f9811b1..c1b142f 100755 --- a/taskoverview.py +++ b/docs/taskoverview.py @@ -1,4 +1,8 @@ #!/usr/bin/python +import sys +import os.path + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) def main(opts): diff --git a/tox.ini b/tox.ini index e229a8d..0ec323d 100644 --- a/tox.ini +++ b/tox.ini @@ -16,11 +16,12 @@ deps = commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive [testenv:docs] +changedir = docs deps = sphinx commands = - ./taskoverview.py --output docs/_static/graph.json - sphinx-build -W -b html -d docs/_build/html/doctrees docs docs/_build/html + ./taskoverview.py --output _static/graph.json + sphinx-build -W -b html -d _build/html/doctrees . _build/html [testenv:docs-serve] changedir = docs/_build/html From 953987fddf8f9dd5d3e7b82ffb76be2ce382fe8f Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:15:49 +0200 Subject: [PATCH 07/19] Combine documentation from github wiki with sphinx This is the first step in the effort of combining all documentation about bootstrap-vz into sphinx while still being able to access it from github (github can parse rst as well) --- bootstrapvz/plugins/admin_user/README.rst | 12 ++++ bootstrapvz/plugins/apt_proxy/README.rst | 21 ++++++ bootstrapvz/plugins/cloud_init/README.rst | 23 ++++++ bootstrapvz/plugins/docker_daemon/README.rst | 18 +++++ bootstrapvz/plugins/image_commands/README.rst | 19 +++++ bootstrapvz/plugins/minimize_size/README.rst | 37 ++++++++++ bootstrapvz/plugins/ntp/README.rst | 12 ++++ bootstrapvz/plugins/opennebula/README.md | 13 ---- bootstrapvz/plugins/opennebula/README.rst | 27 +++++++ bootstrapvz/plugins/pip_install/README.rst | 14 ++++ .../plugins/prebootstrapped/README.rst | 26 +++++++ bootstrapvz/plugins/puppet/README.rst | 24 +++++++ bootstrapvz/plugins/root_password/README.rst | 11 +++ bootstrapvz/plugins/salt/README.rst | 26 +++++++ .../plugins/unattended_upgrades/README.rst | 18 +++++ bootstrapvz/plugins/vagrant/README.rst | 12 ++++ .../providers/azure/{README.md => README.rst} | 28 +++++--- bootstrapvz/providers/ec2/README.rst | 71 +++++++++++++++++++ bootstrapvz/providers/gce/README.rst | 9 +++ bootstrapvz/providers/kvm/README.rst | 8 +++ bootstrapvz/providers/virtualbox/README.rst | 12 ++++ docs/conf.py | 20 ++++++ docs/plugins/.gitignore | 3 + docs/plugins/index.rst | 16 +++++ docs/providers/.gitignore | 3 + docs/providers/index.rst | 16 +++++ 26 files changed, 477 insertions(+), 22 deletions(-) create mode 100644 bootstrapvz/plugins/admin_user/README.rst create mode 100644 bootstrapvz/plugins/apt_proxy/README.rst create mode 100644 bootstrapvz/plugins/cloud_init/README.rst create mode 100644 bootstrapvz/plugins/docker_daemon/README.rst create mode 100644 bootstrapvz/plugins/image_commands/README.rst create mode 100644 bootstrapvz/plugins/minimize_size/README.rst create mode 100644 bootstrapvz/plugins/ntp/README.rst delete mode 100644 bootstrapvz/plugins/opennebula/README.md create mode 100644 bootstrapvz/plugins/opennebula/README.rst create mode 100644 bootstrapvz/plugins/pip_install/README.rst create mode 100644 bootstrapvz/plugins/prebootstrapped/README.rst create mode 100644 bootstrapvz/plugins/puppet/README.rst create mode 100644 bootstrapvz/plugins/root_password/README.rst create mode 100644 bootstrapvz/plugins/salt/README.rst create mode 100644 bootstrapvz/plugins/unattended_upgrades/README.rst create mode 100644 bootstrapvz/plugins/vagrant/README.rst rename bootstrapvz/providers/azure/{README.md => README.rst} (67%) create mode 100644 bootstrapvz/providers/ec2/README.rst create mode 100644 bootstrapvz/providers/gce/README.rst create mode 100644 bootstrapvz/providers/kvm/README.rst create mode 100644 bootstrapvz/providers/virtualbox/README.rst create mode 100644 docs/plugins/.gitignore create mode 100644 docs/providers/.gitignore diff --git a/bootstrapvz/plugins/admin_user/README.rst b/bootstrapvz/plugins/admin_user/README.rst new file mode 100644 index 0000000..3506f92 --- /dev/null +++ b/bootstrapvz/plugins/admin_user/README.rst @@ -0,0 +1,12 @@ +Admin user +---------- + +This plugin creates a user with passwordless sudo privileges. It also +disables the SSH root login. If the EC2 init scripts are installed, the +script for fetching the SSH authorized keys will be adjust to match the +username specified. + +Settings +~~~~~~~~ + +- ``username``: The username of the account to create. *``required``* diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst new file mode 100644 index 0000000..33711a7 --- /dev/null +++ b/bootstrapvz/plugins/apt_proxy/README.rst @@ -0,0 +1,21 @@ +APT Proxy +--------- + +This plugin creates a proxy configuration file for APT, so you could +enjoy the benefits of using cached packages instead of downloading them +from the mirror every time. You could just install ``apt-cacher-ng`` on +the host machine and then add ``"address": "127.0.0.1"`` and +``"port": 3142`` to the manifest file. + +Settings +~~~~~~~~ + +- ``address``: The IP or host of the proxy server. + *``required``* +- ``port``: The port (integer) of the proxy server. + *``required``* +- ``persistent``: Whether the proxy configuration file should remain on + the machine or not. + Valid values: true, false + Default: ``false`` + *``optional``* diff --git a/bootstrapvz/plugins/cloud_init/README.rst b/bootstrapvz/plugins/cloud_init/README.rst new file mode 100644 index 0000000..fa36e1f --- /dev/null +++ b/bootstrapvz/plugins/cloud_init/README.rst @@ -0,0 +1,23 @@ +cloud-init +---------- + +This plugin installs and configures +`cloud-init `__ +on the system. Depending on the release it installs it from either +backports or the main repository. + +cloud-init is only compatible with Debian wheezy and upwards. + +Settings +~~~~~~~~ + +- ``username``: The username of the account to create. + *``required``* +- ``disable_modules``: A list of strings specifying which cloud-init + modules should be disabled. + *``optional``* +- ``metadata_sources``: A string that sets the + `datasources `__ + that cloud-init should try fetching metadata from. The source is + automatically set when using the ec2 provider. + *``optional``* diff --git a/bootstrapvz/plugins/docker_daemon/README.rst b/bootstrapvz/plugins/docker_daemon/README.rst new file mode 100644 index 0000000..e210435 --- /dev/null +++ b/bootstrapvz/plugins/docker_daemon/README.rst @@ -0,0 +1,18 @@ +Docker daemon +------------- + +Install `docker `__ daemon in the image. Uses +init scripts for the official repository. + +This plugin can only be used if the distribution being bootstrapped is +at least ``wheezy``, as Docker needs a kernel version ``3.8`` or higher, +which is available at the ``wheezy-backports`` repository. There's also +an architecture requirement, as it runs only on ``amd64``. + +Settings +~~~~~~~~ + +- ``version``: Selects the docker version to install. To select the + latest version simply omit this setting. + Default: ``latest`` + *``optional``* diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/image_commands/README.rst new file mode 100644 index 0000000..106ad22 --- /dev/null +++ b/bootstrapvz/plugins/image_commands/README.rst @@ -0,0 +1,19 @@ +Image commands +-------------- + +The image commands plugin allows you to run arbitrary commands during +the bootstrap process. The commands are run at an indeterminate point +*after* packages have been installed, but *before* the volume has been +unmounted. + +Settings +~~~~~~~~ + +- ``commands``: A list of lists containing strings. Each top-level item + is a single command, while the strings inside each list comprise + parts of a command. This allows for proper shell argument escaping + (to circumvent this, simply put the entire command in a single + string). In addition to the manifest variables ``{root}`` is also + available. It points at the root of the image volume. + *``required``* + *``manifest vars``* diff --git a/bootstrapvz/plugins/minimize_size/README.rst b/bootstrapvz/plugins/minimize_size/README.rst new file mode 100644 index 0000000..e275237 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/README.rst @@ -0,0 +1,37 @@ +minimize size +------------- + +This plugin can be used to reduce the size of the resulting image. Often +virtual volumes are much smaller than their reported size until any data +is written to them. During the bootstrapping process temporary data like +the aptitude cache is written to the volume only to be removed again. + +The minimize size plugin employs three different strategies to keep a +low volume footprint: + +- Mount folders from the host into key locations of the image volume to + avoid any unneccesary disk writes. +- Use `zerofree `__ to + deallocate unused sectors on the volume. On an unpartitioned volume + this will be done for the entire volume, while it will only happen on + the root partition for partitioned volumes. +- Use + `vmware-vdiskmanager `__ + to shrink the real volume size (only applicable when using vmdk + backing). The tool is part of the `VMWare + Workstation `__ + package. + +Settings +~~~~~~~~ + +- ``zerofree``: Specifies if it should mark unallocated blocks as + zeroes, so the volume could be better shrunk after this. + Valid values: true, false + Default: false + *``optional``* +- ``shrink``: Whether the volume should be shrunk. This setting works + best in conjunction with the zerofree tool. + Valid values: true, false + Default: false + *``optional``* diff --git a/bootstrapvz/plugins/ntp/README.rst b/bootstrapvz/plugins/ntp/README.rst new file mode 100644 index 0000000..40421b1 --- /dev/null +++ b/bootstrapvz/plugins/ntp/README.rst @@ -0,0 +1,12 @@ +NTP +--- + +This plugins installs the Network Time Protocol daemon and optionally +defines which time servers it should use. + +Settings +~~~~~~~~ + +- ``servers``: A list of strings specifying which servers should be + used to synchronize the machine clock. + *``optional``* diff --git a/bootstrapvz/plugins/opennebula/README.md b/bootstrapvz/plugins/opennebula/README.md deleted file mode 100644 index 49ab0c9..0000000 --- a/bootstrapvz/plugins/opennebula/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Open Nebula provider - -This provider adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong). - -It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like: - - ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] - ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] - ETH0_IP $NIC[IP, NETWORK_ID=2] - ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] - ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] - FILES path_to_my_ssh_public_key.pub - diff --git a/bootstrapvz/plugins/opennebula/README.rst b/bootstrapvz/plugins/opennebula/README.rst new file mode 100644 index 0000000..a29d7ae --- /dev/null +++ b/bootstrapvz/plugins/opennebula/README.rst @@ -0,0 +1,27 @@ +Open Nebula +----------- + +This plugin adds `OpenNebula +contextualization `__ +to the image, which sets up the network configuration and SSH keys. + +The virtual machine context should be configured as follows: + +:: + + ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] + ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] + ETH0_IP $NIC[IP, NETWORK_ID=2] + ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] + ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] + FILES path_to_my_ssh_public_key.pub + +The plugin will install all *.pub* files in the root authorized\_keys +file. When using the ec2 provider, the USER\_EC2\_DATA will be executed +if present. + +Settings +~~~~~~~~ + +This plugin has no settings. To enable it add ``"opennebula":{}`` to the +plugin section of the manifest. diff --git a/bootstrapvz/plugins/pip_install/README.rst b/bootstrapvz/plugins/pip_install/README.rst new file mode 100644 index 0000000..3bed629 --- /dev/null +++ b/bootstrapvz/plugins/pip_install/README.rst @@ -0,0 +1,14 @@ +Pip install +----------- + +Install packages from the Python Package Index via pip. + +Installs ``build-essential`` and ``python-dev`` debian packages, so +Python extension modules can be built. + +Settings +~~~~~~~~ + +- ``packages``: Python packages to install, a list of strings. The list + can contain anything that ``pip install`` would accept as an + argument, for example ``awscli==1.3.13``. diff --git a/bootstrapvz/plugins/prebootstrapped/README.rst b/bootstrapvz/plugins/prebootstrapped/README.rst new file mode 100644 index 0000000..bc30de8 --- /dev/null +++ b/bootstrapvz/plugins/prebootstrapped/README.rst @@ -0,0 +1,26 @@ +prebootstrapped +--------------- + +When developing for bootstrap-vz, testing can be quite tedious since the +bootstrapping process can take a while. The prebootstrapped plugin +solves that problem by creating a snapshot of your volume right after +all the software has been installed. The next time bootstrap-vz is run, +the plugin replaces all volume preparation and bootstrapping tasks and +recreates the volume from the snapshot instead. + +The plugin assumes that the users knows what he is doing (e.g. it +doesn't check whether bootstrap-vz is being run with a partitioned +volume configuration, while the snapshot is unpartitioned). + +When no snapshot or image is specified the plugin creates one and +outputs its ID/path. Specifying an ID/path enables the second mode of +operation which recreates the volume from the specified snapshot instead +of creating it from scratch. + +Settings +~~~~~~~~ + +- ``snapshot``: ID of the EBS snapshot to use. This setting only works + with EBS backed EC2 configurations. +- ``image``: Path to the loopbackvolume snapshot. This setting works + with all configurable volume backings except EBS. diff --git a/bootstrapvz/plugins/puppet/README.rst b/bootstrapvz/plugins/puppet/README.rst new file mode 100644 index 0000000..203e0b3 --- /dev/null +++ b/bootstrapvz/plugins/puppet/README.rst @@ -0,0 +1,24 @@ +Puppet +------ + +Installs `puppet `__ and optionally applies a +manifest inside the chroot. You can also have it copy your puppet +configuration into the image so it is readily available once the image +is booted. + +Keep in mind that when applying a manifest, the system is in a chrooted +environment. This can prevent daemons from running properly (e.g. +listening to ports), they will also need to be shut down gracefully +(which bootstrap-vz cannot do) before unmounting the volume. It is +advisable to avoid starting any daemons inside the chroot at all. + +Settings +~~~~~~~~ + +- ``manifest``: Path to the puppet manifest that should be applied. + *``optional``* +- ``assets``: Path to puppet assets. The contents will be copied into + ``/etc/puppet`` on the image. Any existing files will be overwritten. + *``optional``* +- ``enable_agent``: Whether the puppet agent daemon should be enabled. + *``optional``* diff --git a/bootstrapvz/plugins/root_password/README.rst b/bootstrapvz/plugins/root_password/README.rst new file mode 100644 index 0000000..a24d904 --- /dev/null +++ b/bootstrapvz/plugins/root_password/README.rst @@ -0,0 +1,11 @@ +root password +------------- + +Sets the root password. This plugin removes the task that disables the +SSH password authentication. + +Settings +~~~~~~~~ + +- ``password``: The password for the root user. + *``required``* diff --git a/bootstrapvz/plugins/salt/README.rst b/bootstrapvz/plugins/salt/README.rst new file mode 100644 index 0000000..acb71ab --- /dev/null +++ b/bootstrapvz/plugins/salt/README.rst @@ -0,0 +1,26 @@ +Salt +---- + +Install `salt `__ minion in the image. Uses +`salt-bootstrap `__ script +to install. + +Settings +~~~~~~~~ + +- ``install_source``: Source to install salt codebase from. ``stable`` + for current stable, ``daily`` for installing the daily build, and + ``git`` to install from git repository. + *``required``* +- ``version``: Only needed if you are installing from ``git``. + \ ``develop`` to install current development head, or provide any tag + name or commit hash from `salt + repo `__ + *``optional``* +- ``master``: Salt master FQDN or IP + *``optional``* +- ``grains``: Set `salt + grains `__ + for this minion. Accepts a map with grain name as key and the grain + data as value. + *``optional``* diff --git a/bootstrapvz/plugins/unattended_upgrades/README.rst b/bootstrapvz/plugins/unattended_upgrades/README.rst new file mode 100644 index 0000000..3339b7b --- /dev/null +++ b/bootstrapvz/plugins/unattended_upgrades/README.rst @@ -0,0 +1,18 @@ +Unattended upgrades +------------------- + +Enables the `unattended update/upgrade +feature `__ in +aptitude. Enable it to have your system automatically download and +install security updates automatically with a set interval. + +Settings +~~~~~~~~ + +- ``update_interval``: Days between running ``apt-get update``. + *``required``* +- ``download_interval``: Days between running + ``apt-get upgrade --download-only`` + *``required``* +- ``upgrade_interval``: Days between installing any security upgrades. + *``required``* diff --git a/bootstrapvz/plugins/vagrant/README.rst b/bootstrapvz/plugins/vagrant/README.rst new file mode 100644 index 0000000..a40be4a --- /dev/null +++ b/bootstrapvz/plugins/vagrant/README.rst @@ -0,0 +1,12 @@ +Vagrant +------- + +Vagrant is a tool to quickly create virtualized environments. It uses +"boxes" to make downloading and sharing those environments easier. A box +is a tarball containing a virtual volumes accompanied by an `OVF +specification `__ +of the virtual machine. + +This plugin creates a vagrant box that is ready to be shared or +deployed. At the moment it is only compatible with the VirtualBox +provider and doesn't requires any additional settings. diff --git a/bootstrapvz/providers/azure/README.md b/bootstrapvz/providers/azure/README.rst similarity index 67% rename from bootstrapvz/providers/azure/README.md rename to bootstrapvz/providers/azure/README.rst index c03009e..165b557 100644 --- a/bootstrapvz/providers/azure/README.md +++ b/bootstrapvz/providers/azure/README.rst @@ -1,22 +1,28 @@ -Azure provider -=========== +Azure +===== -This provider generates raw images for Microsoft Azure computing platform. +This provider generates raw images for Microsoft Azure computing +platform. Setup -===== +----- qemu-img >= 1.7.0 required to convert raw image to vhd fixed size disk. This release is available in wheezy-backports. *wget* must be installed on local computer. - -Manifest must use the *raw* format, provider will automatically transform the disk to a vhd disk format. +Manifest must use the *raw* format, provider will automatically +transform the disk to a vhd disk format. Do not create swap space on the OS disk: -The Windows Azure Linux Agent can automatically configure swap space using the local resource disk that is attached to the VM after provisioning on Azure. Modify the following parameters in /etc/waagent.conf appropriately: +The Windows Azure Linux Agent can automatically configure swap space +using the local resource disk that is attached to the VM after +provisioning on Azure. Modify the following parameters in +/etc/waagent.conf appropriately: + +:: ResourceDisk.Format=y ResourceDisk.Filesystem=ext4 @@ -24,7 +30,10 @@ The Windows Azure Linux Agent can automatically configure swap space using the l ResourceDisk.EnableSwap=y ResourceDisk.SwapSizeMB=2048 ## NOTE: set this to whatever you need it to be. -You can specify a waagent.conf file to replace the default one in the manifest in the azure/waagent section of the provider: +You can specify a waagent.conf file to replace the default one in the +manifest in the azure/waagent section of the provider: + +:: "system" : { "waagent" : { @@ -33,4 +42,5 @@ You can specify a waagent.conf file to replace the default one in the manifest i } }, ... -Waagent versions are available at: https://github.com/Azure/WALinuxAgent/releases +Waagent versions are available at: +https://github.com/Azure/WALinuxAgent/releases diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst new file mode 100644 index 0000000..7c79201 --- /dev/null +++ b/bootstrapvz/providers/ec2/README.rst @@ -0,0 +1,71 @@ +EC2 +=== + +The `EC2 `__ provider automatically creates +a volume for bootstrapping (be it EBS or S3), makes a snapshot of it +once it is done and registers it as an AMI. EBS volume backing only +works on an EC2 host while S3 backed volumes *should* work locally (at +this time however they do not, a fix is in the works). + +Unless `the cloud-init plugin `__ is used, +special startup scripts will be installed that automatically fetch the +configured authorized\_key from the instance metadata and save or run +any userdata supplied (if the userdata begins with ``#!`` it will be +run). + +Credentials +----------- + +The AWS credentials can be configured in two ways: Via the manifest or +through environment variables. To bootstrap S3 backed instances you will +need a user certificate and a private key in addition to the access key +and secret key, which are needed for bootstraping EBS backed instances. + +The settings describes below should be placed in the ``credentials`` key +under the ``provider`` section. + +- **``access-key``**: AWS access-key. + May also be supplied via the environment variable + ``$AWS_ACCESS_KEY`` + *``required for EBS & S3 backing``* +- **``secret-key``**: AWS secret-key. + May also be supplied via the environment variable + ``$AWS_SECRET_KEY`` + *``required for EBS & S3 backing``* +- **``certificate``**: Path to the AWS user certificate. Used for + uploading the image to an S3 bucket. + May also be supplied via the environment variable + ``$AWS_CERTIFICATE`` + *``required for S3 backing``* +- **``private-key``**: Path to the AWS private key. Used for uploading + the image to an S3 bucket. + May also be supplied via the environment variable + ``$AWS_PRIVATE_KEY`` + *``required for S3 backing``* +- **``user-id``**: AWS user ID. Used for uploading the image to an S3 + bucket. + May also be supplied via the environment variable ``$AWS_USER_ID`` + *``required for S3 backing``* + +Example: + +.. code:: yaml + + --- + provider: + name: ec2 + virtualization: hvm + enhanced_networking: simple + credentials: + access-key: AFAKEACCESSKEYFORAWS + secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva + +Dependencies +------------ + +To communicate with the AWS API `boto `__ +is required (version 2.14.0 or higher) you can install boto with +``pip install boto`` (on wheezy, the packaged version is too low). S3 +images are chopped up and uploaded using +`euca2ools `__ (install with +``apt-get install euca2ools``). diff --git a/bootstrapvz/providers/gce/README.rst b/bootstrapvz/providers/gce/README.rst new file mode 100644 index 0000000..e760120 --- /dev/null +++ b/bootstrapvz/providers/gce/README.rst @@ -0,0 +1,9 @@ +Google Compute Engine +--------------------- + +The `GCE `__ provider +can creates image as expected by GCE - i.e. raw disk image in \*.tar.gz +file. It can upload created images to Google Storage Engine (to URI +provided in manifest by ``gcs_destination``) and can register image to +be used by Google Compute Engine to project provided in manifest by +``gce_project``. Both of those functionalities are not fully tested yet. diff --git a/bootstrapvz/providers/kvm/README.rst b/bootstrapvz/providers/kvm/README.rst new file mode 100644 index 0000000..57e60d4 --- /dev/null +++ b/bootstrapvz/providers/kvm/README.rst @@ -0,0 +1,8 @@ +KVM +--- + +The `KVM `__ provider creates +virtual images for Linux Kernel-based Virtual Machines. It supports the +installation of `virtio kernel +modules `__ (paravirtualized +drivers for IO operations). diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst new file mode 100644 index 0000000..b3e3d16 --- /dev/null +++ b/bootstrapvz/providers/virtualbox/README.rst @@ -0,0 +1,12 @@ +VirtualBox +---------- + +The `VirtualBox `__ provider can bootstrap +to both .vdi and .vmdk images (raw images are also supported but do not +run in VirtualBox). It's advisable to always use vmdk images for +interoperability (e.g. +`OVF `__ files +*should* support vdi files, but since they have no identifier URL not +even VirtualBox itself can import them). VirtualBox Guest Additions can +be installed automatically if the ISO is `provided in the +manifest `__. diff --git a/docs/conf.py b/docs/conf.py index 6eb7717..5abdcfc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,8 @@ import sys import os +import glob +import os.path # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -261,3 +263,21 @@ texinfo_documents = [('index', 'bootstrap-vz', u'bootstrap-vz Documentation', # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False + + +# -- Link to rst files scattered throughout the project ------------------- + +for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'): + provider_name = os.path.basename(os.path.dirname(readme_path)) + include_path = os.path.join('providers', provider_name + '.rst') + path_to_readme = os.path.join('../../bootstrapvz/providers', provider_name, 'README.rst') + with open(include_path, 'w') as include: + include.write('.. include:: ' + path_to_readme) + + +for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'): + plugin_name = os.path.basename(os.path.dirname(readme_path)) + include_path = os.path.join('plugins', plugin_name + '.rst') + path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst') + with open(include_path, 'w') as include: + include.write('.. include:: ' + path_to_readme) diff --git a/docs/plugins/.gitignore b/docs/plugins/.gitignore new file mode 100644 index 0000000..50db996 --- /dev/null +++ b/docs/plugins/.gitignore @@ -0,0 +1,3 @@ +* +!index.rst +!.gitignore diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 690ccbc..208c1d3 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -1,3 +1,19 @@ Plugins ======= + +Plugins are a key feature of bootstrap-vz. Despite their small size +(most plugins do not exceed 100 source lines of code) they can modify +the behavior of bootstrapped systems to a great extent. + +Below you will find documentation for all plugins available for +bootstrap-vz. If you cannot find what you are looking for, consider +`developing it yourself `__ and +contribute to this list! + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/providers/.gitignore b/docs/providers/.gitignore new file mode 100644 index 0000000..50db996 --- /dev/null +++ b/docs/providers/.gitignore @@ -0,0 +1,3 @@ +* +!index.rst +!.gitignore diff --git a/docs/providers/index.rst b/docs/providers/index.rst index ef70c42..fb991f2 100644 --- a/docs/providers/index.rst +++ b/docs/providers/index.rst @@ -1,3 +1,19 @@ Providers ========= + +Plugins are a key feature of bootstrap-vz. Despite their small size +(most plugins do not exceed 100 source lines of code) they can modify +the behavior of bootstrapped systems to a great extent. + +Below you will find documentation for all plugins available for +bootstrap-vz. If you cannot find what you are looking for, consider +`developing it yourself `__ and +contribute to this list! + + +.. toctree:: + :maxdepth: 1 + :glob: + + * From fc29266c9788ef3d9d9b4164da518eea790d5ca7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:17:50 +0200 Subject: [PATCH 08/19] Move API docs into api/ --- docs/{ => api}/base/fs.rst | 0 docs/{ => api}/base/index.rst | 4 ++-- docs/{ => api}/base/pkg.rst | 0 docs/{ => api}/common/fs.rst | 0 docs/{ => api}/common/index.rst | 0 docs/{ => api}/common/tasks/index.rst | 0 docs/api/index.rst | 9 +++++++++ docs/index.rst | 6 +++--- 8 files changed, 14 insertions(+), 5 deletions(-) rename docs/{ => api}/base/fs.rst (100%) rename docs/{ => api}/base/index.rst (98%) rename docs/{ => api}/base/pkg.rst (100%) rename docs/{ => api}/common/fs.rst (100%) rename docs/{ => api}/common/index.rst (100%) rename docs/{ => api}/common/tasks/index.rst (100%) create mode 100644 docs/api/index.rst diff --git a/docs/base/fs.rst b/docs/api/base/fs.rst similarity index 100% rename from docs/base/fs.rst rename to docs/api/base/fs.rst diff --git a/docs/base/index.rst b/docs/api/base/index.rst similarity index 98% rename from docs/base/index.rst rename to docs/api/base/index.rst index d80e702..c17d336 100644 --- a/docs/base/index.rst +++ b/docs/api/base/index.rst @@ -7,9 +7,9 @@ and handles the gather, sorting and running of tasks. .. toctree:: :maxdepth: 2 + :glob: - fs - pkg + * Bootstrap information --------------------- diff --git a/docs/base/pkg.rst b/docs/api/base/pkg.rst similarity index 100% rename from docs/base/pkg.rst rename to docs/api/base/pkg.rst diff --git a/docs/common/fs.rst b/docs/api/common/fs.rst similarity index 100% rename from docs/common/fs.rst rename to docs/api/common/fs.rst diff --git a/docs/common/index.rst b/docs/api/common/index.rst similarity index 100% rename from docs/common/index.rst rename to docs/api/common/index.rst diff --git a/docs/common/tasks/index.rst b/docs/api/common/tasks/index.rst similarity index 100% rename from docs/common/tasks/index.rst rename to docs/api/common/tasks/index.rst diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 0000000..1c7c3c7 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,9 @@ +API +=== + + +.. toctree:: + :maxdepth: 2 + + base/index + common/index diff --git a/docs/index.rst b/docs/index.rst index 5413191..a27b43c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,10 +11,10 @@ Contents: .. toctree:: :maxdepth: 2 - base/index - common/index - plugins/index providers/index + plugins/index + + api/index guidelines taskoverview howitworks From c089301f79dab9bdd35e2d0343a640adb36cf040 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:28:22 +0200 Subject: [PATCH 09/19] Generate task graph data through sphinx conf.py --- docs/__init__.py | 0 docs/_static/.gitignore | 1 + docs/_static/graph.json | 1 - docs/conf.py | 13 +++++++++++-- docs/taskoverview.py | 9 ++++----- tox.ini | 1 - 6 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 docs/__init__.py create mode 100644 docs/_static/.gitignore delete mode 100644 docs/_static/graph.json diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore new file mode 100644 index 0000000..4dd81db --- /dev/null +++ b/docs/_static/.gitignore @@ -0,0 +1 @@ +graph.json diff --git a/docs/_static/graph.json b/docs/_static/graph.json deleted file mode 100644 index 3485725..0000000 --- a/docs/_static/graph.json +++ /dev/null @@ -1 +0,0 @@ -{"modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.extlinux"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.grub"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.kernel"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.docker_daemon.tasks"}, {"name": "bootstrapvz.plugins.file_copy.tasks"}, {"name": "bootstrapvz.plugins.google_cloud_sdk.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.pip_install.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.initd"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "links": [{"target": 46, "definer": 15, "source": 15}, {"target": 53, "definer": 27, "source": 27}, {"target": 46, "definer": 28, "source": 28}, {"target": 41, "definer": 39, "source": 39}, {"target": 79, "definer": 41, "source": 41}, {"target": 53, "definer": 45, "source": 45}, {"target": 49, "definer": 47, "source": 47}, {"target": 49, "definer": 50, "source": 50}, {"target": 66, "definer": 51, "source": 51}, {"target": 46, "definer": 57, "source": 57}, {"target": 77, "definer": 58, "source": 58}, {"target": 66, "definer": 65, "source": 65}, {"target": 46, "definer": 67, "source": 67}, {"target": 35, "definer": 68, "source": 68}, {"target": 79, "definer": 70, "source": 70}, {"target": 49, "definer": 73, "source": 73}, {"target": 81, "definer": 78, "source": 78}, {"target": 5, "definer": 89, "source": 89}, {"target": 5, "definer": 95, "source": 95}, {"target": 49, "definer": 99, "source": 99}, {"target": 44, "definer": 100, "source": 100}, {"target": 45, "definer": 100, "source": 100}, {"target": 46, "definer": 109, "source": 109}, {"target": 4, "definer": 110, "source": 110}, {"target": 70, "definer": 112, "source": 112}, {"target": 79, "definer": 112, "source": 112}, {"target": 66, "definer": 113, "source": 113}, {"target": 77, "definer": 119, "source": 119}, {"target": 155, "definer": 120, "source": 120}, {"target": 81, "definer": 141, "source": 141}, {"target": 44, "definer": 143, "source": 143}, {"target": 45, "definer": 143, "source": 143}, {"target": 81, "definer": 150, "source": 150}, {"target": 46, "definer": 159, "source": 159}, {"target": 49, "definer": 162, "source": 162}, {"target": 53, "definer": 165, "source": 165}, {"target": 4, "definer": 167, "source": 167}, {"target": 61, "definer": 167, "source": 167}, {"target": 12, "definer": 168, "source": 168}, {"target": 44, "definer": 170, "source": 170}, {"target": 45, "definer": 170, "source": 170}, {"target": 66, "definer": 183, "source": 183}, {"target": 0, "definer": 0, "source": 1}, {"target": 1, "definer": 1, "source": 3}, {"target": 5, "definer": 5, "source": 54}, {"target": 5, "definer": 5, "source": 12}, {"target": 5, "definer": 5, "source": 11}, {"target": 6, "definer": 6, "source": 5}, {"target": 6, "definer": 6, "source": 7}, {"target": 11, "definer": 11, "source": 12}, {"target": 12, "definer": 12, "source": 9}, {"target": 16, "definer": 16, "source": 19}, {"target": 23, "definer": 23, "source": 1}, {"target": 26, "definer": 26, "source": 34}, {"target": 26, "definer": 26, "source": 24}, {"target": 27, "definer": 27, "source": 34}, {"target": 27, "definer": 27, "source": 25}, {"target": 29, "definer": 29, "source": 1}, {"target": 30, "definer": 30, "source": 38}, {"target": 31, "definer": 31, "source": 37}, {"target": 33, "definer": 33, "source": 41}, {"target": 36, "definer": 36, "source": 31}, {"target": 37, "definer": 37, "source": 32}, {"target": 38, "definer": 38, "source": 16}, {"target": 40, "definer": 40, "source": 35}, {"target": 42, "definer": 42, "source": 1}, {"target": 43, "definer": 43, "source": 34}, {"target": 44, "definer": 44, "source": 34}, {"target": 45, "definer": 45, "source": 34}, {"target": 48, "definer": 48, "source": 49}, {"target": 52, "definer": 52, "source": 66}, {"target": 64, "definer": 64, "source": 1}, {"target": 65, "definer": 65, "source": 5}, {"target": 66, "definer": 66, "source": 6}, {"target": 68, "definer": 68, "source": 69}, {"target": 70, "definer": 70, "source": 41}, {"target": 72, "definer": 72, "source": 1}, {"target": 82, "definer": 82, "source": 1}, {"target": 83, "definer": 83, "source": 49}, {"target": 90, "definer": 90, "source": 1}, {"target": 93, "definer": 93, "source": 1}, {"target": 93, "definer": 93, "source": 0}, {"target": 95, "definer": 95, "source": 54}, {"target": 100, "definer": 100, "source": 43}, {"target": 100, "definer": 100, "source": 170}, {"target": 101, "definer": 101, "source": 97}, {"target": 102, "definer": 102, "source": 103}, {"target": 108, "definer": 108, "source": 16}, {"target": 111, "definer": 111, "source": 79}, {"target": 112, "definer": 112, "source": 41}, {"target": 115, "definer": 115, "source": 1}, {"target": 115, "definer": 115, "source": 0}, {"target": 116, "definer": 116, "source": 1}, {"target": 118, "definer": 118, "source": 66}, {"target": 118, "definer": 118, "source": 185}, {"target": 121, "definer": 121, "source": 66}, {"target": 121, "definer": 121, "source": 185}, {"target": 122, "definer": 122, "source": 1}, {"target": 123, "definer": 123, "source": 126}, {"target": 129, "definer": 129, "source": 66}, {"target": 130, "definer": 130, "source": 1}, {"target": 132, "definer": 132, "source": 1}, {"target": 134, "definer": 134, "source": 138}, {"target": 135, "definer": 135, "source": 1}, {"target": 137, "definer": 137, "source": 80}, {"target": 137, "definer": 137, "source": 136}, {"target": 145, "definer": 145, "source": 1}, {"target": 146, "definer": 146, "source": 66}, {"target": 147, "definer": 147, "source": 153}, {"target": 149, "definer": 149, "source": 157}, {"target": 149, "definer": 149, "source": 151}, {"target": 151, "definer": 151, "source": 148}, {"target": 153, "definer": 153, "source": 154}, {"target": 153, "definer": 153, "source": 160}, {"target": 153, "definer": 153, "source": 161}, {"target": 155, "definer": 155, "source": 156}, {"target": 163, "definer": 163, "source": 1}, {"target": 166, "definer": 166, "source": 1}, {"target": 168, "definer": 168, "source": 9}, {"target": 169, "definer": 169, "source": 3}, {"target": 169, "definer": 169, "source": 0}, {"target": 172, "definer": 172, "source": 60}, {"target": 174, "definer": 174, "source": 59}, {"target": 175, "definer": 175, "source": 176}, {"target": 176, "definer": 176, "source": 174}, {"target": 177, "definer": 177, "source": 47}, {"target": 177, "definer": 177, "source": 48}, {"target": 178, "definer": 178, "source": 1}, {"target": 179, "definer": 179, "source": 178}, {"target": 180, "definer": 180, "source": 1}, {"target": 180, "definer": 180, "source": 0}, {"target": 180, "definer": 180, "source": 178}, {"target": 181, "definer": 181, "source": 1}, {"target": 185, "definer": 185, "source": 66}, {"target": 186, "definer": 186, "source": 1}], "nodes": [{"name": "AddBackports", "module": 0, "phase": 0}, {"name": "AddDefaultSources", "module": 0, "phase": 0}, {"name": "AddManifestPreferences", "module": 0, "phase": 0}, {"name": "AddManifestSources", "module": 0, "phase": 0}, {"name": "AptClean", "module": 0, "phase": 7}, {"name": "AptUpdate", "module": 0, "phase": 5}, {"name": "AptUpgrade", "module": 0, "phase": 5}, {"name": "DisableDaemonAutostart", "module": 0, "phase": 5}, {"name": "EnableDaemonAutostart", "module": 0, "phase": 7}, {"name": "InstallTrustedKeys", "module": 0, "phase": 5}, {"name": "PurgeUnusedPackages", "module": 0, "phase": 7}, {"name": "WritePreferences", "module": 0, "phase": 5}, {"name": "WriteSources", "module": 0, "phase": 5}, {"name": "BlackListModules", "module": 1, "phase": 6}, {"name": "DisableGetTTYs", "module": 1, "phase": 6}, {"name": "AddRequiredCommands", "module": 2, "phase": 0}, {"name": "Bootstrap", "module": 2, "phase": 4}, {"name": "ExcludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "IncludePackagesInBootstrap", "module": 2, "phase": 0}, {"name": "MakeTarball", "module": 2, "phase": 4}, {"name": "CleanTMP", "module": 3, "phase": 7}, {"name": "ClearMOTD", "module": 3, "phase": 7}, {"name": "TriggerRollback", "module": 4, "phase": 10}, {"name": "AddExtlinuxPackage", "module": 5, "phase": 0}, {"name": "ConfigureExtlinux", "module": 5, "phase": 6}, {"name": "ConfigureExtlinuxJessie", "module": 5, "phase": 6}, {"name": "InstallExtlinux", "module": 5, "phase": 6}, {"name": "InstallExtlinuxJessie", "module": 5, "phase": 6}, {"name": "AddRequiredCommands", "module": 6, "phase": 0}, {"name": "AddXFSProgs", "module": 6, "phase": 0}, {"name": "CopyMountTable", "module": 6, "phase": 4}, {"name": "CreateBootMountDir", "module": 6, "phase": 3}, {"name": "CreateMountDir", "module": 6, "phase": 3}, {"name": "DeleteMountDir", "module": 6, "phase": 8}, {"name": "FStab", "module": 6, "phase": 6}, {"name": "Format", "module": 6, "phase": 2}, {"name": "MountBoot", "module": 6, "phase": 3}, {"name": "MountRoot", "module": 6, "phase": 3}, {"name": "MountSpecials", "module": 6, "phase": 4}, {"name": "RemoveMountTable", "module": 6, "phase": 8}, {"name": "TuneVolumeFS", "module": 6, "phase": 2}, {"name": "UnmountRoot", "module": 6, "phase": 8}, {"name": "AddGrubPackage", "module": 7, "phase": 0}, {"name": "ConfigureGrub", "module": 7, "phase": 6}, {"name": "InstallGrub_1_99", "module": 7, "phase": 6}, {"name": "InstallGrub_2", "module": 7, "phase": 6}, {"name": "CheckExternalCommands", "module": 8, "phase": 0}, {"name": "AddExpandRoot", "module": 9, "phase": 6}, {"name": "AdjustExpandRootScript", "module": 9, "phase": 6}, {"name": "InstallInitScripts", "module": 9, "phase": 6}, {"name": "RemoveHWClock", "module": 9, "phase": 6}, {"name": "AddDKMSPackages", "module": 10, "phase": 5}, {"name": "DetermineKernelVersion", "module": 10, "phase": 5}, {"name": "UpdateInitramfs", "module": 10, "phase": 6}, {"name": "GenerateLocale", "module": 11, "phase": 5}, {"name": "LocaleBootstrapPackage", "module": 11, "phase": 0}, {"name": "SetTimezone", "module": 11, "phase": 6}, {"name": "AddRequiredCommands", "module": 12, "phase": 0}, {"name": "Create", "module": 12, "phase": 1}, {"name": "MoveImage", "module": 12, "phase": 9}, {"name": "ConfigureNetworkIF", "module": 13, "phase": 6}, {"name": "RemoveDNSInfo", "module": 13, "phase": 7}, {"name": "RemoveHostname", "module": 13, "phase": 7}, {"name": "SetHostname", "module": 13, "phase": 6}, {"name": "AddManifestPackages", "module": 14, "phase": 0}, {"name": "AddTaskselStandardPackages", "module": 14, "phase": 5}, {"name": "InstallPackages", "module": 14, "phase": 5}, {"name": "AddRequiredCommands", "module": 15, "phase": 0}, {"name": "MapPartitions", "module": 15, "phase": 2}, {"name": "PartitionVolume", "module": 15, "phase": 2}, {"name": "UnmapPartitions", "module": 15, "phase": 8}, {"name": "EnableShadowConfig", "module": 16, "phase": 6}, {"name": "AddOpenSSHPackage", "module": 17, "phase": 0}, {"name": "AddSSHKeyGeneration", "module": 17, "phase": 6}, {"name": "DisableSSHDNSLookup", "module": 17, "phase": 6}, {"name": "DisableSSHPasswordAuthentication", "module": 17, "phase": 6}, {"name": "ShredHostkeys", "module": 17, "phase": 7}, {"name": "Attach", "module": 18, "phase": 1}, {"name": "Delete", "module": 18, "phase": 10}, {"name": "Detach", "module": 18, "phase": 8}, {"name": "CreateWorkspace", "module": 19, "phase": 0}, {"name": "DeleteWorkspace", "module": 19, "phase": 10}, {"name": "AddSudoPackage", "module": 20, "phase": 0}, {"name": "AdminUserCredentials", "module": 20, "phase": 6}, {"name": "CreateAdminUser", "module": 20, "phase": 6}, {"name": "DisableRootLogin", "module": 20, "phase": 6}, {"name": "PasswordlessSudo", "module": 20, "phase": 6}, {"name": "CheckAptProxy", "module": 21, "phase": 0}, {"name": "RemoveAptProxy", "module": 21, "phase": 7}, {"name": "SetAptProxy", "module": 21, "phase": 5}, {"name": "AddPackages", "module": 22, "phase": 0}, {"name": "CheckAssetsPath", "module": 22, "phase": 0}, {"name": "CopyChefAssets", "module": 22, "phase": 6}, {"name": "AddCloudInitPackages", "module": 23, "phase": 0}, {"name": "DisableModules", "module": 23, "phase": 6}, {"name": "SetMetadataSource", "module": 23, "phase": 5}, {"name": "SetUsername", "module": 23, "phase": 6}, {"name": "AddDockerBinary", "module": 24, "phase": 6}, {"name": "AddDockerDeps", "module": 24, "phase": 5}, {"name": "AddDockerInit", "module": 24, "phase": 6}, {"name": "EnableMemoryCgroup", "module": 24, "phase": 6}, {"name": "PullDockerImages", "module": 24, "phase": 6}, {"name": "FileCopyCommand", "module": 25, "phase": 6}, {"name": "MkdirCommand", "module": 25, "phase": 6}, {"name": "ValidateSourcePaths", "module": 25, "phase": 0}, {"name": "InstallCloudSDK", "module": 26, "phase": 6}, {"name": "RemoveCloudSDKTarball", "module": 26, "phase": 7}, {"name": "ImageExecuteCommand", "module": 27, "phase": 6}, {"name": "AddFolderMounts", "module": 28, "phase": 4}, {"name": "AddRequiredCommands", "module": 28, "phase": 0}, {"name": "RemoveFolderMounts", "module": 28, "phase": 7}, {"name": "ShrinkVolume", "module": 28, "phase": 8}, {"name": "Zerofree", "module": 28, "phase": 8}, {"name": "AddNtpPackage", "module": 29, "phase": 5}, {"name": "SetNtpServers", "module": 29, "phase": 6}, {"name": "AddONEContextPackage", "module": 30, "phase": 0}, {"name": "AddPipPackage", "module": 31, "phase": 0}, {"name": "PipInstallCommand", "module": 31, "phase": 6}, {"name": "CopyImage", "module": 32, "phase": 5}, {"name": "CreateFromImage", "module": 32, "phase": 1}, {"name": "CreateFromSnapshot", "module": 32, "phase": 1}, {"name": "Snapshot", "module": 32, "phase": 5}, {"name": "AddPackages", "module": 33, "phase": 0}, {"name": "ApplyPuppetManifest", "module": 33, "phase": 6}, {"name": "CheckAssetsPath", "module": 33, "phase": 0}, {"name": "CheckManifestPath", "module": 33, "phase": 0}, {"name": "CopyPuppetAssets", "module": 33, "phase": 6}, {"name": "EnableAgent", "module": 33, "phase": 6}, {"name": "SetRootPassword", "module": 34, "phase": 6}, {"name": "BootstrapSaltMinion", "module": 35, "phase": 5}, {"name": "InstallSaltDependencies", "module": 35, "phase": 0}, {"name": "SetSaltGrains", "module": 35, "phase": 6}, {"name": "AddUnattendedUpgradesPackage", "module": 36, "phase": 0}, {"name": "EnablePeriodicUpgrades", "module": 36, "phase": 6}, {"name": "AddInsecurePublicKey", "module": 37, "phase": 6}, {"name": "AddPackages", "module": 37, "phase": 0}, {"name": "CheckBoxPath", "module": 37, "phase": 0}, {"name": "CreateVagrantBoxDir", "module": 37, "phase": 0}, {"name": "CreateVagrantUser", "module": 37, "phase": 6}, {"name": "PackageBox", "module": 37, "phase": 9}, {"name": "PasswordlessSudo", "module": 37, "phase": 6}, {"name": "RemoveVagrantBoxDir", "module": 37, "phase": 10}, {"name": "SetRootPassword", "module": 37, "phase": 6}, {"name": "ConfigureGrub", "module": 38, "phase": 6}, {"name": "ConvertToVhd", "module": 39, "phase": 9}, {"name": "DefaultPackages", "module": 40, "phase": 0}, {"name": "Waagent", "module": 40, "phase": 5}, {"name": "AMIName", "module": 41, "phase": 0}, {"name": "BundleImage", "module": 41, "phase": 9}, {"name": "RegisterAMI", "module": 41, "phase": 9}, {"name": "RemoveBundle", "module": 41, "phase": 10}, {"name": "UploadImage", "module": 41, "phase": 9}, {"name": "ConfigurePVGrub", "module": 42, "phase": 6}, {"name": "Connect", "module": 43, "phase": 0}, {"name": "GetCredentials", "module": 43, "phase": 0}, {"name": "Attach", "module": 44, "phase": 1}, {"name": "Create", "module": 44, "phase": 1}, {"name": "Snapshot", "module": 44, "phase": 9}, {"name": "S3FStab", "module": 45, "phase": 6}, {"name": "AddExternalCommands", "module": 46, "phase": 0}, {"name": "GetInstanceMetadata", "module": 46, "phase": 0}, {"name": "SetRegion", "module": 46, "phase": 0}, {"name": "AddEC2InitScripts", "module": 47, "phase": 6}, {"name": "AddBuildEssentialPackage", "module": 48, "phase": 0}, {"name": "EnableDHCPCDDNS", "module": 48, "phase": 6}, {"name": "InstallEnhancedNetworking", "module": 48, "phase": 6}, {"name": "DefaultPackages", "module": 49, "phase": 0}, {"name": "CleanGoogleRepositoriesAndKeys", "module": 50, "phase": 7}, {"name": "ImportGoogleKey", "module": 50, "phase": 5}, {"name": "SetPackageRepositories", "module": 50, "phase": 0}, {"name": "ConfigureGrub", "module": 51, "phase": 6}, {"name": "GatherReleaseInformation", "module": 52, "phase": 6}, {"name": "DisableIPv6", "module": 53, "phase": 6}, {"name": "InstallHostnameHook", "module": 53, "phase": 6}, {"name": "CreateTarball", "module": 54, "phase": 9}, {"name": "RegisterImage", "module": 54, "phase": 9}, {"name": "UploadImage", "module": 54, "phase": 9}, {"name": "AdjustExpandRootDev", "module": 55, "phase": 6}, {"name": "DefaultPackages", "module": 56, "phase": 0}, {"name": "GooglePackages", "module": 56, "phase": 0}, {"name": "ReleasePackages", "module": 56, "phase": 0}, {"name": "DefaultPackages", "module": 57, "phase": 0}, {"name": "VirtIO", "module": 58, "phase": 6}, {"name": "AddGuestAdditionsPackages", "module": 59, "phase": 5}, {"name": "CheckGuestAdditionsPath", "module": 59, "phase": 0}, {"name": "InstallGuestAdditions", "module": 59, "phase": 5}, {"name": "DefaultPackages", "module": 60, "phase": 0}], "phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}]} \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 5abdcfc..5e1d3e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,6 @@ import sys import os -import glob -import os.path # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -267,6 +265,9 @@ texinfo_documents = [('index', 'bootstrap-vz', u'bootstrap-vz Documentation', # -- Link to rst files scattered throughout the project ------------------- +import glob +import os.path + for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'): provider_name = os.path.basename(os.path.dirname(readme_path)) include_path = os.path.join('providers', provider_name + '.rst') @@ -281,3 +282,11 @@ for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'): path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst') with open(include_path, 'w') as include: include.write('.. include:: ' + path_to_readme) + + +# -- Create task overview graph data -------------------------------------- + +from docs import taskoverview + +data = taskoverview.generate_graph_data() +taskoverview.write_data(data, '_static/graph.json') diff --git a/docs/taskoverview.py b/docs/taskoverview.py index c1b142f..97ecbb0 100755 --- a/docs/taskoverview.py +++ b/docs/taskoverview.py @@ -5,7 +5,7 @@ import os.path sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -def main(opts): +def generate_graph_data(): from bootstrapvz.base.tasklist import get_all_tasks tasks = get_all_tasks() @@ -49,13 +49,11 @@ def main(opts): link[key] = tasks.index(link[key]) return link - data = {'phases': map(mk_phase, phases.order), + return {'phases': map(mk_phase, phases.order), 'modules': map(mk_module, modules), 'nodes': map(mk_node, tasks), 'links': map(mk_link, task_links)} - write_data(data, opts.get('--output', None)) - def write_data(data, output_path=None): import json @@ -77,4 +75,5 @@ if __name__ == '__main__' and __package__ is None: """ opts = docopt(usage) - main(opts) + data = generate_graph_data() + write_data(data, opts.get('--output', None)) diff --git a/tox.ini b/tox.ini index 0ec323d..e538b91 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,6 @@ changedir = docs deps = sphinx commands = - ./taskoverview.py --output _static/graph.json sphinx-build -W -b html -d _build/html/doctrees . _build/html [testenv:docs-serve] From f9b234e3e64037285fb06bdb8d9e6752d9b634de Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:28:39 +0200 Subject: [PATCH 10/19] Remove docs-serve testenv again It can cause some weird errors when running two tox invocations at the same time --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index e538b91..64ef939 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,3 @@ deps = sphinx commands = sphinx-build -W -b html -d _build/html/doctrees . _build/html - -[testenv:docs-serve] -changedir = docs/_build/html -commands = python -m SimpleHTTPServer 8080 From 34a87acf1696081ae1f88a62496ff6ccc24e8b34 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:40:16 +0200 Subject: [PATCH 11/19] Add documentation about the manifest --- docs/index.rst | 1 + docs/manifest.rst | 1 + manifests/README.rst | 241 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 docs/manifest.rst create mode 100644 manifests/README.rst diff --git a/docs/index.rst b/docs/index.rst index a27b43c..16a5568 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Contents: .. toctree:: :maxdepth: 2 + manifest providers/index plugins/index diff --git a/docs/manifest.rst b/docs/manifest.rst new file mode 100644 index 0000000..d8b3cef --- /dev/null +++ b/docs/manifest.rst @@ -0,0 +1 @@ +.. include:: ../manifests/README.rst diff --git a/manifests/README.rst b/manifests/README.rst new file mode 100644 index 0000000..3d5c1b5 --- /dev/null +++ b/manifests/README.rst @@ -0,0 +1,241 @@ +Manifest +======== + +| The manifest file is the primary way to interact with bootstrap-vz. +| Every configuration and customization of a Debian installation is + specified in this file. + +The manifest format is YAML or JSON. It is near impossible to run the +bootstrapper with an invalid configuration, since every part of the +framework supplies a `json-schema `__ that +specifies exactly which configuration settings are valid in different +situations. + +Manifest variables +------------------ + +Many of the settings in the example manifests use strings like +``debian-{system.release}-{system.architecture}-{{"{%y"}}}{{"{%m"}}}{{"{%d"}}}``. +These strings make use of manifest variables, which can cross reference +other settings in the manifest or specific values supplied by the +bootstrapper (e.g. all python date formatting variables are available). + +Any reference uses dots to specify a path to the desired manifest +setting. Not all settings support this though, to see whether embedding +a manifest variable in a setting is possible, look for the +*``manifest vars``* label. + +Sections +-------- + +The manifest is split into 7 sections. + +Provider +~~~~~~~~ + +The provider section contains all provider specific settings and the +name of the provider itself. + +- **``name``**: target virtualization platform of the installation + *``required``* + +Consult the `providers `__ section of the documentation +for a list of valid values. + +Bootstrapper +~~~~~~~~~~~~ + +This section concerns the bootstrapper itself and its behavior. There +are 4 possible settings: + +- **``workspace``**: Path to where the bootstrapper should place images + and intermediate files. Any volumes will be mounted under that path. + *``required``* +- **``tarball``**: debootstrap has the option to download all the + software and pack it up in a tarball. When starting the actual + bootstrapping process, debootstrap can then be pointed at that + tarball and use it instead of downloading anything from the internet. + If you plan on running the bootstrapper multiple times, this option + can save you a lot of bandwidth and time. This option just specifies + whether it should create a new tarball or not. It will search for and + use an available tarball if it already exists, regardless of this + setting. + *``optional``* + Valid values: ``true, false`` + Default: ``false`` +- **``mirror``**: The mirror debootstrap should download software from. + It is advisable to specify a mirror close to your location (or the + location of the host you are bootstrapping on), to decrease latency + and improve bandwidth. If not specified, `the configured aptitude + mirror URL <#packages>`__ is used. + *``optional``* +- **``include_packages``**: Extra packages to be installed during + bootstrap. Accepts a list of package names. + *``optional``* +- **``exclude_packages``**: Packages to exclude during bootstrap phase. + Accepts a list of package names. + *``optional``* +- **``guest_additions``**: This setting is only relevant for the + `virtualbox provider `__. It specifies the + path to the VirtualBox Guest Additions ISO, which, when specified, + will be mounted and used to install the VirtualBox Guest Additions. + *``optional``* + +Image +~~~~~ + +The image section configures anything pertaining directly to the image +that will be created. + +- **``name``**: The name of the resulting image. + When bootstrapping cloud images, this would be the name visible in + the interface when booting up new instances. + When bootstrapping for VirtualBox or kvm, it's the filename of the + image. + *``required``* + *``manifest vars``* +- **``description``**: Description of the image. Where this setting is + used depends highly on which provider is set. At the moment it is + only used for AWS images. + **``required for ec2 provider``** + *``manifest vars``* +- **``bucket``**: When bootstrapping an S3 backed image for AWS, this + will be the bucket where the image is uploaded to. + **``required for S3 backing``** +- **``region``**: Region in which the AMI should be registered. + **``required for S3 backing``** + +System +~~~~~~ + +This section defines anything that pertains directly to the bootstrapped +system and does not fit under any other section. + +- **``architecture``**: The architecture of the system. + Valid values: ``i386, amd64`` + *``required``* +- **``bootloader``**: The bootloader for the system. Depending on the + bootmethod of the virtualization platform, the options may be + restricted. + Valid values: ``grub, extlinux, pv-grub`` + *``required``* +- **``charmap``**: The default charmap of the system. + Valid values: Any valid charmap like ``UTF-8``, ``ISO-8859-*`` or + ``GBK``. + *``required``* +- **``hostname``**: hostname to preconfigure the system with. + *``optional``* +- **``locale``**: The default locale of the system. + Valid values: Any locale mentioned in ``/etc/locale.gen`` + *``required``* +- **``release``**: Defines which debian release should be bootstrapped. + Valid values: ``squeeze``, ``wheezy``, ``jessie``, ``sid``, + ``oldstable``, ``stable``, ``testing``, ``unstable`` + *``required``* +- **``timezone``**: Timezone of the system. + Valid values: Any filename from ``/usr/share/zoneinfo`` + *``required``* + +Packages +~~~~~~~~ + +The packages section allows you to install custom packages from a +variety of sources. + +- **``install``**: A list of strings that specify which packages should + be installed. Valid values: package names optionally followed by a + ``/target`` or paths to local ``.deb`` files. +- **``install_standard``**: Defines if the packages of the + ``"Standard System Utilities"`` option of the Debian installer, + provided by `tasksel `__, should be + installed or not. The problem is that with just ``debootstrap``, the + system ends up with very basic commands. This is not a problem for a + machine that will not be used interactively, but otherwise it is nice + to have at hand tools like ``bash-completion``, ``less``, ``locate``, + etc. + *``optional``* + Valid values: ``true``, ``false`` + Default: ``false`` +- **``mirror``**: The default aptitude mirror. + *``optional``* + Default: ``http://http.debian.net/debian`` +- **``sources``**: A map of additional sources that should be added to + the aptitude sources list. The key becomes the filename in + ``/etc/apt/sources.list.d/`` (with ``.list`` appended to it), while + the value is an array with each entry being a line. + *``optional``* +- **``components``**: A list of components that should be added to the + default apt sources. For example ``contrib`` or ``non-free`` + *``optional``* + Default: ``['main']`` +- **``trusted-keys``**: List of paths to ``.gpg`` keyrings that should + be added to the aptitude keyring of trusted signatures for + repositories. + *``optional``* +- **``preferences``**: Allows you to pin packages through `apt + preferences `__. The setting + is an object where the key is the preference filename in + ``/etc/apt/preferences.d/``. The key ``main`` is special and refers + to the file ``/etc/apt/preferences``, which will be overwritten if + specified. + *``optional``* + The values are objects with three keys: +- **``package``**: The package to pin (wildcards allowed) +- **``pin``**: The release to pin the package to. +- **``pin-priority``**: The priority of this pin. + +Volume +~~~~~~ + +bootstrap-vz allows a wide range of options for configuring the disk +layout of the system. It can create unpartitioned as well as partitioned +volumes using either the gpt or msdos scheme. At most, there are only +three partitions with predefined roles configurable though. They are +boot, root and swap. + +- **``backing``**: Specifies the volume backing. This setting is very + provider specific. + Valid values: ``ebs``, ``s3``, ``vmdk``, ``vdi``, ``raw`` + *``required``* +- **``partitions``**: A map of the partitions that should be created on + the volume. +- **``type``**: The partitioning scheme to use. When using ``none``, + only root can be specified as a partition. + Valid values: ``none``, ``gpt``, ``msdos`` + *``required``* +- **``root``**: Configuration of the root partition. *``required``* + + - **``size``**: The size of the partition. Valid values: Any + datasize specification up to TB (e.g. 5KiB, 1MB, 6TB). + *``required``* + - **``filesystem``**: The filesystem of the partition. When choosing + ``xfs``, the ``xfsprogs`` package will need to be installed. + Valid values: ``ext2``, ``ext3``, ``ext4``, ``xfs`` + *``required``* + - **``format_command``**: Command to format the partition with. This + optional setting overrides the command bootstrap-vz would normally + use to format the partition. The command is specified as a string + array where each option/argument is an item in that array (much + like the `image\_commands `__ plugin). + *``optional``* The following variables are available: + - **``{fs}``**: The filesystem of the partition. + - **``{device_path}``**: The device path of the partition. + - **``{size}``**: The size of the partition. + + The default command used by boostrap-vz is + ``['mkfs.{fs}', '{device_path}']``. + + - **``boot``**: Configuration of the boot partition. The three + settings equal those of the root partition. + *``optional``* + - **``swap``**: Configuration of the swap partition. Since the swap + partition has its own filesystem you can only specify the size for + this partition. + *``optional``* + +Plugins +~~~~~~~ + +The plugins section is a map of plugin names to whatever configuration a +plugin requires. Go to the `plugin section `__ of the +documentation, to see the configuration for a specific plugin. From 79ba5577778a1df62f277885687753599d9f2ba7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 16:58:49 +0200 Subject: [PATCH 12/19] Convert README.md into rst and include from the docs --- README.md | 45 ------------ README.rst | 145 +++++++++++++++++++++++++++++++++++++++ docs/index.rst | 16 ++--- docs/intro.rst | 1 + docs/providers/index.rst | 1 - 5 files changed, 150 insertions(+), 58 deletions(-) delete mode 100644 README.md create mode 100644 README.rst create mode 100644 docs/intro.rst diff --git a/README.md b/README.md deleted file mode 100644 index 8eb57fe..0000000 --- a/README.md +++ /dev/null @@ -1,45 +0,0 @@ -bootstrap-vz -=========================================== -bootstrap-vz is a bootstrapping framework for Debian. -It is is specifically targeted at bootstrapping systems for virtualized environments. -bootstrap-vz runs without any user intervention and generates ready-to-boot images for -[a number of virtualization platforms](http://andsens.github.io/bootstrap-vz/providers.html). -Its aim is to provide a reproducable bootstrapping process using [manifests](http://andsens.github.io/bootstrap-vz/manifest.html) as well as supporting a high degree of customizability through plugins. - -bootstrap-vz was coded from scratch in python once the bash script architecture that was used in the -[build-debian-cloud](https://github.com/andsens/build-debian-cloud) bootstrapper reached its -limits. - -Documentation -------------- -The end-user documentation for bootstrap-vz is available -at [andsens.github.io/bootstrap-vz](http://andsens.github.io/bootstrap-vz). -There, you can discover [what the dependencies](http://andsens.github.io/bootstrap-vz/#dependencies) -for a specific cloud provider are, [see a list of available plugins](http://andsens.github.io/bootstrap-vz/plugins.html) -and learn [how you create a manifest](http://andsens.github.io/bootstrap-vz/manifest.html). - -Installation ------------- - -bootstrap-vz has a master branch for stable releases and a development for, well, development. -After checking out the branch of your choice you can install the python dependencies by running -`python setup.py install`. However, depending on what kind of image you'd like to bootstrap, -there are other debian package dependencies as well, at the very least you will need `debootstrap`. -[The documentation](http://andsens.github.io/bootstrap-vz/) explains this in more detail. - -Note that bootstrap-vz will tell you which tools it requires when they aren't -present (the different packages are mentioned in the error message), so you can -simply run bootstrap-vz once to get a list of the packages, install them, -and then re-run. - -Developers ----------- -The API documentation, development guidelines and an explanation of bootstrap-vz internals -can be found at [bootstrap-vz.readthedocs.org](http://bootstrap-vz.readthedocs.org). - -Contributing ------------- - -Contribution guidelines are described on the [CONTRIBUTING](CONTRIBUTING.md) file. There's also a -[topic on the documentation](http://bootstrap-vz.readthedocs.org/en/development/guidelines.html#coding-style) -regarding the coding style. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..93c699f --- /dev/null +++ b/README.rst @@ -0,0 +1,145 @@ +Introduction +============ + +bootstrap-vz is a bootstrapping framework for Debian. +It is is specifically targeted at bootstrapping systems for virtualized environments. +bootstrap-vz runs without any user intervention and +generates ready-to-boot images for a number of virtualization +platforms. +Its aim is to provide a reproducable bootstrapping process using +`manifests `__ as +well as supporting a high degree of customizability through plugins. + +bootstrap-vz was coded from scratch in python once the bash script +architecture that was used in the +`build-debian-cloud `__ +bootstrapper reached its limits. + +Documentation +------------- + +The end-user documentation for bootstrap-vz is available at +`andsens.github.io/bootstrap-vz `__. +There, you can discover `what the +dependencies `__ +for a specific cloud provider are, `see a list of available +plugins `__ and +learn `how you create a +manifest `__. + +Installation +------------ + +bootstrap-vz has a master branch for stable releases and a development +for, well, development. + +After checking out the branch of your choice you can install the +python dependencies by running ``python setup.py install``. However, +depending on what kind of image you'd like to bootstrap, there are +other debian package dependencies as well, at the very least you will +need ``debootstrap``. +`The documentation `__ +explains this in more detail. + +Note that bootstrap-vz will tell you which tools it requires when they +aren't present (the different packages are mentioned in the error +message), so you can simply run bootstrap-vz once to get a list of the +packages, install them, and then re-run. + +Quick start +----------- + +Here are a few quickstart tutorials for the most common images. +If you plan on partitioning your volume, you will need the ``parted`` +package and ``kpartx``: + +.. code:: sh + + root@host:~# apt-get install parted kpartx + +VirtualBox Vagrant +~~~~~~~~~~~~~~~~~~ + +:: + + user@host:~$ sudo -i # become root + root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo + root@host:~# apt-get install qemu-utils debootstrap python-pip # Install dependencies from aptitude + root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies + root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml + +If you want to use the `minimize\_size `__ +plugin, you will have to install the ``zerofree`` package and `VMWare +Workstation `__ +as well. + +Amazon EC2 EBS backed AMI +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: sh + + user@host:~$ sudo -i # become root + root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo + root@host:~# apt-get install debootstrap python-pip # Install dependencies from aptitude + root@host:~# pip install termcolor jsonschema fysom docopt pyyaml boto # Install python dependencies + root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml + +To bootstrap S3 backed AMIs, bootstrap-vz will also need the +``euca2ools`` package. However, version 3.2.0 is required meaning you +must however install it directly from the eucalyptus repository like +this: + +.. code:: sh + + apt-get install --no-install-recommends python-dev libxml2-dev libxslt-dev gcc + pip install git+git://github.com/eucalyptus/euca2ools.git@v3.2.0 + +Cleanup +------- + +bootstrap-vz tries very hard to clean up after itself both if a run was +successful but also if it failed. This ensures that you are not left +with volumes still attached to the host which are useless. If an error +occurred you can simply correct the problem that caused it and rerun +everything, there will be no leftovers from the previous run (as always +there are of course rare/unlikely exceptions to that rule). The error +messages should always give you a strong hint at what is wrong, if that +is not the case please consider `opening an +issue `__ and attach +both the error message and your manifest (preferably as a gist or +similar). + +Dependencies +------------ + +bootstrap-vz has a number of dependencies depending on the target +platform and `the selected plugins `__. At a bare minimum +the following python libraries are needed: \* +`termcolor `__ \* +`fysom `__ \* +`jsonschema `__ \* +`docopt `__ \* +`pyyaml `__ To bootstrap Debian +itself +`debootstrap `__ is +needed as well. + +Any other requirements are dependent upon the manifest configuration +and are detailed in the corresponding sections of the documentation. +bootstrap-vz will however warn you if a requirement has not been met, +before the bootstrapping process begins. + +Developers +---------- + +The API documentation, development guidelines and an explanation of +bootstrap-vz internals can be found at +`bootstrap-vz.readthedocs.org `__. + +Contributing +------------ + +Contribution guidelines are described on the +`CONTRIBUTING `__ file. There's also a `topic on the +documentation `__ +regarding the coding style. diff --git a/docs/index.rst b/docs/index.rst index 16a5568..9944ea4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,14 +3,14 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to bootstrap-vz's documentation! -======================================== -Contents: +bootstrap-vz +============ .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + intro manifest providers/index plugins/index @@ -21,11 +21,3 @@ Contents: howitworks switches logging - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/providers/index.rst b/docs/providers/index.rst index fb991f2..f5b187b 100644 --- a/docs/providers/index.rst +++ b/docs/providers/index.rst @@ -1,4 +1,3 @@ - Providers ========= From 619fed52754e38fc82f83b11dfb74c542118c36f Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 17:04:09 +0200 Subject: [PATCH 13/19] Merge dev guidelines and contribution info into single doc --- CONTRIBUTING.md | 42 --------------------- docs/guidelines.rst => CONTRIBUTING.rst | 49 ++++++++++++++++++++++++- docs/contributing.rst | 1 + docs/index.rst | 2 +- 4 files changed, 50 insertions(+), 44 deletions(-) delete mode 100644 CONTRIBUTING.md rename docs/guidelines.rst => CONTRIBUTING.rst (69%) create mode 100644 docs/contributing.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 0cbba9d..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,42 +0,0 @@ -Contributing -============ - -Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow: - -* Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html) -* Fork this repository. -* Make any changes you want/need. -* Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8` - and fix any warnings that may appear. - This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz) - once you send a pull request, so it's better if you check this beforehand. -* If the change is significant (e.g. a new plugin, manifest setting or security fix) - add your name and contribution to the [CHANGELOG](CHANGELOG). -* Commit your changes. -* Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units - of work, but there's no reason to have many little commits just because of corrected typos. -* Push to your fork, preferably on a topic branch. - -From here on there are two paths to consider: - -If your patch is a new feature, e.g.: plugin, provider, etc. then: - -* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make - sure that the code is stable. - -If it is a bug/security fix: - -* Send a pull request to the `master` branch. - --- - -Please try to be very descriptive about your changes when you write a pull request, stating what it does, why -it is needed, which use cases this change covers etc. -You may be asked to rebase your work on the current branch state, so it can be merged cleanly. -If you push a new commit to your pull request you will have to add a new comment to the PR, -provided that you want us notified. Github will otherwise not send a notification. - -Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they -concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`. - -Happy hacking! :-) diff --git a/docs/guidelines.rst b/CONTRIBUTING.rst similarity index 69% rename from docs/guidelines.rst rename to CONTRIBUTING.rst index 0d4f81d..7554a35 100644 --- a/docs/guidelines.rst +++ b/CONTRIBUTING.rst @@ -1,6 +1,53 @@ +Contributing +============ + +Sending pull requests +--------------------- + +Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow: + ++ Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html) ++ Fork this repository. ++ Make any changes you want/need. ++ Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8` + and fix any warnings that may appear. + This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz) + once you send a pull request, so it's better if you check this beforehand. ++ If the change is significant (e.g. a new plugin, manifest setting or security fix) + add your name and contribution to the [CHANGELOG](CHANGELOG). ++ Commit your changes. ++ Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units + of work, but there's no reason to have many little commits just because of corrected typos. ++ Push to your fork, preferably on a topic branch. + +From here on there are two paths to consider: + +If your patch is a new feature, e.g.: plugin, provider, etc. then: + +* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make + sure that the code is stable. + +If it is a bug/security fix: + +* Send a pull request to the `master` branch. + +-- + +Please try to be very descriptive about your changes when you write a pull request, stating what it does, why +it is needed, which use cases this change covers etc. +You may be asked to rebase your work on the current branch state, so it can be merged cleanly. +If you push a new commit to your pull request you will have to add a new comment to the PR, +provided that you want us notified. Github will otherwise not send a notification. + +Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they +concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`. + +Happy hacking! :-) + Development guidelines -====================== +---------------------- + The following guidelines should serve as general advice when developing providers or plugins for bootstrap-vz. Keep in mind that these guidelines are not rules , they are advice on how to better add diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 9944ea4..d17042b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,7 +16,7 @@ bootstrap-vz plugins/index api/index - guidelines + contributing taskoverview howitworks switches From f27d622e2cfb68ed771ba6ffcefb1b7daffa83be Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sat, 11 Apr 2015 17:12:31 +0200 Subject: [PATCH 14/19] Include changelog in docs (converted it to rst) --- CHANGELOG | 48 ----------------------------- CHANGELOG.rst | 77 ++++++++++++++++++++++++++++++++++++++++++++++ docs/changelog.rst | 1 + docs/index.rst | 1 + 4 files changed, 79 insertions(+), 48 deletions(-) delete mode 100644 CHANGELOG create mode 100644 CHANGELOG.rst create mode 100644 docs/changelog.rst diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 9d5b2de..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,48 +0,0 @@ -2014-11-23: - Noah Fontes: - * Add support for enhanced networking on EC2 images -2014-07-12: - Tiago Ilieve: - * Fixes #96: AddBackports is now a common task -2014-07-09: - Anders Ingemann: - * Allow passing data into the manifest - * Refactor logging setup to be more modular - * Convert every JSON file to YAML - * Convert "provider" into provider specific section -2014-07-02: - Vladimir Vitkov: - * Improve grub options to work better with virtual machines -2014-06-30: - Tomasz Rybak: - * Return information about created image -2014-06-22: - Victor Marmol: - * Enable the memory cgroup for the Docker plugin -2014-06-19: - Tiago Ilieve: - * Fixes #94: allow stable/oldstable as release name on manifest - Vladimir Vitkov: - * Improve ami listing performance -2014-06-07: - Tiago Ilieve: - * Download `gsutil` tarball to workspace instead of working directory - * Fixes #97: remove raw disk image created by GCE after build -2014-06-06: - Ilya Margolin: - * pip_install plugin -2014-05-23: - Tiago Ilieve: - * Fixes #95: check if the specified APT proxy server can be reached -2014-05-04: - Dhananjay Balan: - * Salt minion installation & configuration plugin - * Expose debootstrap --include-packages and --exclude-packages options to manifest -2014-05-03: - Anders Ingemann: - * Require hostname setting for vagrant plugin - * Fixes #14: S3 images can now be bootstrapped outside EC2. - * Added enable_agent option to puppet plugin -2014-05-02: - Tomasz Rybak: - * Added Google Compute Engine Provider diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..420540c --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,77 @@ +Changelog +========= + +2014-11-23: + +Noah Fontes: + + Add support for enhanced networking on EC2 images + +2014-07-12: + +Tiago Ilieve: + + Fixes #96: AddBackports is now a common task + +2014-07-09: + +Anders Ingemann: + + Allow passing data into the manifest + + Refactor logging setup to be more modular + + Convert every JSON file to YAML + + Convert "provider" into provider specific section + +2014-07-02: + +Vladimir Vitkov: + + Improve grub options to work better with virtual machines + +2014-06-30: + +Tomasz Rybak: + + Return information about created image + +2014-06-22: + +Victor Marmol: + + Enable the memory cgroup for the Docker plugin + +2014-06-19: + +Tiago Ilieve: + + Fixes #94: allow stable/oldstable as release name on manifest + +Vladimir Vitkov: + + Improve ami listing performance + +2014-06-07: + +Tiago Ilieve: + + Download `gsutil` tarball to workspace instead of working directory + + Fixes #97: remove raw disk image created by GCE after build + +2014-06-06: + +Ilya Margolin: + + pip_install plugin + +2014-05-23: + +Tiago Ilieve: + + Fixes #95: check if the specified APT proxy server can be reached + +2014-05-04: + +Dhananjay Balan: + + Salt minion installation & configuration plugin + + Expose debootstrap --include-packages and --exclude-packages options to manifest + +2014-05-03: + +Anders Ingemann: + + Require hostname setting for vagrant plugin + + Fixes #14: S3 images can now be bootstrapped outside EC2. + + Added enable_agent option to puppet plugin + +2014-05-02: + +Tomasz Rybak: + + Added Google Compute Engine Provider diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..565b052 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1 @@ +.. include:: ../CHANGELOG.rst diff --git a/docs/index.rst b/docs/index.rst index d17042b..f10ebe0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ bootstrap-vz manifest providers/index plugins/index + changelog api/index contributing From 3e129b594b25546dfde22c10cf6e1851ea33dc3c Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Apr 2015 11:15:27 +0200 Subject: [PATCH 15/19] Fix unit testing, don't try parsing README.rst as manifest --- tests/integration/manifests_tests.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/manifests_tests.py b/tests/integration/manifests_tests.py index fd2b4b6..a3e0152 100644 --- a/tests/integration/manifests_tests.py +++ b/tests/integration/manifests_tests.py @@ -1,12 +1,6 @@ -import os from nose.tools import assert_true from bootstrapvz.base.manifest import Manifest -MANIFEST_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - '../../manifests' -) - def test_manifest_generator(): """ @@ -15,10 +9,16 @@ def test_manifest_generator(): Loops through the manifests directory and tests that each file can successfully be loaded and validated. """ - for fobj in os.listdir(MANIFEST_DIR): - path = os.path.join(os.path.abspath(MANIFEST_DIR), fobj) - - yield validate_manifests, path + import os.path + import glob + manifests = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../manifests' + ) + for manifest_path in glob.glob(manifests + '/*.yml'): + yield validate_manifests, manifest_path + for manifest_path in glob.glob(manifests + '/*.json'): + yield validate_manifests, manifest_path def validate_manifests(path): From 62b87f22d599b458f83b4a2f0b3ca510e69170a7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Apr 2015 11:54:50 +0200 Subject: [PATCH 16/19] Link to code in github rather than embedding it --- docs/conf.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 5e1d3e4..7f412db 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ sys.path.insert(0, os.path.abspath(os.pardir)) # ones. extensions = ['sphinx.ext.coverage', 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', + 'sphinx.ext.linkcode', ] # Add any paths that contain templates here, relative to this directory. @@ -290,3 +290,57 @@ from docs import taskoverview data = taskoverview.generate_graph_data() taskoverview.write_data(data, '_static/graph.json') + + +# -- Substitute links for github with relative links in readthedocs ------- + + +if on_rtd: + pass + +# Snatched from here: +# https://sourcegraph.com/github.com/Gallopsled/pwntools@master/.PipPackage/pwntools/.def/docs/source/conf/linkcode_resolve/lines +baseurl = 'https://github.com/andsens/bootstrap-vz' + +import subprocess +try: + git_head = subprocess.check_output('git describe --tags 2>/dev/null', shell=True) +except subprocess.CalledProcessError: + try: + git_head = subprocess.check_output('git rev-parse HEAD', shell=True).strip()[:10] + except subprocess.CalledProcessError: + pass + + +def linkcode_resolve(domain, info): + if domain != 'py': + return None + if not info['module']: + return None + + filepath = info['module'].replace('.', '/') + '.py' + fmt_args = {'baseurl': baseurl, + 'commit': git_head, + 'path': filepath} + + import importlib + import inspect + import types + module = importlib.import_module(info['module']) + value = module + for part in info['fullname'].split('.'): + value = getattr(value, part, None) + if value is None: + break + valid_types = (types.ModuleType, types.ClassType, types.MethodType, + types.FunctionType, types.TracebackType, + types.FrameType, types.CodeType) + if isinstance(value, valid_types): + try: + lines, first = inspect.getsourcelines(value) + fmt_args['linestart'] = first + fmt_args['lineend'] = first + len(lines) - 1 + return '{baseurl}/blob/{commit}/{path}#L{linestart}-L{lineend}'.format(**fmt_args) + except IOError: + pass + return '{baseurl}/blob/{commit}/{path}'.format(**fmt_args) From 6980dd6517e0211380e89c91e23e11a20f90f7b4 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Apr 2015 13:18:10 +0200 Subject: [PATCH 17/19] Add sphinx extension that replaces absolut RTD urls This makes it possible to add proper links when showing an rst on github, while also resolving it to relative links on readthedocs --- README.rst | 4 ++-- docs/conf.py | 1 + docs/replace_rtd_links.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 docs/replace_rtd_links.py diff --git a/README.rst b/README.rst index 93c699f..9a54e5c 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,8 @@ bootstrap-vz runs without any user intervention and generates ready-to-boot images for a number of virtualization platforms. Its aim is to provide a reproducable bootstrapping process using -`manifests `__ as -well as supporting a high degree of customizability through plugins. +`manifests `__ +as well as supporting a high degree of customizability through plugins. bootstrap-vz was coded from scratch in python once the bash script architecture that was used in the diff --git a/docs/conf.py b/docs/conf.py index 7f412db..f6deef4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ sys.path.insert(0, os.path.abspath(os.pardir)) extensions = ['sphinx.ext.coverage', 'sphinx.ext.autodoc', 'sphinx.ext.linkcode', + 'docs.replace_rtd_links', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/replace_rtd_links.py b/docs/replace_rtd_links.py new file mode 100644 index 0000000..c18d1c4 --- /dev/null +++ b/docs/replace_rtd_links.py @@ -0,0 +1,19 @@ + +def setup(app): + app.connect('doctree-resolved', replace_rtd_links) + + return {'version': '0.1'} + + +def replace_rtd_links(app, doctree, fromdocname): + from docutils import nodes + import re + + rtd_baseurl = 'http://bootstrap-vz.readthedocs.org/en/master/' + search = re.compile('^' + re.escape(rtd_baseurl) + '(.*)$') + for node in doctree.traverse(nodes.reference): + if 'refuri' not in node: + continue + if not node['refuri'].startswith(rtd_baseurl): + continue + node['refuri'] = re.sub(search, r'\1', node['refuri']) From 65ddee99d735414812549ac82529b47d2eaa7005 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Apr 2015 14:32:47 +0200 Subject: [PATCH 18/19] Fix some links --- CONTRIBUTING.rst | 14 +++---- README.rst | 43 +++++++++++---------- bootstrapvz/providers/ec2/README.rst | 4 +- bootstrapvz/providers/virtualbox/README.rst | 2 +- manifests/README.rst | 12 +++--- 5 files changed, 39 insertions(+), 36 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7554a35..9c853f5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -6,15 +6,15 @@ Sending pull requests Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow: -+ Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html) ++ Read the `development guidelines `__ + Fork this repository. + Make any changes you want/need. -+ Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8` ++ Check the coding style of your changes using `tox `__ by running `tox -e flake8` and fix any warnings that may appear. - This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz) + This check will be repeated by `Travis CI `__ once you send a pull request, so it's better if you check this beforehand. + If the change is significant (e.g. a new plugin, manifest setting or security fix) - add your name and contribution to the [CHANGELOG](CHANGELOG). + add your name and contribution to the `changelog `__. + Commit your changes. + Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units of work, but there's no reason to have many little commits just because of corrected typos. @@ -61,7 +61,7 @@ value to the bootstrap-vz codebase. This allows others to easily reproduce any setup other people are running and makes it possible to share manifests. `The official debian EC2 images `_ + profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5>`__ for example can be reproduced using the manifests available in the manifest directory of bootstrap-vz. @@ -107,7 +107,7 @@ value to the bootstrap-vz codebase. This allows other tasks to interleave with the control-flow and add extended functionality (e.g. because volume creation and mounting are two separate tasks, `the prebootstrapped plugin - `_ + `__ can replace the volume creation task with a task of its own that creates a volume from a snapshot instead, but still reuse the mount task). @@ -152,7 +152,7 @@ guidelines. There however a few exceptions: + Ignore ``W191``: Indent with tabs not spaces The codebase can be checked for any violations quite easily, since those rules are already specified in the -`tox `_ configuration file. +`tox `__ configuration file. :: tox -e flake8 diff --git a/README.rst b/README.rst index 9a54e5c..1e08507 100644 --- a/README.rst +++ b/README.rst @@ -18,14 +18,14 @@ bootstrapper reached its limits. Documentation ------------- -The end-user documentation for bootstrap-vz is available at -`andsens.github.io/bootstrap-vz `__. +The documentation for bootstrap-vz is available at +`bootstrap-vz.readthedocs.org `__. There, you can discover `what the -dependencies `__ +dependencies `__ for a specific cloud provider are, `see a list of available -plugins `__ and +plugins `__ and learn `how you create a -manifest `__. +manifest `__. Installation ------------ @@ -38,7 +38,7 @@ python dependencies by running ``python setup.py install``. However, depending on what kind of image you'd like to bootstrap, there are other debian package dependencies as well, at the very least you will need ``debootstrap``. -`The documentation `__ +`The documentation `__ explains this in more detail. Note that bootstrap-vz will tell you which tools it requires when they @@ -68,7 +68,7 @@ VirtualBox Vagrant root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml -If you want to use the `minimize\_size `__ +If you want to use the `minimize\_size `__ plugin, you will have to install the ``zerofree`` package and `VMWare Workstation `__ as well. @@ -113,16 +113,18 @@ Dependencies ------------ bootstrap-vz has a number of dependencies depending on the target -platform and `the selected plugins `__. At a bare minimum -the following python libraries are needed: \* -`termcolor `__ \* -`fysom `__ \* -`jsonschema `__ \* -`docopt `__ \* -`pyyaml `__ To bootstrap Debian -itself -`debootstrap `__ is -needed as well. +platform and `the selected plugins `__. +At a bare minimum the following python libraries are needed: + +* `termcolor `__ +* `fysom `__ +* `jsonschema `__ +* `docopt `__ +* `pyyaml `__ + +To bootstrap Debian itself +`debootstrap `__ +is needed as well. Any other requirements are dependent upon the manifest configuration and are detailed in the corresponding sections of the documentation. @@ -139,7 +141,8 @@ bootstrap-vz internals can be found at Contributing ------------ -Contribution guidelines are described on the -`CONTRIBUTING `__ file. There's also a `topic on the -documentation `__ +Contribution guidelines are described in the documentation under +`Contributing `__. +There's also +`a topic `__ regarding the coding style. diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst index 7c79201..05b63e8 100644 --- a/bootstrapvz/providers/ec2/README.rst +++ b/bootstrapvz/providers/ec2/README.rst @@ -7,8 +7,8 @@ once it is done and registers it as an AMI. EBS volume backing only works on an EC2 host while S3 backed volumes *should* work locally (at this time however they do not, a fix is in the works). -Unless `the cloud-init plugin `__ is used, -special startup scripts will be installed that automatically fetch the +Unless `the cloud-init plugin `__ +is used, special startup scripts will be installed that automatically fetch the configured authorized\_key from the instance metadata and save or run any userdata supplied (if the userdata begins with ``#!`` it will be run). diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst index b3e3d16..154199b 100644 --- a/bootstrapvz/providers/virtualbox/README.rst +++ b/bootstrapvz/providers/virtualbox/README.rst @@ -9,4 +9,4 @@ interoperability (e.g. *should* support vdi files, but since they have no identifier URL not even VirtualBox itself can import them). VirtualBox Guest Additions can be installed automatically if the ISO is `provided in the -manifest `__. +manifest `__. diff --git a/manifests/README.rst b/manifests/README.rst index 3d5c1b5..d644979 100644 --- a/manifests/README.rst +++ b/manifests/README.rst @@ -39,7 +39,7 @@ name of the provider itself. - **``name``**: target virtualization platform of the installation *``required``* -Consult the `providers `__ section of the documentation +Consult the `providers `__ section of the documentation for a list of valid values. Bootstrapper @@ -76,8 +76,8 @@ are 4 possible settings: Accepts a list of package names. *``optional``* - **``guest_additions``**: This setting is only relevant for the - `virtualbox provider `__. It specifies the - path to the VirtualBox Guest Additions ISO, which, when specified, + `virtualbox provider `__. + It specifies the path to the VirtualBox Guest Additions ISO, which, when specified, will be mounted and used to install the VirtualBox Guest Additions. *``optional``* @@ -216,7 +216,7 @@ boot, root and swap. optional setting overrides the command bootstrap-vz would normally use to format the partition. The command is specified as a string array where each option/argument is an item in that array (much - like the `image\_commands `__ plugin). + like the `image\_commands `__ plugin). *``optional``* The following variables are available: - **``{fs}``**: The filesystem of the partition. - **``{device_path}``**: The device path of the partition. @@ -237,5 +237,5 @@ Plugins ~~~~~~~ The plugins section is a map of plugin names to whatever configuration a -plugin requires. Go to the `plugin section `__ of the -documentation, to see the configuration for a specific plugin. +plugin requires. Go to the `plugin section `__ +of the documentation, to see the configuration for a specific plugin. From 188672404bb90dc965e2dd3691566d0e623ff9c7 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Sun, 12 Apr 2015 15:12:37 +0200 Subject: [PATCH 19/19] Move developer doc stuff into developers/ subfolder --- README.rst | 6 ++--- docs/_static/taskoverview.coffee | 2 +- docs/contributing.rst | 1 - docs/{ => developers}/api/base/fs.rst | 0 docs/{ => developers}/api/base/index.rst | 0 docs/{ => developers}/api/base/pkg.rst | 0 docs/{ => developers}/api/common/fs.rst | 0 docs/{ => developers}/api/common/index.rst | 0 .../api/common/tasks/index.rst | 0 docs/{ => developers}/api/index.rst | 0 docs/developers/contributing.rst | 1 + docs/{howitworks.rst => developers/index.rst} | 27 +++++++++++++------ docs/{ => developers}/switches.rst | 0 docs/{ => developers}/taskoverview.rst | 4 +-- docs/index.rst | 15 ++--------- 15 files changed, 28 insertions(+), 28 deletions(-) delete mode 100644 docs/contributing.rst rename docs/{ => developers}/api/base/fs.rst (100%) rename docs/{ => developers}/api/base/index.rst (100%) rename docs/{ => developers}/api/base/pkg.rst (100%) rename docs/{ => developers}/api/common/fs.rst (100%) rename docs/{ => developers}/api/common/index.rst (100%) rename docs/{ => developers}/api/common/tasks/index.rst (100%) rename docs/{ => developers}/api/index.rst (100%) create mode 100644 docs/developers/contributing.rst rename docs/{howitworks.rst => developers/index.rst} (80%) rename docs/{ => developers}/switches.rst (100%) rename docs/{ => developers}/taskoverview.rst (76%) diff --git a/README.rst b/README.rst index 1e08507..ac2ce78 100644 --- a/README.rst +++ b/README.rst @@ -136,13 +136,13 @@ Developers The API documentation, development guidelines and an explanation of bootstrap-vz internals can be found at -`bootstrap-vz.readthedocs.org `__. +`bootstrap-vz.readthedocs.org `__. Contributing ------------ Contribution guidelines are described in the documentation under -`Contributing `__. +`Contributing `__. There's also -`a topic `__ +`a topic `__ regarding the coding style. diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee index ccae87d..f4e1f25 100644 --- a/docs/_static/taskoverview.coffee +++ b/docs/_static/taskoverview.coffee @@ -26,7 +26,7 @@ class window.TaskOverview constructor: ({@selector}) -> @svg = d3.select(@selector) .attr('viewBox', "0 0 #{viewBoxWidth} #{viewBoxHeight}") - d3.json '_static/graph.json', @buildGraph + d3.json '../_static/graph.json', @buildGraph buildGraph: (error, @data) => @createDefinitions() diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053..0000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/api/base/fs.rst b/docs/developers/api/base/fs.rst similarity index 100% rename from docs/api/base/fs.rst rename to docs/developers/api/base/fs.rst diff --git a/docs/api/base/index.rst b/docs/developers/api/base/index.rst similarity index 100% rename from docs/api/base/index.rst rename to docs/developers/api/base/index.rst diff --git a/docs/api/base/pkg.rst b/docs/developers/api/base/pkg.rst similarity index 100% rename from docs/api/base/pkg.rst rename to docs/developers/api/base/pkg.rst diff --git a/docs/api/common/fs.rst b/docs/developers/api/common/fs.rst similarity index 100% rename from docs/api/common/fs.rst rename to docs/developers/api/common/fs.rst diff --git a/docs/api/common/index.rst b/docs/developers/api/common/index.rst similarity index 100% rename from docs/api/common/index.rst rename to docs/developers/api/common/index.rst diff --git a/docs/api/common/tasks/index.rst b/docs/developers/api/common/tasks/index.rst similarity index 100% rename from docs/api/common/tasks/index.rst rename to docs/developers/api/common/tasks/index.rst diff --git a/docs/api/index.rst b/docs/developers/api/index.rst similarity index 100% rename from docs/api/index.rst rename to docs/developers/api/index.rst diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst new file mode 100644 index 0000000..ac7b6bc --- /dev/null +++ b/docs/developers/contributing.rst @@ -0,0 +1 @@ +.. include:: ../../CONTRIBUTING.rst diff --git a/docs/howitworks.rst b/docs/developers/index.rst similarity index 80% rename from docs/howitworks.rst rename to docs/developers/index.rst index cca75c9..5f1cdea 100644 --- a/docs/howitworks.rst +++ b/docs/developers/index.rst @@ -1,6 +1,17 @@ +Developers +============ + +.. toctree:: + :maxdepth: 1 + + contributing + switches + api/index + taskoverview + How bootstrap-vz works -====================== +---------------------- Tasks ~~~~~ @@ -15,14 +26,14 @@ via attributes. Here is an example: :: class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - predecessors = [PartitionVolume] - successors = [filesystem.Format] + description = 'Mapping volume partitions' + phase = phases.volume_preparation + predecessors = [PartitionVolume] + successors = [filesystem.Format] - @classmethod - def run(cls, info): - info.volume.partition_map.map(info.volume) + @classmethod + def run(cls, info): + info.volume.partition_map.map(info.volume) In this case the attributes define that the task at hand should run after the ``PartitionVolume`` task — i.e. after volume has been diff --git a/docs/switches.rst b/docs/developers/switches.rst similarity index 100% rename from docs/switches.rst rename to docs/developers/switches.rst diff --git a/docs/taskoverview.rst b/docs/developers/taskoverview.rst similarity index 76% rename from docs/taskoverview.rst rename to docs/developers/taskoverview.rst index e84680e..2e76e19 100644 --- a/docs/taskoverview.rst +++ b/docs/developers/taskoverview.rst @@ -5,11 +5,11 @@ Taskoverview .. raw:: html - + - + diff --git a/docs/index.rst b/docs/index.rst index f10ebe0..4056a36 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,9 +1,3 @@ -.. bootstrap-vz documentation master file, created by - sphinx-quickstart on Sun Mar 23 16:17:28 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - - bootstrap-vz ============ @@ -14,11 +8,6 @@ bootstrap-vz manifest providers/index plugins/index - changelog - - api/index - contributing - taskoverview - howitworks - switches logging + changelog + developers/index