diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 9d5b2de..0000000 --- a/CHANGELOG +++ /dev/null @@ -1,48 +0,0 @@ -2014-11-23: - Noah Fontes: - * Add support for enhanced networking on EC2 images -2014-07-12: - Tiago Ilieve: - * Fixes #96: AddBackports is now a common task -2014-07-09: - Anders Ingemann: - * Allow passing data into the manifest - * Refactor logging setup to be more modular - * Convert every JSON file to YAML - * Convert "provider" into provider specific section -2014-07-02: - Vladimir Vitkov: - * Improve grub options to work better with virtual machines -2014-06-30: - Tomasz Rybak: - * Return information about created image -2014-06-22: - Victor Marmol: - * Enable the memory cgroup for the Docker plugin -2014-06-19: - Tiago Ilieve: - * Fixes #94: allow stable/oldstable as release name on manifest - Vladimir Vitkov: - * Improve ami listing performance -2014-06-07: - Tiago Ilieve: - * Download `gsutil` tarball to workspace instead of working directory - * Fixes #97: remove raw disk image created by GCE after build -2014-06-06: - Ilya Margolin: - * pip_install plugin -2014-05-23: - Tiago Ilieve: - * Fixes #95: check if the specified APT proxy server can be reached -2014-05-04: - Dhananjay Balan: - * Salt minion installation & configuration plugin - * Expose debootstrap --include-packages and --exclude-packages options to manifest -2014-05-03: - Anders Ingemann: - * Require hostname setting for vagrant plugin - * Fixes #14: S3 images can now be bootstrapped outside EC2. - * Added enable_agent option to puppet plugin -2014-05-02: - Tomasz Rybak: - * Added Google Compute Engine Provider diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..420540c --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,77 @@ +Changelog +========= + +2014-11-23: + +Noah Fontes: + + Add support for enhanced networking on EC2 images + +2014-07-12: + +Tiago Ilieve: + + Fixes #96: AddBackports is now a common task + +2014-07-09: + +Anders Ingemann: + + Allow passing data into the manifest + + Refactor logging setup to be more modular + + Convert every JSON file to YAML + + Convert "provider" into provider specific section + +2014-07-02: + +Vladimir Vitkov: + + Improve grub options to work better with virtual machines + +2014-06-30: + +Tomasz Rybak: + + Return information about created image + +2014-06-22: + +Victor Marmol: + + Enable the memory cgroup for the Docker plugin + +2014-06-19: + +Tiago Ilieve: + + Fixes #94: allow stable/oldstable as release name on manifest + +Vladimir Vitkov: + + Improve ami listing performance + +2014-06-07: + +Tiago Ilieve: + + Download `gsutil` tarball to workspace instead of working directory + + Fixes #97: remove raw disk image created by GCE after build + +2014-06-06: + +Ilya Margolin: + + pip_install plugin + +2014-05-23: + +Tiago Ilieve: + + Fixes #95: check if the specified APT proxy server can be reached + +2014-05-04: + +Dhananjay Balan: + + Salt minion installation & configuration plugin + + Expose debootstrap --include-packages and --exclude-packages options to manifest + +2014-05-03: + +Anders Ingemann: + + Require hostname setting for vagrant plugin + + Fixes #14: S3 images can now be bootstrapped outside EC2. + + Added enable_agent option to puppet plugin + +2014-05-02: + +Tomasz Rybak: + + Added Google Compute Engine Provider diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 0cbba9d..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,42 +0,0 @@ -Contributing -============ - -Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow: - -* Read the [development guidelines](http://bootstrap-vz.readthedocs.org/en/master/guidelines.html) -* Fork this repository. -* Make any changes you want/need. -* Check the coding style of your changes using [tox](http://tox.readthedocs.org/) by running `tox -e flake8` - and fix any warnings that may appear. - This check will be repeated by [Travis CI](https://travis-ci.org/andsens/bootstrap-vz) - once you send a pull request, so it's better if you check this beforehand. -* If the change is significant (e.g. a new plugin, manifest setting or security fix) - add your name and contribution to the [CHANGELOG](CHANGELOG). -* Commit your changes. -* Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units - of work, but there's no reason to have many little commits just because of corrected typos. -* Push to your fork, preferably on a topic branch. - -From here on there are two paths to consider: - -If your patch is a new feature, e.g.: plugin, provider, etc. then: - -* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make - sure that the code is stable. - -If it is a bug/security fix: - -* Send a pull request to the `master` branch. - --- - -Please try to be very descriptive about your changes when you write a pull request, stating what it does, why -it is needed, which use cases this change covers etc. -You may be asked to rebase your work on the current branch state, so it can be merged cleanly. -If you push a new commit to your pull request you will have to add a new comment to the PR, -provided that you want us notified. Github will otherwise not send a notification. - -Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they -concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`. - -Happy hacking! :-) diff --git a/docs/guidelines.rst b/CONTRIBUTING.rst similarity index 65% rename from docs/guidelines.rst rename to CONTRIBUTING.rst index 0d4f81d..9c853f5 100644 --- a/docs/guidelines.rst +++ b/CONTRIBUTING.rst @@ -1,6 +1,53 @@ +Contributing +============ + +Sending pull requests +--------------------- + +Do you want to contribute to the bootstrap-vz project? Nice! Here is the basic workflow: + ++ Read the `development guidelines `__ ++ Fork this repository. ++ Make any changes you want/need. ++ Check the coding style of your changes using `tox `__ by running `tox -e flake8` + and fix any warnings that may appear. + This check will be repeated by `Travis CI `__ + once you send a pull request, so it's better if you check this beforehand. ++ If the change is significant (e.g. a new plugin, manifest setting or security fix) + add your name and contribution to the `changelog `__. ++ Commit your changes. ++ Squash the commits if needed. For instance, it is fine if you have multiple commits describing atomic units + of work, but there's no reason to have many little commits just because of corrected typos. ++ Push to your fork, preferably on a topic branch. + +From here on there are two paths to consider: + +If your patch is a new feature, e.g.: plugin, provider, etc. then: + +* Send a pull request to the `development` branch. It will be merged into the `master` branch when we can make + sure that the code is stable. + +If it is a bug/security fix: + +* Send a pull request to the `master` branch. + +-- + +Please try to be very descriptive about your changes when you write a pull request, stating what it does, why +it is needed, which use cases this change covers etc. +You may be asked to rebase your work on the current branch state, so it can be merged cleanly. +If you push a new commit to your pull request you will have to add a new comment to the PR, +provided that you want us notified. Github will otherwise not send a notification. + +Be aware that your modifications need to be properly documented and pushed to the `gh-pages` branch, if they +concern anything done on `master`. Otherwise, they should be sent to the `gh-pages-dev`. + +Happy hacking! :-) + Development guidelines -====================== +---------------------- + The following guidelines should serve as general advice when developing providers or plugins for bootstrap-vz. Keep in mind that these guidelines are not rules , they are advice on how to better add @@ -14,7 +61,7 @@ value to the bootstrap-vz codebase. This allows others to easily reproduce any setup other people are running and makes it possible to share manifests. `The official debian EC2 images `_ + profile?id=890be55d-32d8-4bc8-9042-2b4fd83064d5>`__ for example can be reproduced using the manifests available in the manifest directory of bootstrap-vz. @@ -60,7 +107,7 @@ value to the bootstrap-vz codebase. This allows other tasks to interleave with the control-flow and add extended functionality (e.g. because volume creation and mounting are two separate tasks, `the prebootstrapped plugin - `_ + `__ can replace the volume creation task with a task of its own that creates a volume from a snapshot instead, but still reuse the mount task). @@ -105,7 +152,7 @@ guidelines. There however a few exceptions: + Ignore ``W191``: Indent with tabs not spaces The codebase can be checked for any violations quite easily, since those rules are already specified in the -`tox `_ configuration file. +`tox `__ configuration file. :: tox -e flake8 diff --git a/README.md b/README.md deleted file mode 100644 index 8eb57fe..0000000 --- a/README.md +++ /dev/null @@ -1,45 +0,0 @@ -bootstrap-vz -=========================================== -bootstrap-vz is a bootstrapping framework for Debian. -It is is specifically targeted at bootstrapping systems for virtualized environments. -bootstrap-vz runs without any user intervention and generates ready-to-boot images for -[a number of virtualization platforms](http://andsens.github.io/bootstrap-vz/providers.html). -Its aim is to provide a reproducable bootstrapping process using [manifests](http://andsens.github.io/bootstrap-vz/manifest.html) as well as supporting a high degree of customizability through plugins. - -bootstrap-vz was coded from scratch in python once the bash script architecture that was used in the -[build-debian-cloud](https://github.com/andsens/build-debian-cloud) bootstrapper reached its -limits. - -Documentation -------------- -The end-user documentation for bootstrap-vz is available -at [andsens.github.io/bootstrap-vz](http://andsens.github.io/bootstrap-vz). -There, you can discover [what the dependencies](http://andsens.github.io/bootstrap-vz/#dependencies) -for a specific cloud provider are, [see a list of available plugins](http://andsens.github.io/bootstrap-vz/plugins.html) -and learn [how you create a manifest](http://andsens.github.io/bootstrap-vz/manifest.html). - -Installation ------------- - -bootstrap-vz has a master branch for stable releases and a development for, well, development. -After checking out the branch of your choice you can install the python dependencies by running -`python setup.py install`. However, depending on what kind of image you'd like to bootstrap, -there are other debian package dependencies as well, at the very least you will need `debootstrap`. -[The documentation](http://andsens.github.io/bootstrap-vz/) explains this in more detail. - -Note that bootstrap-vz will tell you which tools it requires when they aren't -present (the different packages are mentioned in the error message), so you can -simply run bootstrap-vz once to get a list of the packages, install them, -and then re-run. - -Developers ----------- -The API documentation, development guidelines and an explanation of bootstrap-vz internals -can be found at [bootstrap-vz.readthedocs.org](http://bootstrap-vz.readthedocs.org). - -Contributing ------------- - -Contribution guidelines are described on the [CONTRIBUTING](CONTRIBUTING.md) file. There's also a -[topic on the documentation](http://bootstrap-vz.readthedocs.org/en/development/guidelines.html#coding-style) -regarding the coding style. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..ac2ce78 --- /dev/null +++ b/README.rst @@ -0,0 +1,148 @@ +Introduction +============ + +bootstrap-vz is a bootstrapping framework for Debian. +It is is specifically targeted at bootstrapping systems for virtualized environments. +bootstrap-vz runs without any user intervention and +generates ready-to-boot images for a number of virtualization +platforms. +Its aim is to provide a reproducable bootstrapping process using +`manifests `__ +as well as supporting a high degree of customizability through plugins. + +bootstrap-vz was coded from scratch in python once the bash script +architecture that was used in the +`build-debian-cloud `__ +bootstrapper reached its limits. + +Documentation +------------- + +The documentation for bootstrap-vz is available at +`bootstrap-vz.readthedocs.org `__. +There, you can discover `what the +dependencies `__ +for a specific cloud provider are, `see a list of available +plugins `__ and +learn `how you create a +manifest `__. + +Installation +------------ + +bootstrap-vz has a master branch for stable releases and a development +for, well, development. + +After checking out the branch of your choice you can install the +python dependencies by running ``python setup.py install``. However, +depending on what kind of image you'd like to bootstrap, there are +other debian package dependencies as well, at the very least you will +need ``debootstrap``. +`The documentation `__ +explains this in more detail. + +Note that bootstrap-vz will tell you which tools it requires when they +aren't present (the different packages are mentioned in the error +message), so you can simply run bootstrap-vz once to get a list of the +packages, install them, and then re-run. + +Quick start +----------- + +Here are a few quickstart tutorials for the most common images. +If you plan on partitioning your volume, you will need the ``parted`` +package and ``kpartx``: + +.. code:: sh + + root@host:~# apt-get install parted kpartx + +VirtualBox Vagrant +~~~~~~~~~~~~~~~~~~ + +:: + + user@host:~$ sudo -i # become root + root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo + root@host:~# apt-get install qemu-utils debootstrap python-pip # Install dependencies from aptitude + root@host:~# pip install termcolor jsonschema fysom docopt pyyaml # Install python dependencies + root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/virtualbox-vagrant.manifest.yml + +If you want to use the `minimize\_size `__ +plugin, you will have to install the ``zerofree`` package and `VMWare +Workstation `__ +as well. + +Amazon EC2 EBS backed AMI +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: sh + + user@host:~$ sudo -i # become root + root@host:~# git clone https://github.com/andsens/bootstrap-vz.git # Clone the repo + root@host:~# apt-get install debootstrap python-pip # Install dependencies from aptitude + root@host:~# pip install termcolor jsonschema fysom docopt pyyaml boto # Install python dependencies + root@host:~# bootstrap-vz/bootstrap-vz bootstrap-vz/manifests/ec2-ebs-debian-official-amd64-pvm.manifest.yml + +To bootstrap S3 backed AMIs, bootstrap-vz will also need the +``euca2ools`` package. However, version 3.2.0 is required meaning you +must however install it directly from the eucalyptus repository like +this: + +.. code:: sh + + apt-get install --no-install-recommends python-dev libxml2-dev libxslt-dev gcc + pip install git+git://github.com/eucalyptus/euca2ools.git@v3.2.0 + +Cleanup +------- + +bootstrap-vz tries very hard to clean up after itself both if a run was +successful but also if it failed. This ensures that you are not left +with volumes still attached to the host which are useless. If an error +occurred you can simply correct the problem that caused it and rerun +everything, there will be no leftovers from the previous run (as always +there are of course rare/unlikely exceptions to that rule). The error +messages should always give you a strong hint at what is wrong, if that +is not the case please consider `opening an +issue `__ and attach +both the error message and your manifest (preferably as a gist or +similar). + +Dependencies +------------ + +bootstrap-vz has a number of dependencies depending on the target +platform and `the selected plugins `__. +At a bare minimum the following python libraries are needed: + +* `termcolor `__ +* `fysom `__ +* `jsonschema `__ +* `docopt `__ +* `pyyaml `__ + +To bootstrap Debian itself +`debootstrap `__ +is needed as well. + +Any other requirements are dependent upon the manifest configuration +and are detailed in the corresponding sections of the documentation. +bootstrap-vz will however warn you if a requirement has not been met, +before the bootstrapping process begins. + +Developers +---------- + +The API documentation, development guidelines and an explanation of +bootstrap-vz internals can be found at +`bootstrap-vz.readthedocs.org `__. + +Contributing +------------ + +Contribution guidelines are described in the documentation under +`Contributing `__. +There's also +`a topic `__ +regarding the coding style. diff --git a/bootstrapvz/base/fs/volume.py b/bootstrapvz/base/fs/volume.py index 0a17f61..043094a 100644 --- a/bootstrapvz/base/fs/volume.py +++ b/bootstrapvz/base/fs/volume.py @@ -70,6 +70,7 @@ class Volume(FSMProxy): rather than a loopback device or a network block device. :param _e_obj e: Event object containing arguments to create() + Keyword arguments to link_dm_node() are: :param int logical_start_sector: The sector the volume should start at in the new volume diff --git a/bootstrapvz/plugins/admin_user/README.rst b/bootstrapvz/plugins/admin_user/README.rst new file mode 100644 index 0000000..3506f92 --- /dev/null +++ b/bootstrapvz/plugins/admin_user/README.rst @@ -0,0 +1,12 @@ +Admin user +---------- + +This plugin creates a user with passwordless sudo privileges. It also +disables the SSH root login. If the EC2 init scripts are installed, the +script for fetching the SSH authorized keys will be adjust to match the +username specified. + +Settings +~~~~~~~~ + +- ``username``: The username of the account to create. *``required``* diff --git a/bootstrapvz/plugins/apt_proxy/README.rst b/bootstrapvz/plugins/apt_proxy/README.rst new file mode 100644 index 0000000..33711a7 --- /dev/null +++ b/bootstrapvz/plugins/apt_proxy/README.rst @@ -0,0 +1,21 @@ +APT Proxy +--------- + +This plugin creates a proxy configuration file for APT, so you could +enjoy the benefits of using cached packages instead of downloading them +from the mirror every time. You could just install ``apt-cacher-ng`` on +the host machine and then add ``"address": "127.0.0.1"`` and +``"port": 3142`` to the manifest file. + +Settings +~~~~~~~~ + +- ``address``: The IP or host of the proxy server. + *``required``* +- ``port``: The port (integer) of the proxy server. + *``required``* +- ``persistent``: Whether the proxy configuration file should remain on + the machine or not. + Valid values: true, false + Default: ``false`` + *``optional``* diff --git a/bootstrapvz/plugins/cloud_init/README.rst b/bootstrapvz/plugins/cloud_init/README.rst new file mode 100644 index 0000000..fa36e1f --- /dev/null +++ b/bootstrapvz/plugins/cloud_init/README.rst @@ -0,0 +1,23 @@ +cloud-init +---------- + +This plugin installs and configures +`cloud-init `__ +on the system. Depending on the release it installs it from either +backports or the main repository. + +cloud-init is only compatible with Debian wheezy and upwards. + +Settings +~~~~~~~~ + +- ``username``: The username of the account to create. + *``required``* +- ``disable_modules``: A list of strings specifying which cloud-init + modules should be disabled. + *``optional``* +- ``metadata_sources``: A string that sets the + `datasources `__ + that cloud-init should try fetching metadata from. The source is + automatically set when using the ec2 provider. + *``optional``* diff --git a/bootstrapvz/plugins/docker_daemon/README.rst b/bootstrapvz/plugins/docker_daemon/README.rst new file mode 100644 index 0000000..e210435 --- /dev/null +++ b/bootstrapvz/plugins/docker_daemon/README.rst @@ -0,0 +1,18 @@ +Docker daemon +------------- + +Install `docker `__ daemon in the image. Uses +init scripts for the official repository. + +This plugin can only be used if the distribution being bootstrapped is +at least ``wheezy``, as Docker needs a kernel version ``3.8`` or higher, +which is available at the ``wheezy-backports`` repository. There's also +an architecture requirement, as it runs only on ``amd64``. + +Settings +~~~~~~~~ + +- ``version``: Selects the docker version to install. To select the + latest version simply omit this setting. + Default: ``latest`` + *``optional``* diff --git a/bootstrapvz/plugins/image_commands/README.rst b/bootstrapvz/plugins/image_commands/README.rst new file mode 100644 index 0000000..106ad22 --- /dev/null +++ b/bootstrapvz/plugins/image_commands/README.rst @@ -0,0 +1,19 @@ +Image commands +-------------- + +The image commands plugin allows you to run arbitrary commands during +the bootstrap process. The commands are run at an indeterminate point +*after* packages have been installed, but *before* the volume has been +unmounted. + +Settings +~~~~~~~~ + +- ``commands``: A list of lists containing strings. Each top-level item + is a single command, while the strings inside each list comprise + parts of a command. This allows for proper shell argument escaping + (to circumvent this, simply put the entire command in a single + string). In addition to the manifest variables ``{root}`` is also + available. It points at the root of the image volume. + *``required``* + *``manifest vars``* diff --git a/bootstrapvz/plugins/minimize_size/README.rst b/bootstrapvz/plugins/minimize_size/README.rst new file mode 100644 index 0000000..e275237 --- /dev/null +++ b/bootstrapvz/plugins/minimize_size/README.rst @@ -0,0 +1,37 @@ +minimize size +------------- + +This plugin can be used to reduce the size of the resulting image. Often +virtual volumes are much smaller than their reported size until any data +is written to them. During the bootstrapping process temporary data like +the aptitude cache is written to the volume only to be removed again. + +The minimize size plugin employs three different strategies to keep a +low volume footprint: + +- Mount folders from the host into key locations of the image volume to + avoid any unneccesary disk writes. +- Use `zerofree `__ to + deallocate unused sectors on the volume. On an unpartitioned volume + this will be done for the entire volume, while it will only happen on + the root partition for partitioned volumes. +- Use + `vmware-vdiskmanager `__ + to shrink the real volume size (only applicable when using vmdk + backing). The tool is part of the `VMWare + Workstation `__ + package. + +Settings +~~~~~~~~ + +- ``zerofree``: Specifies if it should mark unallocated blocks as + zeroes, so the volume could be better shrunk after this. + Valid values: true, false + Default: false + *``optional``* +- ``shrink``: Whether the volume should be shrunk. This setting works + best in conjunction with the zerofree tool. + Valid values: true, false + Default: false + *``optional``* diff --git a/bootstrapvz/plugins/ntp/README.rst b/bootstrapvz/plugins/ntp/README.rst new file mode 100644 index 0000000..40421b1 --- /dev/null +++ b/bootstrapvz/plugins/ntp/README.rst @@ -0,0 +1,12 @@ +NTP +--- + +This plugins installs the Network Time Protocol daemon and optionally +defines which time servers it should use. + +Settings +~~~~~~~~ + +- ``servers``: A list of strings specifying which servers should be + used to synchronize the machine clock. + *``optional``* diff --git a/bootstrapvz/plugins/opennebula/README.md b/bootstrapvz/plugins/opennebula/README.md deleted file mode 100644 index 49ab0c9..0000000 --- a/bootstrapvz/plugins/opennebula/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Open Nebula provider - -This provider adds OpenNebula contextualization to the virtual image (see http://opennebula.org/documentation:rel4.2:cong). - -It set ups the network and ssh keys. TO do so you should configure your virtual machine context with something like: - - ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] - ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] - ETH0_IP $NIC[IP, NETWORK_ID=2] - ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] - ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] - FILES path_to_my_ssh_public_key.pub - diff --git a/bootstrapvz/plugins/opennebula/README.rst b/bootstrapvz/plugins/opennebula/README.rst new file mode 100644 index 0000000..a29d7ae --- /dev/null +++ b/bootstrapvz/plugins/opennebula/README.rst @@ -0,0 +1,27 @@ +Open Nebula +----------- + +This plugin adds `OpenNebula +contextualization `__ +to the image, which sets up the network configuration and SSH keys. + +The virtual machine context should be configured as follows: + +:: + + ETH0_DNS $NETWORK[DNS, NETWORK_ID=2] + ETH0_GATEWAY $NETWORK[GATEWAY, NETWORK_ID=2] + ETH0_IP $NIC[IP, NETWORK_ID=2] + ETH0_MASK $NETWORK[MASK, NETWORK_ID=2] + ETH0_NETWORK $NETWORK[NETWORK, NETWORK_ID=2] + FILES path_to_my_ssh_public_key.pub + +The plugin will install all *.pub* files in the root authorized\_keys +file. When using the ec2 provider, the USER\_EC2\_DATA will be executed +if present. + +Settings +~~~~~~~~ + +This plugin has no settings. To enable it add ``"opennebula":{}`` to the +plugin section of the manifest. diff --git a/bootstrapvz/plugins/pip_install/README.rst b/bootstrapvz/plugins/pip_install/README.rst new file mode 100644 index 0000000..3bed629 --- /dev/null +++ b/bootstrapvz/plugins/pip_install/README.rst @@ -0,0 +1,14 @@ +Pip install +----------- + +Install packages from the Python Package Index via pip. + +Installs ``build-essential`` and ``python-dev`` debian packages, so +Python extension modules can be built. + +Settings +~~~~~~~~ + +- ``packages``: Python packages to install, a list of strings. The list + can contain anything that ``pip install`` would accept as an + argument, for example ``awscli==1.3.13``. diff --git a/bootstrapvz/plugins/prebootstrapped/README.rst b/bootstrapvz/plugins/prebootstrapped/README.rst new file mode 100644 index 0000000..bc30de8 --- /dev/null +++ b/bootstrapvz/plugins/prebootstrapped/README.rst @@ -0,0 +1,26 @@ +prebootstrapped +--------------- + +When developing for bootstrap-vz, testing can be quite tedious since the +bootstrapping process can take a while. The prebootstrapped plugin +solves that problem by creating a snapshot of your volume right after +all the software has been installed. The next time bootstrap-vz is run, +the plugin replaces all volume preparation and bootstrapping tasks and +recreates the volume from the snapshot instead. + +The plugin assumes that the users knows what he is doing (e.g. it +doesn't check whether bootstrap-vz is being run with a partitioned +volume configuration, while the snapshot is unpartitioned). + +When no snapshot or image is specified the plugin creates one and +outputs its ID/path. Specifying an ID/path enables the second mode of +operation which recreates the volume from the specified snapshot instead +of creating it from scratch. + +Settings +~~~~~~~~ + +- ``snapshot``: ID of the EBS snapshot to use. This setting only works + with EBS backed EC2 configurations. +- ``image``: Path to the loopbackvolume snapshot. This setting works + with all configurable volume backings except EBS. diff --git a/bootstrapvz/plugins/puppet/README.rst b/bootstrapvz/plugins/puppet/README.rst new file mode 100644 index 0000000..203e0b3 --- /dev/null +++ b/bootstrapvz/plugins/puppet/README.rst @@ -0,0 +1,24 @@ +Puppet +------ + +Installs `puppet `__ and optionally applies a +manifest inside the chroot. You can also have it copy your puppet +configuration into the image so it is readily available once the image +is booted. + +Keep in mind that when applying a manifest, the system is in a chrooted +environment. This can prevent daemons from running properly (e.g. +listening to ports), they will also need to be shut down gracefully +(which bootstrap-vz cannot do) before unmounting the volume. It is +advisable to avoid starting any daemons inside the chroot at all. + +Settings +~~~~~~~~ + +- ``manifest``: Path to the puppet manifest that should be applied. + *``optional``* +- ``assets``: Path to puppet assets. The contents will be copied into + ``/etc/puppet`` on the image. Any existing files will be overwritten. + *``optional``* +- ``enable_agent``: Whether the puppet agent daemon should be enabled. + *``optional``* diff --git a/bootstrapvz/plugins/root_password/README.rst b/bootstrapvz/plugins/root_password/README.rst new file mode 100644 index 0000000..a24d904 --- /dev/null +++ b/bootstrapvz/plugins/root_password/README.rst @@ -0,0 +1,11 @@ +root password +------------- + +Sets the root password. This plugin removes the task that disables the +SSH password authentication. + +Settings +~~~~~~~~ + +- ``password``: The password for the root user. + *``required``* diff --git a/bootstrapvz/plugins/salt/README.rst b/bootstrapvz/plugins/salt/README.rst new file mode 100644 index 0000000..acb71ab --- /dev/null +++ b/bootstrapvz/plugins/salt/README.rst @@ -0,0 +1,26 @@ +Salt +---- + +Install `salt `__ minion in the image. Uses +`salt-bootstrap `__ script +to install. + +Settings +~~~~~~~~ + +- ``install_source``: Source to install salt codebase from. ``stable`` + for current stable, ``daily`` for installing the daily build, and + ``git`` to install from git repository. + *``required``* +- ``version``: Only needed if you are installing from ``git``. + \ ``develop`` to install current development head, or provide any tag + name or commit hash from `salt + repo `__ + *``optional``* +- ``master``: Salt master FQDN or IP + *``optional``* +- ``grains``: Set `salt + grains `__ + for this minion. Accepts a map with grain name as key and the grain + data as value. + *``optional``* diff --git a/bootstrapvz/plugins/unattended_upgrades/README.rst b/bootstrapvz/plugins/unattended_upgrades/README.rst new file mode 100644 index 0000000..3339b7b --- /dev/null +++ b/bootstrapvz/plugins/unattended_upgrades/README.rst @@ -0,0 +1,18 @@ +Unattended upgrades +------------------- + +Enables the `unattended update/upgrade +feature `__ in +aptitude. Enable it to have your system automatically download and +install security updates automatically with a set interval. + +Settings +~~~~~~~~ + +- ``update_interval``: Days between running ``apt-get update``. + *``required``* +- ``download_interval``: Days between running + ``apt-get upgrade --download-only`` + *``required``* +- ``upgrade_interval``: Days between installing any security upgrades. + *``required``* diff --git a/bootstrapvz/plugins/vagrant/README.rst b/bootstrapvz/plugins/vagrant/README.rst new file mode 100644 index 0000000..a40be4a --- /dev/null +++ b/bootstrapvz/plugins/vagrant/README.rst @@ -0,0 +1,12 @@ +Vagrant +------- + +Vagrant is a tool to quickly create virtualized environments. It uses +"boxes" to make downloading and sharing those environments easier. A box +is a tarball containing a virtual volumes accompanied by an `OVF +specification `__ +of the virtual machine. + +This plugin creates a vagrant box that is ready to be shared or +deployed. At the moment it is only compatible with the VirtualBox +provider and doesn't requires any additional settings. diff --git a/bootstrapvz/providers/azure/README.md b/bootstrapvz/providers/azure/README.rst similarity index 67% rename from bootstrapvz/providers/azure/README.md rename to bootstrapvz/providers/azure/README.rst index c03009e..165b557 100644 --- a/bootstrapvz/providers/azure/README.md +++ b/bootstrapvz/providers/azure/README.rst @@ -1,22 +1,28 @@ -Azure provider -=========== +Azure +===== -This provider generates raw images for Microsoft Azure computing platform. +This provider generates raw images for Microsoft Azure computing +platform. Setup -===== +----- qemu-img >= 1.7.0 required to convert raw image to vhd fixed size disk. This release is available in wheezy-backports. *wget* must be installed on local computer. - -Manifest must use the *raw* format, provider will automatically transform the disk to a vhd disk format. +Manifest must use the *raw* format, provider will automatically +transform the disk to a vhd disk format. Do not create swap space on the OS disk: -The Windows Azure Linux Agent can automatically configure swap space using the local resource disk that is attached to the VM after provisioning on Azure. Modify the following parameters in /etc/waagent.conf appropriately: +The Windows Azure Linux Agent can automatically configure swap space +using the local resource disk that is attached to the VM after +provisioning on Azure. Modify the following parameters in +/etc/waagent.conf appropriately: + +:: ResourceDisk.Format=y ResourceDisk.Filesystem=ext4 @@ -24,7 +30,10 @@ The Windows Azure Linux Agent can automatically configure swap space using the l ResourceDisk.EnableSwap=y ResourceDisk.SwapSizeMB=2048 ## NOTE: set this to whatever you need it to be. -You can specify a waagent.conf file to replace the default one in the manifest in the azure/waagent section of the provider: +You can specify a waagent.conf file to replace the default one in the +manifest in the azure/waagent section of the provider: + +:: "system" : { "waagent" : { @@ -33,4 +42,5 @@ You can specify a waagent.conf file to replace the default one in the manifest i } }, ... -Waagent versions are available at: https://github.com/Azure/WALinuxAgent/releases +Waagent versions are available at: +https://github.com/Azure/WALinuxAgent/releases diff --git a/bootstrapvz/providers/ec2/README.rst b/bootstrapvz/providers/ec2/README.rst new file mode 100644 index 0000000..05b63e8 --- /dev/null +++ b/bootstrapvz/providers/ec2/README.rst @@ -0,0 +1,71 @@ +EC2 +=== + +The `EC2 `__ provider automatically creates +a volume for bootstrapping (be it EBS or S3), makes a snapshot of it +once it is done and registers it as an AMI. EBS volume backing only +works on an EC2 host while S3 backed volumes *should* work locally (at +this time however they do not, a fix is in the works). + +Unless `the cloud-init plugin `__ +is used, special startup scripts will be installed that automatically fetch the +configured authorized\_key from the instance metadata and save or run +any userdata supplied (if the userdata begins with ``#!`` it will be +run). + +Credentials +----------- + +The AWS credentials can be configured in two ways: Via the manifest or +through environment variables. To bootstrap S3 backed instances you will +need a user certificate and a private key in addition to the access key +and secret key, which are needed for bootstraping EBS backed instances. + +The settings describes below should be placed in the ``credentials`` key +under the ``provider`` section. + +- **``access-key``**: AWS access-key. + May also be supplied via the environment variable + ``$AWS_ACCESS_KEY`` + *``required for EBS & S3 backing``* +- **``secret-key``**: AWS secret-key. + May also be supplied via the environment variable + ``$AWS_SECRET_KEY`` + *``required for EBS & S3 backing``* +- **``certificate``**: Path to the AWS user certificate. Used for + uploading the image to an S3 bucket. + May also be supplied via the environment variable + ``$AWS_CERTIFICATE`` + *``required for S3 backing``* +- **``private-key``**: Path to the AWS private key. Used for uploading + the image to an S3 bucket. + May also be supplied via the environment variable + ``$AWS_PRIVATE_KEY`` + *``required for S3 backing``* +- **``user-id``**: AWS user ID. Used for uploading the image to an S3 + bucket. + May also be supplied via the environment variable ``$AWS_USER_ID`` + *``required for S3 backing``* + +Example: + +.. code:: yaml + + --- + provider: + name: ec2 + virtualization: hvm + enhanced_networking: simple + credentials: + access-key: AFAKEACCESSKEYFORAWS + secret-key: thes3cr3tkeyf0ryourawsaccount/FS4d8Qdva + +Dependencies +------------ + +To communicate with the AWS API `boto `__ +is required (version 2.14.0 or higher) you can install boto with +``pip install boto`` (on wheezy, the packaged version is too low). S3 +images are chopped up and uploaded using +`euca2ools `__ (install with +``apt-get install euca2ools``). diff --git a/bootstrapvz/providers/gce/README.rst b/bootstrapvz/providers/gce/README.rst new file mode 100644 index 0000000..e760120 --- /dev/null +++ b/bootstrapvz/providers/gce/README.rst @@ -0,0 +1,9 @@ +Google Compute Engine +--------------------- + +The `GCE `__ provider +can creates image as expected by GCE - i.e. raw disk image in \*.tar.gz +file. It can upload created images to Google Storage Engine (to URI +provided in manifest by ``gcs_destination``) and can register image to +be used by Google Compute Engine to project provided in manifest by +``gce_project``. Both of those functionalities are not fully tested yet. diff --git a/bootstrapvz/providers/kvm/README.rst b/bootstrapvz/providers/kvm/README.rst new file mode 100644 index 0000000..57e60d4 --- /dev/null +++ b/bootstrapvz/providers/kvm/README.rst @@ -0,0 +1,8 @@ +KVM +--- + +The `KVM `__ provider creates +virtual images for Linux Kernel-based Virtual Machines. It supports the +installation of `virtio kernel +modules `__ (paravirtualized +drivers for IO operations). diff --git a/bootstrapvz/providers/virtualbox/README.rst b/bootstrapvz/providers/virtualbox/README.rst new file mode 100644 index 0000000..154199b --- /dev/null +++ b/bootstrapvz/providers/virtualbox/README.rst @@ -0,0 +1,12 @@ +VirtualBox +---------- + +The `VirtualBox `__ provider can bootstrap +to both .vdi and .vmdk images (raw images are also supported but do not +run in VirtualBox). It's advisable to always use vmdk images for +interoperability (e.g. +`OVF `__ files +*should* support vdi files, but since they have no identifier URL not +even VirtualBox itself can import them). VirtualBox Guest Additions can +be installed automatically if the ISO is `provided in the +manifest `__. diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 66d46ad..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/bootstrap-vz.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/bootstrap-vz.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/bootstrap-vz" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/bootstrap-vz" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docs/_static/.gitignore b/docs/_static/.gitignore new file mode 100644 index 0000000..4dd81db --- /dev/null +++ b/docs/_static/.gitignore @@ -0,0 +1 @@ +graph.json diff --git a/docs/_static/graph.json b/docs/_static/graph.json deleted file mode 100644 index 3caaa2b..0000000 --- a/docs/_static/graph.json +++ /dev/null @@ -1 +0,0 @@ -{"phases": [{"name": "Preparation", "description": "Initializing connections, fetching data etc."}, {"name": "Volume creation", "description": "Creating the volume to bootstrap onto"}, {"name": "Volume preparation", "description": "Formatting the bootstrap volume"}, {"name": "Volume mounting", "description": "Mounting bootstrap volume"}, {"name": "OS installation", "description": "Installing the operating system"}, {"name": "Package installation", "description": "Installing software"}, {"name": "System modification", "description": "Modifying configuration files, adding resources, etc."}, {"name": "System cleaning", "description": "Removing sensitive data, temporary files and other leftovers"}, {"name": "Volume unmounting", "description": "Unmounting the bootstrap volume"}, {"name": "Image registration", "description": "Uploading/Registering with the provider"}, {"name": "Cleaning", "description": "Removing temporary files"}], "modules": [{"name": "bootstrapvz.common.tasks.apt"}, {"name": "bootstrapvz.common.tasks.boot"}, {"name": "bootstrapvz.common.tasks.bootstrap"}, {"name": "bootstrapvz.common.tasks.cleanup"}, {"name": "bootstrapvz.common.tasks.development"}, {"name": "bootstrapvz.common.tasks.filesystem"}, {"name": "bootstrapvz.common.tasks.host"}, {"name": "bootstrapvz.common.tasks.initd"}, {"name": "bootstrapvz.common.tasks.locale"}, {"name": "bootstrapvz.common.tasks.loopback"}, {"name": "bootstrapvz.common.tasks.network"}, {"name": "bootstrapvz.common.tasks.packages"}, {"name": "bootstrapvz.common.tasks.partitioning"}, {"name": "bootstrapvz.common.tasks.security"}, {"name": "bootstrapvz.common.tasks.ssh"}, {"name": "bootstrapvz.common.tasks.volume"}, {"name": "bootstrapvz.common.tasks.workspace"}, {"name": "bootstrapvz.plugins.admin_user.tasks"}, {"name": "bootstrapvz.plugins.apt_proxy.tasks"}, {"name": "bootstrapvz.plugins.chef.tasks"}, {"name": "bootstrapvz.plugins.cloud_init.tasks"}, {"name": "bootstrapvz.plugins.image_commands.tasks"}, {"name": "bootstrapvz.plugins.minimize_size.tasks"}, {"name": "bootstrapvz.plugins.ntp.tasks"}, {"name": "bootstrapvz.plugins.opennebula.tasks"}, {"name": "bootstrapvz.plugins.prebootstrapped.tasks"}, {"name": "bootstrapvz.plugins.puppet.tasks"}, {"name": "bootstrapvz.plugins.root_password.tasks"}, {"name": "bootstrapvz.plugins.salt.tasks"}, {"name": "bootstrapvz.plugins.unattended_upgrades.tasks"}, {"name": "bootstrapvz.plugins.vagrant.tasks"}, {"name": "bootstrapvz.providers.azure.tasks.boot"}, {"name": "bootstrapvz.providers.azure.tasks.image"}, {"name": "bootstrapvz.providers.azure.tasks.packages"}, {"name": "bootstrapvz.providers.ec2.tasks.ami"}, {"name": "bootstrapvz.providers.ec2.tasks.boot"}, {"name": "bootstrapvz.providers.ec2.tasks.connection"}, {"name": "bootstrapvz.providers.ec2.tasks.ebs"}, {"name": "bootstrapvz.providers.ec2.tasks.filesystem"}, {"name": "bootstrapvz.providers.ec2.tasks.host"}, {"name": "bootstrapvz.providers.ec2.tasks.initd"}, {"name": "bootstrapvz.providers.ec2.tasks.network"}, {"name": "bootstrapvz.providers.ec2.tasks.packages"}, {"name": "bootstrapvz.providers.gce.tasks.apt"}, {"name": "bootstrapvz.providers.gce.tasks.boot"}, {"name": "bootstrapvz.providers.gce.tasks.configuration"}, {"name": "bootstrapvz.providers.gce.tasks.host"}, {"name": "bootstrapvz.providers.gce.tasks.image"}, {"name": "bootstrapvz.providers.gce.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.packages"}, {"name": "bootstrapvz.providers.kvm.tasks.virtio"}, {"name": "bootstrapvz.providers.virtualbox.tasks.guest_additions"}, {"name": "bootstrapvz.providers.virtualbox.tasks.packages"}], "nodes": [{"phase": 0, "name": "AddDefaultSources", "module": 0}, {"phase": 0, "name": "AddManifestPreferences", "module": 0}, {"phase": 0, "name": "AddManifestSources", "module": 0}, {"phase": 7, "name": "AptClean", "module": 0}, {"phase": 5, "name": "AptUpdate", "module": 0}, {"phase": 5, "name": "AptUpgrade", "module": 0}, {"phase": 5, "name": "DisableDaemonAutostart", "module": 0}, {"phase": 7, "name": "EnableDaemonAutostart", "module": 0}, {"phase": 5, "name": "InstallTrustedKeys", "module": 0}, {"phase": 7, "name": "PurgeUnusedPackages", "module": 0}, {"phase": 5, "name": "WritePreferences", "module": 0}, {"phase": 5, "name": "WriteSources", "module": 0}, {"phase": 0, "name": "AddExtlinuxPackage", "module": 1}, {"phase": 0, "name": "AddGrubPackage", "module": 1}, {"phase": 6, "name": "BlackListModules", "module": 1}, {"phase": 6, "name": "ConfigureGrub", "module": 1}, {"phase": 6, "name": "DisableGetTTYs", "module": 1}, {"phase": 6, "name": "InstallExtLinux", "module": 1}, {"phase": 6, "name": "InstallGrub", "module": 1}, {"phase": 0, "name": "AddRequiredCommands", "module": 2}, {"phase": 4, "name": "Bootstrap", "module": 2}, {"phase": 0, "name": "ExcludePackagesInBootstrap", "module": 2}, {"phase": 0, "name": "IncludePackagesInBootstrap", "module": 2}, {"phase": 4, "name": "MakeTarball", "module": 2}, {"phase": 7, "name": "CleanTMP", "module": 3}, {"phase": 7, "name": "ClearMOTD", "module": 3}, {"phase": 10, "name": "TriggerRollback", "module": 4}, {"phase": 0, "name": "AddRequiredCommands", "module": 5}, {"phase": 0, "name": "AddXFSProgs", "module": 5}, {"phase": 3, "name": "CreateBootMountDir", "module": 5}, {"phase": 3, "name": "CreateMountDir", "module": 5}, {"phase": 8, "name": "DeleteMountDir", "module": 5}, {"phase": 6, "name": "FStab", "module": 5}, {"phase": 2, "name": "Format", "module": 5}, {"phase": 3, "name": "MountBoot", "module": 5}, {"phase": 3, "name": "MountRoot", "module": 5}, {"phase": 4, "name": "MountSpecials", "module": 5}, {"phase": 2, "name": "TuneVolumeFS", "module": 5}, {"phase": 8, "name": "UnmountRoot", "module": 5}, {"phase": 0, "name": "CheckExternalCommands", "module": 6}, {"phase": 6, "name": "AddExpandRoot", "module": 7}, {"phase": 6, "name": "AdjustExpandRootScript", "module": 7}, {"phase": 6, "name": "InstallInitScripts", "module": 7}, {"phase": 6, "name": "RemoveHWClock", "module": 7}, {"phase": 5, "name": "GenerateLocale", "module": 8}, {"phase": 0, "name": "LocaleBootstrapPackage", "module": 8}, {"phase": 6, "name": "SetTimezone", "module": 8}, {"phase": 0, "name": "AddRequiredCommands", "module": 9}, {"phase": 1, "name": "Create", "module": 9}, {"phase": 9, "name": "MoveImage", "module": 9}, {"phase": 6, "name": "ConfigureNetworkIF", "module": 10}, {"phase": 6, "name": "RemoveDNSInfo", "module": 10}, {"phase": 6, "name": "RemoveHostname", "module": 10}, {"phase": 6, "name": "SetHostname", "module": 10}, {"phase": 0, "name": "AddManifestPackages", "module": 11}, {"phase": 5, "name": "AddTaskselStandardPackages", "module": 11}, {"phase": 5, "name": "InstallPackages", "module": 11}, {"phase": 0, "name": "AddRequiredCommands", "module": 12}, {"phase": 2, "name": "MapPartitions", "module": 12}, {"phase": 2, "name": "PartitionVolume", "module": 12}, {"phase": 8, "name": "UnmapPartitions", "module": 12}, {"phase": 6, "name": "EnableShadowConfig", "module": 13}, {"phase": 0, "name": "AddOpenSSHPackage", "module": 14}, {"phase": 6, "name": "AddSSHKeyGeneration", "module": 14}, {"phase": 6, "name": "DisableSSHDNSLookup", "module": 14}, {"phase": 6, "name": "DisableSSHPasswordAuthentication", "module": 14}, {"phase": 7, "name": "ShredHostkeys", "module": 14}, {"phase": 1, "name": "Attach", "module": 15}, {"phase": 10, "name": "Delete", "module": 15}, {"phase": 8, "name": "Detach", "module": 15}, {"phase": 0, "name": "CreateWorkspace", "module": 16}, {"phase": 10, "name": "DeleteWorkspace", "module": 16}, {"phase": 0, "name": "AddSudoPackage", "module": 17}, {"phase": 6, "name": "AdminUserCredentials", "module": 17}, {"phase": 6, "name": "CreateAdminUser", "module": 17}, {"phase": 6, "name": "DisableRootLogin", "module": 17}, {"phase": 6, "name": "PasswordlessSudo", "module": 17}, {"phase": 7, "name": "RemoveAptProxy", "module": 18}, {"phase": 5, "name": "SetAptProxy", "module": 18}, {"phase": 0, "name": "AddPackages", "module": 19}, {"phase": 0, "name": "CheckAssetsPath", "module": 19}, {"phase": 6, "name": "CopyChefAssets", "module": 19}, {"phase": 0, "name": "AddBackports", "module": 20}, {"phase": 0, "name": "AddCloudInitPackages", "module": 20}, {"phase": 6, "name": "DisableModules", "module": 20}, {"phase": 5, "name": "SetMetadataSource", "module": 20}, {"phase": 6, "name": "SetUsername", "module": 20}, {"phase": 6, "name": "ImageExecuteCommand", "module": 21}, {"phase": 4, "name": "AddFolderMounts", "module": 22}, {"phase": 0, "name": "AddRequiredCommands", "module": 22}, {"phase": 7, "name": "RemoveFolderMounts", "module": 22}, {"phase": 8, "name": "ShrinkVolume", "module": 22}, {"phase": 8, "name": "Zerofree", "module": 22}, {"phase": 5, "name": "AddNtpPackage", "module": 23}, {"phase": 6, "name": "SetNtpServers", "module": 23}, {"phase": 0, "name": "AddBackports", "module": 24}, {"phase": 0, "name": "AddONEContextPackage", "module": 24}, {"phase": 5, "name": "CopyImage", "module": 25}, {"phase": 1, "name": "CreateFromImage", "module": 25}, {"phase": 1, "name": "CreateFromSnapshot", "module": 25}, {"phase": 5, "name": "Snapshot", "module": 25}, {"phase": 0, "name": "AddPackages", "module": 26}, {"phase": 6, "name": "ApplyPuppetManifest", "module": 26}, {"phase": 0, "name": "CheckAssetsPath", "module": 26}, {"phase": 0, "name": "CheckManifestPath", "module": 26}, {"phase": 6, "name": "CopyPuppetAssets", "module": 26}, {"phase": 6, "name": "EnableAgent", "module": 26}, {"phase": 6, "name": "SetRootPassword", "module": 27}, {"phase": 5, "name": "BootstrapSaltMinion", "module": 28}, {"phase": 5, "name": "InstallSaltDependencies", "module": 28}, {"phase": 6, "name": "SetSaltGrains", "module": 28}, {"phase": 0, "name": "AddUnattendedUpgradesPackage", "module": 29}, {"phase": 6, "name": "EnablePeriodicUpgrades", "module": 29}, {"phase": 6, "name": "AddInsecurePublicKey", "module": 30}, {"phase": 0, "name": "AddPackages", "module": 30}, {"phase": 0, "name": "CheckBoxPath", "module": 30}, {"phase": 0, "name": "CreateVagrantBoxDir", "module": 30}, {"phase": 6, "name": "CreateVagrantUser", "module": 30}, {"phase": 9, "name": "PackageBox", "module": 30}, {"phase": 6, "name": "PasswordlessSudo", "module": 30}, {"phase": 10, "name": "RemoveVagrantBoxDir", "module": 30}, {"phase": 6, "name": "SetRootPassword", "module": 30}, {"phase": 6, "name": "ConfigureGrub", "module": 31}, {"phase": 9, "name": "ConvertToVhd", "module": 32}, {"phase": 0, "name": "DefaultPackages", "module": 33}, {"phase": 5, "name": "Waagent", "module": 33}, {"phase": 0, "name": "AMIName", "module": 34}, {"phase": 9, "name": "BundleImage", "module": 34}, {"phase": 9, "name": "RegisterAMI", "module": 34}, {"phase": 10, "name": "RemoveBundle", "module": 34}, {"phase": 9, "name": "UploadImage", "module": 34}, {"phase": 6, "name": "ConfigurePVGrub", "module": 35}, {"phase": 0, "name": "Connect", "module": 36}, {"phase": 0, "name": "GetCredentials", "module": 36}, {"phase": 1, "name": "Attach", "module": 37}, {"phase": 1, "name": "Create", "module": 37}, {"phase": 9, "name": "Snapshot", "module": 37}, {"phase": 6, "name": "S3FStab", "module": 38}, {"phase": 0, "name": "AddExternalCommands", "module": 39}, {"phase": 0, "name": "GetInstanceMetadata", "module": 39}, {"phase": 0, "name": "SetRegion", "module": 39}, {"phase": 6, "name": "AddEC2InitScripts", "module": 40}, {"phase": 0, "name": "AddBuildEssentialPackage", "module": 41}, {"phase": 6, "name": "EnableDHCPCDDNS", "module": 41}, {"phase": 5, "name": "InstallEnhancedNetworking", "module": 41}, {"phase": 0, "name": "DefaultPackages", "module": 42}, {"phase": 7, "name": "CleanGoogleRepositoriesAndKeys", "module": 43}, {"phase": 5, "name": "ImportGoogleKey", "module": 43}, {"phase": 0, "name": "SetPackageRepositories", "module": 43}, {"phase": 6, "name": "ConfigureGrub", "module": 44}, {"phase": 6, "name": "GatherReleaseInformation", "module": 45}, {"phase": 6, "name": "DisableIPv6", "module": 46}, {"phase": 6, "name": "SetHostname", "module": 46}, {"phase": 9, "name": "CreateTarball", "module": 47}, {"phase": 9, "name": "RegisterImage", "module": 47}, {"phase": 9, "name": "UploadImage", "module": 47}, {"phase": 0, "name": "DefaultPackages", "module": 48}, {"phase": 0, "name": "GooglePackages", "module": 48}, {"phase": 5, "name": "InstallGSUtil", "module": 48}, {"phase": 0, "name": "DefaultPackages", "module": 49}, {"phase": 6, "name": "VirtIO", "module": 50}, {"phase": 5, "name": "AddGuestAdditionsPackages", "module": 51}, {"phase": 0, "name": "CheckGuestAdditionsPath", "module": 51}, {"phase": 5, "name": "InstallGuestAdditions", "module": 51}, {"phase": 0, "name": "DefaultPackages", "module": 52}], "links": [{"source": 19, "target": 39, "definer": 19}, {"source": 21, "target": 20, "definer": 21}, {"source": 22, "target": 20, "definer": 22}, {"source": 27, "target": 39, "definer": 27}, {"source": 38, "target": 69, "definer": 38}, {"source": 40, "target": 42, "definer": 40}, {"source": 43, "target": 42, "definer": 43}, {"source": 47, "target": 39, "definer": 47}, {"source": 48, "target": 67, "definer": 48}, {"source": 55, "target": 56, "definer": 55}, {"source": 57, "target": 39, "definer": 57}, {"source": 58, "target": 33, "definer": 58}, {"source": 60, "target": 69, "definer": 60}, {"source": 63, "target": 42, "definer": 63}, {"source": 68, "target": 71, "definer": 68}, {"source": 78, "target": 4, "definer": 78}, {"source": 85, "target": 4, "definer": 85}, {"source": 89, "target": 39, "definer": 89}, {"source": 90, "target": 3, "definer": 90}, {"source": 92, "target": 60, "definer": 92}, {"source": 92, "target": 69, "definer": 92}, {"source": 93, "target": 56, "definer": 93}, {"source": 98, "target": 67, "definer": 98}, {"source": 99, "target": 134, "definer": 99}, {"source": 102, "target": 52, "definer": 102}, {"source": 102, "target": 51, "definer": 102}, {"source": 120, "target": 71, "definer": 120}, {"source": 122, "target": 18, "definer": 122}, {"source": 129, "target": 71, "definer": 129}, {"source": 138, "target": 39, "definer": 138}, {"source": 141, "target": 42, "definer": 141}, {"source": 146, "target": 3, "definer": 146}, {"source": 147, "target": 11, "definer": 147}, {"source": 148, "target": 2, "definer": 148}, {"source": 149, "target": 18, "definer": 149}, {"source": 161, "target": 56, "definer": 161}, {"source": 2, "target": 0, "definer": 0}, {"source": 44, "target": 4, "definer": 4}, {"source": 11, "target": 4, "definer": 4}, {"source": 4, "target": 5, "definer": 5}, {"source": 6, "target": 5, "definer": 5}, {"source": 11, "target": 10, "definer": 10}, {"source": 8, "target": 11, "definer": 11}, {"source": 0, "target": 12, "definer": 12}, {"source": 0, "target": 13, "definer": 13}, {"source": 32, "target": 15, "definer": 15}, {"source": 32, "target": 17, "definer": 17}, {"source": 32, "target": 18, "definer": 18}, {"source": 23, "target": 20, "definer": 20}, {"source": 0, "target": 28, "definer": 28}, {"source": 35, "target": 29, "definer": 29}, {"source": 38, "target": 31, "definer": 31}, {"source": 29, "target": 34, "definer": 34}, {"source": 30, "target": 35, "definer": 35}, {"source": 20, "target": 36, "definer": 36}, {"source": 33, "target": 37, "definer": 37}, {"source": 42, "target": 41, "definer": 41}, {"source": 0, "target": 54, "definer": 54}, {"source": 4, "target": 55, "definer": 55}, {"source": 5, "target": 56, "definer": 56}, {"source": 59, "target": 58, "definer": 58}, {"source": 38, "target": 60, "definer": 60}, {"source": 0, "target": 62, "definer": 62}, {"source": 0, "target": 72, "definer": 72}, {"source": 42, "target": 73, "definer": 73}, {"source": 0, "target": 79, "definer": 79}, {"source": 0, "target": 83, "definer": 83}, {"source": 82, "target": 83, "definer": 83}, {"source": 44, "target": 85, "definer": 85}, {"source": 20, "target": 88, "definer": 88}, {"source": 69, "target": 91, "definer": 91}, {"source": 38, "target": 92, "definer": 92}, {"source": 0, "target": 96, "definer": 96}, {"source": 95, "target": 96, "definer": 96}, {"source": 56, "target": 97, "definer": 97}, {"source": 163, "target": 97, "definer": 97}, {"source": 56, "target": 100, "definer": 100}, {"source": 163, "target": 100, "definer": 100}, {"source": 0, "target": 101, "definer": 101}, {"source": 105, "target": 102, "definer": 102}, {"source": 56, "target": 108, "definer": 108}, {"source": 0, "target": 109, "definer": 109}, {"source": 0, "target": 111, "definer": 111}, {"source": 117, "target": 113, "definer": 113}, {"source": 0, "target": 114, "definer": 114}, {"source": 70, "target": 116, "definer": 116}, {"source": 115, "target": 116, "definer": 116}, {"source": 0, "target": 124, "definer": 124}, {"source": 56, "target": 125, "definer": 125}, {"source": 132, "target": 126, "definer": 126}, {"source": 136, "target": 128, "definer": 128}, {"source": 130, "target": 128, "definer": 128}, {"source": 127, "target": 130, "definer": 130}, {"source": 133, "target": 132, "definer": 132}, {"source": 139, "target": 132, "definer": 132}, {"source": 140, "target": 132, "definer": 132}, {"source": 135, "target": 134, "definer": 134}, {"source": 0, "target": 142, "definer": 142}, {"source": 0, "target": 145, "definer": 145}, {"source": 8, "target": 147, "definer": 147}, {"source": 50, "target": 151, "definer": 151}, {"source": 49, "target": 153, "definer": 153}, {"source": 155, "target": 154, "definer": 154}, {"source": 153, "target": 155, "definer": 155}, {"source": 0, "target": 156, "definer": 156}, {"source": 156, "target": 157, "definer": 157}, {"source": 0, "target": 159, "definer": 159}, {"source": 56, "target": 163, "definer": 163}, {"source": 0, "target": 164, "definer": 164}]} \ No newline at end of file diff --git a/docs/_static/taskoverview.coffee b/docs/_static/taskoverview.coffee index d9d3b94..f4e1f25 100644 --- a/docs/_static/taskoverview.coffee +++ b/docs/_static/taskoverview.coffee @@ -1,10 +1,10 @@ class window.TaskOverview viewBoxHeight = 800 - viewBoxWidth = 200 + viewBoxWidth = 800 margins = - top: 100 + top: 200 left: 50 - bottom: 100 + bottom: 200 right: 50 gravity = lateral: .1 @@ -26,7 +26,7 @@ class window.TaskOverview constructor: ({@selector}) -> @svg = d3.select(@selector) .attr('viewBox', "0 0 #{viewBoxWidth} #{viewBoxHeight}") - d3.json '_static/graph.json', @buildGraph + d3.json '../_static/graph.json', @buildGraph buildGraph: (error, @data) => @createDefinitions() @@ -118,15 +118,23 @@ class window.TaskOverview .selectAll('line').data(layout.links()).enter() .append('line').attr('marker-end', 'url(#right-arrowhead)') + mouseOver = (d) -> + labels.classed 'hover', (l) -> d is l + nodes.classed 'highlight', (n) -> d.module is n.module + + mouseOut = (d) -> + labels.classed 'hover', no + nodes.classed 'highlight', no + nodes = @svg.append('g').attr('class', 'nodes') .selectAll('g.partition').data(groups).enter() .append('g').attr('class', 'partition') .selectAll('circle').data((d) -> d.values).enter() .append('circle').attr('r', (d) -> d.radius) - .style('fill', (d, i) -> nodeColors(d[nodeColorKey])) + .style('fill', (d) -> nodeColors(d[nodeColorKey])) .call(layout.drag) - .on('mouseover', (d) -> (labels.filter (l) -> d is l).classed 'hover', true) - .on('mouseout', (d) -> (labels.filter (l) -> d is l).classed 'hover', false) + .on('mouseover', mouseOver) + .on('mouseout', mouseOut) labels = @svg.append('g').attr('class', 'node-labels') .selectAll('g.partition').data(groups).enter() diff --git a/docs/_static/taskoverview.less b/docs/_static/taskoverview.less index 2903816..8c1e67c 100644 --- a/docs/_static/taskoverview.less +++ b/docs/_static/taskoverview.less @@ -6,6 +6,11 @@ } g.nodes circle { stroke: #000000; + &.highlight { + stroke: #555599; + stroke-width: 2.5px; + fill: #EEAAAA !important; + } opacity: .9; stroke-width: 1.5px; } diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..565b052 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1 @@ +.. include:: ../CHANGELOG.rst diff --git a/docs/conf.py b/docs/conf.py index 6eb7717..f6deef4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,8 @@ sys.path.insert(0, os.path.abspath(os.pardir)) # ones. extensions = ['sphinx.ext.coverage', 'sphinx.ext.autodoc', - 'sphinx.ext.viewcode', + 'sphinx.ext.linkcode', + 'docs.replace_rtd_links', ] # Add any paths that contain templates here, relative to this directory. @@ -261,3 +262,86 @@ texinfo_documents = [('index', 'bootstrap-vz', u'bootstrap-vz Documentation', # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False + + +# -- Link to rst files scattered throughout the project ------------------- + +import glob +import os.path + +for readme_path in glob.glob('../bootstrapvz/providers/*/README.rst'): + provider_name = os.path.basename(os.path.dirname(readme_path)) + include_path = os.path.join('providers', provider_name + '.rst') + path_to_readme = os.path.join('../../bootstrapvz/providers', provider_name, 'README.rst') + with open(include_path, 'w') as include: + include.write('.. include:: ' + path_to_readme) + + +for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'): + plugin_name = os.path.basename(os.path.dirname(readme_path)) + include_path = os.path.join('plugins', plugin_name + '.rst') + path_to_readme = os.path.join('../../bootstrapvz/plugins', plugin_name, 'README.rst') + with open(include_path, 'w') as include: + include.write('.. include:: ' + path_to_readme) + + +# -- Create task overview graph data -------------------------------------- + +from docs import taskoverview + +data = taskoverview.generate_graph_data() +taskoverview.write_data(data, '_static/graph.json') + + +# -- Substitute links for github with relative links in readthedocs ------- + + +if on_rtd: + pass + +# Snatched from here: +# https://sourcegraph.com/github.com/Gallopsled/pwntools@master/.PipPackage/pwntools/.def/docs/source/conf/linkcode_resolve/lines +baseurl = 'https://github.com/andsens/bootstrap-vz' + +import subprocess +try: + git_head = subprocess.check_output('git describe --tags 2>/dev/null', shell=True) +except subprocess.CalledProcessError: + try: + git_head = subprocess.check_output('git rev-parse HEAD', shell=True).strip()[:10] + except subprocess.CalledProcessError: + pass + + +def linkcode_resolve(domain, info): + if domain != 'py': + return None + if not info['module']: + return None + + filepath = info['module'].replace('.', '/') + '.py' + fmt_args = {'baseurl': baseurl, + 'commit': git_head, + 'path': filepath} + + import importlib + import inspect + import types + module = importlib.import_module(info['module']) + value = module + for part in info['fullname'].split('.'): + value = getattr(value, part, None) + if value is None: + break + valid_types = (types.ModuleType, types.ClassType, types.MethodType, + types.FunctionType, types.TracebackType, + types.FrameType, types.CodeType) + if isinstance(value, valid_types): + try: + lines, first = inspect.getsourcelines(value) + fmt_args['linestart'] = first + fmt_args['lineend'] = first + len(lines) - 1 + return '{baseurl}/blob/{commit}/{path}#L{linestart}-L{lineend}'.format(**fmt_args) + except IOError: + pass + return '{baseurl}/blob/{commit}/{path}'.format(**fmt_args) diff --git a/docs/base/fs.rst b/docs/developers/api/base/fs.rst similarity index 100% rename from docs/base/fs.rst rename to docs/developers/api/base/fs.rst diff --git a/docs/base/index.rst b/docs/developers/api/base/index.rst similarity index 98% rename from docs/base/index.rst rename to docs/developers/api/base/index.rst index d80e702..c17d336 100644 --- a/docs/base/index.rst +++ b/docs/developers/api/base/index.rst @@ -7,9 +7,9 @@ and handles the gather, sorting and running of tasks. .. toctree:: :maxdepth: 2 + :glob: - fs - pkg + * Bootstrap information --------------------- diff --git a/docs/base/pkg.rst b/docs/developers/api/base/pkg.rst similarity index 96% rename from docs/base/pkg.rst rename to docs/developers/api/base/pkg.rst index 1a8db8b..4625358 100644 --- a/docs/base/pkg.rst +++ b/docs/developers/api/base/pkg.rst @@ -16,7 +16,7 @@ Sources list :private-members: Preferences list ------------- +---------------- .. automodule:: bootstrapvz.base.pkg.preferenceslist :members: :private-members: diff --git a/docs/common/fs.rst b/docs/developers/api/common/fs.rst similarity index 100% rename from docs/common/fs.rst rename to docs/developers/api/common/fs.rst diff --git a/docs/common/index.rst b/docs/developers/api/common/index.rst similarity index 100% rename from docs/common/index.rst rename to docs/developers/api/common/index.rst diff --git a/docs/common/tasks/index.rst b/docs/developers/api/common/tasks/index.rst similarity index 100% rename from docs/common/tasks/index.rst rename to docs/developers/api/common/tasks/index.rst diff --git a/docs/developers/api/index.rst b/docs/developers/api/index.rst new file mode 100644 index 0000000..1c7c3c7 --- /dev/null +++ b/docs/developers/api/index.rst @@ -0,0 +1,9 @@ +API +=== + + +.. toctree:: + :maxdepth: 2 + + base/index + common/index diff --git a/docs/developers/contributing.rst b/docs/developers/contributing.rst new file mode 100644 index 0000000..ac7b6bc --- /dev/null +++ b/docs/developers/contributing.rst @@ -0,0 +1 @@ +.. include:: ../../CONTRIBUTING.rst diff --git a/docs/howitworks.rst b/docs/developers/index.rst similarity index 80% rename from docs/howitworks.rst rename to docs/developers/index.rst index cca75c9..5f1cdea 100644 --- a/docs/howitworks.rst +++ b/docs/developers/index.rst @@ -1,6 +1,17 @@ +Developers +============ + +.. toctree:: + :maxdepth: 1 + + contributing + switches + api/index + taskoverview + How bootstrap-vz works -====================== +---------------------- Tasks ~~~~~ @@ -15,14 +26,14 @@ via attributes. Here is an example: :: class MapPartitions(Task): - description = 'Mapping volume partitions' - phase = phases.volume_preparation - predecessors = [PartitionVolume] - successors = [filesystem.Format] + description = 'Mapping volume partitions' + phase = phases.volume_preparation + predecessors = [PartitionVolume] + successors = [filesystem.Format] - @classmethod - def run(cls, info): - info.volume.partition_map.map(info.volume) + @classmethod + def run(cls, info): + info.volume.partition_map.map(info.volume) In this case the attributes define that the task at hand should run after the ``PartitionVolume`` task — i.e. after volume has been diff --git a/docs/switches.rst b/docs/developers/switches.rst similarity index 100% rename from docs/switches.rst rename to docs/developers/switches.rst diff --git a/docs/taskoverview.rst b/docs/developers/taskoverview.rst similarity index 76% rename from docs/taskoverview.rst rename to docs/developers/taskoverview.rst index e84680e..2e76e19 100644 --- a/docs/taskoverview.rst +++ b/docs/developers/taskoverview.rst @@ -5,11 +5,11 @@ Taskoverview .. raw:: html - + - + diff --git a/docs/index.rst b/docs/index.rst index 5413191..4056a36 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,30 +1,13 @@ -.. bootstrap-vz documentation master file, created by - sphinx-quickstart on Sun Mar 23 16:17:28 2014. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to bootstrap-vz's documentation! -======================================== - -Contents: +bootstrap-vz +============ .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - base/index - common/index - plugins/index + intro + manifest providers/index - guidelines - taskoverview - howitworks - switches + plugins/index logging - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - + changelog + developers/index diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/manifest.rst b/docs/manifest.rst new file mode 100644 index 0000000..d8b3cef --- /dev/null +++ b/docs/manifest.rst @@ -0,0 +1 @@ +.. include:: ../manifests/README.rst diff --git a/docs/plugins/.gitignore b/docs/plugins/.gitignore new file mode 100644 index 0000000..50db996 --- /dev/null +++ b/docs/plugins/.gitignore @@ -0,0 +1,3 @@ +* +!index.rst +!.gitignore diff --git a/docs/plugins/index.rst b/docs/plugins/index.rst index 690ccbc..208c1d3 100644 --- a/docs/plugins/index.rst +++ b/docs/plugins/index.rst @@ -1,3 +1,19 @@ Plugins ======= + +Plugins are a key feature of bootstrap-vz. Despite their small size +(most plugins do not exceed 100 source lines of code) they can modify +the behavior of bootstrapped systems to a great extent. + +Below you will find documentation for all plugins available for +bootstrap-vz. If you cannot find what you are looking for, consider +`developing it yourself `__ and +contribute to this list! + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/providers/.gitignore b/docs/providers/.gitignore new file mode 100644 index 0000000..50db996 --- /dev/null +++ b/docs/providers/.gitignore @@ -0,0 +1,3 @@ +* +!index.rst +!.gitignore diff --git a/docs/providers/index.rst b/docs/providers/index.rst index ef70c42..f5b187b 100644 --- a/docs/providers/index.rst +++ b/docs/providers/index.rst @@ -1,3 +1,18 @@ - Providers ========= + +Plugins are a key feature of bootstrap-vz. Despite their small size +(most plugins do not exceed 100 source lines of code) they can modify +the behavior of bootstrapped systems to a great extent. + +Below you will find documentation for all plugins available for +bootstrap-vz. If you cannot find what you are looking for, consider +`developing it yourself `__ and +contribute to this list! + + +.. toctree:: + :maxdepth: 1 + :glob: + + * diff --git a/docs/replace_rtd_links.py b/docs/replace_rtd_links.py new file mode 100644 index 0000000..c18d1c4 --- /dev/null +++ b/docs/replace_rtd_links.py @@ -0,0 +1,19 @@ + +def setup(app): + app.connect('doctree-resolved', replace_rtd_links) + + return {'version': '0.1'} + + +def replace_rtd_links(app, doctree, fromdocname): + from docutils import nodes + import re + + rtd_baseurl = 'http://bootstrap-vz.readthedocs.org/en/master/' + search = re.compile('^' + re.escape(rtd_baseurl) + '(.*)$') + for node in doctree.traverse(nodes.reference): + if 'refuri' not in node: + continue + if not node['refuri'].startswith(rtd_baseurl): + continue + node['refuri'] = re.sub(search, r'\1', node['refuri']) diff --git a/taskoverview.py b/docs/taskoverview.py similarity index 90% rename from taskoverview.py rename to docs/taskoverview.py index f9811b1..97ecbb0 100755 --- a/taskoverview.py +++ b/docs/taskoverview.py @@ -1,7 +1,11 @@ #!/usr/bin/python +import sys +import os.path + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) -def main(opts): +def generate_graph_data(): from bootstrapvz.base.tasklist import get_all_tasks tasks = get_all_tasks() @@ -45,13 +49,11 @@ def main(opts): link[key] = tasks.index(link[key]) return link - data = {'phases': map(mk_phase, phases.order), + return {'phases': map(mk_phase, phases.order), 'modules': map(mk_module, modules), 'nodes': map(mk_node, tasks), 'links': map(mk_link, task_links)} - write_data(data, opts.get('--output', None)) - def write_data(data, output_path=None): import json @@ -73,4 +75,5 @@ if __name__ == '__main__' and __package__ is None: """ opts = docopt(usage) - main(opts) + data = generate_graph_data() + write_data(data, opts.get('--output', None)) diff --git a/manifests/README.rst b/manifests/README.rst new file mode 100644 index 0000000..d644979 --- /dev/null +++ b/manifests/README.rst @@ -0,0 +1,241 @@ +Manifest +======== + +| The manifest file is the primary way to interact with bootstrap-vz. +| Every configuration and customization of a Debian installation is + specified in this file. + +The manifest format is YAML or JSON. It is near impossible to run the +bootstrapper with an invalid configuration, since every part of the +framework supplies a `json-schema `__ that +specifies exactly which configuration settings are valid in different +situations. + +Manifest variables +------------------ + +Many of the settings in the example manifests use strings like +``debian-{system.release}-{system.architecture}-{{"{%y"}}}{{"{%m"}}}{{"{%d"}}}``. +These strings make use of manifest variables, which can cross reference +other settings in the manifest or specific values supplied by the +bootstrapper (e.g. all python date formatting variables are available). + +Any reference uses dots to specify a path to the desired manifest +setting. Not all settings support this though, to see whether embedding +a manifest variable in a setting is possible, look for the +*``manifest vars``* label. + +Sections +-------- + +The manifest is split into 7 sections. + +Provider +~~~~~~~~ + +The provider section contains all provider specific settings and the +name of the provider itself. + +- **``name``**: target virtualization platform of the installation + *``required``* + +Consult the `providers `__ section of the documentation +for a list of valid values. + +Bootstrapper +~~~~~~~~~~~~ + +This section concerns the bootstrapper itself and its behavior. There +are 4 possible settings: + +- **``workspace``**: Path to where the bootstrapper should place images + and intermediate files. Any volumes will be mounted under that path. + *``required``* +- **``tarball``**: debootstrap has the option to download all the + software and pack it up in a tarball. When starting the actual + bootstrapping process, debootstrap can then be pointed at that + tarball and use it instead of downloading anything from the internet. + If you plan on running the bootstrapper multiple times, this option + can save you a lot of bandwidth and time. This option just specifies + whether it should create a new tarball or not. It will search for and + use an available tarball if it already exists, regardless of this + setting. + *``optional``* + Valid values: ``true, false`` + Default: ``false`` +- **``mirror``**: The mirror debootstrap should download software from. + It is advisable to specify a mirror close to your location (or the + location of the host you are bootstrapping on), to decrease latency + and improve bandwidth. If not specified, `the configured aptitude + mirror URL <#packages>`__ is used. + *``optional``* +- **``include_packages``**: Extra packages to be installed during + bootstrap. Accepts a list of package names. + *``optional``* +- **``exclude_packages``**: Packages to exclude during bootstrap phase. + Accepts a list of package names. + *``optional``* +- **``guest_additions``**: This setting is only relevant for the + `virtualbox provider `__. + It specifies the path to the VirtualBox Guest Additions ISO, which, when specified, + will be mounted and used to install the VirtualBox Guest Additions. + *``optional``* + +Image +~~~~~ + +The image section configures anything pertaining directly to the image +that will be created. + +- **``name``**: The name of the resulting image. + When bootstrapping cloud images, this would be the name visible in + the interface when booting up new instances. + When bootstrapping for VirtualBox or kvm, it's the filename of the + image. + *``required``* + *``manifest vars``* +- **``description``**: Description of the image. Where this setting is + used depends highly on which provider is set. At the moment it is + only used for AWS images. + **``required for ec2 provider``** + *``manifest vars``* +- **``bucket``**: When bootstrapping an S3 backed image for AWS, this + will be the bucket where the image is uploaded to. + **``required for S3 backing``** +- **``region``**: Region in which the AMI should be registered. + **``required for S3 backing``** + +System +~~~~~~ + +This section defines anything that pertains directly to the bootstrapped +system and does not fit under any other section. + +- **``architecture``**: The architecture of the system. + Valid values: ``i386, amd64`` + *``required``* +- **``bootloader``**: The bootloader for the system. Depending on the + bootmethod of the virtualization platform, the options may be + restricted. + Valid values: ``grub, extlinux, pv-grub`` + *``required``* +- **``charmap``**: The default charmap of the system. + Valid values: Any valid charmap like ``UTF-8``, ``ISO-8859-*`` or + ``GBK``. + *``required``* +- **``hostname``**: hostname to preconfigure the system with. + *``optional``* +- **``locale``**: The default locale of the system. + Valid values: Any locale mentioned in ``/etc/locale.gen`` + *``required``* +- **``release``**: Defines which debian release should be bootstrapped. + Valid values: ``squeeze``, ``wheezy``, ``jessie``, ``sid``, + ``oldstable``, ``stable``, ``testing``, ``unstable`` + *``required``* +- **``timezone``**: Timezone of the system. + Valid values: Any filename from ``/usr/share/zoneinfo`` + *``required``* + +Packages +~~~~~~~~ + +The packages section allows you to install custom packages from a +variety of sources. + +- **``install``**: A list of strings that specify which packages should + be installed. Valid values: package names optionally followed by a + ``/target`` or paths to local ``.deb`` files. +- **``install_standard``**: Defines if the packages of the + ``"Standard System Utilities"`` option of the Debian installer, + provided by `tasksel `__, should be + installed or not. The problem is that with just ``debootstrap``, the + system ends up with very basic commands. This is not a problem for a + machine that will not be used interactively, but otherwise it is nice + to have at hand tools like ``bash-completion``, ``less``, ``locate``, + etc. + *``optional``* + Valid values: ``true``, ``false`` + Default: ``false`` +- **``mirror``**: The default aptitude mirror. + *``optional``* + Default: ``http://http.debian.net/debian`` +- **``sources``**: A map of additional sources that should be added to + the aptitude sources list. The key becomes the filename in + ``/etc/apt/sources.list.d/`` (with ``.list`` appended to it), while + the value is an array with each entry being a line. + *``optional``* +- **``components``**: A list of components that should be added to the + default apt sources. For example ``contrib`` or ``non-free`` + *``optional``* + Default: ``['main']`` +- **``trusted-keys``**: List of paths to ``.gpg`` keyrings that should + be added to the aptitude keyring of trusted signatures for + repositories. + *``optional``* +- **``preferences``**: Allows you to pin packages through `apt + preferences `__. The setting + is an object where the key is the preference filename in + ``/etc/apt/preferences.d/``. The key ``main`` is special and refers + to the file ``/etc/apt/preferences``, which will be overwritten if + specified. + *``optional``* + The values are objects with three keys: +- **``package``**: The package to pin (wildcards allowed) +- **``pin``**: The release to pin the package to. +- **``pin-priority``**: The priority of this pin. + +Volume +~~~~~~ + +bootstrap-vz allows a wide range of options for configuring the disk +layout of the system. It can create unpartitioned as well as partitioned +volumes using either the gpt or msdos scheme. At most, there are only +three partitions with predefined roles configurable though. They are +boot, root and swap. + +- **``backing``**: Specifies the volume backing. This setting is very + provider specific. + Valid values: ``ebs``, ``s3``, ``vmdk``, ``vdi``, ``raw`` + *``required``* +- **``partitions``**: A map of the partitions that should be created on + the volume. +- **``type``**: The partitioning scheme to use. When using ``none``, + only root can be specified as a partition. + Valid values: ``none``, ``gpt``, ``msdos`` + *``required``* +- **``root``**: Configuration of the root partition. *``required``* + + - **``size``**: The size of the partition. Valid values: Any + datasize specification up to TB (e.g. 5KiB, 1MB, 6TB). + *``required``* + - **``filesystem``**: The filesystem of the partition. When choosing + ``xfs``, the ``xfsprogs`` package will need to be installed. + Valid values: ``ext2``, ``ext3``, ``ext4``, ``xfs`` + *``required``* + - **``format_command``**: Command to format the partition with. This + optional setting overrides the command bootstrap-vz would normally + use to format the partition. The command is specified as a string + array where each option/argument is an item in that array (much + like the `image\_commands `__ plugin). + *``optional``* The following variables are available: + - **``{fs}``**: The filesystem of the partition. + - **``{device_path}``**: The device path of the partition. + - **``{size}``**: The size of the partition. + + The default command used by boostrap-vz is + ``['mkfs.{fs}', '{device_path}']``. + + - **``boot``**: Configuration of the boot partition. The three + settings equal those of the root partition. + *``optional``* + - **``swap``**: Configuration of the swap partition. Since the swap + partition has its own filesystem you can only specify the size for + this partition. + *``optional``* + +Plugins +~~~~~~~ + +The plugins section is a map of plugin names to whatever configuration a +plugin requires. Go to the `plugin section `__ +of the documentation, to see the configuration for a specific plugin. diff --git a/tests/integration/manifests_tests.py b/tests/integration/manifests_tests.py index fd2b4b6..a3e0152 100644 --- a/tests/integration/manifests_tests.py +++ b/tests/integration/manifests_tests.py @@ -1,12 +1,6 @@ -import os from nose.tools import assert_true from bootstrapvz.base.manifest import Manifest -MANIFEST_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - '../../manifests' -) - def test_manifest_generator(): """ @@ -15,10 +9,16 @@ def test_manifest_generator(): Loops through the manifests directory and tests that each file can successfully be loaded and validated. """ - for fobj in os.listdir(MANIFEST_DIR): - path = os.path.join(os.path.abspath(MANIFEST_DIR), fobj) - - yield validate_manifests, path + import os.path + import glob + manifests = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + '../../manifests' + ) + for manifest_path in glob.glob(manifests + '/*.yml'): + yield validate_manifests, manifest_path + for manifest_path in glob.glob(manifests + '/*.json'): + yield validate_manifests, manifest_path def validate_manifests(path): diff --git a/tox.ini b/tox.ini index b4a76b4..64ef939 100644 --- a/tox.ini +++ b/tox.ini @@ -14,3 +14,10 @@ deps = nose nose-cov commands = nosetests -v tests/integration --with-coverage --cover-package=bootstrapvz --cover-inclusive + +[testenv:docs] +changedir = docs +deps = + sphinx +commands = + sphinx-build -W -b html -d _build/html/doctrees . _build/html