From f8118b3f3e237df7f76d7acd606321c5d30a4729 Mon Sep 17 00:00:00 2001 From: Anders Ingemann Date: Thu, 3 Mar 2016 23:32:44 +0100 Subject: [PATCH] Rename integration tests to system tests, since they cover the entire system *cough e82bdf4 *cough --- CHANGELOG.rst | 5 +++++ bootstrapvz/remote/README.rst | 4 ++-- docs/conf.py | 6 +++--- docs/testing/index.rst | 4 ++-- docs/testing/integration_test_providers/index.rst | 10 ---------- docs/testing/integration_tests.rst | 1 - .../.gitignore | 0 docs/testing/system_test_providers/index.rst | 10 ++++++++++ docs/testing/system_tests.rst | 1 + tests/integration/providers/docker/README.rst | 7 ------- tests/integration/providers/ec2/README.rst | 7 ------- tests/{integration => system}/README.rst | 14 +++++++------- tests/{integration => system}/__init__.py | 0 tests/{integration => system}/docker_tests.py | 0 tests/{integration => system}/ec2_ebs_hvm_tests.py | 0 tests/{integration => system}/ec2_ebs_pvm_tests.py | 0 tests/{integration => system}/ec2_s3_pvm_tests.py | 0 .../{integration => system}/manifests/__init__.py | 0 tests/{integration => system}/manifests/base.yml | 0 .../{integration => system}/manifests/extlinux.yml | 0 tests/{integration => system}/manifests/gpt.yml | 0 tests/{integration => system}/manifests/grub.yml | 0 tests/{integration => system}/manifests/msdos.yml | 0 .../manifests/oldstable64.yml | 0 .../manifests/partitioned.yml | 0 .../manifests/root_password.yml | 0 .../manifests/single_partition.yml | 0 .../{integration => system}/manifests/stable64.yml | 0 .../{integration => system}/manifests/stable86.yml | 0 .../manifests/unpartitioned.yml | 0 .../manifests/unstable64.yml | 0 tests/{integration => system}/providers/README.rst | 6 +++--- .../{integration => system}/providers/__init__.py | 0 tests/system/providers/docker/README.rst | 7 +++++++ .../providers/docker/__init__.py | 0 .../providers/docker/image.py | 0 tests/system/providers/ec2/README.rst | 7 +++++++ .../providers/ec2/__init__.py | 2 +- .../providers/ec2/images.py | 0 .../providers/virtualbox/README.rst | 2 +- .../providers/virtualbox/__init__.py | 0 .../providers/virtualbox/image.py | 0 .../providers/virtualbox/instance.py | 4 ++-- tests/{integration => system}/tools/__init__.py | 2 +- tests/{integration => system}/tools/exceptions.py | 0 tests/{integration => system}/virtualbox_tests.py | 0 tox.ini | 4 ++-- 47 files changed, 54 insertions(+), 49 deletions(-) delete mode 100644 docs/testing/integration_test_providers/index.rst delete mode 100644 docs/testing/integration_tests.rst rename docs/testing/{integration_test_providers => system_test_providers}/.gitignore (100%) create mode 100644 docs/testing/system_test_providers/index.rst create mode 100644 docs/testing/system_tests.rst delete mode 100644 tests/integration/providers/docker/README.rst delete mode 100644 tests/integration/providers/ec2/README.rst rename tests/{integration => system}/README.rst (88%) rename tests/{integration => system}/__init__.py (100%) rename tests/{integration => system}/docker_tests.py (100%) rename tests/{integration => system}/ec2_ebs_hvm_tests.py (100%) rename tests/{integration => system}/ec2_ebs_pvm_tests.py (100%) rename tests/{integration => system}/ec2_s3_pvm_tests.py (100%) rename tests/{integration => system}/manifests/__init__.py (100%) rename tests/{integration => system}/manifests/base.yml (100%) rename tests/{integration => system}/manifests/extlinux.yml (100%) rename tests/{integration => system}/manifests/gpt.yml (100%) rename tests/{integration => system}/manifests/grub.yml (100%) rename tests/{integration => system}/manifests/msdos.yml (100%) rename tests/{integration => system}/manifests/oldstable64.yml (100%) rename tests/{integration => system}/manifests/partitioned.yml (100%) rename tests/{integration => system}/manifests/root_password.yml (100%) rename tests/{integration => system}/manifests/single_partition.yml (100%) rename tests/{integration => system}/manifests/stable64.yml (100%) rename tests/{integration => system}/manifests/stable86.yml (100%) rename tests/{integration => system}/manifests/unpartitioned.yml (100%) rename tests/{integration => system}/manifests/unstable64.yml (100%) rename tests/{integration => system}/providers/README.rst (85%) rename tests/{integration => system}/providers/__init__.py (100%) create mode 100644 tests/system/providers/docker/README.rst rename tests/{integration => system}/providers/docker/__init__.py (100%) rename tests/{integration => system}/providers/docker/image.py (100%) create mode 100644 tests/system/providers/ec2/README.rst rename tests/{integration => system}/providers/ec2/__init__.py (99%) rename tests/{integration => system}/providers/ec2/images.py (100%) rename tests/{integration => system}/providers/virtualbox/README.rst (71%) rename tests/{integration => system}/providers/virtualbox/__init__.py (100%) rename tests/{integration => system}/providers/virtualbox/image.py (100%) rename tests/{integration => system}/providers/virtualbox/instance.py (97%) rename tests/{integration => system}/tools/__init__.py (96%) rename tests/{integration => system}/tools/exceptions.py (100%) rename tests/{integration => system}/virtualbox_tests.py (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7e319b7..30c3f52 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +2016-03-03 +---------- +Anders Ingemann: + * Rename integration tests to system tests + 2016-02-23 ---------- Nicolas Braud-Santoni: diff --git a/bootstrapvz/remote/README.rst b/bootstrapvz/remote/README.rst index 977af12..90b35db 100644 --- a/bootstrapvz/remote/README.rst +++ b/bootstrapvz/remote/README.rst @@ -14,7 +14,7 @@ when doing this multiple times SSHing into the machines and copying the manifests can be a hassle. Lastly, the main motivation for supporting remote bootstrapping is the -automation of `integration testing <../../tests/integration>`__. +automation of `system testing <../../tests/system>`__. As you will see `further down <#bootstrap-vz-remote>`__, bootstrap-vz is able to select which build server is required for a specific test and run the bootstrapping procedure on said server. @@ -172,7 +172,7 @@ at ``/root/guest_additions.iso`` on server 1, while server 2 has it at Run settings ~~~~~~~~~~~~~~ The run settings hold information about how to start a bootstrapped image. -This is useful only when running integration tests. +This is useful only when running system tests. .. code:: yaml diff --git a/docs/conf.py b/docs/conf.py index 2134d87..439eb30 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -287,11 +287,11 @@ for readme_path in glob.glob('../bootstrapvz/plugins/*/README.rst'): include.write('.. include:: ' + path_to_readme) -for readme_path in glob.glob('../tests/integration/providers/*/README.rst'): +for readme_path in glob.glob('../tests/system/providers/*/README.rst'): provider_name = os.path.basename(os.path.dirname(readme_path)) - include_path = os.path.join('testing/integration_test_providers', provider_name + '.rst') + include_path = os.path.join('testing/system_test_providers', provider_name + '.rst') if not os.path.exists(include_path): - path_to_readme = os.path.join('../../../tests/integration/providers', provider_name, 'README.rst') + path_to_readme = os.path.join('../../../tests/system/providers', provider_name, 'README.rst') with open(include_path, 'w') as include: include.write('.. include:: ' + path_to_readme) diff --git a/docs/testing/index.rst b/docs/testing/index.rst index f552ef5..7758c91 100644 --- a/docs/testing/index.rst +++ b/docs/testing/index.rst @@ -6,7 +6,7 @@ Testing :hidden: unit_tests - integration_tests - integration_test_providers/index + system_tests + system_test_providers/index .. include:: ../../tests/README.rst diff --git a/docs/testing/integration_test_providers/index.rst b/docs/testing/integration_test_providers/index.rst deleted file mode 100644 index 804c4cb..0000000 --- a/docs/testing/integration_test_providers/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -Integration test providers -========================== - -.. toctree:: - :maxdepth: 1 - :glob: - - * - -.. include:: ../../../tests/integration/providers/README.rst diff --git a/docs/testing/integration_tests.rst b/docs/testing/integration_tests.rst deleted file mode 100644 index 73d05c4..0000000 --- a/docs/testing/integration_tests.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../tests/integration/README.rst diff --git a/docs/testing/integration_test_providers/.gitignore b/docs/testing/system_test_providers/.gitignore similarity index 100% rename from docs/testing/integration_test_providers/.gitignore rename to docs/testing/system_test_providers/.gitignore diff --git a/docs/testing/system_test_providers/index.rst b/docs/testing/system_test_providers/index.rst new file mode 100644 index 0000000..b473afc --- /dev/null +++ b/docs/testing/system_test_providers/index.rst @@ -0,0 +1,10 @@ +System test providers +===================== + +.. toctree:: + :maxdepth: 1 + :glob: + + * + +.. include:: ../../../tests/system/providers/README.rst diff --git a/docs/testing/system_tests.rst b/docs/testing/system_tests.rst new file mode 100644 index 0000000..7d950bf --- /dev/null +++ b/docs/testing/system_tests.rst @@ -0,0 +1 @@ +.. include:: ../../tests/system/README.rst diff --git a/tests/integration/providers/docker/README.rst b/tests/integration/providers/docker/README.rst deleted file mode 100644 index 555c568..0000000 --- a/tests/integration/providers/docker/README.rst +++ /dev/null @@ -1,7 +0,0 @@ -Docker ------- - - -Dependencies -~~~~~~~~~~~~ -The host machine running the integration tests must have docker installed. diff --git a/tests/integration/providers/ec2/README.rst b/tests/integration/providers/ec2/README.rst deleted file mode 100644 index 41bfe9c..0000000 --- a/tests/integration/providers/ec2/README.rst +++ /dev/null @@ -1,7 +0,0 @@ -EC2 ---- - -Dependencies -~~~~~~~~~~~~ -The host machine running the integration tests must have the python package -``boto`` installed (``>= 2.14.0``). diff --git a/tests/integration/README.rst b/tests/system/README.rst similarity index 88% rename from tests/integration/README.rst rename to tests/system/README.rst index a4e1d62..815588e 100644 --- a/tests/integration/README.rst +++ b/tests/system/README.rst @@ -1,11 +1,11 @@ -Integration tests -================= -`Integration tests`__ test +System tests +============ +`System tests`__ test bootstrap-vz in its entirety. This testing includes building images from manifests and creating/booting said images. -__ http://en.wikipedia.org/wiki/Integration_testing +__ http://en.wikipedia.org/wiki/System_testing Since hardcoding manifests for each test, bootstrapping them and booting the resulting images is too much code for a single test, a testing harness has @@ -15,12 +15,12 @@ been developed that reduces each test to it's bare essentials: * Boot an instance from a manifest * Run tests on the booted instance -In order for the integration testing harness to be able to bootstrap it must +In order for the system testing harness to be able to bootstrap it must know about your `build-servers <../../bootstrapvz/remote#build-servers-yml>`__. Depending on the manifest that is bootstrapped, the harness chooses a fitting build-server, connects to it and starts the bootstrapping process. -When running integration tests, the framework will look for ``build-servers.yml`` +When running system tests, the framework will look for ``build-servers.yml`` at the root of the repo and raise an error if it is not found. @@ -65,7 +65,7 @@ randomly generated on load. Missing parts ------------- -The integration testing harness is in no way complete. +The system testing harness is in no way complete. * It still has no support for providers other than Virtualbox, EC2 and Docker. * Creating an SSH connection to a booted instance is cumbersome and does not diff --git a/tests/integration/__init__.py b/tests/system/__init__.py similarity index 100% rename from tests/integration/__init__.py rename to tests/system/__init__.py diff --git a/tests/integration/docker_tests.py b/tests/system/docker_tests.py similarity index 100% rename from tests/integration/docker_tests.py rename to tests/system/docker_tests.py diff --git a/tests/integration/ec2_ebs_hvm_tests.py b/tests/system/ec2_ebs_hvm_tests.py similarity index 100% rename from tests/integration/ec2_ebs_hvm_tests.py rename to tests/system/ec2_ebs_hvm_tests.py diff --git a/tests/integration/ec2_ebs_pvm_tests.py b/tests/system/ec2_ebs_pvm_tests.py similarity index 100% rename from tests/integration/ec2_ebs_pvm_tests.py rename to tests/system/ec2_ebs_pvm_tests.py diff --git a/tests/integration/ec2_s3_pvm_tests.py b/tests/system/ec2_s3_pvm_tests.py similarity index 100% rename from tests/integration/ec2_s3_pvm_tests.py rename to tests/system/ec2_s3_pvm_tests.py diff --git a/tests/integration/manifests/__init__.py b/tests/system/manifests/__init__.py similarity index 100% rename from tests/integration/manifests/__init__.py rename to tests/system/manifests/__init__.py diff --git a/tests/integration/manifests/base.yml b/tests/system/manifests/base.yml similarity index 100% rename from tests/integration/manifests/base.yml rename to tests/system/manifests/base.yml diff --git a/tests/integration/manifests/extlinux.yml b/tests/system/manifests/extlinux.yml similarity index 100% rename from tests/integration/manifests/extlinux.yml rename to tests/system/manifests/extlinux.yml diff --git a/tests/integration/manifests/gpt.yml b/tests/system/manifests/gpt.yml similarity index 100% rename from tests/integration/manifests/gpt.yml rename to tests/system/manifests/gpt.yml diff --git a/tests/integration/manifests/grub.yml b/tests/system/manifests/grub.yml similarity index 100% rename from tests/integration/manifests/grub.yml rename to tests/system/manifests/grub.yml diff --git a/tests/integration/manifests/msdos.yml b/tests/system/manifests/msdos.yml similarity index 100% rename from tests/integration/manifests/msdos.yml rename to tests/system/manifests/msdos.yml diff --git a/tests/integration/manifests/oldstable64.yml b/tests/system/manifests/oldstable64.yml similarity index 100% rename from tests/integration/manifests/oldstable64.yml rename to tests/system/manifests/oldstable64.yml diff --git a/tests/integration/manifests/partitioned.yml b/tests/system/manifests/partitioned.yml similarity index 100% rename from tests/integration/manifests/partitioned.yml rename to tests/system/manifests/partitioned.yml diff --git a/tests/integration/manifests/root_password.yml b/tests/system/manifests/root_password.yml similarity index 100% rename from tests/integration/manifests/root_password.yml rename to tests/system/manifests/root_password.yml diff --git a/tests/integration/manifests/single_partition.yml b/tests/system/manifests/single_partition.yml similarity index 100% rename from tests/integration/manifests/single_partition.yml rename to tests/system/manifests/single_partition.yml diff --git a/tests/integration/manifests/stable64.yml b/tests/system/manifests/stable64.yml similarity index 100% rename from tests/integration/manifests/stable64.yml rename to tests/system/manifests/stable64.yml diff --git a/tests/integration/manifests/stable86.yml b/tests/system/manifests/stable86.yml similarity index 100% rename from tests/integration/manifests/stable86.yml rename to tests/system/manifests/stable86.yml diff --git a/tests/integration/manifests/unpartitioned.yml b/tests/system/manifests/unpartitioned.yml similarity index 100% rename from tests/integration/manifests/unpartitioned.yml rename to tests/system/manifests/unpartitioned.yml diff --git a/tests/integration/manifests/unstable64.yml b/tests/system/manifests/unstable64.yml similarity index 100% rename from tests/integration/manifests/unstable64.yml rename to tests/system/manifests/unstable64.yml diff --git a/tests/integration/providers/README.rst b/tests/system/providers/README.rst similarity index 85% rename from tests/integration/providers/README.rst rename to tests/system/providers/README.rst index 9a2f3be..80c6666 100644 --- a/tests/integration/providers/README.rst +++ b/tests/system/providers/README.rst @@ -1,4 +1,4 @@ -Integration testing providers are implemented on top of the abstraction +System testing providers are implemented on top of the abstraction that is the testing harness. Implementation @@ -21,7 +21,7 @@ resources, once testing is done (or failed, so remember to wrap ``yield`` in a Debugging --------- -When developing an integration test provider, debugging through multiple +When developing a system test provider, debugging through multiple invocations of ``tox`` can be cumbersome. A short test script, which sets up logging and invokes a specific test can be used instead: @@ -32,7 +32,7 @@ Example: #!/usr/bin/env python - from tests.integration.docker_tests import test_stable + from tests.system.docker_tests import test_stable from bootstrapvz.base.main import setup_loggers setup_loggers({'--log': '-', '--color': 'default', '--debug': True}) diff --git a/tests/integration/providers/__init__.py b/tests/system/providers/__init__.py similarity index 100% rename from tests/integration/providers/__init__.py rename to tests/system/providers/__init__.py diff --git a/tests/system/providers/docker/README.rst b/tests/system/providers/docker/README.rst new file mode 100644 index 0000000..1039c89 --- /dev/null +++ b/tests/system/providers/docker/README.rst @@ -0,0 +1,7 @@ +Docker +------ + + +Dependencies +~~~~~~~~~~~~ +The host machine running the system tests must have docker installed. diff --git a/tests/integration/providers/docker/__init__.py b/tests/system/providers/docker/__init__.py similarity index 100% rename from tests/integration/providers/docker/__init__.py rename to tests/system/providers/docker/__init__.py diff --git a/tests/integration/providers/docker/image.py b/tests/system/providers/docker/image.py similarity index 100% rename from tests/integration/providers/docker/image.py rename to tests/system/providers/docker/image.py diff --git a/tests/system/providers/ec2/README.rst b/tests/system/providers/ec2/README.rst new file mode 100644 index 0000000..a6127a0 --- /dev/null +++ b/tests/system/providers/ec2/README.rst @@ -0,0 +1,7 @@ +EC2 +--- + +Dependencies +~~~~~~~~~~~~ +The host machine running the system tests must have the python package +``boto`` installed (``>= 2.14.0``). diff --git a/tests/integration/providers/ec2/__init__.py b/tests/system/providers/ec2/__init__.py similarity index 99% rename from tests/integration/providers/ec2/__init__.py rename to tests/system/providers/ec2/__init__.py index 43b528f..b1288e5 100644 --- a/tests/integration/providers/ec2/__init__.py +++ b/tests/system/providers/ec2/__init__.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from tests.integration.tools import waituntil +from tests.system.tools import waituntil import logging log = logging.getLogger(__name__) diff --git a/tests/integration/providers/ec2/images.py b/tests/system/providers/ec2/images.py similarity index 100% rename from tests/integration/providers/ec2/images.py rename to tests/system/providers/ec2/images.py diff --git a/tests/integration/providers/virtualbox/README.rst b/tests/system/providers/virtualbox/README.rst similarity index 71% rename from tests/integration/providers/virtualbox/README.rst rename to tests/system/providers/virtualbox/README.rst index 96f2bf0..da2e3fd 100644 --- a/tests/integration/providers/virtualbox/README.rst +++ b/tests/system/providers/virtualbox/README.rst @@ -5,5 +5,5 @@ Virtualbox Dependencies ~~~~~~~~~~~~ VirtualBox itself is required on the machine that is -running the integration tests. The same machine also needs to +running the system tests. The same machine also needs to have python package ``vboxapi`` (``>=1.0``) installed. diff --git a/tests/integration/providers/virtualbox/__init__.py b/tests/system/providers/virtualbox/__init__.py similarity index 100% rename from tests/integration/providers/virtualbox/__init__.py rename to tests/system/providers/virtualbox/__init__.py diff --git a/tests/integration/providers/virtualbox/image.py b/tests/system/providers/virtualbox/image.py similarity index 100% rename from tests/integration/providers/virtualbox/image.py rename to tests/system/providers/virtualbox/image.py diff --git a/tests/integration/providers/virtualbox/instance.py b/tests/system/providers/virtualbox/instance.py similarity index 97% rename from tests/integration/providers/virtualbox/instance.py rename to tests/system/providers/virtualbox/instance.py index 5ab28da..d8d84e2 100644 --- a/tests/integration/providers/virtualbox/instance.py +++ b/tests/system/providers/virtualbox/instance.py @@ -1,6 +1,6 @@ import virtualbox from contextlib import contextmanager -from tests.integration.tools import waituntil +from tests.system.tools import waituntil import logging log = logging.getLogger(__name__) @@ -59,7 +59,7 @@ class VirtualBoxInstance(object): def boot(self): log.debug('Booting vbox machine `{name}\''.format(name=self.name)) self.machine.launch_vm_process(self.session, 'headless').wait_for_completion(-1) - from tests.integration.tools import read_from_socket + from tests.system.tools import read_from_socket # Gotta figure out a more reliable way to check when the system is done booting. # Maybe bootstrapped unit test images should have a startup script that issues # a callback to the host. diff --git a/tests/integration/tools/__init__.py b/tests/system/tools/__init__.py similarity index 96% rename from tests/integration/tools/__init__.py rename to tests/system/tools/__init__.py index 1e82230..7f3b995 100644 --- a/tests/integration/tools/__init__.py +++ b/tests/system/tools/__init__.py @@ -20,7 +20,7 @@ def boot_manifest(manifest_data, boot_vars={}): manifest = Manifest(data=manifest_data) import importlib - provider_module = importlib.import_module('tests.integration.providers.' + manifest.provider['name']) + provider_module = importlib.import_module('tests.system.providers.' + manifest.provider['name']) prepare_bootstrap = getattr(provider_module, 'prepare_bootstrap', noop) with prepare_bootstrap(manifest, build_server): diff --git a/tests/integration/tools/exceptions.py b/tests/system/tools/exceptions.py similarity index 100% rename from tests/integration/tools/exceptions.py rename to tests/system/tools/exceptions.py diff --git a/tests/integration/virtualbox_tests.py b/tests/system/virtualbox_tests.py similarity index 100% rename from tests/integration/virtualbox_tests.py rename to tests/system/virtualbox_tests.py diff --git a/tox.ini b/tox.ini index 8590fc4..b462733 100644 --- a/tox.ini +++ b/tox.ini @@ -16,14 +16,14 @@ deps = nose-cov commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --verbose {posargs:tests/unit} -[testenv:integration] +[testenv:system] deps = nose nose-cov nose-htmloutput Pyro4 >= 4.30 pyvbox >= 0.2.0 -commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --with-html --html-file=integration.html --verbose {posargs:tests/integration} +commands = nosetests --with-coverage --cover-package=bootstrapvz --cover-inclusive --with-html --html-file=system.html --verbose {posargs:tests/system} [testenv:docs] changedir = docs