From 4942d2e7f295b4301aac65aa981cc1b6bedb4c1e Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 27 Apr 2017 13:01:03 -0400 Subject: [PATCH] selftests: introduce multiple levels of testing coverage A finer level of granularity may help to extend our testing coverage while reducing the amount of false positives. This introduces the three different levels, mapped like this: * level 0, AKA "make check" * level 2, AKA "make check-full" There are no changes of test assignment, that is, tests previously being run under "check-full" will continue to be run only at that target. The same is true for tests that would run under "make check" before this. The big change is that there's now a middle ground, that can be activated by manually setting the AVOCADO_CHECK_LEVEL variable. Level 1 is intended to be used be used on environments that are halfway between a dedicated machine and a really low powered environment. Signed-off-by: Cleber Rosa --- Makefile | 17 +++++++++-------- selftests/functional/test_basic.py | 4 ++-- selftests/functional/test_interrupt.py | 2 +- selftests/functional/test_loader.py | 2 +- selftests/functional/test_utils.py | 2 +- selftests/run_coverage | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 4bfa001d..b462cb34 100644 --- a/Makefile +++ b/Makefile @@ -35,13 +35,13 @@ MOCK_CONFIG=default all: @echo @echo "Development related targets:" - @echo "check: Runs tree static check, unittests and fast functional tests" - @echo "check-full: Runs tree static check, unittests and all functional tests" - @echo "develop: Runs 'python setup.py --develop on this tree alone" - @echo "link: Runs 'python setup.py --develop' in all subprojects and links the needed resources" - @echo "clean: Get rid of scratch, byte files and removes the links to other subprojects" - @echo "selfcheck: Runs tree static check, unittests and functional tests using Avocado itself" - @echo "spell: Runs spell checker on comments and docstrings (requires python-enchant)" + @echo "check: Runs tree static check, unittests and fast functional tests" + @echo "check-full: Runs tree static check, and all unittests and functional tests" + @echo "develop: Runs 'python setup.py --develop on this tree alone" + @echo "link: Runs 'python setup.py --develop' in all subprojects and links the needed resources" + @echo "clean: Get rid of scratch, byte files and removes the links to other subprojects" + @echo "selfcheck: Runs tree static check, unittests and functional tests using Avocado itself" + @echo "spell: Runs spell checker on comments and docstrings (requires python-enchant)" @echo @echo "Package requirements related targets" @echo "requirements: Install runtime requirements" @@ -158,11 +158,12 @@ smokecheck: clean develop ./scripts/avocado run passtest.py check: clean develop check_cyclical modules_boundaries + # Unless manually set, this is equivalent to AVOCADO_CHECK_LEVEL=0 selftests/checkall selftests/check_tmp_dirs check-full: clean develop check_cyclical modules_boundaries - AVOCADO_CHECK_FULL=1 selftests/checkall + AVOCADO_CHECK_LEVEL=2 selftests/checkall selftests/check_tmp_dirs selfcheck: clean check_cyclical modules_boundaries develop diff --git a/selftests/functional/test_basic.py b/selftests/functional/test_basic.py index b27d092d..1cdae7f9 100644 --- a/selftests/functional/test_basic.py +++ b/selftests/functional/test_basic.py @@ -368,7 +368,7 @@ class RunnerOperationTest(unittest.TestCase): # Ensure no test aborted error messages show up self.assertNotIn("TestAbortedError: Test aborted unexpectedly", output) - @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1", + @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2, "Skipping test that take a long time to run, are " "resource intensive or time sensitve") def test_runner_abort(self): @@ -650,7 +650,7 @@ class RunnerSimpleTest(unittest.TestCase): "Avocado did not return rc %d:\n%s" % (expected_rc, result)) - @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1", + @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2, "Skipping test that take a long time to run, are " "resource intensive or time sensitve") def test_runner_onehundred_fail_timing(self): diff --git a/selftests/functional/test_interrupt.py b/selftests/functional/test_interrupt.py index 08385676..b2ac9bec 100644 --- a/selftests/functional/test_interrupt.py +++ b/selftests/functional/test_interrupt.py @@ -55,7 +55,7 @@ class InterruptTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__) - @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1", + @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2, "Skipping test that take a long time to run, are " "resource intensive or time sensitve") def test_badly_behaved(self): diff --git a/selftests/functional/test_loader.py b/selftests/functional/test_loader.py index ab29518a..a4e99ec0 100644 --- a/selftests/functional/test_loader.py +++ b/selftests/functional/test_loader.py @@ -215,7 +215,7 @@ class LoaderTestFunctional(unittest.TestCase): def test_load_not_a_test_not_exec(self): self._test('notatest.py', NOT_A_TEST, 'NOT_A_TEST') - @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1", + @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2, "Skipping test that take a long time to run, are " "resource intensive or time sensitve") def test_runner_simple_python_like_multiple_files(self): diff --git a/selftests/functional/test_utils.py b/selftests/functional/test_utils.py index 7a5090c1..233a4948 100644 --- a/selftests/functional/test_utils.py +++ b/selftests/functional/test_utils.py @@ -170,7 +170,7 @@ class FileLockTest(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__) - @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1", + @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2, "Skipping test that take a long time to run, are " "resource intensive or time sensitve") def test_filelock(self): diff --git a/selftests/run_coverage b/selftests/run_coverage index cf451555..ab761b76 100755 --- a/selftests/run_coverage +++ b/selftests/run_coverage @@ -7,7 +7,7 @@ coverage erase rm .coverage.* -AVOCADO_CHECK_FULL=1 UNITTEST_AVOCADO_CMD="coverage run -p --include 'avocado/*' ./scripts/avocado" coverage run -p --include "avocado/*" ./selftests/run +AVOCADO_CHECK_LEVEL=2 UNITTEST_AVOCADO_CMD="coverage run -p --include 'avocado/*' ./scripts/avocado" coverage run -p --include "avocado/*" ./selftests/run coverage combine .coverage* echo coverage report -m --include "avocado/core/*" -- GitLab