selftests: introduce multiple levels of testing coverage

A finer level of granularity may help to extend our testing coverage
while reducing the amount of false positives.  This introduces the
three different levels, mapped like this:

   * level 0, AKA "make check"
   * level 2, AKA "make check-full"

There are no changes of test assignment, that is, tests previously
being run under "check-full" will continue to be run only at that
target.  The same is true for tests that would run under "make check"
before this.

The big change is that there's now a middle ground, that can be
activated by manually setting the AVOCADO_CHECK_LEVEL variable.  Level
1 is intended to be used be used on environments that are halfway
between a dedicated machine and a really low powered environment.
Signed-off-by: NCleber Rosa <crosa@redhat.com>

Makefile

@@ -35,13 +35,13 @@ MOCK_CONFIG=default
 all:
 	@echo
 	@echo "Development related targets:"
-	@echo "check:      Runs tree static check, unittests and fast functional tests"
-	@echo "check-full: Runs tree static check, unittests and all functional tests"
-	@echo "develop:    Runs 'python setup.py --develop on this tree alone"
-	@echo "link:       Runs 'python setup.py --develop' in all subprojects and links the needed resources"
-	@echo "clean:      Get rid of scratch, byte files and removes the links to other subprojects"
-	@echo "selfcheck:  Runs tree static check, unittests and functional tests using Avocado itself"
-	@echo "spell:      Runs spell checker on comments and docstrings (requires python-enchant)"
+	@echo "check:          Runs tree static check, unittests and fast functional tests"
+	@echo "check-full:     Runs tree static check, and all unittests and functional tests"
+	@echo "develop:        Runs 'python setup.py --develop on this tree alone"
+	@echo "link:           Runs 'python setup.py --develop' in all subprojects and links the needed resources"
+	@echo "clean:          Get rid of scratch, byte files and removes the links to other subprojects"
+	@echo "selfcheck:      Runs tree static check, unittests and functional tests using Avocado itself"
+	@echo "spell:          Runs spell checker on comments and docstrings (requires python-enchant)"
 	@echo
 	@echo "Package requirements related targets"
 	@echo "requirements:            Install runtime requirements"
@@ -158,11 +158,12 @@ smokecheck: clean develop
 	./scripts/avocado run passtest.py
 
 check: clean develop check_cyclical modules_boundaries
+	# Unless manually set, this is equivalent to AVOCADO_CHECK_LEVEL=0
 	selftests/checkall
 	selftests/check_tmp_dirs
 
 check-full: clean develop check_cyclical modules_boundaries
-	AVOCADO_CHECK_FULL=1 selftests/checkall
+	AVOCADO_CHECK_LEVEL=2 selftests/checkall
 	selftests/check_tmp_dirs
 
 selfcheck: clean check_cyclical modules_boundaries develop

selftests/functional/test_basic.py

@@ -368,7 +368,7 @@ class RunnerOperationTest(unittest.TestCase):
         # Ensure no test aborted error messages show up
         self.assertNotIn("TestAbortedError: Test aborted unexpectedly", output)
 
-    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
+    @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2,
                      "Skipping test that take a long time to run, are "
                      "resource intensive or time sensitve")
     def test_runner_abort(self):
@@ -650,7 +650,7 @@ class RunnerSimpleTest(unittest.TestCase):
                          "Avocado did not return rc %d:\n%s" %
                          (expected_rc, result))
 
-    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
+    @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2,
                      "Skipping test that take a long time to run, are "
                      "resource intensive or time sensitve")
     def test_runner_onehundred_fail_timing(self):

selftests/functional/test_interrupt.py

@@ -55,7 +55,7 @@ class InterruptTest(unittest.TestCase):
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
 
-    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
+    @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2,
                      "Skipping test that take a long time to run, are "
                      "resource intensive or time sensitve")
     def test_badly_behaved(self):

selftests/functional/test_loader.py

@@ -215,7 +215,7 @@ class LoaderTestFunctional(unittest.TestCase):
     def test_load_not_a_test_not_exec(self):
         self._test('notatest.py', NOT_A_TEST, 'NOT_A_TEST')
 
-    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
+    @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2,
                      "Skipping test that take a long time to run, are "
                      "resource intensive or time sensitve")
     def test_runner_simple_python_like_multiple_files(self):

selftests/functional/test_utils.py

@@ -170,7 +170,7 @@ class FileLockTest(unittest.TestCase):
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
 
-    @unittest.skipIf(os.environ.get("AVOCADO_CHECK_FULL") != "1",
+    @unittest.skipIf(int(os.environ.get("AVOCADO_CHECK_LEVEL", 0)) < 2,
                      "Skipping test that take a long time to run, are "
                      "resource intensive or time sensitve")
     def test_filelock(self):

selftests/run_coverage

@@ -7,7 +7,7 @@
 
 coverage erase
 rm .coverage.*
-AVOCADO_CHECK_FULL=1 UNITTEST_AVOCADO_CMD="coverage run -p --include 'avocado/*' ./scripts/avocado" coverage run -p --include "avocado/*" ./selftests/run
+AVOCADO_CHECK_LEVEL=2 UNITTEST_AVOCADO_CMD="coverage run -p --include 'avocado/*' ./scripts/avocado" coverage run -p --include "avocado/*" ./selftests/run
 coverage combine .coverage*
 echo
 coverage report -m --include "avocado/core/*"