loader: Add support to discover individual python unittests

The python unittests should be executable even now as a simple tests or
via external runner + contrib script. Let's create a new type and
discover python unittests as first-class test types in Avocado.

As Avocado tests are in fact python unittests, we need to manually skip
these by ignoring tests that are discovered as Avocado tests, but are
marked as not-to-be-executed, which required the additional change to
_find_avocado_tests method.
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/loader.py

@@ -42,6 +42,10 @@ AVAILABLE = None
 ALL = True
 
 
+# Regexp to find python unittests
+_RE_UNIT_TEST = re.compile(r'test.*')
+
+
 class MissingTest(object):
     """
     Class representing reference which failed to be discovered
@@ -510,7 +514,8 @@ class FileLoader(TestLoader):
                 MissingTest: 'MISSING',
                 BrokenSymlink: 'BROKEN_SYMLINK',
                 AccessDeniedPath: 'ACCESS_DENIED',
-                test.Test: 'INSTRUMENTED'}
+                test.Test: 'INSTRUMENTED',
+                test.PythonUnittest: 'PyUNITTEST'}
 
     @staticmethod
     def get_decorator_mapping():
@@ -519,7 +524,8 @@ class FileLoader(TestLoader):
                 MissingTest: output.TERM_SUPPORT.fail_header_str,
                 BrokenSymlink: output.TERM_SUPPORT.fail_header_str,
                 AccessDeniedPath: output.TERM_SUPPORT.fail_header_str,
-                test.Test: output.TERM_SUPPORT.healthy_str}
+                test.Test: output.TERM_SUPPORT.healthy_str,
+                test.PythonUnittest: output.TERM_SUPPORT.healthy_str}
 
     def discover(self, reference, which_tests=DEFAULT):
         """
@@ -619,9 +625,11 @@ class FileLoader(TestLoader):
         :type path: str
         :param class_name: the specific class to be found
         :type path: str
-        :returns: dict with class name and additional info such as method names
-                  and tags
-        :rtype: dict
+        :returns: tuple where first item is dict with class name and additional
+                  info such as method names and tags; the second item is
+                  set of class names which look like avocado tests but are
+                  force-disabled.
+        :rtype: tuple
         """
         # If only the Test class was imported from the avocado namespace
         test_import = False
@@ -633,6 +641,7 @@ class FileLoader(TestLoader):
         mod_import_name = None
         # The resulting test classes
         result = {}
+        disabled = set()
 
         if os.path.isdir(path):
             path = os.path.join(path, "__init__.py")
@@ -678,6 +687,7 @@ class FileLoader(TestLoader):
                 has_disable = safeloader.check_docstring_directive(docstring,
                                                                    'disable')
                 if (has_disable and class_name is None):
+                    disabled.add(statement.name)
                     continue
 
                 cl_tags = safeloader.get_docstring_directives_tags(docstring)
@@ -708,12 +718,12 @@ class FileLoader(TestLoader):
                         # Looking for a 'class FooTest(Parent)'
                         else:
                             parent_class = parent.id
-
-                        res = self._find_avocado_tests(path, parent_class)
+                        res, dis = self._find_avocado_tests(path, parent_class)
                         if res:
                             parents.remove(parent)
                             for cls in res:
                                 info.extend(res[cls])
+                        disabled.update(dis)
 
                     # If there are parents left to be discovered, they
                     # might be in a different module.
@@ -755,11 +765,12 @@ class FileLoader(TestLoader):
                                         parent_module = node.module
                                     _, ppath, _ = imp.find_module(parent_module,
                                                                   modules_paths)
-                                    res = self._find_avocado_tests(ppath,
-                                                                   parent_class)
+                                    res, dis = self._find_avocado_tests(ppath,
+                                                                        parent_class)
                                     if res:
                                         for cls in res:
                                             info.extend(res[cls])
+                                    disabled.update(dis)
 
                     continue
 
@@ -784,7 +795,7 @@ class FileLoader(TestLoader):
                             result[statement.name] = info
                             continue
 
-        return result
+        return result, disabled
 
     @staticmethod
     def _get_methods_info(statement_body, class_tags):
@@ -802,13 +813,32 @@ class FileLoader(TestLoader):
 
         return methods_info
 
+    def _find_python_unittests(self, test_path, disabled, subtests_filter):
+        result = []
+        class_methods = safeloader.find_class_and_methods(test_path,
+                                                          _RE_UNIT_TEST)
+        for klass, methods in class_methods.iteritems():
+            if klass in disabled:
+                continue
+            if test_path.endswith(".py"):
+                test_path = test_path[:-3]
+            test_module_name = os.path.relpath(test_path)
+            test_module_name = test_module_name.replace(os.path.sep, ".")
+            candidates = ["%s.%s.%s" % (test_module_name, klass, method)
+                          for method in methods]
+            if subtests_filter:
+                result += [_ for _ in candidates if subtests_filter.search(_)]
+            else:
+                result += candidates
+        return result
+
     def _make_existing_file_tests(self, test_path, make_broken,
                                   subtests_filter, test_name=None):
         if test_name is None:
             test_name = test_path
         try:
             # Avocado tests
-            avocado_tests = self._find_avocado_tests(test_path)
+            avocado_tests, disabled = self._find_avocado_tests(test_path)
             if avocado_tests:
                 test_factories = []
                 for test_class, info in avocado_tests.items():
@@ -825,6 +855,21 @@ class FileLoader(TestLoader):
                                                 'tags': tags})
                             test_factories.append(tst)
                 return test_factories
+            # Python unittests
+            old_dir = os.getcwd()
+            try:
+                py_test_dir = os.path.abspath(os.path.dirname(test_path))
+                py_test_name = os.path.basename(test_path)
+                os.chdir(py_test_dir)
+                python_unittests = self._find_python_unittests(py_test_name,
+                                                               disabled,
+                                                               subtests_filter)
+            finally:
+                os.chdir(old_dir)
+            if python_unittests:
+                return [(test.PythonUnittest, {"name": name,
+                                               "test_dir": py_test_dir})
+                        for name in python_unittests]
             else:
                 if os.access(test_path, os.X_OK):
                     # Module does not have an avocado test class inside but

avocado/core/test.py

@@ -973,6 +973,45 @@ class ExternalRunnerTest(SimpleTest):
                 os.chdir(pre_cwd)
 
 
+class PythonUnittest(ExternalRunnerTest):
+    """
+    Python unittest test
+    """
+    def __init__(self, name, params=None, base_logdir=None, job=None,
+                 test_dir=None):
+        runner = "%s -m unittest -q -c" % sys.executable
+        external_runner = ExternalRunnerSpec(runner, "test", test_dir)
+        super(PythonUnittest, self).__init__(name, params, base_logdir, job,
+                                             external_runner=external_runner)
+
+    def _find_result(self, status="OK"):
+        status_line = "[stderr] %s" % status
+        with open(self.logfile) as logfile:
+            lines = iter(logfile)
+            for line in lines:
+                if "[stderr] Ran 1 test in" in line:
+                    break
+            for line in lines:
+                if status_line in line:
+                    return line
+        self.error("Fail to parse status from test result.")
+
+    def test(self):
+        try:
+            super(PythonUnittest, self).test()
+        except exceptions.TestFail:
+            status = self._find_result("FAILED")
+            if "errors" in status:
+                self.error("Unittest reported error(s)")
+            elif "failures" in status:
+                self.fail("Unittest reported failure(s)")
+            else:
+                self.error("Unknown failure executing the unittest")
+        status = self._find_result("OK")
+        if "skipped" in status:
+            self.cancel("Unittest reported skip")
+
+
 class MockingTest(Test):
 
     """

docs/source/Loaders.rst

@@ -104,11 +104,11 @@ please refer to the corresponding loader/plugin documentation.
 File Loader
 -----------
 
-For the File Loader, the loader responsible for discovering INSTRUMENTED
-and SIMPLE tests, the Test Reference is a path/filename of a test file.
+For the File Loader, the loader responsible for discovering INSTRUMENTED,
+PyUNITTEST (classic python unittests) and SIMPLE tests.
 
-If the file corresponds to an INSTRUMENTED test, you can filter the Test
-IDs by adding to the Test Reference a ``:`` followed by a regular
+If the file corresponds to an INSTRUMENTED or PyUNITTEST test, you can filter
+the Test IDs by adding to the Test Reference a ``:`` followed by a regular
 expression.
 
 For instance, if you want to list all tests that are present in the

docs/source/ReferenceGuide.rst

@@ -129,8 +129,8 @@ Example of Test IDs::
 Test Types
 ==========
 
-Avocado at its simplest configuration can run two different types of tests [#f1]_. You can mix
-and match those in a single job.
+Avocado at its simplest configuration can run three different types of tests
+[#f1]_. You can mix and match those in a single job.
 
 Instrumented
 ------------
@@ -148,6 +148,16 @@ Test statuses ``PASS``, ``WARN``, ``START`` and ``SKIP`` are considered as
 successful builds. The ``ABORT``, ``ERROR``, ``FAIL``, ``ALERT``, ``RUNNING``,
 ``NOSTATUS`` and ``INTERRUPTED`` are considered as failed ones.
 
+Python unittest
+---------------
+
+The discovery of classical python unittest is also supported, although unlike
+python unittest we still use static analysis to get individual tests so
+dynamically created cases are not recognized. Also note that test result SKIP
+is reported as CANCEL in Avocado as SKIP test meaning differs from our
+definition. Apart from that there should be no surprises when running
+unittests via Avocado.
+
 Simple
 ------
 

selftests/.data/unittests.py

+#!/usr/bin/env python
+import unittest
+
+
+class First(unittest.TestCase):
+    def test_pass(self):
+        pass
+
+
+class Second(unittest.TestCase):
+    def test_fail(self):
+        self.fail("this is suppose to fail")
+
+    def test_error(self):
+        raise RuntimeError("This is suppose to error")
+
+    @unittest.skip("This is suppose to be skipped")
+    def test_skip(self):
+        pass
+
+
+if __name__ == "__main__":
+    unittest.main()

selftests/functional/test_loader.py

 import os
+import json
 import subprocess
 import time
 import stat
@@ -108,6 +109,9 @@ from avocado import main
 from test2 import *
 
 class BasicTestSuite(SuperTest):
+    '''
+    :avocado: disable
+    '''
 
     def test1(self):
         self.xxx()
@@ -319,6 +323,30 @@ class LoaderTestFunctional(unittest.TestCase):
         self.assertEqual(test, 11, "Number of tests is not 12 (%s):\n%s"
                          % (test, result))
 
+    def test_python_unittest(self):
+        test_path = os.path.join(basedir, "selftests", ".data", "unittests.py")
+        cmd = ("%s run --sysinfo=off --job-results-dir %s --json - -- %s"
+               % (AVOCADO, self.tmpdir, test_path))
+        result = process.run(cmd, ignore_status=True)
+        jres = json.loads(result.stdout)
+        self.assertEqual(result.exit_status, 1, result)
+        exps = [("unittests.Second.test_fail", "FAIL"),
+                ("unittests.Second.test_error", "ERROR"),
+                ("unittests.Second.test_skip", "CANCEL"),
+                ("unittests.First.test_pass", "PASS")]
+        for test in jres["tests"]:
+            for exp in exps:
+                if exp[0] in test["id"]:
+                    self.assertEqual(test["status"], exp[1], "Status of %s not"
+                                     " as expected\n%s" % (exp, result))
+                    exps.remove(exp)
+                    break
+            else:
+                self.fail("No expected result for %s\n%s\n\nexps = %s"
+                          % (test["id"], result, exps))
+        self.assertFalse(exps, "Some expected result not matched to actual"
+                         "results:\n%s\n\nexps = %s" % (result, exps))
+
     def tearDown(self):
         shutil.rmtree(self.tmpdir)
 

selftests/unit/test_loader.py

@@ -8,6 +8,7 @@ import unittest
 
 from avocado.core import test
 from avocado.core import loader
+from avocado.core import test
 from avocado.utils import script
 
 # We need to access protected members pylint: disable=W0212
@@ -238,6 +239,14 @@ class ThirdChild(Test, SecondChild):
         pass
 """
 
+PYTHON_UNITTEST = """#!/usr/bin/env python
+from unittest import TestCase
+
+class SampleTest(TestCase):
+    def test(self):
+        pass
+"""
+
 
 class LoaderTest(unittest.TestCase):
 
@@ -513,7 +522,7 @@ class LoaderTest(unittest.TestCase):
             KEEP_METHODS_ORDER)
         avocado_keep_methods_order.save()
         expected_order = ['test2', 'testA', 'test1', 'testZZZ', 'test']
-        tests = self.loader._find_avocado_tests(avocado_keep_methods_order.path)
+        tests = self.loader._find_avocado_tests(avocado_keep_methods_order.path)[0]
         methods = [method[0] for method in tests['MyClass']]
         self.assertEqual(expected_order, methods)
         avocado_keep_methods_order.remove()
@@ -529,13 +538,29 @@ class LoaderTest(unittest.TestCase):
         avocado_recursive_discovery_test2.save()
 
         sys.path.append(os.path.dirname(avocado_recursive_discovery_test1.path))
-        tests = self.loader._find_avocado_tests(avocado_recursive_discovery_test2.path)
+        tests = self.loader._find_avocado_tests(avocado_recursive_discovery_test2.path)[0]
         expected = {'ThirdChild': [('test_third_child', set([])),
                                    ('test_second_child', set([])),
                                    ('test_first_child', set([])),
                                    ('test_basic', set([]))]}
         self.assertEqual(expected, tests)
 
+    def test_python_unittest(self):
+        disabled_test = script.TemporaryScript("disabled.py",
+                                               AVOCADO_TEST_OK_DISABLED,
+                                               mode=DEFAULT_NON_EXEC_MODE)
+        python_unittest = script.TemporaryScript("python_unittest.py",
+                                                 PYTHON_UNITTEST)
+        disabled_test.save()
+        python_unittest.save()
+        tests = self.loader.discover(disabled_test.path)
+        self.assertEqual(tests, [])
+        tests = self.loader.discover(python_unittest.path)
+        exp = [(test.PythonUnittest,
+                {"name": "python_unittest.SampleTest.test",
+                 "test_dir": os.path.dirname(python_unittest.path)})]
+        self.assertEqual(tests, exp)
+
     def tearDown(self):
         shutil.rmtree(self.tmpdir)