Merge remote-tracking branch 'ldoktor/loader-failures'

Signed-off-by: NCleber Rosa <crosa@redhat.com>

avocado/core/exceptions.py

@@ -75,17 +75,6 @@ class TestError(TestBaseException):
     status = "ERROR"
 
 
-class NotATestError(TestBaseException):
-
-    """
-    Indicates that the file is not a test.
-
-    Causes: Non executable, non python file or python module without
-    an avocado test class in it.
-    """
-    status = "NOT_A_TEST"
-
-
 class TestNotFoundError(TestBaseException):
 
     """

avocado/core/loader.py

@@ -43,6 +43,12 @@ AVAILABLE = None
 ALL = True
 
 
+class MissingTest(object):
+    """
+    Class representing reference which failed to be discovered
+    """
+
+
 def filter_test_tags(test_suite, filter_by_tags, include_empty=False):
     """
     Filter the existing (unfiltered) test suite based on tags
@@ -209,10 +215,10 @@ class TestLoaderProxy(object):
         Update the mappings according the current initialized plugins
         """
         # Plugins are initialized, let's update mappings
-        self._label_mapping = {test.MissingTest: "MISSING"}
+        self._label_mapping = {MissingTest: "MISSING"}
         for plugin in self._initialized_plugins:
             self._label_mapping.update(plugin.get_type_label_mapping())
-        self._decorator_mapping = {test.MissingTest:
+        self._decorator_mapping = {MissingTest:
                                    output.TERM_SUPPORT.fail_header_str}
         for plugin in self._initialized_plugins:
             self._decorator_mapping.update(plugin.get_decorator_mapping())
@@ -275,7 +281,7 @@ class TestLoaderProxy(object):
                     unhandled_references.append(reference)
         if unhandled_references:
             if which_tests:
-                tests.extend([(test.MissingTest, {'name': reference})
+                tests.extend([(MissingTest, {'name': reference})
                               for reference in unhandled_references])
             else:
                 raise LoaderUnhandledReferenceError(unhandled_references,
@@ -453,6 +459,12 @@ def add_loader_options(parser):
                               'run tests from files'))
 
 
+class NotATest(object):
+    """
+    Class representing something that is not a test
+    """
+
+
 class FileLoader(TestLoader):
 
     """
@@ -460,6 +472,8 @@ class FileLoader(TestLoader):
     """
 
     name = 'file'
+    __not_test_str = (": Does not look like an INSTRUMENTED test, nor is "
+                      "it executable")
 
     def __init__(self, args, extra_params):
         test_type = extra_params.pop('allowed_test_types', None)
@@ -469,8 +483,8 @@ class FileLoader(TestLoader):
     @staticmethod
     def get_type_label_mapping():
         return {test.SimpleTest: 'SIMPLE',
-                test.NotATest: 'NOT_A_TEST',
-                test.MissingTest: 'MISSING',
+                NotATest: 'NOT_A_TEST',
+                MissingTest: 'MISSING',
                 BrokenSymlink: 'BROKEN_SYMLINK',
                 AccessDeniedPath: 'ACCESS_DENIED',
                 test.Test: 'INSTRUMENTED'}
@@ -478,8 +492,8 @@ class FileLoader(TestLoader):
     @staticmethod
     def get_decorator_mapping():
         return {test.SimpleTest: output.TERM_SUPPORT.healthy_str,
-                test.NotATest: output.TERM_SUPPORT.warn_header_str,
-                test.MissingTest: output.TERM_SUPPORT.fail_header_str,
+                NotATest: output.TERM_SUPPORT.warn_header_str,
+                MissingTest: output.TERM_SUPPORT.fail_header_str,
                 BrokenSymlink: output.TERM_SUPPORT.fail_header_str,
                 AccessDeniedPath: output.TERM_SUPPORT.fail_header_str,
                 test.Test: output.TERM_SUPPORT.healthy_str}
@@ -705,7 +719,8 @@ class FileLoader(TestLoader):
                 else:
                     # Module does not have an avocado test class inside, and
                     # it's not executable. Not a Test.
-                    return make_broken(test.NotATest, test_path)
+                    return make_broken(NotATest,
+                                       test_path + self.__not_test_str)
 
         # Since a lot of things can happen here, the broad exception is
         # justified. The user will get it unadulterated anyway, and avocado
@@ -718,7 +733,7 @@ class FileLoader(TestLoader):
                 # execute it.
                 return self._make_test(test.SimpleTest, test_path)
             else:
-                return make_broken(test.NotATest, test_path)
+                return make_broken(NotATest, test_path + self.__not_test_str)
 
     @staticmethod
     def _make_test(klass, uid):
@@ -747,7 +762,8 @@ class FileLoader(TestLoader):
         test_name = test_path
         if os.path.exists(test_path):
             if os.access(test_path, os.R_OK) is False:
-                return make_broken(AccessDeniedPath, test_path)
+                return make_broken(AccessDeniedPath, test_path + ": Is not "
+                                   "readable")
             path_analyzer = path.PathInspector(test_path)
             if path_analyzer.is_python():
                 return self._make_avocado_tests(test_path, make_broken,
@@ -757,14 +773,17 @@ class FileLoader(TestLoader):
                     return self._make_test(test.SimpleTest,
                                            test_path)
                 else:
-                    return make_broken(test.NotATest, test_path)
+                    return make_broken(NotATest,
+                                       test_path + self.__not_test_str)
         else:
             if os.path.islink(test_path):
                 try:
                     if not os.path.isfile(os.readlink(test_path)):
-                        return make_broken(BrokenSymlink, test_path)
+                        return make_broken(BrokenSymlink, test_path + ": Is "
+                                           "a broken symlink")
                 except OSError:
-                    return make_broken(AccessDeniedPath, test_path)
+                    return make_broken(AccessDeniedPath, test_path + ": Is "
+                                       "not accessible.")
 
             # Try to resolve test ID (keep compatibility)
             test_path = os.path.join(data_dir.get_test_dir(), test_name)
@@ -781,7 +800,7 @@ class FileLoader(TestLoader):
                         return self._make_avocado_tests(test_path, make_broken,
                                                         subtests_filter,
                                                         test_name)
-        return make_broken(test.MissingTest, test_name)
+        return make_broken(NotATest, test_name + self.__not_test_str)
 
 
 class ExternalLoader(TestLoader):

avocado/core/output.py

@@ -561,6 +561,10 @@ class Paginator(object):
         except Exception:
             pass
 
+    def flush(self):
+        if not self.pipe.closed:
+            self.pipe.flush()
+
 
 def add_log_handler(logger, klass=logging.StreamHandler, stream=sys.stdout,
                     level=logging.INFO, fmt='%(name)s: %(message)s'):

avocado/core/test.py

@@ -931,34 +931,6 @@ class ExternalRunnerTest(SimpleTest):
                 os.chdir(pre_cwd)
 
 
-class MissingTest(Test):
-
-    """
-    Handle when there is no such test module in the test directory.
-    """
-
-    def test(self):
-        e_msg = ('Test %s could not be found in the test dir %s '
-                 '(or test path does not exist)' %
-                 (self.name, data_dir.get_test_dir()))
-        raise exceptions.TestNotFoundError(e_msg)
-
-
-class NotATest(Test):
-
-    """
-    The file is not a test.
-
-    Either a non executable python module with no avocado test class in it,
-    or a regular, non executable file.
-    """
-
-    def test(self):
-        e_msg = ('File %s is not executable and does not contain an avocado '
-                 'test class in it ' % self.name)
-        raise exceptions.NotATestError(e_msg)
-
-
 class MockingTest(Test):
 
     """

optional_plugins/robot/avocado_robot/__init__.py

@@ -67,6 +67,13 @@ class RobotTest(test.SimpleTest):
                       'non-0 exit code (%s)' % result)
 
 
+class NotRobotTest(object):
+
+    """
+    Not a robot test (for reporting purposes)
+    """
+
+
 class RobotLoader(loader.TestLoader):
     """
     Robot loader class
@@ -91,7 +98,9 @@ class RobotLoader(loader.TestLoader):
                                  source=url,
                                  include_suites=SuiteNamePatterns())
             robot_suite = self._find_tests(test_data, test_suite={})
-        except DataError:
+        except Exception as data:
+            if which_tests == loader.ALL:
+                return [(NotRobotTest, {"name": "%s: %s" % (url, data)})]
             return []
 
         for item in robot_suite:
@@ -102,6 +111,9 @@ class RobotLoader(loader.TestLoader):
                 if subtests_filter and not subtests_filter.search(test_name):
                     continue
                 avocado_suite.append((RobotTest, {'name': test_name}))
+        if which_tests is loader.ALL and not avocado_suite:
+            return [(NotRobotTest, {"name": "%s: No robot-like tests found"
+                                    % url})]
         return avocado_suite
 
     def _find_tests(self, data, test_suite):
@@ -115,11 +127,13 @@ class RobotLoader(loader.TestLoader):
 
     @staticmethod
     def get_type_label_mapping():
-        return {RobotTest: 'ROBOT'}
+        return {RobotTest: 'ROBOT',
+                NotRobotTest: "!ROBOT"}
 
     @staticmethod
     def get_decorator_mapping():
-        return {RobotTest: output.TERM_SUPPORT.healthy_str}
+        return {RobotTest: output.TERM_SUPPORT.healthy_str,
+                NotRobotTest: output.TERM_SUPPORT.fail_header_str}
 
 
 class RobotCLI(CLI):

selftests/unit/test_loader.py

@@ -5,7 +5,6 @@ import tempfile
 import unittest
 
 from avocado.core import test
-from avocado.core import exceptions
 from avocado.core import loader
 from avocado.utils import script
 
@@ -235,13 +234,8 @@ class LoaderTest(unittest.TestCase):
                                              'avocado_loader_unittest',
                                              mode=DEFAULT_NON_EXEC_MODE)
         simple_test.save()
-        test_class, test_parameters = (
-            self.loader.discover(simple_test.path, loader.ALL)[0])
-        self.assertTrue(test_class == test.NotATest, test_class)
-        test_parameters['name'] = test.TestName(0, test_parameters['name'])
-        test_parameters['base_logdir'] = self.tmpdir
-        tc = test_class(**test_parameters)
-        self.assertRaises(exceptions.NotATestError, tc.test)
+        test_class, _ = self.loader.discover(simple_test.path, loader.ALL)[0]
+        self.assertTrue(test_class == loader.NotATest, test_class)
         simple_test.remove()
 
     def test_load_pass(self):
@@ -249,8 +243,8 @@ class LoaderTest(unittest.TestCase):
                                                    AVOCADO_TEST_OK,
                                                    'avocado_loader_unittest')
         avocado_pass_test.save()
-        test_class, test_parameters = (
-            self.loader.discover(avocado_pass_test.path, loader.ALL)[0])
+        test_class, _ = self.loader.discover(avocado_pass_test.path,
+                                             loader.ALL)[0]
         self.assertTrue(test_class == 'PassTest', test_class)
         avocado_pass_test.remove()
 
@@ -260,13 +254,9 @@ class LoaderTest(unittest.TestCase):
                                                     'avocado_loader_unittest',
                                                     mode=DEFAULT_NON_EXEC_MODE)
         avocado_not_a_test.save()
-        test_class, test_parameters = (
-            self.loader.discover(avocado_not_a_test.path, loader.ALL)[0])
-        self.assertTrue(test_class == test.NotATest, test_class)
-        test_parameters['name'] = test.TestName(0, test_parameters['name'])
-        test_parameters['base_logdir'] = self.tmpdir
-        tc = test_class(**test_parameters)
-        self.assertRaises(exceptions.NotATestError, tc.test)
+        test_class, _ = self.loader.discover(avocado_not_a_test.path,
+                                             loader.ALL)[0]
+        self.assertTrue(test_class == loader.NotATest, test_class)
         avocado_not_a_test.remove()
 
     def test_load_not_a_test_exec(self):
@@ -304,13 +294,9 @@ class LoaderTest(unittest.TestCase):
                                                      'avocado_loader_unittest',
                                                      mode=DEFAULT_NON_EXEC_MODE)
         avocado_simple_test.save()
-        test_class, test_parameters = (
-            self.loader.discover(avocado_simple_test.path, loader.ALL)[0])
-        self.assertTrue(test_class == test.NotATest)
-        test_parameters['name'] = test.TestName(0, test_parameters['name'])
-        test_parameters['base_logdir'] = self.tmpdir
-        tc = test_class(**test_parameters)
-        self.assertRaises(exceptions.NotATestError, tc.test)
+        test_class, _ = self.loader.discover(avocado_simple_test.path,
+                                             loader.ALL)[0]
+        self.assertTrue(test_class == loader.NotATest)
         avocado_simple_test.remove()
 
     def test_multiple_methods(self):
@@ -373,7 +359,7 @@ class LoaderTest(unittest.TestCase):
         avocado_pass_test.save()
         test_class, test_parameters = (
             self.loader.discover(avocado_pass_test.path, loader.ALL)[0])
-        self.assertTrue(test_class == test.NotATest)
+        self.assertTrue(test_class == loader.NotATest)
         avocado_pass_test.remove()
 
     def test_load_tagged_nested(self):
@@ -382,10 +368,9 @@ class LoaderTest(unittest.TestCase):
                                                      'avocado_loader_unittest',
                                                      DEFAULT_NON_EXEC_MODE)
         avocado_nested_test.save()
-        test_class, test_parameters = (
-            self.loader.discover(avocado_nested_test.path, loader.ALL)[0])
-        results = self.loader.discover(avocado_nested_test.path, loader.ALL)
-        self.assertTrue(test_class == test.NotATest)
+        test_class, _ = self.loader.discover(avocado_nested_test.path,
+                                             loader.ALL)[0]
+        self.assertTrue(test_class == loader.NotATest)
         avocado_nested_test.remove()
 
     def test_load_multiple_imports(self):