avocado: Add NOT_FOUND test status

To avoid confusion, when passing an invalid test ID
(typo, incorrect path, alias that could not be resolved),
introduce a new status, NOT_FOUND inside avocado, as
well as proper handling handling for such situations.

$ scripts/avocado run sbrubles
JOB ID    : 312b690173a8f5f261d9c5cefdb8699ac0798ded
JOB LOG   : /home/lmr/avocado/job-results/job-2014-08-14T10.02-312b690/job.log
TESTS     : 1
(1/1) sbrubles: NOT_FOUND (0.00 s)
PASS      : 0
ERROR     : 0
FAIL      : 0
SKIP      : 0
WARN      : 0
NOT FOUND : 1
TIME      : 0.00 s

Also, add a functional test for this new status.
Signed-off-by: NLucas Meneghel Rodrigues <lmr@redhat.com>

avocado/core/exceptions.py

@@ -75,6 +75,16 @@ class TestError(TestBaseException):
     status = "ERROR"
 
 
+class TestNotFoundError(TestBaseException):
+
+    """
+    Indicates that the test was not found.
+
+    Causes: non existing path or could not resolve alias.
+    """
+    status = "NOT_FOUND"
+
+
 class TestTimeoutError(TestBaseException):
 
     """

avocado/core/output.py

@@ -100,6 +100,7 @@ class TermSupport(object):
         self.SKIP = self.COLOR_YELLOW
         self.FAIL = self.COLOR_RED
         self.ERROR = self.COLOR_RED
+        self.NOT_FOUND = self.COLOR_YELLOW
         self.WARN = self.COLOR_YELLOW
         self.ENDC = self.CONTROL_END
         term = os.environ.get("TERM")
@@ -115,6 +116,7 @@ class TermSupport(object):
         self.SKIP = ''
         self.FAIL = ''
         self.ERROR = ''
+        self.NOT_FOUND = ''
         self.WARN = ''
         self.ENDC = ''
 
@@ -168,12 +170,20 @@ class TermSupport(object):
 
     def error_str(self):
         """
-        Print an error string (red colored).
+        Print a not found string (yellow colored).
 
         If the output does not support colors, just return the original string.
         """
         return self.MOVE_BACK + self.ERROR + 'ERROR' + self.ENDC
 
+    def not_found_str(self):
+        """
+        Print an error string (red colored).
+
+        If the output does not support colors, just return the original string.
+        """
+        return self.MOVE_BACK + self.NOT_FOUND + 'NOT_FOUND' + self.ENDC
+
     def warn_str(self):
         """
         Print an warning string (yellow colored).
@@ -313,6 +323,15 @@ class OutputManager(object):
         normal_error_msg = term_support.error_str() + " (%.2f s)" % t_elapsed
         self.error(normal_error_msg)
 
+    def log_not_found(self, t_elapsed):
+        """
+        Log a NOT_FOUND message.
+
+        :param t_elapsed: Time it took for the operation to complete.
+        """
+        normal_error_msg = term_support.not_found_str() + " (%.2f s)" % t_elapsed
+        self.error(normal_error_msg)
+
     def log_fail(self, t_elapsed):
         """
         Log a FAIL message.

avocado/core/status.py

@@ -19,6 +19,7 @@ a test or a job in avocado PASSed or FAILed.
 mapping = {"TEST_NA": True,
            "ABORT": False,
            "ERROR": False,
+           "NOT_FOUND": False,
            "FAIL": False,
            "WARN": False,
            "PASS": True,

avocado/plugins/vm.py

@@ -196,19 +196,20 @@ class VMTestResult(TestResult):
         Called once before any tests are executed.
         """
         TestResult.start_tests(self)
-        self.stream.log_header("TESTS: %s" % self.tests_total)
+        self.stream.log_header("TESTS     : %s" % self.tests_total)
 
     def end_tests(self):
         """
         Called once after all tests are executed.
         """
         self.stream.log_header("PASS      : %d" % len(self.passed))
-        self.stream.log_header("ERROR: %d" % len(self.errors))
+        self.stream.log_header("ERROR     : %d" % len(self.errors))
+        self.stream.log_header("NOT FOUND : %d" % len(self.not_found))
         self.stream.log_header("FAIL      : %d" % len(self.failed))
         self.stream.log_header("SKIP      : %d" % len(self.skipped))
         self.stream.log_header("WARN      : %d" % len(self.warned))
         self.stream.log_header("TIME      : %.2f s" % self.total_time)
-        self.stream.log_header("DEBUG LOG: %s" % self.stream.logfile)
+        self.stream.log_header("JOB LOG   : %s" % self.stream.logfile)
 
     def start_test(self, test):
         """
@@ -248,6 +249,15 @@ class VMTestResult(TestResult):
         TestResult.add_error(self, test)
         self.stream.log_error(test.time_elapsed)
 
+    def add_not_found(self, test):
+        """
+        Called when a test had a setup error.
+
+        :param test: :class:`avocado.test.Test` instance.
+        """
+        TestResult.add_not_found(self, test)
+        self.stream.log_not_found(test.time_elapsed)
+
     def add_fail(self, test):
         """
         Called when a test fails.

avocado/result.py

@@ -73,6 +73,10 @@ class TestResultProxy(object):
         for output_plugin in self.output_plugins:
             output_plugin.add_error(state)
 
+    def add_not_found(self, state):
+        for output_plugin in self.output_plugins:
+            output_plugin.add_not_found(state)
+
     def add_fail(self, state):
         for output_plugin in self.output_plugins:
             output_plugin.add_fail(state)
@@ -110,6 +114,7 @@ class TestResult(object):
         self.total_time = 0.0
         self.passed = []
         self.errors = []
+        self.not_found = []
         self.failed = []
         self.skipped = []
         self.warned = []
@@ -191,6 +196,17 @@ class TestResult(object):
         """
         self.errors.append(state)
 
+    def add_not_found(self, state):
+        """
+        Called when a test was not found.
+
+        Causes: non existing path or could not resolve alias.
+
+        :param state: result of :class:`avocado.test.Test.get_state`.
+        :type state: dict
+        """
+        self.not_found.append(state)
+
     def add_fail(self, state):
         """
         Called when a test fails.
@@ -225,6 +241,7 @@ class TestResult(object):
         """
         status_map = {'PASS': self.add_pass,
                       'ERROR': self.add_error,
+                      'NOT_FOUND': self.add_not_found,
                       'FAIL': self.add_fail,
                       'TEST_NA': self.add_skip,
                       'WARN': self.add_warn}
@@ -245,7 +262,7 @@ class HumanTestResult(TestResult):
         """
         TestResult.start_tests(self)
         self.stream.log_header("JOB ID    : %s" % self.stream.job_unique_id)
-        self.stream.log_header("JOB LOG: %s" % self.stream.logfile)
+        self.stream.log_header("JOB LOG   : %s" % self.stream.logfile)
         self.stream.log_header("TESTS     : %s" % self.tests_total)
 
     def end_tests(self):
@@ -253,10 +270,11 @@ class HumanTestResult(TestResult):
         Called once after all tests are executed.
         """
         self.stream.log_header("PASS      : %d" % len(self.passed))
-        self.stream.log_header("ERROR: %d" % len(self.errors))
+        self.stream.log_header("ERROR     : %d" % len(self.errors))
         self.stream.log_header("FAIL      : %d" % len(self.failed))
         self.stream.log_header("SKIP      : %d" % len(self.skipped))
         self.stream.log_header("WARN      : %d" % len(self.warned))
+        self.stream.log_header("NOT FOUND : %d" % len(self.not_found))
         self.stream.log_header("TIME      : %.2f s" % self.total_time)
 
     def start_test(self, state):
@@ -300,6 +318,16 @@ class HumanTestResult(TestResult):
         TestResult.add_error(self, state)
         self.stream.log_error(state['time_elapsed'])
 
+    def add_not_found(self, state):
+        """
+        Called when a test was not found.
+
+        :param state: result of :class:`avocado.test.Test.get_state`.
+        :type state: dict
+        """
+        TestResult.add_not_found(self, state)
+        self.stream.log_not_found(state['time_elapsed'])
+
     def add_fail(self, state):
         """
         Called when a test fails.

avocado/test.py

@@ -455,6 +455,7 @@ class MissingTest(Test):
                                           tag=tag, job=job)
 
     def action(self):
-        raise exceptions.TestError('Test %s could not be found in the test '
-                                   'dir %s' %
+        e_msg = ('Test %s could not be found in the test dir %s'
+                 '(or test path does not exist)' %
                  (self.name, data_dir.get_test_dir()))
+        raise exceptions.TestNotFoundError(e_msg)

selftests/all/functional/avocado/basic_tests.py

@@ -162,6 +162,15 @@ class RunnerOperationTest(unittest.TestCase):
         self.assertEqual(result.exit_status, expected_rc)
         self.assertIn(expected_output, result.stderr)
 
+    def test_not_found(self):
+        os.chdir(basedir)
+        cmd_line = './scripts/avocado run sbrubles'
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = 1
+        self.assertEqual(result.exit_status, expected_rc)
+        self.assertIn('NOT_FOUND', result.stderr)
+        self.assertIn('NOT FOUND : 1', result.stderr)
+
 
 class RunnerDropinTest(unittest.TestCase):
 

selftests/all/functional/avocado/multiplex_tests.py

@@ -45,7 +45,7 @@ class MultiplexTests(unittest.TestCase):
         output = result.stdout + result.stderr
         if expected_lines is not None:
             for line in output.splitlines():
-                if 'JOB LOG:' in line:
+                if 'JOB LOG   :' in line:
                     debug_log = line.split()[-1]
                     debug_log_obj = open(debug_log, 'r')
                     job_log_lines = debug_log_obj.readlines()