Merge pull request #49 from avocado-framework/fix-nonexisting-test

Fix assorted bugs, make avocado return codes and functional test more complete

.travis.yml

@@ -18,3 +18,4 @@ script:
     - make -C docs html 2>&1 | grep -E '(ERROR|WARNING)' || test $? -eq 1
     - ./scripts/avocado run "sleeptest sleeptest"
     - ./scripts/avocado run "sleeptest failtest sleeptest" || test $? -eq 1
+    - ./scripts/avocado run "bogustest" || test $? -ne 3

avocado/core/error_codes.py

+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See LICENSE for more details.
+
+"""
+Maps the different avocado failure statuses to unix return codes.
+
+The current return codes are:
+    * AVOCADO_ALL_OK
+        Both job and tests PASSed
+    * AVOCADO_TESTS_FAIL
+        Job went fine, but some tests FAILed or ERRORed
+    * AVOCADO_JOB_FAIL
+        Something went wrong with the Job itself, by explicit
+        :class:`avocado.core.exceptions.JobError` exception.
+    * AVOCADO_CRASH
+        Something else went wrong and avocado plain crashed.
+"""
+
+numeric_status = {"AVOCADO_ALL_OK": 0,
+                  "AVOCADO_TESTS_FAIL": 1,
+                  "AVOCADO_JOB_FAIL": 2,
+                  "AVOCADO_CRASH": 3}

avocado/core/exceptions.py

@@ -17,6 +17,22 @@ Exception classes, useful for tests, and other parts of the framework code.
 """
 
 
+class JobBaseException(Exception):
+
+    """
+    The parent of all job exceptions.
+    """
+    status = "NEVER_RAISE_THIS"
+
+
+class JobError(Exception):
+
+    """
+    A generic error happened during a job execution.
+    """
+    status = "ERROR"
+
+
 class TestBaseException(Exception):
 
     """

avocado/core/output.py

@@ -232,6 +232,17 @@ class OutputManager(object):
                            " (%.2f s)" % t_elapsed)
         self.info(normal_pass_msg)
 
+    def log_error(self, label, t_elapsed):
+        """
+        Log a test ERROR message.
+
+        :param label: Label for the FAIL message (test name + index).
+        :param t_elapsed: Time it took for test to complete.
+        """
+        normal_error_msg = (label + " " + colors.error_str() +
+                            " (%.2f s)" % t_elapsed)
+        self.error(normal_error_msg)
+
     def log_fail(self, label, t_elapsed):
         """
         Log a test FAIL message.

avocado/core/status.py

@@ -16,7 +16,7 @@ This is used by methods and functions to return a cut and dry answer to wether
 a test or a job in avocado PASSed or FAILed.
 """
 
-mapping = {"TEST_NA": False,
+mapping = {"TEST_NA": True,
            "ABORT": False,
            "ERROR": False,
            "FAIL": False,

avocado/job.py

@@ -18,11 +18,15 @@ Class that describes a sequence of automated operations.
 import imp
 import logging
 import os
+import sys
+import traceback
 import uuid
 
 from avocado.core import data_dir
 from avocado.core import output
 from avocado.core import status
+from avocado.core import exceptions
+from avocado.core import error_codes
 from avocado import test
 from avocado import result
 
@@ -59,14 +63,18 @@ class Job(object):
                                        base_logdir=self.debugdir,
                                        job=self)
         else:
-            test_module_dir = os.path.join(self.test_dir, url)
-            f, p, d = imp.find_module(url, [test_module_dir])
-            test_module = imp.load_module(url, f, p, d)
-            f.close()
-            test_class = getattr(test_module, url)
-            test_instance = test_class(name=url,
-                                       base_logdir=self.debugdir,
-                                       job=self)
+            try:
+                test_module_dir = os.path.join(self.test_dir, url)
+                f, p, d = imp.find_module(url, [test_module_dir])
+                test_module = imp.load_module(url, f, p, d)
+                f.close()
+                test_class = getattr(test_module, url)
+            except ImportError:
+                test_class = test.MissingTest
+            finally:
+                test_instance = test_class(name=url,
+                                           base_logdir=self.debugdir,
+                                           job=self)
         return test_instance
 
     def run_test(self, url):
@@ -89,16 +97,17 @@ class Job(object):
                                         self.args)
         return test_result
 
-    def run(self, urls=None):
+    def _run(self, urls=None):
         """
-        Main job method. Runs a list of test URLs to its completion.
+        Unhandled job method. Runs a list of test URLs to its completion.
 
         :param urls: String with tests to run.
 
-        :return: Integer with overall job status
-                 0 - Job passed, all tests passed
-                 1 - Job passed, some/all tests failed
-                 2 - Job failed
+        :return: Integer with overall job status. See
+                 :mod:`avocado.core.error_codes` for more information.
+        :raise: Any exception (avocado crashed), or
+                :class:`avocado.core.exceptions.JobBaseException` errors,
+                that configure a job failure.
         """
         failures = []
         if urls is None:
@@ -119,16 +128,37 @@ class Job(object):
         # Let's clean up test artifacts
         if not self.args.keep_tmp_files:
             data_dir.clean_tmp_files()
-        # Let's assess the overall status:
-        job_status = status.mapping[self.status]
         tests_status = not bool(failures)
-        if job_status:
-            if tests_status:
-                return 0
-            else:
-                return 1
+        if tests_status:
+            return error_codes.numeric_status['AVOCADO_ALL_OK']
         else:
-            return 2
+            return error_codes.numeric_status['AVOCADO_TESTS_FAIL']
+
+    def run(self, urls=None):
+        """
+        Handled main job method. Runs a list of test URLs to its completion.
+
+        :param urls: String with tests to run.
+
+        :return: Integer with overall job status. See
+                 :mod:`avocado.core.error_codes` for more information.
+        """
+        try:
+            return self._run(urls)
+        except exceptions.JobBaseException, details:
+            self.status = details.status
+            fail_class = details.__class__.name
+            self.output_manager.error('Avocado job failed: %s: %s' %
+                                      (fail_class, details))
+            return error_codes.numeric_status['AVOCADO_JOB_FAIL']
+        except Exception, details:
+            self.status = "ERROR"
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            tb_info = traceback.format_exception(exc_type, exc_value,
+                                                 exc_traceback.tb_next)
+            for line in tb_info:
+                self.output_manager.error(line)
+            return error_codes.numeric_status['AVOCADO_CRASH']
 
 
 class TestModuleRunner(object):

avocado/result.py

@@ -184,7 +184,7 @@ class HumanTestResult(TestResult):
         :param test: :class:`avocado.test.Test` instance.
         """
         TestResult.add_error(self, test)
-        self.stream.log_pass(self.test_label, test.time_elapsed)
+        self.stream.log_error(self.test_label, test.time_elapsed)
 
     def add_fail(self, test):
         """

avocado/test.py

@@ -294,3 +294,20 @@ class DropinTest(Test):
         except exceptions.CmdError, details:
             self._log_detailed_cmd_info(details.result)
             raise exceptions.TestFail(details)
+
+
+class MissingTest(Test):
+
+    """
+    Handle when there is no such test module in the test directory.
+    """
+
+    def __init__(self, name=None, base_logdir=None, tag=None, job=None):
+        super(MissingTest, self).__init__(name=name,
+                                          base_logdir=base_logdir,
+                                          tag=tag, job=job)
+
+    def action(self):
+        raise exceptions.TestError('Test %s could not be found in the test '
+                                   'dir %s' %
+                                   (self.name, data_dir.get_test_dir()))