SIMPLE tests: improve status API

Currently simple tests have a limited status API, being able to set only
the WARN status by generating an output string in a very specific and
hard-coded format.

This patch, while respects the current behaviour, creates configuration
keys, allowing users to provide regular expressions to search for in the
test outputs, aiming to set the final test status to either WARN or SKIP
when the test finishes with exit code 0.
Signed-off-by: NAmador Pahim <apahim@redhat.com>

avocado/core/test.py

@@ -1101,9 +1101,6 @@ class SimpleTest(Test):
 
     DATA_SOURCES = ["variant", "file"]
 
-    re_avocado_log = re.compile(r'^\d\d:\d\d:\d\d DEBUG\| \[stdout\]'
-                                r' \d\d:\d\d:\d\d WARN \|')
-
     def __init__(self, name, params=None, base_logdir=None, job=None):
         super(SimpleTest, self).__init__(name=name, params=params,
                                          base_logdir=base_logdir, job=job)
@@ -1146,12 +1143,51 @@ class SimpleTest(Test):
         except process.CmdError as details:
             self._log_detailed_cmd_info(details.result)
             raise exceptions.TestFail(details)
-        with open(self.logfile) as logfile:
-            for line in logfile:
-                if self.re_avocado_log.match(line):
-                    raise exceptions.TestWarn("Test passed but there were warnings"
-                                              " on stdout during execution. Check "
-                                              "the log for details.")
+
+        warn_regex = settings.get_value('simpletests.status',
+                                        'warn_regex',
+                                        key_type='str',
+                                        default=None)
+
+        warn_location = settings.get_value('simpletests.status',
+                                           'warn_location',
+                                           default='all')
+
+        skip_regex = settings.get_value('simpletests.status',
+                                        'skip_regex',
+                                        key_type='str',
+                                        default=None)
+
+        skip_location = settings.get_value('simpletests.status',
+                                           'skip_location',
+                                           default='all')
+
+        # Keeping compatibility with 'avocado_warn' libexec
+        for regex in [warn_regex, r'^\d\d:\d\d:\d\d WARN \|']:
+            warn_msg = ("Test passed but there were warnings on %s during "
+                        "execution. Check the log for details.")
+            if regex is not None:
+                re_warn = re.compile(regex)
+                if warn_location in ['all', 'stdout']:
+                    if re_warn.search(result.stdout):
+                        raise exceptions.TestWarn(warn_msg % 'stdout')
+
+                if warn_location in ['all', 'stderr']:
+                    if re_warn.search(result.stderr):
+                        raise exceptions.TestWarn(warn_msg % 'stderr')
+
+        if skip_regex is not None:
+            re_skip = re.compile(skip_regex)
+            skip_msg = ("Test passed but %s indicates test was skipped. "
+                        "Check the log for details.")
+
+            if skip_location in ['all', 'stdout']:
+                if re_skip.search(result.stdout):
+                    raise exceptions.TestSkipError(skip_msg % 'stdout')
+
+            if warn_location in ['all', 'stderr']:
+                if re_skip.search(result.stderr):
+                    raise exceptions.TestSkipError(skip_msg % 'stderr')
 
     def test(self):
         """

avocado/etc/avocado/avocado.conf

@@ -84,3 +84,24 @@ skip_broken_plugin_notification = []
 # options or test types).
 # The keyword "@DEFAULT" will be replaced with all available unused loaders.
 loaders = ['file', '@DEFAULT']
+
+[simpletests.status]
+# Python regular expression that will make the test
+# status WARN when matched. Defaults to disabled.
+# Reference: http://docs.python.org/2.7/howto/regex.html
+# warn_regex = ^WARN$
+
+# Location to search the regular expression on.
+# Accepted values: all, stdout, stderr.
+# Defaults to all.
+# warn_location = all
+
+# Python regular expression that will make the test
+# status SKIP when matched. Defaults to disabled.
+# Reference: http://docs.python.org/2.7/howto/regex.html
+# skip_regex = ^SKIP$
+
+# Location to search the regular expression on.
+# Accepted values: all, stdout, stderr.
+# Defaults to all.
+# skip_location = all

docs/source/WritingTests.rst

@@ -1753,6 +1753,42 @@ by ``avocado exec-path`` (if any).  Also, the example test
 .. tip:: These extensions may be available as a separate package.  For
          RPM packages, look for the ``bash`` sub-package.
 
+SIMPLE Tests Status
+===================
+
+With SIMPLE tests, Avocado checks the exit code of the test to determine
+whether the test PASSed or FAILed.
+
+If your test exits with exit code 0 but you still want to set a different test
+status in some conditions, Avocado can search a given regular expression in
+the test outputs and, based on that, set the status to WARN or SKIP.
+
+To use that feature, you have to set the proper keys in the configuration
+file. For instance, to set the test status to SKIP when the test outputs
+a line like this: '11:08:24 Test Skipped'::
+
+    [simpletests.output]
+    skip_regex = ^\d\d:\d\d:\d\d Test Skipped$
+
+That configuration will make avocado to search the
+`Python Regular Expression <http://docs.python.org/2.7/howto/regex.html>`__
+on  both stdout and stderr. If you want to limit the search for only one of
+them, there's another key for that configuration, resulting in::
+
+    [simpletests.output]
+    skip_regex = ^\d\d:\d\d:\d\d Test Skipped$
+    skip_location = stderr
+
+The equivalent settings can be present for the WARN status. For instance,
+if you want to set the test status to WARN when the test outputs a line
+starting with string ``WARNING:``, the configuration file will look like this::
+
+    [simpletests.output]
+    skip_regex = ^\d\d:\d\d:\d\d Test Skipped$
+    skip_location = stderr
+    warn_regex = ^WARNING:
+    warn_location = all
+
 Wrap Up
 =======
 

selftests/functional/test_basic.py

@@ -858,6 +858,50 @@ class RunnerSimpleTest(unittest.TestCase):
         shutil.rmtree(self.tmpdir)
 
 
+class RunnerSimpleTestStatus(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp(prefix='avocado_' + __name__)
+
+        self.config_file = script.TemporaryScript('avocado.conf',
+                                                  "[simpletests.status]\n"
+                                                  "warn_regex = ^WARN$\n"
+                                                  "skip_regex = ^SKIP$\n")
+        self.config_file.save()
+        os.chdir(basedir)
+
+    def test_simpletest_status(self):
+        warn_script = script.TemporaryScript('avocado_warn.sh',
+                                             "#!/bin/sh\necho WARN",
+                                             'avocado_simpletest_'
+                                             'functional')
+        warn_script.save()
+        cmd_line = ('%s --config %s run --job-results-dir %s --sysinfo=off'
+                    ' %s --json -' % (AVOCADO, self.config_file.path,
+                                      self.tmpdir, warn_script.path))
+        result = process.system_output(cmd_line, ignore_status=True)
+        json_results = json.loads(result)
+        self.assertEquals(json_results['tests'][0]['status'], 'WARN')
+        warn_script.remove()
+
+        skip_script = script.TemporaryScript('avocado_skip.sh',
+                                             "#!/bin/sh\necho SKIP",
+                                             'avocado_simpletest_'
+                                             'functional')
+        skip_script.save()
+        cmd_line = ('%s --config %s run --job-results-dir %s --sysinfo=off'
+                    ' %s --json -' % (AVOCADO, self.config_file.path,
+                                      self.tmpdir, skip_script.path))
+        result = process.system_output(cmd_line, ignore_status=True)
+        json_results = json.loads(result)
+        self.assertEquals(json_results['tests'][0]['status'], 'SKIP')
+        skip_script.remove()
+
+    def tearDown(self):
+        self.config_file.remove()
+        shutil.rmtree(self.tmpdir)
+
+
 class ExternalRunnerTest(unittest.TestCase):
 
     def setUp(self):