Output check: also support checking against 'output.expected'

Avocado recently introduced the support to record the combined
output generated by a test.  Now, this introduces support for
checking against a previously record combined output.

If an 'output.expected' file exists, it'll take precedence
over the `stdout.expected` and `stderr.expected` files.

Also it's necessary to switch the default operation mode for
the `avocado.utils.process` functions, since to compare with
an `output.expected` file, the record mode for the currently
running test has to be the same.
Signed-off-by: NCleber Rosa <crosa@redhat.com>

avocado/core/test.py

@@ -786,12 +786,22 @@ class Test(unittest.TestCase, TestData):
         """
         Auxiliary method to run_avocado.
         """
+        # If the test contains an output.expected file, it requires
+        # changing the mode of operation of the process.* utility
+        # methods, so that after the test finishes, the output
+        # produced can be compared to the expected one.  This runs in
+        # its own process, so the change should not effect other
+        # components using process.* functions.
+        if self.get_data('output.expected') is not None:
+            process.OUTPUT_CHECK_RECORD_MODE = 'combined'
+
         testMethod = getattr(self, self._testMethodName)
         self._start_logging()
         if self.__sysinfo_enabled:
             self.__sysinfo_logger.start_test_hook()
         test_exception = None
         cleanup_exception = None
+        output_check_exception = None
         stdout_check_exception = None
         stderr_check_exception = None
         skip_test = getattr(testMethod, '__skip_test_decorator__', False)
@@ -877,26 +887,41 @@ class Test(unittest.TestCase, TestData):
 
             if job_standalone or no_record_mode:
                 if not disable_output_check:
+                    output_checked = False
                     try:
-                        self._check_reference(self._stdout_file,
-                                              'stdout.expected',
-                                              'stdout.diff',
-                                              'stdout_diff',
-                                              'Stdout')
-                    except Exception as details:
-                        stacktrace.log_exc_info(sys.exc_info(),
-                                                logger=LOG_JOB)
-                        stdout_check_exception = details
-                    try:
-                        self._check_reference(self._stderr_file,
-                                              'stderr.expected',
-                                              'stderr.diff',
-                                              'stderr_diff',
-                                              'Stderr')
+                        output_checked = self._check_reference(
+                            self._output_file,
+                            'output.expected',
+                            'output.diff',
+                            'output_diff',
+                            'Output')
                     except Exception as details:
+                        # output check was performed (and failed)
+                        output_checked = True
                         stacktrace.log_exc_info(sys.exc_info(),
                                                 logger=LOG_JOB)
-                        stderr_check_exception = details
+                        output_check_exception = details
+                    if not output_checked:
+                        try:
+                            self._check_reference(self._stdout_file,
+                                                  'stdout.expected',
+                                                  'stdout.diff',
+                                                  'stdout_diff',
+                                                  'Stdout')
+                        except Exception as details:
+                            stacktrace.log_exc_info(sys.exc_info(),
+                                                    logger=LOG_JOB)
+                            stdout_check_exception = details
+                        try:
+                            self._check_reference(self._stderr_file,
+                                                  'stderr.expected',
+                                                  'stderr.diff',
+                                                  'stderr_diff',
+                                                  'Stderr')
+                        except Exception as details:
+                            stacktrace.log_exc_info(sys.exc_info(),
+                                                    logger=LOG_JOB)
+                            stderr_check_exception = details
             elif not job_standalone:
                 if output_check_record == 'combined':
                     self._record_reference(self._output_file,
@@ -914,6 +939,8 @@ class Test(unittest.TestCase, TestData):
             raise test_exception
         elif cleanup_exception is not None:
             raise cleanup_exception
+        elif output_check_exception is not None:
+            raise output_check_exception
         elif stdout_check_exception is not None:
             raise stdout_check_exception
         elif stderr_check_exception is not None:

selftests/functional/test_output_check.py

@@ -45,6 +45,19 @@ class RunnerSimpleTest(unittest.TestCase):
         self.assertTrue(os.path.isfile(stdout_file))
         self.assertTrue(os.path.isfile(stderr_file))
 
+    def _check_output_record_combined(self):
+        os.chdir(basedir)
+        cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s '
+                    '--output-check-record combined'
+                    % (AVOCADO, self.tmpdir, self.output_script.path))
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" %
+                         (expected_rc, result))
+        output_file = os.path.join("%s.data/output.expected" % self.output_script)
+        self.assertTrue(os.path.isfile(output_file))
+
     def test_output_record_none(self):
         os.chdir(basedir)
         cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s '
@@ -85,6 +98,16 @@ class RunnerSimpleTest(unittest.TestCase):
                          "Avocado did not return rc %d:\n%s" %
                          (expected_rc, result))
 
+    def test_output_record_and_check_combined(self):
+        self._check_output_record_combined()
+        cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s'
+                    % (AVOCADO, self.tmpdir, self.output_script.path))
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" %
+                         (expected_rc, result))
+
     def test_output_tamper_stdout(self):
         self._check_output_record_all()
         tampered_msg = "I PITY THE FOOL THAT STANDS ON MY WAY!"
@@ -100,6 +123,21 @@ class RunnerSimpleTest(unittest.TestCase):
                          (expected_rc, result))
         self.assertIn(tampered_msg, result.stdout)
 
+    def test_output_tamper_combined(self):
+        self._check_output_record_combined()
+        tampered_msg = "I PITY THE FOOL THAT STANDS ON MY WAY!"
+        output_file = os.path.join("%s.data/output.expected" % self.output_script.path)
+        with open(output_file, 'w') as output_file_obj:
+            output_file_obj.write(tampered_msg)
+        cmd_line = ('%s run --job-results-dir %s --sysinfo=off %s --xunit -'
+                    % (AVOCADO, self.tmpdir, self.output_script.path))
+        result = process.run(cmd_line, ignore_status=True)
+        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" %
+                         (expected_rc, result))
+        self.assertIn(tampered_msg, result.stdout)
+
     def test_output_diff(self):
         self._check_output_record_all()
         tampered_msg_stdout = "I PITY THE FOOL THAT STANDS ON STDOUT!"