Make exit codes a combination of meanings (aka ORable)

Currently, exit codes cover all situations with one single value.

This patch makes exit codes to be ORable, so we can better represent
the variety of things that happens during an Avocado execution.

Reference: https://trello.com/c/SU5fixgHSigned-off-by: NAmador Pahim <apahim@redhat.com>

avocado/core/exit_codes.py

@@ -21,21 +21,22 @@ statuses.
 """
 
 #: Both job and tests PASSed
-AVOCADO_ALL_OK = 0
+AVOCADO_ALL_OK = 0x0000
 
 #: Job went fine, but some tests FAILed or ERRORed
-AVOCADO_TESTS_FAIL = 1
+AVOCADO_TESTS_FAIL = 0x0001
 
 #: Something went wrong with the Job itself, by explicit
 #: :class:`avocado.core.exceptions.JobError` exception.
-AVOCADO_JOB_FAIL = 2
+AVOCADO_JOB_FAIL = 0x0002
 
 #: Something else went wrong and avocado failed (or crashed). Commonly
 #: used on command line validation errors.
-AVOCADO_FAIL = 3
+AVOCADO_FAIL = 0x0004
 
 #: The job was explicitly interrupted. Usually this means that a user
 #: hit CTRL+C while the job was still running.
-AVOCADO_JOB_INTERRUPTED = 4
+AVOCADO_JOB_INTERRUPTED = 0x0008
+
 #: Avocado generic crash
 AVOCADO_GENERIC_CRASH = -1

avocado/core/job.py

@@ -123,6 +123,7 @@ class Job(object):
                                                            _TEST_LOGGER)
         self.stdout_stderr = None
         self.replay_sourcejob = getattr(self.args, 'replay_sourcejob', None)
+        self.exitcode = exit_codes.AVOCADO_ALL_OK
 
     def _setup_job_results(self):
         logdir = getattr(self.args, 'logdir', None)
@@ -296,7 +297,8 @@ class Job(object):
                            'simultaneously', " ".join(op_set_stdout))
             self.log.error('Please set at least one of them to a file to '
                            'avoid conflicts')
-            sys.exit(exit_codes.AVOCADO_JOB_FAIL)
+            self.exitcode |= exit_codes.AVOCADO_JOB_FAIL
+            sys.exit(self.exitcode)
 
         if not op_set_stdout and not self.standalone:
             human_plugin = result.HumanTestResult(self)
@@ -511,13 +513,15 @@ class Job(object):
         _TEST_LOGGER.info('Test results available in %s', self.logdir)
 
         if summary is None:
-            return exit_codes.AVOCADO_JOB_FAIL
-        elif 'INTERRUPTED' in summary:
-            return exit_codes.AVOCADO_JOB_INTERRUPTED
-        elif summary:
-            return exit_codes.AVOCADO_TESTS_FAIL
-        else:
-            return exit_codes.AVOCADO_ALL_OK
+            self.exitcode |= exit_codes.AVOCADO_JOB_FAIL
+            return self.exitcode
+
+        if 'INTERRUPTED' in summary:
+            self.exitcode |= exit_codes.AVOCADO_JOB_INTERRUPTED
+        if 'FAIL' in summary:
+            self.exitcode |= exit_codes.AVOCADO_TESTS_FAIL
+
+        return self.exitcode
 
     def run(self):
         """
@@ -545,10 +549,12 @@ class Job(object):
             self.status = details.status
             fail_class = details.__class__.__name__
             self.log.error('\nAvocado job failed: %s: %s', fail_class, details)
-            return exit_codes.AVOCADO_JOB_FAIL
+            self.exitcode |= exit_codes.AVOCADO_JOB_FAIL
+            return self.exitcode
         except exceptions.OptionValidationError as details:
             self.log.error('\n' + str(details))
-            return exit_codes.AVOCADO_JOB_FAIL
+            self.exitcode |= exit_codes.AVOCADO_JOB_FAIL
+            return self.exitcode
 
         except Exception as details:
             self.status = "ERROR"
@@ -562,7 +568,8 @@ class Job(object):
             self.log.error("Please include the traceback info and command line"
                            " used on your bug report")
             self.log.error('Report bugs visiting %s', _NEW_ISSUE_LINK)
-            return exit_codes.AVOCADO_FAIL
+            self.exitcode |= exit_codes.AVOCADO_FAIL
+            return self.exitcode
         finally:
             if not settings.get_value('runner.behavior', 'keep_tmp_files',
                                       key_type=bool, default=False):

docs/source/ResultFormats.rst

@@ -152,6 +152,26 @@ the program::
 
 That's basically the only rule, and a sane one, that you need to follow.
 
+Exit Codes
+----------
+
+Avocado exit code tries to represent different things that can happen during
+an execution. That means exit codes can be a combination of codes that were
+ORed toghether as a simgle exit code. The final exit code can be debundled so
+users can have a good idea on what happened to the job.
+
+The single individual exit codes are:
+
+* AVOCADO_ALL_OK (0)
+* AVOCADO_TESTS_FAIL (1)
+* AVOCADO_JOB_FAIL (2)
+* AVOCADO_FAIL (4)
+* AVOCADO_JOB_INTERRUPTED (8)
+
+If a job finishes with exit code `9`, for example, it means we had at least
+one test that failed and also we had at some point a job interruption, probably
+due to the job timeout or a `CTRL+C`.
+
 Implementing other result formats
 ---------------------------------
 

selftests/functional/test_job_timeout.py

@@ -108,13 +108,15 @@ class JobTimeOutTest(unittest.TestCase):
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '
                     '--xunit - --job-timeout=1 %s examples/tests/passtest.py' %
                     (self.tmpdir, self.script.path))
-        self.run_and_check(cmd_line, exit_codes.AVOCADO_JOB_INTERRUPTED, 2, 1, 0, 1)
+        self.run_and_check(cmd_line, exit_codes.AVOCADO_JOB_INTERRUPTED,
+                           2, 1, 0, 1)
 
     def test_sleep_short_timeout_with_test_methods(self):
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '
                     '--xunit - --job-timeout=1 %s' %
                     (self.tmpdir, self.py.path))
-        self.run_and_check(cmd_line, exit_codes.AVOCADO_JOB_INTERRUPTED, 3, 1, 0, 2)
+        self.run_and_check(cmd_line, exit_codes.AVOCADO_JOB_INTERRUPTED,
+                           3, 1, 0, 2)
 
     def test_invalid_values(self):
         cmd_line = ('./scripts/avocado run --job-results-dir %s --sysinfo=off '