Merge remote-tracking branch 'ldoktor/mux-per-suite2'

Signed-off-by: NCleber Rosa <crosa@redhat.com>

avocado/core/job.py

@@ -445,10 +445,12 @@ class Job(object):
         jobdata.record(self.args, self.logdir, variant, self.references,
                        sys.argv)
         replay_map = getattr(self.args, 'replay_map', None)
+        execution_order = getattr(self.args, "execution_order", None)
         summary = self.test_runner.run_suite(self.test_suite,
                                              variant,
                                              self.timeout,
-                                             replay_map)
+                                             replay_map,
+                                             execution_order)
         # If it's all good so far, set job status to 'PASS'
         if self.status == 'RUNNING':
             self.status = 'PASS'

avocado/core/runner.py

@@ -489,36 +489,60 @@ class TestRunner(object):
         return True
 
     @staticmethod
-    def _iter_variants(template, variants):
+    def _template_to_factory(template, variant):
         """
-        Iterate through variants and set the params/variants accordingly.
+        Applies test params from variant to the test template
 
-        :param template: test template
-        :param variants: the Mux object containing the variants
-        :return: Yields tuple(test_factory including params, variant id)
-        :raises ValueError: When variant and template declare params.
+        :param template: a test template
+        :param variant: variant to be applied
+        :return: tuple(new_test_factory, applied_variant)
         """
-        for variant in variants.itertests():
-            params = (variant.get("variant"), variant.get("mux_path"))
-            if params:
-                if "params" in template[1]:
-                    msg = ("Unable to use test variants %s, params are already"
-                           " present in test factory: %s"
-                           % (template[0], template[1]))
-                    raise ValueError(msg)
-                factory = [template[0], template[1].copy()]
-                factory[1]["params"] = params
-            else:
-                factory = template
-            yield factory, variant
-
-    def run_suite(self, test_suite, variants, timeout=0, replay_map=None):
+        params = variant.get("variant"), variant.get("mux_path")
+        if params:
+            if "params" in template[1]:
+                msg = ("Unable to use test variants %s, params are already"
+                       " present in test factory: %s"
+                       % (template[0], template[1]))
+                raise ValueError(msg)
+            factory = [template[0], template[1].copy()]
+            factory[1]["params"] = params
+        else:
+            factory = template
+        return factory, variant
+
+    def _iter_suite(self, test_suite, variants, execution_order):
+        """
+        Iterates through test_suite and variants in defined order
+
+        :param test_suite: a list of tests to run
+        :param variants: a varianter object to produce test params
+        :param execution_order: way of iterating through tests/variants
+        :return: generator yielding tuple(test_factory, variant)
+        """
+        if execution_order in ("variants-per-test", None):
+            return (self._template_to_factory(template, variant)
+                    for template in test_suite
+                    for variant in variants.itertests())
+        elif execution_order == "tests-per-variant":
+            return (self._template_to_factory(template, variant)
+                    for variant in variants.itertests()
+                    for template in test_suite)
+        else:
+            raise NotImplementedError("Suite_order %s is not supported"
+                                      % execution_order)
+
+    def run_suite(self, test_suite, variants, timeout=0, replay_map=None,
+                  execution_order=None):
         """
         Run one or more tests and report with test result.
 
         :param test_suite: a list of tests to run.
         :param variants: A varianter iterator to produce test params.
         :param timeout: maximum amount of time (in seconds) to execute.
+        :param replay_map: optional list to override test class based on test
+                           index.
+        :param execution_order: Mode in which we should iterate through tests
+                            resp. variants.
         :return: a set with types of test failures.
         """
         summary = set()
@@ -537,39 +561,33 @@ class TestRunner(object):
         index = -1
         try:
             for test_template in test_suite:
-                test_template[1]['base_logdir'] = self.job.logdir
-                test_template[1]['job'] = self.job
-                break_loop = False
-                for test_factory, variant in self._iter_variants(test_template,
-                                                                 variants):
-                    index += 1
-                    test_parameters = test_factory[1]
-                    name = test_parameters.get("name")
-                    test_parameters["name"] = test.TestName(index + 1, name,
-                                                            variant,
-                                                            no_digits)
-                    if deadline is not None and time.time() > deadline:
-                        summary.add('INTERRUPTED')
-                        if 'methodName' in test_parameters:
-                            del test_parameters['methodName']
-                        test_factory = (test.TimeOutSkipTest, test_parameters)
-                        break_loop = not self.run_test(test_factory, queue,
-                                                       summary)
-                        if break_loop:
-                            break
-                    else:
-                        if (replay_map is not None and
-                                replay_map[index] is not None):
-                            test_parameters["methodName"] = "test"
-                            test_factory = (replay_map[index], test_parameters)
-
-                        break_loop = not self.run_test(test_factory, queue,
-                                                       summary, deadline)
-                        if break_loop:
-                            break
+                test_template[1]["base_logdir"] = self.job.logdir
+                test_template[1]["job"] = self.job
+            for test_factory, variant in self._iter_suite(test_suite, variants,
+                                                          execution_order):
+                index += 1
+                test_parameters = test_factory[1]
+                name = test_parameters.get("name")
+                test_parameters["name"] = test.TestName(index + 1, name,
+                                                        variant,
+                                                        no_digits)
+                if deadline is not None and time.time() > deadline:
+                    summary.add('INTERRUPTED')
+                    if 'methodName' in test_parameters:
+                        del test_parameters['methodName']
+                    test_factory = (test.TimeOutSkipTest, test_parameters)
+                    if not self.run_test(test_factory, queue, summary):
+                        break
+                else:
+                    if (replay_map is not None and
+                            replay_map[index] is not None):
+                        test_parameters["methodName"] = "test"
+                        test_factory = (replay_map[index], test_parameters)
+
+                    if not self.run_test(test_factory, queue, summary,
+                                         deadline):
+                        break
                 runtime.CURRENT_TEST = None
-                if break_loop:
-                    break
         except KeyboardInterrupt:
             TEST_LOG.error('Job interrupted by ctrl+c.')
             summary.add('INTERRUPTED')

avocado/plugins/run.py

@@ -99,6 +99,12 @@ class Run(CLICmd):
                             "system information (hardware details, profilers, "
                             "etc.). Current:  %(default)s")
 
+        parser.add_argument("--execution-order", default="variants-per-test",
+                            choices=("tests-per-variant",
+                                     "variants-per-test"),
+                            help="Defines the order of iterating through test "
+                            "suite and test variants")
+
         parser.output = parser.add_argument_group('output and result format')
 
         parser.output.add_argument('-s', '--silent', action="store_true",

docs/source/TestParameters.rst

@@ -33,8 +33,8 @@ Overall picture of how the params handling works is:
              |  // single variant is passed to Test
              |
        +-----------+
-       |  Runner   |  // iterates through tests and runs each test with
-       +-----^-----+  // all variants supplied by Varianter
+       |  Runner   |  // iterates through tests and variants to run all
+       +-----^-----+  // desired combinations specified by "--execution-order"
              |
              |
    +-------------------+ provide variants +-----------------------+

man/avocado.rst

@@ -90,6 +90,8 @@ Options for subcommand `run` (`avocado run --help`)::
                             debugging). Defaults to off.
       --sysinfo {on,off}    Enable or disable system information (hardware
                             details, profilers, etc.). Current: on
+      --execution-order {tests-per-variant,variants-per-test}
+                            How to iterate through test suite and variants
 
     output and result format:
       -s, --silent          Silence stdout

optional_plugins/runner_remote/avocado_runner_remote/__init__.py

@@ -448,7 +448,8 @@ class RemoteTestRunner(TestRunner):
 
         return json_result
 
-    def run_suite(self, test_suite, variants, timeout=0, replay_map=None):
+    def run_suite(self, test_suite, variants, timeout=0, replay_map=None,
+                  suite_order="variants-per-test"):
         """
         Run one or more tests and report with test result.
 
@@ -459,6 +460,10 @@ class RemoteTestRunner(TestRunner):
         """
         del test_suite     # using self.job.references instead
         del variants            # we're not using multiplexation here
+        if suite_order != "variants-per-test" and suite_order is not None:
+            raise exceptions.JobError("execution-order %s is not supported "
+                                      "for remote execution." % suite_order)
+        del suite_order     # suite_order is ignored for now
         if not timeout:     # avoid timeout = 0
             timeout = None
         summary = set()

selftests/functional/test_multiplex.py

@@ -92,7 +92,24 @@ class MultiplexTests(unittest.TestCase):
                     'examples/tests/sleeptest.py.data/sleeptest.yaml'
                     % (AVOCADO, self.tmpdir))
         expected_rc = exit_codes.AVOCADO_TESTS_FAIL
-        self.run_and_check(cmd_line, expected_rc, (4, 4))
+        result = self.run_and_check(cmd_line, expected_rc, (4, 4))
+        self.assertIn("(1/8) passtest.py:PassTest.test;short", result.stdout)
+        self.assertIn("(2/8) passtest.py:PassTest.test;medium", result.stdout)
+        self.assertIn("(8/8) failtest.py:FailTest.test;longest",
+                      result.stdout)
+
+    def test_run_mplex_failtest_tests_per_variant(self):
+        cmd_line = ("%s run --job-results-dir %s --sysinfo=off "
+                    "passtest.py failtest.py -m "
+                    "examples/tests/sleeptest.py.data/sleeptest.yaml "
+                    "--execution-order tests-per-variant"
+                    % (AVOCADO, self.tmpdir))
+        expected_rc = exit_codes.AVOCADO_TESTS_FAIL
+        result = self.run_and_check(cmd_line, expected_rc, (4, 4))
+        self.assertIn("(1/8) passtest.py:PassTest.test;short", result.stdout)
+        self.assertIn("(2/8) failtest.py:FailTest.test;short", result.stdout)
+        self.assertIn("(8/8) failtest.py:FailTest.test;longest",
+                      result.stdout)
 
     def test_run_double_mplex(self):
         cmd_line = ('%s run --job-results-dir %s --sysinfo=off '