avocado.core.multiplexer: Define multiplexer API

This patch is a 1st step to independent multiplex plugin. It unifies the
multiplexation, which is handled exclusively by the
avocado.core.multiplexer.Mux object, which is initialized directly in
argument parser as `args.mux`, instead of the
`parser.default_avocado_params`.

The idea is to provide unified database with simple interface:

 * data_inject - inject key/value[/path] data
 * data_merge - merge node/tree into the data

Additionally it supports internal API to generate the params:

 * is_parsed - whether it was already initialized/filtered/...
 * get_number_of_tests - reports number of test*variants
 * itertests - iterate through tests

All those changes are pickle-safe from the old object to the new object
(not vice versa), therefor it's possible to use old Mux objects stored
in `jobdata` to replay tests.

This API is not 100% stable yet, but we plan to keep the one sided
pickle compatibility into the future (no guarantee, but we'll do our
best).
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/job.py

@@ -446,14 +446,15 @@ class Job(object):
                          "command.")
             raise exceptions.OptionValidationError(e_msg)
 
-        if isinstance(getattr(self.args, 'multiplex_files', None),
-                      multiplexer.Mux):
-            mux = self.args.multiplex_files     # pylint: disable=E1101
-        else:
+        mux = getattr(self.args, "mux", None)
+        if mux is None:
+            mux = multiplexer.Mux()
+        if not mux.is_parsed():   # Mux not yet parsed, apply args
             try:
-                mux = multiplexer.Mux(self.args)
+                mux.parse(self.args)
             except (IOError, ValueError) as details:
-                raise exceptions.OptionValidationError(details)
+                raise exceptions.OptionValidationError("Unable to parse mux: "
+                                                       "%s" % details)
         self.args.test_result_total = mux.get_number_of_tests(self.test_suite)
 
         self._make_test_result()

avocado/core/jobdata.py

@@ -100,7 +100,7 @@ def retrieve_urls(resultsdir):
 
 def retrieve_mux(resultsdir):
     """
-    Retrieves the job multiplex from the results directory.
+    Retrieves the job Mux object from the results directory.
     """
     recorded_mux = _retrieve(resultsdir, MUX_FILENAME)
     if recorded_mux is None:

avocado/core/multiplexer.py

@@ -377,28 +377,104 @@ class AvocadoParam(object):
                 yield (leaf.environment_origin[key].path, key, value)
 
 
+def _report_mux_already_parsed(self, *args, **kwargs):
+    """
+    Raises exception describing that `self.data` alteration is restricted
+    """
+    raise RuntimeError("Mux already parsed, altering is restricted. %s %s"
+                       % (args, kwargs))
+
+
 class Mux(object):
 
     """
     This is a multiplex object which multiplexes the test_suite.
     """
 
-    def __init__(self, args):
+    def __init__(self):
         self._has_multiple_variants = None
-        mux_files = getattr(args, 'multiplex_files', None)
+        self.variants = None
+        self.data = tree.TreeNode()
+        self._mux_path = None
+
+    def parse(self, args, debug=False):
+        """
+        Apply options defined on the cmdline
+
+        :param args: Parsed cmdline arguments
+        :param debug: Whether to debug the mux parsing
+        """
         filter_only = getattr(args, 'filter_only', None)
         filter_out = getattr(args, 'filter_out', None)
-        if mux_files:
-            mux_tree = yaml2tree(mux_files)
-        else:   # no variants
-            mux_tree = tree.TreeNode()
-        if getattr(args, 'default_avocado_params', None):
-            mux_tree.merge(args.default_avocado_params)
-        mux_tree = tree.apply_filters(mux_tree, filter_only, filter_out)
+        self._parse_basic_injects(args, debug)
+        mux_tree = tree.apply_filters(self.data, filter_only, filter_out)
         self.variants = MuxTree(mux_tree)
         self._mux_path = getattr(args, 'mux_path', None)
         if self._mux_path is None:
             self._mux_path = ['/run/*']
+        # disable data alteration (and remove data as they are not useful)
+        self.data = None
+        self.data_inject = _report_mux_already_parsed
+        self.data_merge = _report_mux_already_parsed
+
+    def _parse_basic_injects(self, args, debug):
+        """
+        Inject data from the basic injects defined by Mux
+
+        :param args: Parsed cmdline arguments
+        :param debug: Whether to debug the mux parsing
+        """
+        # Merge the multiplex_files
+        multiplex_files = getattr(args, "multiplex_files", None)
+        if multiplex_files:
+            self.data_merge(yaml2tree(multiplex_files, debug=debug))
+
+        # FIXME: Backward compatibility params, to be removed when 36 LTS is
+        # discontinued
+        if (not getattr(args, "mux_skip_defaults", False) and
+                hasattr(args, "default_avocado_params")):
+            self.data_merge(args.default_avocado_params)
+
+        # Extend default multiplex tree of --mux-inject values
+        for inject in getattr(args, "mux_inject", []):
+            entry = inject.split(':', 3)
+            if len(entry) < 2:
+                raise ValueError("key:entry pairs required, found only %s"
+                                 % (entry))
+            elif len(entry) == 2:   # key, entry
+                self.data_inject(*entry)
+            else:                   # path, key, entry
+                self.data_inject(key=entry[1], value=entry[2], path=entry[0])
+
+    def is_parsed(self):
+        """
+        Reports whether the tree was already multiplexed
+        """
+        return self.variants is not None
+
+    def data_inject(self, key, value, path=None):
+        """
+        Inject entry to the mux tree (params database)
+
+        :param key: Key to which we'd like to assign the value
+        :param value: The key's value
+        :param path: Optional path to the node to which we assign the value,
+                     by default '/'.
+        """
+        if path:
+            node = self.data.get_node(path, True)
+        else:
+            node = self.data
+        node.value[key] = value
+
+    def data_merge(self, tree):
+        """
+        Merge tree into the mux tree (params database)
+
+        :param tree: Tree to be merged into this database.
+        :type tree: :class:`avocado.core.tree.TreeNode`
+        """
+        self.data.merge(tree)
 
     def get_number_of_tests(self, test_suite):
         """

avocado/core/parser.py

@@ -12,7 +12,6 @@
 # Copyright: Red Hat Inc. 2013-2014
 # Author: Ruda Moura <rmoura@redhat.com>
 
-
 """
 Avocado application command line parsing.
 """
@@ -21,8 +20,9 @@ import argparse
 import logging
 
 from . import exit_codes
-from . import tree
+from . import multiplexer
 from . import settings
+from . import tree
 from .output import BUILTIN_STREAMS, BUILTIN_STREAM_SETS
 from .version import VERSION
 
@@ -125,6 +125,9 @@ class Parser(object):
             dest='subcommand')
 
         # Allow overriding default params by plugins
+        self.args.mux = multiplexer.Mux()
+        # FIXME: Backward compatibility params, to be removed when 36 LTS is
+        # discontinued
         self.args.default_avocado_params = tree.TreeNode()
 
     def finish(self):

avocado/plugins/multiplex.py

@@ -33,7 +33,6 @@ class Multiplex(CLICmd):
 
     def __init__(self, *args, **kwargs):
         super(Multiplex, self).__init__(*args, **kwargs)
-        self._from_args_tree = tree.TreeNode()
 
     def configure(self, parser):
         if multiplexer.MULTIPLEX_CAPABLE is False:
@@ -48,7 +47,8 @@ class Multiplex(CLICmd):
 
         parser.add_argument('--filter-out', nargs='*', default=[],
                             help='Filter out path(s) from multiplexing')
-        parser.add_argument('--system-wide', action='store_true',
+        parser.add_argument('--system-wide', action='store_false',
+                            default=True, dest="mux-skip-defaults",
                             help="Combine the files with the default "
                             "tree.")
         parser.add_argument('-c', '--contents', action='store_true',
@@ -68,21 +68,7 @@ class Multiplex(CLICmd):
         tree_parser.add_argument('-i', '--inherit', action="store_true",
                                  help="Show the inherited values")
 
-    def _activate(self, args):
-        # Extend default multiplex tree of --env values
-        for value in getattr(args, "mux_inject", []):
-            value = value.split(':', 2)
-            if len(value) < 2:
-                raise ValueError("key:value pairs required, found only %s"
-                                 % (value))
-            elif len(value) == 2:
-                self._from_args_tree.value[value[0]] = value[1]
-            else:
-                node = self._from_args_tree.get_node(value[0], True)
-                node.value[value[1]] = value[2]
-
     def run(self, args):
-        self._activate(args)
         log = logging.getLogger("avocado.app")
         err = None
         if args.tree and args.debug:
@@ -92,16 +78,13 @@ class Multiplex(CLICmd):
         if err:
             log.error(err)
             sys.exit(exit_codes.AVOCADO_FAIL)
+        mux = multiplexer.Mux()
+        root = mux.data
         try:
-            mux_tree = multiplexer.yaml2tree(args.multiplex_files,
-                                             args.filter_only, args.filter_out,
-                                             args.debug)
-        except IOError as details:
-            log.error(details.strerror)
+            mux.parse(args, debug=args.debug)
+        except (IOError, ValueError) as details:
+            log.error("Unable to parse mux: %s", details)
             sys.exit(exit_codes.AVOCADO_JOB_FAIL)
-        if args.system_wide:
-            mux_tree.merge(args.default_avocado_params)
-        mux_tree.merge(self._from_args_tree)
         if args.tree:
             if args.contents:
                 verbose = 1
@@ -111,12 +94,11 @@ class Multiplex(CLICmd):
                 verbose += 2
             use_utf8 = settings.get_value("runner.output", "utf8",
                                           key_type=bool, default=None)
-            log.debug(tree.tree_view(mux_tree, verbose, use_utf8))
+            log.debug(tree.tree_view(root, verbose, use_utf8))
             sys.exit(exit_codes.AVOCADO_ALL_OK)
 
-        variants = multiplexer.MuxTree(mux_tree)
         log.info('Variants generated:')
-        for (index, tpl) in enumerate(variants):
+        for (index, tpl) in enumerate(mux.variants):
             if not args.debug:
                 paths = ', '.join([x.path for x in tpl])
             else:

avocado/plugins/replay.py

@@ -217,7 +217,7 @@ class Replay(CLI):
                     log.error('Source job multiplex data not found. Aborting.')
                     sys.exit(exit_codes.AVOCADO_JOB_FAIL)
                 else:
-                    setattr(args, "multiplex_files", mux)
+                    setattr(args, "mux", mux)
 
         if args.replay_teststatus:
             replay_map = self._create_replay_map(resultsdir,

avocado/plugins/run.py

@@ -150,19 +150,6 @@ class Run(CLICmd):
                          help="Inject [path:]key:node values into the "
                          "final multiplex tree.")
 
-    def _activate(self, args):
-        # Extend default multiplex tree of --mux_inject values
-        for value in getattr(args, "mux_inject", []):
-            value = value.split(':', 2)
-            if len(value) < 2:
-                raise ValueError("key:value pairs required, found only %s"
-                                 % (value))
-            elif len(value) == 2:
-                args.default_avocado_params.value[value[0]] = value[1]
-            else:
-                node = args.default_avocado_params.get_node(value[0], True)
-                node.value[value[1]] = value[2]
-
     def run(self, args):
         """
         Run test modules or simple tests.
@@ -170,7 +157,6 @@ class Run(CLICmd):
         :param args: Command line args received from the run subparser.
         """
         log = logging.getLogger("avocado.app")
-        self._activate(args)
         if args.unique_job_id is not None:
             try:
                 int(args.unique_job_id, 16)