avocado.multiplexer: Support for modifying multiplex tree on cmdline

This patch adds "--env" argument, which allows one to specify
[path:]key:value records which are injected into the multiplex tree
parsed from yaml files.

It also demonstrate the way plugins can override multiplex tree
therefor modify test params during activate/run phase.

There are few intrusive changes. It reuses the existing namespace
during "avocado.core.parser.Parser.finish" part to keep the values set
during "Plugin.activate" phase and it adds
"self.args.default_multiplex_tree" argument which is reserved for
multiplex tree to be merged into tree made from yaml files.
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/parser.py

@@ -20,6 +20,7 @@ Avocado application command line parsing.
 import sys
 import argparse
 
+from avocado.core import tree
 from avocado.version import VERSION
 
 PROG = 'avocado'
@@ -76,6 +77,9 @@ class Parser(object):
         self.args, _ = self.application.parse_known_args(args=default_args)
         if not hasattr(self.args, 'dispatch'):
             self.application.set_defaults(dispatch=self.application.print_help)
+        if tree.MULTIPLEX_CAPABLE:
+            # Allow overriding multiplex variants by plugins args
+            self.args.default_multiplex_tree = tree.TreeNode()
 
     def finish(self):
         """
@@ -83,7 +87,7 @@ class Parser(object):
 
         Side effect: set the final value for attribute `args`.
         """
-        self.args = self.application.parse_args()
+        self.args = self.application.parse_args(namespace=self.args)
 
     def take_action(self):
         """

avocado/core/plugins/multiplexer.py

@@ -56,33 +56,40 @@ class Multiplexer(plugin.Plugin):
         self.parser.add_argument('-d', '--debug', action='store_true',
                                  default=False, help="Debug multiplexed "
                                  "files.")
+        self.parser.add_argument('--env', default=[], nargs='*')
         super(Multiplexer, self).configure(self.parser)
 
+    def activate(self, args):
+        # Extend default multiplex tree of --env values
+        for value in getattr(args, "env", []):
+            value = value.split(':', 2)
+            if len(value) < 2:
+                raise ValueError("key:value pairs required, found only %s"
+                                 % (value))
+            elif len(value) == 2:
+                args.default_multiplex_tree.value[value[0]] = value[1]
+            else:
+                node = args.default_multiplex_tree.get_node(value[0], True)
+                node.value[value[1]] = value[2]
+
     def run(self, args):
         view = output.View(app_args=args)
-        multiplex_files = args.multiplex_files
-        if args.tree:
-            view.notify(event='message', msg='Config file tree structure:')
-            try:
-                t = tree.create_from_yaml(multiplex_files)
-            except IOError, details:
-                view.notify(event='error', msg=details.strerror)
-                sys.exit(exit_codes.AVOCADO_JOB_FAIL)
-            t = tree.apply_filters(t, args.filter_only, args.filter_out)
-            view.notify(event='minor',
-                        msg=t.get_ascii(attributes=args.attr))
-            sys.exit(exit_codes.AVOCADO_ALL_OK)
-
         try:
-            variants = multiplexer.multiplex_yamls(multiplex_files,
-                                                   args.filter_only,
-                                                   args.filter_out,
-                                                   args.debug)
+            mux_tree = multiplexer.yaml2tree(args.multiplex_files,
+                                             args.filter_only, args.filter_out,
+                                             args.debug)
         except IOError, details:
             view.notify(event='error',
                         msg=details.strerror)
             sys.exit(exit_codes.AVOCADO_JOB_FAIL)
+        mux_tree.merge(args.default_multiplex_tree)
+        if args.tree:
+            view.notify(event='message', msg='Config file tree structure:')
+            view.notify(event='minor',
+                        msg=mux_tree.get_ascii(attributes=args.attr))
+            sys.exit(exit_codes.AVOCADO_ALL_OK)
 
+        variants = multiplexer.MuxTree(mux_tree)
         view.notify(event='message', msg='Variants generated:')
         for (index, tpl) in enumerate(variants):
             if not args.debug:

avocado/core/plugins/runner.py

@@ -126,11 +126,24 @@ class TestRunner(plugin.Plugin):
                              help='Filter out path(s) from multiplexing')
             mux.add_argument('--mux-entry', nargs='*', default=None,
                              help="Multiplex entry point(s)")
-
+            mux.add_argument('--env', default=[], nargs='*')
         super(TestRunner, self).configure(self.parser)
         # Export the test runner parser back to the main parser
         parser.runner = self.parser
 
+    def activate(self, args):
+        # Extend default multiplex tree of --env values
+        for value in getattr(args, "env", []):
+            value = value.split(':', 2)
+            if len(value) < 2:
+                raise ValueError("key:value pairs required, found only %s"
+                                 % (value))
+            elif len(value) == 2:
+                args.default_multiplex_tree.value[value[0]] = value[1]
+            else:
+                node = args.default_multiplex_tree.get_node(value[0], True)
+                node.value[value[1]] = value[2]
+
     def _validate_job_timeout(self, raw_timeout):
         units = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
         mult = 1

avocado/multiplexer.py

@@ -90,8 +90,8 @@ class MuxTree(object):
             yield ret
 
 
-def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
-                    debug=False):
+def yaml2tree(input_yamls, filter_only=None, filter_out=None,
+              debug=False):
     if filter_only is None:
         filter_only = []
     if filter_out is None:
@@ -99,8 +99,7 @@ def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
     input_tree = tree.create_from_yaml(input_yamls, debug)
     # TODO: Process filters and multiplex simultaneously
     final_tree = tree.apply_filters(input_tree, filter_only, filter_out)
-    result = MuxTree(final_tree)
-    return result
+    return final_tree
 
 
 # TODO: Create multiplexer plugin and split these functions into multiple files
@@ -396,9 +395,12 @@ class Mux(object):
         filter_only = getattr(args, 'filter_only', None)
         filter_out = getattr(args, 'filter_out', None)
         if mux_files:
-            self.variants = multiplex_yamls(mux_files, filter_only, filter_out)
+            mux_tree = yaml2tree(mux_files, filter_only, filter_out)
         else:   # no variants
-            self.variants = None
+            mux_tree = tree.TreeNode()
+        if getattr(args, 'default_multiplex_tree', None):
+            mux_tree.merge(args.default_multiplex_tree)
+        self.variants = MuxTree(mux_tree)
         self._mux_entry = getattr(args, 'mux_entry', None)
         if self._mux_entry is None:
             self._mux_entry = ['/run/*']

selftests/all/unit/avocado/multiplexer_unittest.py

@@ -43,24 +43,30 @@ class TestMultiplex(unittest.TestCase):
         self.assertEqual(len(self.mux_full), 12)
 
     def test_create_variants(self):
-        from_file = multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'])
+        from_file = multiplexer.yaml2tree(
+            ["/:" + PATH_PREFIX + 'examples/mux-selftest.yaml'])
+        from_file = multiplexer.MuxTree(from_file)
         self.assertEqual(self.mux_full, tuple(from_file))
 
     # Filters are tested in tree_unittests, only verify `multiplex_yamls` calls
     def test_filter_only(self):
         exp = (['intel', 'scsi'], ['intel', 'virtio'])
-        act = tuple(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'],
-                                                ('/hw/cpu/intel',
-                                                 '/distro/fedora',
-                                                 '/hw')))
+        act = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                     'examples/mux-selftest.yaml'],
+                                    ('/hw/cpu/intel',
+                                     '/distro/fedora',
+                                     '/hw'))
+        act = tuple(multiplexer.MuxTree(act))
         self.assertEqual(act, exp)
 
     def test_filter_out(self):
-        act = tuple(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'],
-                                                None,
-                                                ('/hw/cpu/intel',
-                                                 '/distro/fedora',
-                                                 '/distro')))
+        act = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                     'examples/mux-selftest.yaml'],
+                                    None,
+                                    ('/hw/cpu/intel',
+                                     '/distro/fedora',
+                                     '/distro'))
+        act = tuple(multiplexer.MuxTree(act))
         self.assertEqual(len(act), 4)
         self.assertEqual(len(act[0]), 3)
         str_act = str(act)
@@ -71,8 +77,9 @@ class TestMultiplex(unittest.TestCase):
 
 
 class TestAvocadoParams(unittest.TestCase):
-    yamls = iter(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest-params.'
-                                              'yaml']))
+    yamls = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                   'examples/mux-selftest-params.yaml'])
+    yamls = iter(multiplexer.MuxTree(yamls))
     params1 = multiplexer.AvocadoParams(yamls.next(), 'Unittest1', 1,
                                         ['/ch0/*', '/ch1/*'], {})
     yamls.next()    # Skip 2nd