Merge pull request #633 from ldoktor/mux-args2tree3

avocado.multiplexer: Support for modifying multiplex tree on cmdline [v3]

avocado/core/parser.py

@@ -20,6 +20,7 @@ Avocado application command line parsing.
 import sys
 import argparse
 
+from avocado.core import tree
 from avocado.version import VERSION
 
 PROG = 'avocado'
@@ -33,6 +34,8 @@ class Parser(object):
     """
 
     def __init__(self):
+        self.args = None
+        self.subcommands = None
         self.application = argparse.ArgumentParser(
             prog=PROG,
             add_help=False,  # see parent parsing
@@ -71,9 +74,12 @@ class Parser(object):
         """
         # Inject --help if no arguments is present
         default_args = ['--help'] if not sys.argv[1:] else None
-        self.args, rest = self.application.parse_known_args(args=default_args)
+        self.args, _ = self.application.parse_known_args(args=default_args)
         if not hasattr(self.args, 'dispatch'):
             self.application.set_defaults(dispatch=self.application.print_help)
+        if tree.MULTIPLEX_CAPABLE:
+            # Allow overriding multiplex variants by plugins args
+            self.args.default_multiplex_tree = tree.TreeNode()
 
     def finish(self):
         """
@@ -81,7 +87,7 @@ class Parser(object):
 
         Side effect: set the final value for attribute `args`.
         """
-        self.args = self.application.parse_args()
+        self.args = self.application.parse_args(namespace=self.args)
 
     def take_action(self):
         """

avocado/core/plugins/multiplexer.py

@@ -12,7 +12,6 @@
 # Copyright: Red Hat Inc. 2013-2014
 # Author: Lucas Meneghel Rodrigues <lmr@redhat.com>
 
-import os
 import sys
 
 from avocado.core.plugins import plugin
@@ -57,33 +56,40 @@ class Multiplexer(plugin.Plugin):
         self.parser.add_argument('-d', '--debug', action='store_true',
                                  default=False, help="Debug multiplexed "
                                  "files.")
+        self.parser.add_argument('--env', default=[], nargs='*')
         super(Multiplexer, self).configure(self.parser)
 
+    def activate(self, args):
+        # Extend default multiplex tree of --env values
+        for value in getattr(args, "env", []):
+            value = value.split(':', 2)
+            if len(value) < 2:
+                raise ValueError("key:value pairs required, found only %s"
+                                 % (value))
+            elif len(value) == 2:
+                args.default_multiplex_tree.value[value[0]] = value[1]
+            else:
+                node = args.default_multiplex_tree.get_node(value[0], True)
+                node.value[value[1]] = value[2]
+
     def run(self, args):
         view = output.View(app_args=args)
-        multiplex_files = args.multiplex_files
-        if args.tree:
-            view.notify(event='message', msg='Config file tree structure:')
-            try:
-                t = tree.create_from_yaml(multiplex_files)
-            except IOError, details:
-                view.notify(event='error', msg=details.strerror)
-                sys.exit(exit_codes.AVOCADO_JOB_FAIL)
-            t = tree.apply_filters(t, args.filter_only, args.filter_out)
-            view.notify(event='minor',
-                        msg=t.get_ascii(attributes=args.attr))
-            sys.exit(exit_codes.AVOCADO_ALL_OK)
-
         try:
-            variants = multiplexer.multiplex_yamls(multiplex_files,
-                                                   args.filter_only,
-                                                   args.filter_out,
-                                                   args.debug)
+            mux_tree = multiplexer.yaml2tree(args.multiplex_files,
+                                             args.filter_only, args.filter_out,
+                                             args.debug)
         except IOError, details:
             view.notify(event='error',
                         msg=details.strerror)
             sys.exit(exit_codes.AVOCADO_JOB_FAIL)
+        mux_tree.merge(args.default_multiplex_tree)
+        if args.tree:
+            view.notify(event='message', msg='Config file tree structure:')
+            view.notify(event='minor',
+                        msg=mux_tree.get_ascii(attributes=args.attr))
+            sys.exit(exit_codes.AVOCADO_ALL_OK)
 
+        variants = multiplexer.MuxTree(mux_tree)
         view.notify(event='message', msg='Variants generated:')
         for (index, tpl) in enumerate(variants):
             if not args.debug:
@@ -91,7 +97,10 @@ class Multiplexer(plugin.Plugin):
             else:
                 color = output.term_support.LOWLIGHT
                 cend = output.term_support.ENDC
-                paths = ', '.join(["%s%s@%s%s" % (_.name, color, _.yaml, cend)
+                paths = ', '.join(["%s%s@%s%s" % (_.name, color,
+                                                  getattr(_, 'yaml',
+                                                          "Unknown"),
+                                                  cend)
                                    for _ in tpl])
             view.notify(event='minor', msg='%sVariant %s:    %s' %
                         (('\n' if args.contents else ''), index + 1, paths))

avocado/core/plugins/runner.py

@@ -126,11 +126,24 @@ class TestRunner(plugin.Plugin):
                              help='Filter out path(s) from multiplexing')
             mux.add_argument('--mux-entry', nargs='*', default=None,
                              help="Multiplex entry point(s)")
-
+            mux.add_argument('--env', default=[], nargs='*')
         super(TestRunner, self).configure(self.parser)
         # Export the test runner parser back to the main parser
         parser.runner = self.parser
 
+    def activate(self, args):
+        # Extend default multiplex tree of --env values
+        for value in getattr(args, "env", []):
+            value = value.split(':', 2)
+            if len(value) < 2:
+                raise ValueError("key:value pairs required, found only %s"
+                                 % (value))
+            elif len(value) == 2:
+                args.default_multiplex_tree.value[value[0]] = value[1]
+            else:
+                node = args.default_multiplex_tree.get_node(value[0], True)
+                node.value[value[1]] = value[2]
+
     def _validate_job_timeout(self, raw_timeout):
         units = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400}
         mult = 1

avocado/core/tree.py

@@ -89,7 +89,7 @@ class TreeNode(object):
         self._environment = None
         self.environment_origin = {}
         self.ctrl = []
-        self.multiplex = False
+        self.multiplex = None
         for child in children:
             self.add_child(child)
 
@@ -159,7 +159,10 @@ class TreeNode(object):
                             remove.append(key)
                     for key in remove:
                         self.value.pop(key, None)
-        self.multiplex = other.multiplex
+        if other.multiplex is True:
+            self.multiplex = True
+        elif other.multiplex is False:
+            self.multiplex = False
         self.value.update(other.value)
         for child in other.children:
             self.add_child(child)
@@ -245,6 +248,29 @@ class TreeNode(object):
             child.set_environment_dirty()
         self._environment = None
 
+    def get_node(self, path, create=False):
+        """
+        :param path: Path of the desired node (relative to this node)
+        :param create: Create the node (and intermediary ones) when not present
+        :return: the node associated with this path
+        :raise ValueError: When path doesn't exist and create not set
+        """
+        node = self
+        for name in path.split('/'):
+            if not name:
+                continue
+            try:
+                node = node.children[node.children.index(name)]
+            except ValueError:
+                if create:
+                    child = node.__class__(name)
+                    node.add_child(child)
+                    node = child
+                else:
+                    raise ValueError("Path %s does not exists in this tree\n%s"
+                                     % (path, self.get_ascii()))
+        return node
+
     def iter_children_preorder(self):
         """ Iterate through children """
         queue = collections.deque()
@@ -670,7 +696,7 @@ class TreeNodeDebug(TreeNode):  # only container pylint: disable=R0903
         Override origin with the one from other tree. Updated/Newly set values
         are going to use this location as origin.
         """
-        if hasattr(other, 'yaml'):
+        if hasattr(other, 'yaml') and other.yaml:
             srcyaml = os.path.relpath(other.yaml)
             # when we use TreeNodeDebug, value is always ValueDict
             self.value.yaml_per_key.update(other.value.yaml_per_key)    # pylint: disable=E1101

avocado/multiplexer.py

@@ -90,8 +90,8 @@ class MuxTree(object):
             yield ret
 
 
-def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
-                    debug=False):
+def yaml2tree(input_yamls, filter_only=None, filter_out=None,
+              debug=False):
     if filter_only is None:
         filter_only = []
     if filter_out is None:
@@ -99,8 +99,7 @@ def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
     input_tree = tree.create_from_yaml(input_yamls, debug)
     # TODO: Process filters and multiplex simultaneously
     final_tree = tree.apply_filters(input_tree, filter_only, filter_out)
-    result = MuxTree(final_tree)
-    return result
+    return final_tree
 
 
 # TODO: Create multiplexer plugin and split these functions into multiple files
@@ -396,9 +395,12 @@ class Mux(object):
         filter_only = getattr(args, 'filter_only', None)
         filter_out = getattr(args, 'filter_out', None)
         if mux_files:
-            self.variants = multiplex_yamls(mux_files, filter_only, filter_out)
+            mux_tree = yaml2tree(mux_files, filter_only, filter_out)
         else:   # no variants
-            self.variants = None
+            mux_tree = tree.TreeNode()
+        if getattr(args, 'default_multiplex_tree', None):
+            mux_tree.merge(args.default_multiplex_tree)
+        self.variants = MuxTree(mux_tree)
         self._mux_entry = getattr(args, 'mux_entry', None)
         if self._mux_entry is None:
             self._mux_entry = ['/run/*']

selftests/all/unit/avocado/multiplexer_unittest.py

@@ -43,24 +43,30 @@ class TestMultiplex(unittest.TestCase):
         self.assertEqual(len(self.mux_full), 12)
 
     def test_create_variants(self):
-        from_file = multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'])
+        from_file = multiplexer.yaml2tree(
+            ["/:" + PATH_PREFIX + 'examples/mux-selftest.yaml'])
+        from_file = multiplexer.MuxTree(from_file)
         self.assertEqual(self.mux_full, tuple(from_file))
 
     # Filters are tested in tree_unittests, only verify `multiplex_yamls` calls
     def test_filter_only(self):
         exp = (['intel', 'scsi'], ['intel', 'virtio'])
-        act = tuple(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'],
-                                                ('/hw/cpu/intel',
-                                                 '/distro/fedora',
-                                                 '/hw')))
+        act = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                     'examples/mux-selftest.yaml'],
+                                    ('/hw/cpu/intel',
+                                     '/distro/fedora',
+                                     '/hw'))
+        act = tuple(multiplexer.MuxTree(act))
         self.assertEqual(act, exp)
 
     def test_filter_out(self):
-        act = tuple(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest.yaml'],
-                                                None,
-                                                ('/hw/cpu/intel',
-                                                 '/distro/fedora',
-                                                 '/distro')))
+        act = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                     'examples/mux-selftest.yaml'],
+                                    None,
+                                    ('/hw/cpu/intel',
+                                     '/distro/fedora',
+                                     '/distro'))
+        act = tuple(multiplexer.MuxTree(act))
         self.assertEqual(len(act), 4)
         self.assertEqual(len(act[0]), 3)
         str_act = str(act)
@@ -71,8 +77,9 @@ class TestMultiplex(unittest.TestCase):
 
 
 class TestAvocadoParams(unittest.TestCase):
-    yamls = iter(multiplexer.multiplex_yamls(['/:' + PATH_PREFIX + 'examples/mux-selftest-params.'
-                                              'yaml']))
+    yamls = multiplexer.yaml2tree(["/:" + PATH_PREFIX +
+                                   'examples/mux-selftest-params.yaml'])
+    yamls = iter(multiplexer.MuxTree(yamls))
     params1 = multiplexer.AvocadoParams(yamls.next(), 'Unittest1', 1,
                                         ['/ch0/*', '/ch1/*'], {})
     yamls.next()    # Skip 2nd

selftests/all/unit/avocado/tree_unittest.py

@@ -171,20 +171,20 @@ class TestTree(unittest.TestCase):
         self.assertEqual({'new_value': 'something'},
                          oldroot.children[3].children[0].children[0].value)
         # multiplex root (always True)
-        self.assertEqual(tree2.multiplex, False)
+        self.assertEqual(tree2.multiplex, None)
         # multiplex /virt/
-        self.assertEqual(tree2.children[0].multiplex, False)
+        self.assertEqual(tree2.children[0].multiplex, None)
         # multiplex /virt/hw
-        self.assertEqual(tree2.children[0].children[0].multiplex, False)
+        self.assertEqual(tree2.children[0].children[0].multiplex, None)
         # multiplex /virt/distro
         self.assertEqual(tree2.children[0].children[1].multiplex, True)
         # multiplex /virt/env
         self.assertEqual(tree2.children[0].children[2].multiplex, True)
         # multiplex /virt/absolutly
-        self.assertEqual(tree2.children[0].children[3].multiplex, False)
+        self.assertEqual(tree2.children[0].children[3].multiplex, None)
         # multiplex /virt/distro/fedora
         self.assertEqual(tree2.children[0].children[1].children[0].multiplex,
-                         False)
+                         None)
 
 
 class TestPathParent(unittest.TestCase):