Merge pull request #394 from ldoktor/mux_domains4

avocado.multiplexer: Modify multiplexation mechanism [v4]

avocado/core/tree.py

@@ -51,6 +51,7 @@ YAML_INCLUDE = 0
 YAML_USING = 1
 YAML_REMOVE_NODE = 2
 YAML_REMOVE_VALUE = 3
+YAML_JOIN = 4
 
 
 class Control(object):  # Few methods pylint: disable=R0903
@@ -79,6 +80,7 @@ class TreeNode(object):
         self.children = []
         self._environment = None
         self.ctrl = []
+        self.multiplex = True
         for child in children:
             self.add_child(child)
 
@@ -148,6 +150,7 @@ class TreeNode(object):
                             remove.append(key)
                     for key in remove:
                         self.value.pop(key, None)
+        self.multiplex &= other.multiplex
         self.value.update(other.value)
         for child in other.children:
             self.add_child(child)
@@ -331,6 +334,14 @@ class Value(tuple):     # Few methods pylint: disable=R0903
     pass
 
 
+class ListOfNodeObjects(list):     # Few methods pylint: disable=R0903
+
+    """
+    Used to mark list as list of objects from whose node is going to be created
+    """
+    pass
+
+
 def _create_from_yaml(path, cls_node=TreeNode):
     """ Create tree structure from yaml stream """
     def tree_node_from_values(name, values):
@@ -362,27 +373,31 @@ def _create_from_yaml(path, cls_node=TreeNode):
                 elif value[0].code == YAML_REMOVE_VALUE:
                     value[0].value = value[1]   # set the name
                     node.ctrl.append(value[0])
+                elif value[0].code == YAML_JOIN:
+                    node.multiplex = False
             else:
                 node.value[value[0]] = value[1]
         if using:
-            for name in using.split('/')[::-1]:
-                node = cls_node(name, children=[node])
+            if name is not '':
+                for name in using.split('/')[::-1]:
+                    node = cls_node(name, children=[node])
+            else:
+                using = using.split('/')[::-1]
+                node.name = using.pop()
+                while True:
+                    if not using:
+                        break
+                    name = using.pop()  # 'using' is list pylint: disable=E1101
+                    node = cls_node(name, children=[node])
+                node = cls_node('', children=[node])
         return node
 
     def mapping_to_tree_loader(loader, node):
         """ Maps yaml mapping tag to TreeNode structure """
-        def is_node(values):
-            """ Whether these values represent node or just random values """
-            if (isinstance(values, list) and values
-                    and isinstance(values[0], (Value, TreeNode))):
-                # When any value is TreeNode or Value, all of them are already
-                # parsed and we can wrap them into self
-                return True
-
         _value = loader.construct_pairs(node)
-        objects = []
+        objects = ListOfNodeObjects()
         for name, values in _value:
-            if is_node(values):    # New node
+            if isinstance(values, ListOfNodeObjects):   # New node from list
                 objects.append(tree_node_from_values(name, values))
             elif values is None:            # Empty node
                 objects.append(cls_node(str(name)))
@@ -390,6 +405,17 @@ def _create_from_yaml(path, cls_node=TreeNode):
                 objects.append(Value((name, values)))
         return objects
 
+    def join_loader(loader, obj):
+        """
+        Special !join loader which allows to tag node as 'multiplex = False'.
+        """
+        if not isinstance(obj, yaml.ScalarNode):
+            objects = mapping_to_tree_loader(loader, obj)
+        else:   # This means it's empty node. Don't call mapping_to_tree_loader
+            objects = ListOfNodeObjects()
+        objects.append((Control(YAML_JOIN), None))
+        return objects
+
     Loader.add_constructor(u'!include',
                            lambda loader, node: Control(YAML_INCLUDE))
     Loader.add_constructor(u'!using',
@@ -398,6 +424,7 @@ def _create_from_yaml(path, cls_node=TreeNode):
                            lambda loader, node: Control(YAML_REMOVE_NODE))
     Loader.add_constructor(u'!remove_value',
                            lambda loader, node: Control(YAML_REMOVE_VALUE))
+    Loader.add_constructor(u'!join', join_loader)
     Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                            mapping_to_tree_loader)
 
@@ -433,6 +460,9 @@ def create_from_yaml(paths, debug=False):
         for path in paths:
             merge(data, path)
     except (yaml.scanner.ScannerError, yaml.parser.ParserError) as err:
+        if 'mapping values are not allowed in this context' in str(err):
+            err = ("%s\n\nMake sure !tags and colons are separated by a space "
+                   "(eg. !include :)" % err)
         raise SyntaxError(err)
     return data
 

avocado/multiplexer.py

@@ -19,62 +19,48 @@
 Multiplex and create variants.
 """
 
-import collections
+import itertools
 
 from avocado.core import tree
 
 
-def any_sibling(*nodes):
+def tree2pools(node, mux=True):
     """
-    Check if there is any sibling.
-
-    :param nodes: the nodes to check.
-    :return: `True` if there is any sibling or `False`.
+    Process tree and flattens the structure to remaining leaves and
+    list of lists of leaves per each multiplex group.
+    :param node: Node to start with
+    :return: tuple(`leaves`, `pools`), where `leaves` are directly inherited
+    leaves of this node (no other multiplex in the middle). `pools` is list of
+    lists of directly inherited leaves of the nested multiplex domains.
     """
-    if len(nodes) < 2:
-        return False
-    parents = set(node.parent for node in nodes)
-    return len(nodes) != len(parents)
-
-
-def multiplex(*args):
     leaves = []
-    parents = collections.OrderedDict()
-    # filter args and create a set of parents
-    for arg in args[0]:
-        leaves.append(arg)
-        parents[arg.parent] = True
-
     pools = []
-    for p in parents.keys():
-        pools.append(leaves)
-        leaves = [x for x in leaves if x.parent != p]
-
-    result = [[]]
-    result_prev = [[]]
-    for pool in pools:
-
-        # second level of filtering above should use the filter strings
-        # extracted from the node being worked on
-        items = []
-        for x in result:
-            for y in pool:
-                item = x + [y]
-                if any_sibling(*item) is False:
-                    items.append(item)
-        result = items
-
-        # if a pool gets totally filtered out above, result will be empty
-        if len(result) == 0:
-            result = result_prev
-        else:
-            result_prev = result
-
-    if result == [[]]:
-        return
-
-    for prod in result:
-        yield tuple(prod)
+    if mux:
+        # TODO: Get this multiplex leaves filters and store them in this pool
+        # to support 2nd level filtering
+        new_leaves = []
+        for child in node.children:
+            if child.is_leaf:
+                new_leaves.append(child)
+            else:
+                _leaves, _pools = tree2pools(child, node.multiplex)
+                new_leaves.extend(_leaves)
+                # TODO: For 2nd level filters store this separately in case
+                # this branch is filtered out
+                pools.extend(_pools)
+        if new_leaves:
+            # TODO: Filter the new_leaves (and new_pools) before merging
+            # into pools
+            pools.append(new_leaves)
+    else:
+        for child in node.children:
+            if child.is_leaf:
+                leaves.append(child)
+            else:
+                _leaves, _pools = tree2pools(child, node.multiplex)
+                leaves.extend(_leaves)
+                pools.extend(_pools)
+    return leaves, pools
 
 
 def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
@@ -84,7 +70,9 @@ def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
     if filter_out is None:
         filter_out = []
     input_tree = tree.create_from_yaml(input_yamls, debug)
+    # TODO: Process filters and multiplex simultaneously
     final_tree = tree.apply_filters(input_tree, filter_only, filter_out)
-    leaves = (x for x in final_tree.iter_leaves() if x.parent is not None)
-    variants = multiplex(leaves)
-    return variants
+    leaves, pools = tree2pools(final_tree, final_tree.multiplex)
+    if leaves:  # Add remaining leaves (they are not variants, only endpoints
+        pools.extend(leaves)
+    return itertools.product(*pools)    # *magic required pylint: disable=W0142

docs/source/MultiplexConfig.rst

@@ -28,6 +28,7 @@ Here is how a simple and valid multiplex configuration looks like::
         sleep_length: 600
 
 The key concepts here are ``nodes`` (provides context and scope), ``keys`` (think of variables) and ``values`` (scalar or lists).
+
 In the next section, we will describe these concepts in more details.
 
 .. _nodes:
@@ -66,9 +67,17 @@ The ending nodes (the leafs on the tree) will become part of all lower-level
 However, the precedence is evaluated in top-down or ``last defined`` order.
 In other words, the last parsed has precedence over earlier definitions.
 
-It's also possible to remove node using python's regexp, which can be useful
-when extending upstream file using downstream yaml files. This is done by
-`!remove_node : $value_name` directive::
+When you provide multiple files they are processed and merged together using
+the common root (`/`). When certain paths overlap (`$file1:/my/path`,
+`$file2:/my/path`), we first create the tree of `$file1` and then process
+`$file2`. This means all children of `/my/path` of the first file are in
+correct order and `$file2` either updates values or appends new children
+as next ones. This of course happens recursively so you update valures and add
+children of all the nodes beneath.
+
+During this merge it's also possible to remove nodes using python regular
+expressions, which can be useful when extending upstream file using downstream
+yaml files. This is done by `!remove_node : $value_name` directive::
 
     os:
         fedora:
@@ -265,10 +274,99 @@ Whole file is __merged__ into the node where it's defined.
 Variants
 ========
 
-To be written.
+When tree parsing and filtering is finished, we create set of variants.
+Each variant uses one leaf of each sibling group. For example::
+
+    cpu:
+        intel:
+        amd:
+        arm:
+    fmt:
+        qcow2:
+        raw:
+
+Produces 2 groups `[intel, amd, arm]` and `[qcow2, raw]`, which results in
+6 variants (all combinations; product of the groups)
+
+It's also possible to join current node and its children by `!join` tag::
+
+    fmt: !join
+        qcow:
+            2:
+            2v3:
+        raw:
+
+Without the join this would produce 2 groups `[2, 2v3]` and `[raw]` resulting
+in 2 variants `[2, raw]` and `[2v3, raw]`, which is really not useful.
+But we said that `fmt` children should join this sibling group
+so it results in one group `[qcow/2, qcow/2v3, raw]` resulting in 3 variants
+each of different fmt. This is useful when some
+of the variants share some common key. These keys are set inside the
+parent, for example here `qcow2.0` and `qcow2.2v3` share the same key
+`type: qcow2` and `qcow2.2v3` adds `extra_params` into his params::
+
+    fmt:
+        qcow2:
+            type: qcow2
+            0:
+            v3:
+                extra_params: "compat=1.1"
+        raw:
+            type: raw
+
+Complete example::
+
+    hw:
+        cpu:
+            intel:
+            amd:
+            arm:
+        fmt: !join
+            qcow:
+                qcow2:
+                qcow2v3:
+            raw:
+    os: !join
+        linux: !join
+            Fedora:
+                19:
+            Gentoo:
+        windows:
+            3.11:
 
-Avocado comes equipped with a plugin to parse multiplex files. The appropriate
-subcommand is::
+While preserving names and environment values. Then all combinations are
+created resulting into 27 unique variants covering all possible combinations
+of given tree::
+
+    Variant 1:    /hw/cpu/intel, /hw/fmt/qcow/qcow2, /os/linux/Fedora/19
+	Variant 2:    /hw/cpu/intel, /hw/fmt/qcow/qcow2, /os/linux/Gentoo
+	Variant 3:    /hw/cpu/intel, /hw/fmt/qcow/qcow2, /os/windows/3.11
+	Variant 4:    /hw/cpu/intel, /hw/fmt/qcow/qcow2v3, /os/linux/Fedora/19
+	Variant 5:    /hw/cpu/intel, /hw/fmt/qcow/qcow2v3, /os/linux/Gentoo
+	Variant 6:    /hw/cpu/intel, /hw/fmt/qcow/qcow2v3, /os/windows/3.11
+	Variant 7:    /hw/cpu/intel, /hw/fmt/raw, /os/linux/Fedora/19
+	Variant 8:    /hw/cpu/intel, /hw/fmt/raw, /os/linux/Gentoo
+	Variant 9:    /hw/cpu/intel, /hw/fmt/raw, /os/windows/3.11
+	Variant 10:    /hw/cpu/amd, /hw/fmt/qcow/qcow2, /os/linux/Fedora/19
+	Variant 11:    /hw/cpu/amd, /hw/fmt/qcow/qcow2, /os/linux/Gentoo
+	Variant 12:    /hw/cpu/amd, /hw/fmt/qcow/qcow2, /os/windows/3.11
+	Variant 13:    /hw/cpu/amd, /hw/fmt/qcow/qcow2v3, /os/linux/Fedora/19
+	Variant 14:    /hw/cpu/amd, /hw/fmt/qcow/qcow2v3, /os/linux/Gentoo
+	Variant 15:    /hw/cpu/amd, /hw/fmt/qcow/qcow2v3, /os/windows/3.11
+	Variant 16:    /hw/cpu/amd, /hw/fmt/raw, /os/linux/Fedora/19
+	Variant 17:    /hw/cpu/amd, /hw/fmt/raw, /os/linux/Gentoo
+	Variant 18:    /hw/cpu/amd, /hw/fmt/raw, /os/windows/3.11
+	Variant 19:    /hw/cpu/arm, /hw/fmt/qcow/qcow2, /os/linux/Fedora/19
+	Variant 20:    /hw/cpu/arm, /hw/fmt/qcow/qcow2, /os/linux/Gentoo
+	Variant 21:    /hw/cpu/arm, /hw/fmt/qcow/qcow2, /os/windows/3.11
+	Variant 22:    /hw/cpu/arm, /hw/fmt/qcow/qcow2v3, /os/linux/Fedora/19
+	Variant 23:    /hw/cpu/arm, /hw/fmt/qcow/qcow2v3, /os/linux/Gentoo
+	Variant 24:    /hw/cpu/arm, /hw/fmt/qcow/qcow2v3, /os/windows/3.11
+	Variant 25:    /hw/cpu/arm, /hw/fmt/raw, /os/linux/Fedora/19
+	Variant 26:    /hw/cpu/arm, /hw/fmt/raw, /os/linux/Gentoo
+	Variant 27:    /hw/cpu/arm, /hw/fmt/raw, /os/windows/3.11
+
+You can generate this list yourself by executing::
 
     avocado multiplex /path/to/multiplex.yaml [-c]
 
@@ -276,19 +374,5 @@ Note that there's no need to put extensions to a multiplex file, although
 doing so helps with organization. The optional -c param is used to provide
 the contents of the dictionaries generated, not only their shortnames.
 
-``avocado multiplex`` against the content above produces the following
-combinations and names::
-
-    Dictionaries generated:
-        dict 1:    four.one
-        dict 2:    four.two
-        dict 3:    four.three
-        dict 4:    five.one
-        dict 5:    five.two
-        dict 6:    five.three
-        dict 7:    six.one
-        dict 8:    six.two
-        dict 9:    six.three
-
 With Nodes, Keys, Values & Filters, we have most of what you
 actually need to construct most multiplex files.

examples/mux-selftest-advanced.yaml

@@ -19,6 +19,15 @@ distro:
         # And this removes the original 'is_cool'
         # Setting happens after ctrl so it should be created'
         !remove_value : is_cool
+# Following node is an empty node with only Control object. During merge
+# it setls /env node as !join (disable multiplexation)
+env: !join
+distro: !join
+    # Set !join here, it won't be overwritten below as it's defined as
+    # &=.
+    mint:   # This won't change anything
+distro:
+    gentoo:     # This won't change anything
 # This creates new branch the usual way
 new_node:
     # Put this new_node into /absolutely/fresh/ ('/' are automatically

examples/mux-selftest.yaml

@@ -7,7 +7,6 @@
 #          multiple files and checks that the node ordering works fine.
 # /env/opt_CFLAGS: Should be present in merged node
 # /env/prod/opt_CFLAGS: value should be overridden by latter node
-
 hw:
     cpu:
         joinlist:
@@ -27,7 +26,7 @@ hw:
             disk_type: 'scsi'
         virtio:
     corruptlist: ['upper_node_list']
-distro:
+distro:     # This node is set as !multiplex below
     fedora:
         init: 'systemd'
 env:

selftests/all/unit/avocado/multiplexer_unittest.py

@@ -2,52 +2,31 @@ import unittest
 
 from avocado import multiplexer
 from avocado.core import tree
+import itertools
 
 
 TREE = tree.create_from_yaml(['examples/mux-selftest.yaml'])
 
 
-class TestAnySibling(unittest.TestCase):
-    # /hw/cpu/{intel,amd,arm}
-    tree = TREE
-    sibl_a_1 = tree.children[0].children[0].children[0]
-    sibl_a_2 = tree.children[0].children[0].children[1]
-    sibl_a_3 = tree.children[0].children[0].children[2]
-    # /hw/{cpu,disk}
-    sibl_b_1 = tree.children[1].children[0]
-    sibl_b_2 = tree.children[1].children[1]
-
-    def test_empty(self):
-        self.assertFalse(multiplexer.any_sibling())
-
-    def test_one_node(self):
-        single_node = self.tree.children[2].children[0]
-        self.assertFalse(multiplexer.any_sibling(single_node))
-
-    def test_all_siblings(self):
-        self.assertTrue(multiplexer.any_sibling(self.sibl_b_1, self.sibl_b_2))
-        self.assertTrue(multiplexer.any_sibling(self.sibl_a_1, self.sibl_a_2,
-                                                self.sibl_a_3))
-
-    def test_mixed(self):
-        self.assertTrue(multiplexer.any_sibling(self.sibl_a_1, self.sibl_a_2,
-                                                self.sibl_b_1))
-
-    def test_no_relation(self):
-        self.assertFalse(multiplexer.any_sibling(self.sibl_a_1, self.sibl_b_1))
+def combine(leaves_pools):
+    ''' Joins remaining leaves and pools and create product '''
+    if leaves_pools[0]:
+        leaves_pools[1].extend(leaves_pools[0])
+    return itertools.product(*leaves_pools[1])
 
 
 class TestMultiplex(unittest.TestCase):
     tree = TREE
-    mux_full = tuple(multiplexer.multiplex(tree))
+    mux_full = tuple(combine(multiplexer.tree2pools(tree)))
 
     def test_empty(self):
-        self.assertEqual(tuple(multiplexer.multiplex([])), tuple())
+        act = tuple(combine(multiplexer.tree2pools(tree.TreeNode())))
+        self.assertEqual(act, ((),))
 
     def test_partial(self):
         exp = (('intel', 'scsi'), ('intel', 'virtio'), ('amd', 'scsi'),
                ('amd', 'virtio'), ('arm', 'scsi'), ('arm', 'virtio'))
-        act = tuple(multiplexer.multiplex(self.tree.children[0]))
+        act = tuple(combine(multiplexer.tree2pools(self.tree.children[0])))
         self.assertEqual(act, exp)
 
     def test_full(self):

selftests/all/unit/avocado/tree_unittest.py

@@ -155,6 +155,7 @@ class TestTree(unittest.TestCase):
         act = tree2.get_leaves()
         oldroot = tree2.children[0]
         self.assertEqual(exp, act)
+        self.assertEqual(tree2.children[0].children[0].path, "/virt/hw")
         self.assertEqual({'enterprise': True},
                          oldroot.children[1].children[1].value)
         self.assertEqual({'new_init': 'systemd'},
@@ -163,6 +164,21 @@ class TestTree(unittest.TestCase):
                          oldroot.children[1].children[2].value)
         self.assertEqual({'new_value': 'something'},
                          oldroot.children[3].children[0].children[0].value)
+        # multiplex root (always True)
+        self.assertEqual(tree2.multiplex, True)
+        # multiplex /virt/
+        self.assertEqual(tree2.children[0].multiplex, True)
+        # multiplex /virt/hw
+        self.assertEqual(tree2.children[0].children[0].multiplex, True)
+        # multiplex /virt/distro
+        self.assertEqual(tree2.children[0].children[1].multiplex, False)
+        # multiplex /virt/env
+        self.assertEqual(tree2.children[0].children[2].multiplex, False)
+        # multiplex /virt/absolutly
+        self.assertEqual(tree2.children[0].children[3].multiplex, True)
+        # multiplex /virt/distro/fedora
+        self.assertEqual(tree2.children[0].children[1].children[0].multiplex,
+                         True)
 
 
 class TestPathParent(unittest.TestCase):