avocado.multiplexer: Use !mux and support for recursive mux

This patch modifies the multiplexer logic to support full recursive
multiplexation, unlike the old version which only flattened the
multiplexed nodes. This allows greater flexibility.

Additionally it removes the !join flag and inverts the logic to use !mux
instead. By default leaves are not multiplexed and all are part of the
current params. Users can specify exact nodes which get multiplexed by
using !mux keyword.
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/tree.py

@@ -54,7 +54,7 @@ YAML_INCLUDE = 0
 YAML_USING = 1
 YAML_REMOVE_NODE = 2
 YAML_REMOVE_VALUE = 3
-YAML_JOIN = 4
+YAML_MUX = 4
 
 __RE_FILE_SPLIT = re.compile(r'(?<!\\):')   # split by ':' but not '\\:'
 __RE_FILE_SUBS = re.compile(r'(?<!\\)\\:')  # substitute '\\:' but not '\\\\:'
@@ -87,7 +87,7 @@ class TreeNode(object):
         self._environment = None
         self.environment_origin = {}
         self.ctrl = []
-        self.multiplex = True
+        self.multiplex = False
         for child in children:
             self.add_child(child)
 
@@ -157,7 +157,7 @@ class TreeNode(object):
                             remove.append(key)
                     for key in remove:
                         self.value.pop(key, None)
-        self.multiplex &= other.multiplex
+        self.multiplex = other.multiplex
         self.value.update(other.value)
         for child in other.children:
             self.add_child(child)
@@ -384,8 +384,8 @@ def _create_from_yaml(path, cls_node=TreeNode):
                 elif value[0].code == YAML_REMOVE_VALUE:
                     value[0].value = value[1]   # set the name
                     node.ctrl.append(value[0])
-                elif value[0].code == YAML_JOIN:
-                    node.multiplex = False
+                elif value[0].code == YAML_MUX:
+                    node.multiplex = True
             else:
                 node.value[value[0]] = value[1]
         if using:
@@ -416,15 +416,15 @@ def _create_from_yaml(path, cls_node=TreeNode):
                 objects.append(Value((name, values)))
         return objects
 
-    def join_loader(loader, obj):
+    def mux_loader(loader, obj):
         """
-        Special !join loader which allows to tag node as 'multiplex = False'.
+        Special !mux loader which allows to tag node as 'multiplex = True'.
         """
         if not isinstance(obj, yaml.ScalarNode):
             objects = mapping_to_tree_loader(loader, obj)
         else:   # This means it's empty node. Don't call mapping_to_tree_loader
             objects = ListOfNodeObjects()
-        objects.append((Control(YAML_JOIN), None))
+        objects.append((Control(YAML_MUX), None))
         return objects
 
     Loader.add_constructor(u'!include',
@@ -435,7 +435,7 @@ def _create_from_yaml(path, cls_node=TreeNode):
                            lambda loader, node: Control(YAML_REMOVE_NODE))
     Loader.add_constructor(u'!remove_value',
                            lambda loader, node: Control(YAML_REMOVE_VALUE))
-    Loader.add_constructor(u'!join', join_loader)
+    Loader.add_constructor(u'!mux', mux_loader)
     Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                            mapping_to_tree_loader)
 

avocado/multiplexer.py

@@ -19,6 +19,7 @@
 Multiplex and create variants.
 """
 
+import collections
 import itertools
 import logging
 import re
@@ -29,47 +30,68 @@ from avocado.core import tree
 MULTIPLEX_CAPABLE = tree.MULTIPLEX_CAPABLE
 
 
-def tree2pools(node, mux=True):
+class MuxTree(object):
+
     """
-    Process tree and flattens the structure to remaining leaves and
-    list of lists of leaves per each multiplex group.
-    :param node: Node to start with
-    :return: tuple(`leaves`, `pools`), where `leaves` are directly inherited
-    leaves of this node (no other multiplex in the middle). `pools` is list of
-    lists of directly inherited leaves of the nested multiplex domains.
+    Object representing part of the tree from the root to leaves or another
+    multiplex domain. Recursively it creates multiplexed variants of the full
+    tree.
     """
-    leaves = []
-    pools = []
-    if mux:
-        # TODO: Get this multiplex leaves filters and store them in this pool
-        # to support 2nd level filtering
-        new_leaves = []
-        for child in node.children:
-            if child.is_leaf:
-                new_leaves.append(child)
+
+    def __init__(self, root):
+        """
+        :param root: Root of this tree slice
+        """
+        self.pools = []
+        for node in self._iter_mux_leaves(root):
+            if node.is_leaf:
+                self.pools.append(node)
+            else:
+                pools = []
+                for mux_child in node.children:
+                    pools.append(MuxTree(mux_child))
+                self.pools.append(pools)
+
+    @staticmethod
+    def _iter_mux_leaves(node):
+        """ yield leaves or muxes of the tree """
+        queue = collections.deque()
+        while node is not None:
+            if node.is_leaf or node.multiplex:
+                yield node
             else:
-                _leaves, _pools = tree2pools(child, node.multiplex)
-                new_leaves.extend(_leaves)
-                # TODO: For 2nd level filters store this separately in case
-                # this branch is filtered out
-                pools.extend(_pools)
-        if new_leaves:
-            # TODO: Filter the new_leaves (and new_pools) before merging
-            # into pools
-            pools.append(new_leaves)
-    else:
-        for child in node.children:
-            if child.is_leaf:
-                leaves.append(child)
+                queue.extendleft(reversed(node.children))
+            try:
+                node = queue.popleft()
+            except IndexError:
+                raise StopIteration
+
+    def __iter__(self):
+        """
+        Iterates through variants
+        """
+        pools = []
+        for pool in self.pools:
+            if isinstance(pool, list):
+                pools.append(itertools.chain(*pool))
             else:
-                _leaves, _pools = tree2pools(child, node.multiplex)
-                leaves.extend(_leaves)
-                pools.extend(_pools)
-    return leaves, pools
+                pools.append([pool])
+        pools = itertools.product(*pools)
+        while True:
+            # TODO: Implement 2nd level filteres here
+            # TODO: This part takes most of the time, optimize it
+            dirty = pools.next()
+            ret = []
+            for pool in dirty:
+                if isinstance(pool, list):
+                    ret.extend(pool)
+                else:
+                    ret.append(pool)
+            yield ret
 
 
-def parse_yamls(input_yamls, filter_only=None, filter_out=None,
-                debug=False):
+def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
+                    debug=False):
     if filter_only is None:
         filter_only = []
     if filter_out is None:
@@ -77,20 +99,8 @@ def parse_yamls(input_yamls, filter_only=None, filter_out=None,
     input_tree = tree.create_from_yaml(input_yamls, debug)
     # TODO: Process filters and multiplex simultaneously
     final_tree = tree.apply_filters(input_tree, filter_only, filter_out)
-    leaves, pools = tree2pools(final_tree, final_tree.multiplex)
-    if leaves:  # Add remaining leaves (they are not variants, only endpoints
-        pools.extend(leaves)
-    return pools
-
-
-def multiplex_pools(pools):
-    return itertools.product(*pools)
-
-
-def multiplex_yamls(input_yamls, filter_only=None, filter_out=None,
-                    debug=False):
-    pools = parse_yamls(input_yamls, filter_only, filter_out, debug)
-    return multiplex_pools(pools)
+    result = MuxTree(final_tree)
+    return result
 
 
 # TODO: Create multiplexer plugin and split these functions into multiple files
@@ -179,7 +189,8 @@ class AvocadoParams(object):
 
     def log(self, key, path, default, value):
         """ Predefined format for displaying params query """
-        self._log("PARAMS (key=%s, path=%s, default=%s) => %r", key, path, default, value)
+        self._log("PARAMS (key=%s, path=%s, default=%s) => %r", key, path,
+                  default, value)
 
     def _get_matching_leaves(self, path, leaves):
         """
@@ -438,9 +449,9 @@ class Mux(object):
         filter_only = getattr(args, 'filter_only', None)
         filter_out = getattr(args, 'filter_out', None)
         if mux_files:
-            self.pools = parse_yamls(mux_files, filter_only, filter_out)
+            self.variants = multiplex_yamls(mux_files, filter_only, filter_out)
         else:   # no variants
-            self.pools = None
+            self.variants = None
         self._mux_entry = getattr(args, 'mux_entry', None)
         if self._mux_entry is None:
             self._mux_entry = ['/test/*']
@@ -450,9 +461,9 @@ class Mux(object):
         :return: overall number of tests * multiplex variants
         """
         # Currently number of tests is symetrical
-        if self.pools:
+        if self.variants:
             return (len(test_suite) *
-                    sum(1 for _ in multiplex_pools(self.pools)))
+                    sum(1 for _ in self.variants))
         else:
             return len(test_suite)
 
@@ -460,9 +471,9 @@ class Mux(object):
         """
         Processes the template and yields test definition with proper params
         """
-        if self.pools:  # Copy template and modify it's params
+        if self.variants:  # Copy template and modify it's params
             i = None
-            for i, variant in enumerate(multiplex_pools(self.pools)):
+            for i, variant in enumerate(self.variants):
                 test_factory = [template[0], template[1].copy()]
                 test_factory[1]['params'] = (variant, self._mux_entry)
                 yield test_factory

examples/mux-environment.yaml

 hw:
-    cpu:
+    cpu: !mux
         intel:
             cpu_CFLAGS: '-march=core2'
         amd:
             cpu_CFLAGS: '-march=athlon64'
         arm:
             cpu_CFLAGS: '-mabi=apcs-gnu -march=armv8-a -mtune=arm8'
-    disk:
+    disk: !mux
         scsi:
             disk_type: 'scsi'
         virtio:
             disk_type: 'virtio'
-distro:
+distro: !mux
     fedora:
         init: 'systemd'
     mint:
         init: 'systemv'
-env:
+env: !mux
     debug:
         opt_CFLAGS: '-O0 -g'
     prod:

examples/mux-selftest-advanced.yaml

@@ -2,7 +2,7 @@
 !using : virt
 # Following line makes it look exactly as mux-selftest.yaml
 !include : mux-selftest.yaml
-distro:
+distro: !mux
     # This line extends the distro branch using include
     !include : mux-selftest-distro.yaml
     # remove node called "mint"
@@ -19,14 +19,7 @@ distro:
         # And this removes the original 'is_cool'
         # Setting happens after ctrl so it should be created'
         !remove_value : is_cool
-# Following node is an empty node with only Control object. During merge
-# it setls /env node as !join (disable multiplexation)
-env: !join
-distro: !join
-    # Set !join here, it won't be overwritten below as it's defined as
-    # &=.
-    mint:   # This won't change anything
-distro:
+distro: !mux
     gentoo:     # This won't change anything
 # This creates new branch the usual way
 new_node:

examples/mux-selftest-distro.yaml

-RHEL:
+!mux
+RHEL: !mux
     enterprise: true
     6:
         init: 'systemv'
     7:
         init: 'systemd'
-gentoo:
+gentoo: !mux
     is_cool: False

examples/mux-selftest-params.yaml

-!join
 root: "root"
 cache_test: 'cache'
 diff_domain: "text1"
@@ -7,12 +6,13 @@ ch0:
     clash1: "equal"
     clash3: "also equal"
     ch0.1:
-        ch0.1.1:
+        ch0.1.1: !mux
             ch0.1.1.1:
                 unique1: "unique1"
                 clash1: "equal"
             ch0.1.1.2:
                 unique1: "unique1-2"
+    ch0.1b: !mux
         ch0.1.2:
             unique3: "unique3"
             clash3: "also equal"

examples/mux-selftest.yaml

@@ -8,7 +8,7 @@
 # /env/opt_CFLAGS: Should be present in merged node
 # /env/prod/opt_CFLAGS: value should be overridden by latter node
 hw:
-    cpu:
+    cpu: !mux
         joinlist:
             - first_item
         intel:
@@ -18,7 +18,7 @@ hw:
             cpu_CFLAGS: '-march=athlon64'
         arm:
             cpu_CFLAGS: '-mabi=apcs-gnu -march=armv8-a -mtune=arm8'
-    disk:
+    disk: !mux
         disk_type: 'virtio'
         corruptlist: 'nonlist'
         scsi:
@@ -26,17 +26,17 @@ hw:
             disk_type: 'scsi'
         virtio:
     corruptlist: ['upper_node_list']
-distro:     # This node is set as !multiplex below
+distro: !mux     # This node is set as !multiplex below
     fedora:
         init: 'systemd'
-env:
+env: !mux
     opt_CFLAGS: '-Os'
     prod:
         opt_CFLAGS: 'THIS SHOULD GET OVERWRITTEN'
-env:
+env: !mux
     prod:
         opt_CFLAGS: '-O2'
-distro:
+distro: !mux
     mint:
         init: 'systemv'
 

examples/tests/doublefree_nasty.py.data/iterations.yaml

-iterations:
+iterations: !mux
     1:
     2:
     3:

examples/tests/env_variables.sh.data/env_variables.yaml

+!mux
 short:
     CUSTOM_VARIABLE: A
 medium:

examples/tests/multiplextest.py.data/multiplextest.yaml

-env:
+env: !mux
     production:
        malloc_perturb:  no
        gcc_flags:  -O3
@@ -8,11 +8,11 @@ env:
 
 host:
     kernel_config:
-        page_size:
+        page_size: !mux
             default:
             huge_pages:
                 huge_pages:  yes
-        numa:
+        numa: !mux
             default:
             numa_ballance_aggressive:
                 numa_balancing:  1
@@ -24,8 +24,8 @@ host:
                 numa_balancing_scan_size_mb:  32
 
 guest:
-    os: !join
-        windows:
+    os: !mux
+        windows: !mux
             os_type: windows
             xp:
                 win: xp
@@ -33,7 +33,7 @@ guest:
                 win: 2k12
             7:
                 win: 7
-        linux:
+        linux: !mux
             os_type:  linux
             fedora:
                 distro:  fedora
@@ -41,12 +41,12 @@ guest:
                 distro:  ubuntu
 
 hardware:
-    disks:
+    disks: !mux
         ide:
             drive_format: ide
         scsi:
             drive_format: scsi
-    network:
+    network: !mux
         rtl_8139:
             nic_model: rtl8139
         e1000:
@@ -55,15 +55,15 @@ hardware:
             nic_model: virtio
             enable_msix_vectors: yes
 
-tests:
-    sync_test:
+tests: !mux
+    sync_test: !mux
         standard:
             sync_timeout: 30
             sync_tries:   10
         aggressive:
             sync_timeout: 10
             sync_tries:   20
-    ping_test:
+    ping_test: !mux
         standard:
             ping_tries:   10
             ping_timeout: 20

examples/tests/raise.py.data/raise.yaml

+!mux
 sigint:
     signal_number: 2
 siguser1:

examples/tests/sleeptenmin.py.data/sleeptenmin.mplx

-variants:
-    - sleeptenmin:
-        variants:
-            - builtin:
-                sleep_method = builtin
-            - shell:
-                sleep_method = shell
-        variants:
-            - one_cycle:
-                sleep_cycles = 1
-                sleep_length = 600
-            - six_cycles:
-                sleep_cycles = 6
-                sleep_length = 100
-            - one_hundred_cycles:
-                sleep_cycles = 100
-                sleep_length = 6
-            - six_hundred_cycles:
-                sleep_cycles = 600
-                sleep_length = 1
+sleeptenmin: !mux
+    builtin:
+        sleep_method: builtin
+    shell:
+        sleep_method: shell
+variants: !mux
+    one_cycle:
+        sleep_cycles: 1
+        sleep_length: 600
+    six_cycles:
+        sleep_cycles: 6
+        sleep_length: 100
+    one_hundred_cycles:
+        sleep_cycles: 100
+        sleep_length: 6
+    six_hundred_cycles:
+        sleep_cycles: 600
+        sleep_length: 1

examples/tests/sleeptest.py.data/sleeptest.yaml

+!mux
 !using : test
 short:
     sleep_length: 0.5

examples/tests/synctest.py.data/synctest.yaml

 sync_tarball: synctest.tar.bz2
-loop:
+loop: !mux
     short:
         sync_loop:  10
     medium:
         sync_loop: 50
     long:
         sync_loop: 100
-length:
+length: !mux
     short:
         sync_length: 100
     medium:

examples/tests/whiteboard.py.data/whiteboard.yaml

-source:
+source: !mux
     string:
         whiteboard_data_text: 'foo bar foo baz'
     urandom:
         whiteboard_data_file: '/dev/urandom'
 
-iterations:
+iterations: !mux
     single:
         whiteboard_writes: 1
     dozen:
         whiteboard_writes: 12
-size:
+size: !mux
     onekilo:
         whiteboard_data_size: 1024
     onemega:

selftests/all/unit/avocado/multiplexer_unittest.py

@@ -9,9 +9,12 @@ if sys.version_info[:2] == (2, 6):
     import unittest2 as unittest
 else:
     import unittest
+if __name__ == "__main__":
+    PATH_PREFIX = "../../../../"
+else:
+    PATH_PREFIX = ""
 
-
-TREE = tree.create_from_yaml(['examples/mux-selftest.yaml'])
+TREE = tree.create_from_yaml([PATH_PREFIX + 'examples/mux-selftest.yaml'])
 
 
 def combine(leaves_pools):
@@ -23,36 +26,36 @@ def combine(leaves_pools):
 
 class TestMultiplex(unittest.TestCase):
     tree = TREE
-    mux_full = tuple(combine(multiplexer.tree2pools(tree)))
+    mux_full = tuple(multiplexer.MuxTree(tree))
 
     def test_empty(self):
-        act = tuple(combine(multiplexer.tree2pools(tree.TreeNode())))
-        self.assertEqual(act, ((),))
+        act = tuple(multiplexer.MuxTree(tree.TreeNode()))
+        self.assertEqual(act, (['', ],))
 
     def test_partial(self):
-        exp = (('intel', 'scsi'), ('intel', 'virtio'), ('amd', 'scsi'),
-               ('amd', 'virtio'), ('arm', 'scsi'), ('arm', 'virtio'))
-        act = tuple(combine(multiplexer.tree2pools(self.tree.children[0])))
+        exp = (['intel', 'scsi'], ['intel', 'virtio'], ['amd', 'scsi'],
+               ['amd', 'virtio'], ['arm', 'scsi'], ['arm', 'virtio'])
+        act = tuple(multiplexer.MuxTree(self.tree.children[0]))
         self.assertEqual(act, exp)
 
     def test_full(self):
         self.assertEqual(len(self.mux_full), 12)
 
     def test_create_variants(self):
-        from_file = multiplexer.multiplex_yamls(['examples/mux-selftest.yaml'])
+        from_file = multiplexer.multiplex_yamls([PATH_PREFIX + 'examples/mux-selftest.yaml'])
         self.assertEqual(self.mux_full, tuple(from_file))
 
     # Filters are tested in tree_unittests, only verify `multiplex_yamls` calls
     def test_filter_only(self):
-        exp = (('intel', 'scsi'), ('intel', 'virtio'))
-        act = tuple(multiplexer.multiplex_yamls(['examples/mux-selftest.yaml'],
+        exp = (['intel', 'scsi'], ['intel', 'virtio'])
+        act = tuple(multiplexer.multiplex_yamls([PATH_PREFIX + 'examples/mux-selftest.yaml'],
                                                 ('/hw/cpu/intel',
                                                  '/distro/fedora',
                                                  '/hw')))
         self.assertEqual(act, exp)
 
     def test_filter_out(self):
-        act = tuple(multiplexer.multiplex_yamls(['examples/mux-selftest.yaml'],
+        act = tuple(multiplexer.multiplex_yamls([PATH_PREFIX + 'examples/mux-selftest.yaml'],
                                                 None,
                                                 ('/hw/cpu/intel',
                                                  '/distro/fedora',
@@ -67,8 +70,8 @@ class TestMultiplex(unittest.TestCase):
 
 
 class TestAvocadoParams(unittest.TestCase):
-    yamls = multiplexer.multiplex_yamls(['examples/mux-selftest-params.'
-                                         'yaml'])
+    yamls = iter(multiplexer.multiplex_yamls([PATH_PREFIX + 'examples/mux-selftest-params.'
+                                              'yaml']))
     params1 = multiplexer.AvocadoParams(yamls.next(), 'Unittest1', 1,
                                         ['/ch0/*', '/ch1/*'], {})
     yamls.next()    # Skip 2nd
@@ -171,7 +174,7 @@ class TestAvocadoParams(unittest.TestCase):
         self.assertEqual(self.params1.get('clash2', path='/ch11/*'), 'equal')
         # simple clash in params1
         self.assertRaisesRegexp(ValueError, r"'clash3'.* \['/ch0=>also equal',"
-                                r" '/ch0/ch0.1/ch0.1.2=>also equal'\]",
+                                r" '/ch0/ch0.1b/ch0.1.2=>also equal'\]",
                                 self.params1.get, 'clash3',
                                 default='nnn')
         # params2 is sliced the other way around so it returns before the clash

selftests/all/unit/avocado/tree_unittest.py

@@ -170,20 +170,20 @@ class TestTree(unittest.TestCase):
         self.assertEqual({'new_value': 'something'},
                          oldroot.children[3].children[0].children[0].value)
         # multiplex root (always True)
-        self.assertEqual(tree2.multiplex, True)
+        self.assertEqual(tree2.multiplex, False)
         # multiplex /virt/
-        self.assertEqual(tree2.children[0].multiplex, True)
+        self.assertEqual(tree2.children[0].multiplex, False)
         # multiplex /virt/hw
-        self.assertEqual(tree2.children[0].children[0].multiplex, True)
+        self.assertEqual(tree2.children[0].children[0].multiplex, False)
         # multiplex /virt/distro
-        self.assertEqual(tree2.children[0].children[1].multiplex, False)
+        self.assertEqual(tree2.children[0].children[1].multiplex, True)
         # multiplex /virt/env
-        self.assertEqual(tree2.children[0].children[2].multiplex, False)
+        self.assertEqual(tree2.children[0].children[2].multiplex, True)
         # multiplex /virt/absolutly
-        self.assertEqual(tree2.children[0].children[3].multiplex, True)
+        self.assertEqual(tree2.children[0].children[3].multiplex, False)
         # multiplex /virt/distro/fedora
         self.assertEqual(tree2.children[0].children[1].children[0].multiplex,
-                         True)
+                         False)
 
 
 class TestPathParent(unittest.TestCase):