Merge pull request #319 from ldoktor/mux_custom_mapping

avocado.multiplexer: Add support for including files and removing node/values [v1]

avocado/core/tree.py

@@ -34,6 +34,8 @@ original base tree code and re-license under GPLv2+, given that GPLv3 and GPLv2
 """
 
 import collections
+import os
+import re
 
 import yaml
 
@@ -44,6 +46,22 @@ except ImportError:
     from yaml import Loader
 
 
+# Mapping for yaml flags
+YAML_INCLUDE = 0
+YAML_USING = 1
+YAML_REMOVE_NODE = 2
+YAML_REMOVE_VALUE = 3
+
+
+class Control(object):  # Few methods pylint: disable=R0903
+
+    """ Container used to identify node vs. control sequence """
+
+    def __init__(self, code, value=None):
+        self.code = code
+        self.value = value
+
+
 class TreeNode(object):
 
     """
@@ -60,6 +78,7 @@ class TreeNode(object):
         self.parent = parent
         self.children = []
         self._environment = None
+        self.ctrl = []
         for child in children:
             self.add_child(child)
 
@@ -111,6 +130,24 @@ class TreeNode(object):
         added as children (recursively they get either appended at the end
         or merged into existing node in the previous position.
         """
+        for ctrl in other.ctrl:
+            if isinstance(ctrl, Control):
+                if ctrl.code == YAML_REMOVE_NODE:
+                    remove = []
+                    regexp = re.compile(ctrl.value)
+                    for child in self.children:
+                        if regexp.match(child.name):
+                            remove.append(child)
+                    for child in remove:
+                        self.children.remove(child)
+                elif ctrl.code == YAML_REMOVE_VALUE:
+                    remove = []
+                    regexp = re.compile(ctrl.value)
+                    for key in self.value.iterkeys():
+                        if regexp.match(key):
+                            remove.append(key)
+                    for key in remove:
+                        self.value.pop(key, None)
         self.value.update(other.value)
         for child in other.children:
             self.add_child(child)
@@ -127,7 +164,7 @@ class TreeNode(object):
 
     def get_root(self):
         """ Get root of this tree """
-        root = None
+        root = self
         for root in self.iter_parents():
             pass
         return root
@@ -298,14 +335,39 @@ def _create_from_yaml(path, cls_node=TreeNode):
     """ Create tree structure from yaml stream """
     def tree_node_from_values(name, values):
         """ Create `name` node and add values  """
-        node_children = []
-        node_values = []
+        node = cls_node(str(name))
+        using = ''
         for value in values:
             if isinstance(value, TreeNode):
-                node_children.append(value)
+                node.add_child(value)
+            elif isinstance(value[0], Control):
+                if value[0].code == YAML_INCLUDE:
+                    # Include file
+                    ypath = value[1]
+                    if not os.path.isabs(ypath):
+                        ypath = os.path.join(os.path.dirname(path), ypath)
+                    node.merge(_create_from_yaml(ypath, cls_node))
+                elif value[0].code == YAML_USING:
+                    if using:
+                        raise ValueError("!using can be used only once per "
+                                         "node! (%s:%s)" % (path, name))
+                    using = value[1]
+                    if using[0] == '/':
+                        using = using[1:]
+                    if using[-1] == '/':
+                        using = using[:-1]
+                elif value[0].code == YAML_REMOVE_NODE:
+                    value[0].value = value[1]   # set the name
+                    node.ctrl.append(value[0])    # add "blue pill" of death
+                elif value[0].code == YAML_REMOVE_VALUE:
+                    value[0].value = value[1]   # set the name
+                    node.ctrl.append(value[0])
             else:
-                node_values.append(value)
-        return cls_node(name, dict(node_values), children=node_children)
+                node.value[value[0]] = value[1]
+        if using:
+            for name in using.split('/')[::-1]:
+                node = cls_node(name, children=[node])
+        return node
 
     def mapping_to_tree_loader(loader, node):
         """ Maps yaml mapping tag to TreeNode structure """
@@ -323,10 +385,19 @@ def _create_from_yaml(path, cls_node=TreeNode):
             if is_node(values):    # New node
                 objects.append(tree_node_from_values(name, values))
             elif values is None:            # Empty node
-                objects.append(cls_node(name))
+                objects.append(cls_node(str(name)))
             else:                           # Values
                 objects.append(Value((name, values)))
         return objects
+
+    Loader.add_constructor(u'!include',
+                           lambda loader, node: Control(YAML_INCLUDE))
+    Loader.add_constructor(u'!using',
+                           lambda loader, node: Control(YAML_USING))
+    Loader.add_constructor(u'!remove_node',
+                           lambda loader, node: Control(YAML_REMOVE_NODE))
+    Loader.add_constructor(u'!remove_value',
+                           lambda loader, node: Control(YAML_REMOVE_VALUE))
     Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                            mapping_to_tree_loader)
 

docs/source/MultiplexConfig.rst

@@ -66,6 +66,79 @@ The ending nodes (the leafs on the tree) will become part of all lower-level
 However, the precedence is evaluated in top-down or ``last defined`` order.
 In other words, the last parsed has precedence over earlier definitions.
 
+It's also possible to remove node using python's regexp, which can be useful
+when extending upstream file using downstream yaml files. This is done by
+`!remove_node : $value_name` directive::
+
+    os:
+        fedora:
+        windows:
+            3.11:
+            95:
+    os:
+        !remove_node : windows
+        windows:
+            win3.11:
+            win95:
+
+Removes the `windows` node from structure. It's different from `filter-out`
+as it really removes the node (and all children) from the tree and
+it can be replaced by you new structure as shown in the example. It removes
+`windows` with all children and then replaces this structure with slightly
+modified version.
+
+As `!remove_node` is processed during merge, when you reverse the order,
+windows is not removed and you end-up with `/windows/{win3.11,win95,3.11,95}`
+nodes.
+
+Due to yaml nature, it's __mandatory__ to put space between `!remove_node`
+and `:`!
+
+Additionally you can prepend multiple nodes to the given node by using
+`!using : $prepended/path`. This is useful when extending complex structure,
+for example imagine having distro variants in separate ymal files. In the
+end you want to merge them into the `/os` node. The main file can be simply::
+
+    # main.yaml
+    os:
+        !include : os/fedora/21.yaml
+        ....
+
+And each file can look either like this::
+
+    # fedora/21.yaml
+    fedora:
+        21:
+            some: value
+
+or you can use `!using` which prepends the `fedora/21`::
+
+    # fedora/21.yaml
+    !using : /fedora/21
+    some: value
+
+To be precise there is a way to define the structure in the main yaml file::
+
+    # main.yaml
+    os:
+        fedora:
+            21:
+                !include : fedora_21.yaml
+
+Or use recursive `!include` (slower)::
+
+    # main.yaml
+    os:
+        fedora:
+            !include : os/fedora.yaml
+    # os/fedora.yaml
+    21:
+        !include : fedora/21.yaml
+    # os/fedora/21.yaml
+    some: value
+
+Due to yaml nature, it's __mandatory__ to put space between `!using` and `:`!
+
 .. _keys_and_values:
 
 Keys and Values
@@ -90,6 +163,22 @@ And lists::
 
 The list above will become ``['-O2', '-g', '-Wall']`` to Python. In fact,
 YAML is compatible to JSON.
+
+It's also possible to remove key using python's regexp, which can be useful
+when extending upstream file using downstream yaml files. This is done by
+`!remove_value : $value_name` directive::
+
+    debug:
+        CFLAGS: '-O0 -g'
+    debug:
+        !remove_value: CFLAGS
+
+removes the CFLAGS value completely from the debug node. This happens during
+the merge and only once. So if you switch the two, CFLAGS would be defined.
+
+Due to yaml nature, it's __mandatory__ to put space between `!remove_value`
+and `:`!
+
 .. _environment:
 
 Environment
@@ -155,6 +244,21 @@ results in::
     fast:
         CFLAGS: '-Ofast'    # appended
 
+It's also possilbe to include existing file into other file's node. This
+is done by `!include : $path` directive::
+
+    os:
+        fedora:
+            !include : fedora.yaml
+        gentoo:
+            !include : gentoo.yaml
+
+Due to yaml nature, it's __mandatory__ to put space between `!include` and `:`!
+
+The file location can be either absolute path or relative path to the yaml
+file where the `!include` is called (even when it's nested).
+
+Whole file is __merged__ into the node where it's defined.
 
 .. _variants:
 

examples/mux-selftest-advanced.yaml

+# Put everything into /virt
+!using : virt
+# Following line makes it look exactly as mux-selftest.yaml
+!include : mux-selftest.yaml
+distro:
+    # This line extends the distro branch using include
+    !include : mux-selftest-distro.yaml
+    # remove node called "mint"
+    !remove_node : mi.*nt
+    # This is a new /distro/mint appended as latest child
+    mint:
+        new_mint: True
+    fedora:
+        !remove_value : init.*
+        new_init: systemd
+    gentoo:
+        # This modifies the value of 'is_cool'
+        is_cool: True
+        # And this removes the original 'is_cool'
+        # Setting happens after ctrl so it should be created'
+        !remove_value : is_cool
+# This creates new branch the usual way
+new_node:
+    # Put this new_node into /absolutely/fresh/ ('/' are automatically
+    # removed during parse time, absolute location is not supported and
+    # not even planned)
+    !using : /absolutely/fresh/
+    new_value: "something"

examples/mux-selftest-distro.yaml

+RHEL:
+    enterprise: true
+    6:
+        init: 'systemv'
+    7:
+        init: 'systemd'
+gentoo:
+    is_cool: False

selftests/all/unit/avocado/tree_unittest.py

@@ -52,8 +52,6 @@ class TestTree(unittest.TestCase):
         # Add_child existing
         tree3.add_child(tree2.children[0])
         self.assertEqual(tree3, tree2)
-        # Add_child incorrect class
-        self.assertRaises(ValueError, tree3.add_child, 'probably_bad_type')
 
     def test_links(self):
         """ Verify child->parent links """
@@ -152,6 +150,22 @@ class TestTree(unittest.TestCase):
         self.assertEqual({'nic': 'virtio'},
                          tree2.children[0].children[2].children[1].value)
 
+    def test_advanced_yaml(self):
+        tree2 = tree.create_from_yaml(['examples/mux-selftest-advanced.yaml'])
+        exp = ['intel', 'amd', 'arm', 'scsi', 'virtio', 'fedora', '6',
+               '7', 'gentoo', 'mint', 'prod', 'new_node']
+        act = tree2.get_leaves()
+        oldroot = tree2.children[0]
+        self.assertEqual(exp, act)
+        self.assertEqual({'enterprise': True},
+                         oldroot.children[1].children[1].value)
+        self.assertEqual({'new_init': 'systemd'},
+                         oldroot.children[1].children[0].value)
+        self.assertEqual({'is_cool': True},
+                         oldroot.children[1].children[2].value)
+        self.assertEqual({'new_value': 'something'},
+                         oldroot.children[3].children[0].children[0].value)
+
 
 class TestPathParent(unittest.TestCase):