From 8b7122f56c1cd99eb9eb16d40347c06644e074a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Fri, 28 Nov 2014 17:08:14 +0100 Subject: [PATCH] avocado.multiplexer: Construct the tree directly from yaml file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of creating OrderedDicts and then converting it into tree structure, this modifies the `mapping` constructor to create the tree structure directly. This way we only walk through the yaml file once. Also remove the `tree_unittest.TestReadYAML` selftest as it lost it's meaning. Valures are checked in the TestTreeNode selftest. Signed-off-by: Lukáš Doktor --- avocado/core/tree.py | 101 +++++++++++------- avocado/plugins/multiplexer.py | 3 +- .../all/unit/avocado/multiplexer_unittest.py | 5 +- selftests/all/unit/avocado/tree_unittest.py | 38 ++----- 4 files changed, 75 insertions(+), 72 deletions(-) diff --git a/avocado/core/tree.py b/avocado/core/tree.py index d13486bb..92064bfe 100644 --- a/avocado/core/tree.py +++ b/avocado/core/tree.py @@ -34,7 +34,6 @@ original base tree code and re-license under GPLv2+, given that GPLv3 and GPLv2 """ import collections - import yaml @@ -65,8 +64,21 @@ class TreeNode(object): def __iter__(self): return self.iter_leaves() + def __eq__(self, other): + if isinstance(other, str): # Compare names + if self.name == other: + return True + elif isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return False + def add_child(self, node): if isinstance(node, self.__class__): + if node.name in self.children: + raise NotImplementedError('Adding children with the same ' + 'name is not implemented yet.' + '\nnode: %s\nchildren: %s' + % (node, self.children)) node.parent = self self.children.append(node) else: @@ -117,7 +129,7 @@ class TreeNode(object): def get_environment(self): def update_or_extend(target, source): for k, _ in source.items(): - if target.has_key(k) and isinstance(target[k], list): + if k in target and isinstance(target[k], list): target[k].extend(source[k]) else: if isinstance(source[k], list): @@ -207,50 +219,63 @@ class TreeNode(object): return self -def ordered_load(stream, Loader=yaml.Loader, - object_pairs_hook=collections.OrderedDict): - class OrderedLoader(Loader): - pass - OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - lambda loader, node: - object_pairs_hook(loader.construct_pairs(node))) - return yaml.load(stream, OrderedLoader) +def _create_from_yaml(stream): + """ Create tree structure from yaml stream """ + class Value(tuple): + """ Used to mark values to simplify checking for node vs. value """ + pass -def read_ordered_yaml(fileobj): + def tree_node_from_values(name, values): + """ Create $name node and add values """ + node_children = [] + node_values = [] + for value in values: + if isinstance(value, TreeNode): + node_children.append(value) + else: + node_values.append(value) + return TreeNode(name, dict(node_values), children=node_children) + + def mapping_to_tree_loader(loader, node): + def is_node(values): + if isinstance(values, dict): # dicts are always nodes + return True + elif (isinstance(values, list) and values + and isinstance(values[0], (Value, TreeNode))): + # When any value is TreeNode or Value, all of them are already + # parsed and we can wrap them into self + return True + + _value = loader.construct_pairs(node) + objects = [] + for name, values in _value: + if is_node(values): # New node + objects.append(tree_node_from_values(name, values)) + elif values is None: # Empty node + objects.append(TreeNode(name)) + else: # Values + objects.append(Value((name, values))) + return objects + yaml.Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + mapping_to_tree_loader) + return tree_node_from_values('', yaml.load(stream, yaml.Loader)) + + +def create_from_yaml(fileobj): + """ + Create tree structure from yaml-like file + :param fileobj: File object to be processed + :raise SyntaxError: When yaml-file is corrupted + :return: Root of the created tree structure + """ try: - data = ordered_load(fileobj.read()) + data = _create_from_yaml(fileobj.read()) except (yaml.scanner.ScannerError, yaml.parser.ParserError) as err: raise SyntaxError(err) return data -def create_from_ordered_data(data, tree=None, root=None, name=''): - if tree is None: - tree = TreeNode(name) - if root is None: - root = tree - if isinstance(data, dict): - for key, value in data.items(): - if isinstance(value, dict): - node = TreeNode(key) - tree.add_child(node) - create_from_ordered_data(value, node, root) - elif value is None: - # Leaf without variable - node = TreeNode(key) - tree.add_child(node) - else: - # Node/leaf with variable - tree.value[key] = value - return root - - -def create_from_yaml(input_yaml): - data = read_ordered_yaml(input_yaml) - return create_from_ordered_data(data) - - def path_parent(path): """ From a given path, return its parent path. diff --git a/avocado/plugins/multiplexer.py b/avocado/plugins/multiplexer.py index bf6e874b..79cf1f8f 100644 --- a/avocado/plugins/multiplexer.py +++ b/avocado/plugins/multiplexer.py @@ -67,8 +67,7 @@ class Multiplexer(plugin.Plugin): if args.tree: view.notify(event='message', msg='Config file tree structure:') - data = tree.read_ordered_yaml(open(multiplex_file)) - t = tree.create_from_ordered_data(data) + t = tree.create_from_yaml(open(multiplex_file)) t = tree.apply_filters(t, args.filter_only, args.filter_out) view.notify(event='minor', msg=t.get_ascii()) sys.exit(error_codes.numeric_status['AVOCADO_ALL_OK']) diff --git a/selftests/all/unit/avocado/multiplexer_unittest.py b/selftests/all/unit/avocado/multiplexer_unittest.py index 0f542316..b4522de0 100644 --- a/selftests/all/unit/avocado/multiplexer_unittest.py +++ b/selftests/all/unit/avocado/multiplexer_unittest.py @@ -25,7 +25,7 @@ class TestAnySibling(unittest.TestCase): os = t.add_child(TreeNode('os')) linux = os.add_child(TreeNode('linux')) self.mint = linux.add_child(TreeNode('mint')) - self.fedora = linux.add_child(TreeNode('mint')) + self.fedora = linux.add_child(TreeNode('fedora')) win = os.add_child(TreeNode('win')) self.winxp = win.add_child(TreeNode('winxp')) self.win7 = win.add_child(TreeNode('win7')) @@ -53,7 +53,6 @@ class TestAnySibling(unittest.TestCase): class TestMultiplex(unittest.TestCase): def setUp(self): - t1 = TreeNode() e = t1.add_child(TreeNode('env')) e.add_child(TreeNode('production')) @@ -71,7 +70,7 @@ class TestMultiplex(unittest.TestCase): st = t2.add_child(TreeNode('tests')).add_child(TreeNode('sync_test')) st.add_child(TreeNode('standard')) st.add_child(TreeNode('aggressive')) - pt = t2.add_child(TreeNode('tests')).add_child(TreeNode('ping_test')) + pt = t2.children[1].add_child(TreeNode('ping_test')) pt.add_child(TreeNode('standard')) pt.add_child(TreeNode('aggressive')) self.tree2 = t2 diff --git a/selftests/all/unit/avocado/tree_unittest.py b/selftests/all/unit/avocado/tree_unittest.py index f1265a30..a53fdab1 100644 --- a/selftests/all/unit/avocado/tree_unittest.py +++ b/selftests/all/unit/avocado/tree_unittest.py @@ -37,42 +37,22 @@ os: """ -class TestReadYAML(unittest.TestCase): - - def setUp(self): - self.yaml = StringIO.StringIO(source_yaml) - - def test_read_yaml_values(self): - data = read_ordered_yaml(self.yaml) - self.assertIsInstance(data, dict) - self.assertIn('hw', data) - self.assertIn('os', data) - self.assertIn('cpu', data['hw']) - self.assertIn('intel', data['hw']['cpu']) - self.assertIsInstance(data['hw']['cpu']['intel']['arch'], list) - self.assertIn('arm', data['hw']['cpu']) - self.assertIsInstance(data['hw']['cpu']['arm']['arch'], list) - self.assertIn('os', data) - self.assertIn('linux', data['os']) - self.assertIn('dev-tools', data['os']['linux']) - self.assertIn('fedora', data['os']['linux']) - self.assertIn('mint', data['os']['linux']) - self.assertIn('centos', data['os']['linux']) - self.assertIn('win', data['os']) - self.assertIn('dev-tools', data['os']['win']) - self.assertIn('win7', data['os']['win']) - self.assertIn('win8', data['os']['win']) - - class TestTreeNode(unittest.TestCase): def setUp(self): - self.data = read_ordered_yaml(StringIO.StringIO(source_yaml)) - self.treenode = create_from_ordered_data(self.data) + self.treenode = create_from_yaml(StringIO.StringIO(source_yaml)) def test_create_treenode(self): self.assertIsInstance(self.treenode, TreeNode) + def test_comparison(self): + self.assertEqual(self.treenode.children[0], self.treenode.children[0]) + self.assertEqual(self.treenode.children[0], "hw") + self.assertEqual("hw", self.treenode.children[0]) + self.assertNotEqual(self.treenode.children[0], + self.treenode.children[1]) + self.assertNotEqual(self.treenode.children[0], "nothw") + def test_node_order(self): self.assertEqual(self.treenode.children[0].name, 'hw') self.assertEqual(self.treenode.children[0].children[0].name, 'cpu') -- GitLab