avocado: Support for multiple multiplex yaml files

This patch adds support for multiple yaml files by merging the values
together. Merge overrides the nodes value and appends missing children
iteratively. It's deterministic AND order dependant (yaml1 yaml2 migh
not provide the same results as yaml2 yaml1!).
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/tree.py

@@ -65,7 +65,7 @@ class TreeNode(object):
         return '%s: %s' % (self.path, ', '.join(variables))
 
     def __len__(self):
-        return len(self.get_leaves())
+        return len(tuple(self.iter_leaves()))
 
     def __iter__(self):
         return self.iter_leaves()
@@ -85,15 +85,18 @@ class TreeNode(object):
     def add_child(self, node):
         if isinstance(node, self.__class__):
             if node.name in self.children:
-                raise NotImplementedError('Adding children with the same '
-                                          'name is not implemented yet.'
-                                          '\nnode: %s\nchildren: %s'
-                                          % (node, self.children))
-            node.parent = self
-            self.children.append(node)
+                self.children[self.children.index(node.name)].merge(node)
+            else:
+                node.parent = self
+                self.children.append(node)
         else:
             raise ValueError('Bad node type.')
-        return node
+
+    def merge(self, other):
+        """ Merges $other node into this one (doesn't check the name) """
+        self.value.update(other.value)
+        for child in other.children:
+            self.add_child(child)
 
     @property
     def is_leaf(self):
@@ -272,15 +275,17 @@ def _create_from_yaml(stream):
     return tree_node_from_values('', yaml.load(stream, Loader))
 
 
-def create_from_yaml(fileobj):
+def create_from_yaml(paths):
     """
     Create tree structure from yaml-like file
     :param fileobj: File object to be processed
     :raise SyntaxError: When yaml-file is corrupted
     :return: Root of the created tree structure
     """
+    data = TreeNode()
     try:
-        data = _create_from_yaml(fileobj.read())
+        for path in paths:
+            data.merge(_create_from_yaml(open(path).read()))
     except (yaml.scanner.ScannerError, yaml.parser.ParserError) as err:
         raise SyntaxError(err)
     return data

avocado/job.py

@@ -328,12 +328,12 @@ class Job(object):
                 self.loglevel = mapping[raw_log_level]
             else:
                 self.loglevel = logging.DEBUG
-            self.multiplex_file = args.multiplex_file
+            self.multiplex_files = args.multiplex_files
             self.show_job_log = args.show_job_log
             self.silent = args.silent
         else:
             self.loglevel = logging.DEBUG
-            self.multiplex_file = None
+            self.multiplex_files = None
             self.show_job_log = False
             self.silent = False
         if self.show_job_log:
@@ -422,12 +422,12 @@ class Job(object):
             human_plugin = result.HumanTestResult(self.view, self.args)
             self.result_proxy.add_output_plugin(human_plugin)
 
-    def _run(self, urls=None, multiplex_file=None):
+    def _run(self, urls=None, multiplex_files=None):
         """
         Unhandled job method. Runs a list of test URLs to its completion.
 
         :param urls: String with tests to run.
-        :param multiplex_file: File that multiplexes a given test url.
+        :param multiplex_files: File that multiplexes a given test url.
 
         :return: Integer with overall job status. See
                  :mod:`avocado.core.error_codes` for more information.
@@ -450,18 +450,22 @@ class Job(object):
             e_msg = "Empty test ID. A test path or alias must be provided"
             raise exceptions.OptionValidationError(e_msg)
 
-        if multiplex_file is None:
-            if self.args and self.args.multiplex_file is not None:
-                multiplex_file = os.path.abspath(self.args.multiplex_file)
+        if multiplex_files is None:
+            if self.args and self.args.multiplex_files is not None:
+                multiplex_files = self.args.multiplex_files
         else:
-            multiplex_file = os.path.abspath(multiplex_file)
+            multiplex_files = multiplex_files
 
-        if multiplex_file is not None:
+        if multiplex_files is not None:
+            for mux_file in multiplex_files:
+                if not os.path.exists(mux_file):
+                    e_msg = "Multiplex file %s doesn't exist." % (mux_file)
+                    raise exceptions.OptionValidationError(e_msg)
             params_list = []
             if urls is not None:
                 for url in urls:
                     try:
-                        variants = multiplexer.create_variants_from_yaml(open(multiplex_file),
+                        variants = multiplexer.create_variants_from_yaml(multiplex_files,
                                                                          self.args.filter_only,
                                                                          self.args.filter_out)
                     except SyntaxError:
@@ -512,7 +516,7 @@ class Job(object):
         else:
             return error_codes.numeric_status['AVOCADO_TESTS_FAIL']
 
-    def run(self, urls=None, multiplex_file=None):
+    def run(self, urls=None, multiplex_files=None):
         """
         Handled main job method. Runs a list of test URLs to its completion.
 
@@ -520,22 +524,22 @@ class Job(object):
 
         * If urls is provided alone, just make a simple list with no specific
           params (all tests use default params).
-        * If urls and multiplex_file are provided, multiplex provides params
+        * If urls and multiplex_files are provided, multiplex provides params
           and variants to all tests it can.
-        * If multiplex_file is provided alone, just use the matrix produced by
-          the file
+        * If multiplex_files are provided alone, just use the matrix produced
+          by the file
 
         The test runner figures out which tests need to be run on an empty urls
         list by assuming the first component of the shortname is the test url.
 
         :param urls: String with tests to run.
-        :param multiplex_file: File that multiplexes a given test url.
+        :param multiplex_files: File that multiplexes a given test url.
 
         :return: Integer with overall job status. See
                  :mod:`avocado.core.error_codes` for more information.
         """
         try:
-            return self._run(urls, multiplex_file)
+            return self._run(urls, multiplex_files)
         except exceptions.JobBaseException, details:
             self.status = details.status
             fail_class = details.__class__.__name__

avocado/multiplexer.py

@@ -37,7 +37,6 @@ def any_sibling(*nodes):
     return len(nodes) != len(parents)
 
 
-
 def multiplex(*args):
     leaves = []
     parents = collections.OrderedDict()
@@ -78,8 +77,8 @@ def multiplex(*args):
         yield tuple(prod)
 
 
-def create_variants_from_yaml(input_yaml, filter_only=[], filter_out=[]):
-    input_tree = tree.create_from_yaml(input_yaml)
+def create_variants_from_yaml(input_yamls, filter_only=[], filter_out=[]):
+    input_tree = tree.create_from_yaml(input_yamls)
     final_tree = tree.apply_filters(input_tree, filter_only, filter_out)
     leaves = (x for x in final_tree.iter_leaves() if x.parent is not None)
     variants = multiplex(leaves)

avocado/plugins/multiplexer.py

@@ -36,8 +36,8 @@ class Multiplexer(plugin.Plugin):
         self.parser = parser.subcommands.add_parser(
             'multiplex',
             help='Generate a list of dictionaries with params from a multiplex file')
-        self.parser.add_argument('multiplex_file', type=str, nargs='?', default=None,
-                                 help='Path to a multiplex file')
+        self.parser.add_argument('multiplex_files', nargs='+',
+                                 help='Path(s) to a multiplex file(s)')
 
         self.parser.add_argument('--filter-only', nargs='*', default=[],
                                  help='Filter only path(s) from multiplexing')
@@ -54,32 +54,30 @@ class Multiplexer(plugin.Plugin):
 
     def run(self, args):
         view = output.View(app_args=args)
-
-        if not args.multiplex_file:
-            view.notify(event='error', msg='A multiplex file is required, aborting...')
-            sys.exit(error_codes.numeric_status['AVOCADO_JOB_FAIL'])
-
-        multiplex_file = os.path.abspath(args.multiplex_file)
-
-        if not os.path.isfile(multiplex_file):
-            view.notify(event='error', msg='Invalid multiplex file %s' % multiplex_file)
-            sys.exit(error_codes.numeric_status['AVOCADO_JOB_FAIL'])
+        multiplex_files = tuple(os.path.abspath(_)
+                                for _ in args.multiplex_files)
+        for path in multiplex_files:
+            if not os.path.isfile(path):
+                view.notify(event='error',
+                            msg='Invalid multiplex file %s' % path)
+                sys.exit(error_codes.numeric_status['AVOCADO_JOB_FAIL'])
 
         if args.tree:
             view.notify(event='message', msg='Config file tree structure:')
-            t = tree.create_from_yaml(open(multiplex_file))
+            t = tree.create_from_yaml(multiplex_files)
             t = tree.apply_filters(t, args.filter_only, args.filter_out)
             view.notify(event='minor', msg=t.get_ascii())
             sys.exit(error_codes.numeric_status['AVOCADO_ALL_OK'])
 
-        variants = multiplexer.create_variants_from_yaml(open(multiplex_file),
+        variants = multiplexer.create_variants_from_yaml(multiplex_files,
                                                          args.filter_only,
                                                          args.filter_out)
 
         view.notify(event='message', msg='Variants generated:')
         for (index, tpl) in enumerate(variants):
             paths = ', '.join([x.path for x in tpl])
-            view.notify(event='minor', msg='Variant %s:    %s' % (index+1, paths))
+            view.notify(event='minor', msg='Variant %s:    %s' %
+                        (index + 1, paths))
             if args.contents:
                 env = collections.OrderedDict()
                 for node in tpl:

avocado/plugins/runner.py

@@ -97,8 +97,8 @@ class TestRunner(plugin.Plugin):
                                      'Default: False (output check enabled)'))
 
         mux = self.parser.add_argument_group('multiplex arguments')
-        mux.add_argument('-m', '--multiplex-file', type=str, default=None,
-                         help='Path to an avocado multiplex (.yaml) file')
+        mux.add_argument('-m', '--multiplex-files', nargs='*', default=None,
+                         help='Path(s) to a avocado multiplex (.yaml) file(s)')
         mux.add_argument('--filter-only', nargs='*', default=[],
                          help='Filter only path(s) from multiplexing')
         mux.add_argument('--filter-out', nargs='*', default=[],

docs/source/MultiplexConfig.rst

@@ -125,6 +125,37 @@ The environment created for the nodes ``fedora`` and ``osx`` are:
 - Node ``//devtools/fedora`` environment ``compiler: 'gcc'``, ``flags: ['-O2', '-Wall']``
 - None ``//devtools/osx`` environment ``compiler: 'clang'``, ``flags: ['-O2', '-arch i386', '-arch x86_64']``
 
+.. _multiple_files:
+
+Multiple files
+==============
+
+You can provide multiple files. In such scenario final tree is a combination
+of the provided files where later nodes with the same name override values of
+the precending corresponding node. New nodes are appended as new children::
+
+    file-1.yaml:
+        debug:
+            CFLAGS: '-O0 -g'
+        prod:
+            CFLAGS: '-O2'
+
+    file-2.yaml:
+        prod:
+            CFLAGS: '-Os'
+        fast:
+            CFLAGS: '-Ofast'
+
+results in::
+
+    debug:
+        CFLAGS: '-O0 -g'
+    prod:
+        CFLAGS: '-Os'       # overriden
+    fast:
+        CFLAGS: '-Ofast'    # appended
+
+
 .. _variants:
 
 Variants

man/avocado.rst

@@ -35,7 +35,7 @@ of avocado subcommands::
  run         Run one or more tests (test module in .py, test alias or dropin)
  list        List available test modules
  sysinfo     Collect system information
- multiplex   Generate a list of dictionaries with params from a multiplex file
+ multiplex   Generate a list of dictionaries with params from multiplex file(s)
  plugins     List all plugins loaded
  datadir     List all relevant directories used by avocado
 
@@ -45,10 +45,10 @@ To get usage instructions for a given subcommand, run it with `--help`. Example:
  usage: avocado multiplex [-h] [--filter-only [FILTER_ONLY [FILTER_ONLY ...]]]
                           [--filter-out [FILTER_OUT [FILTER_OUT ...]]] [-t]
                           [-c]
-                          [multiplex_file]
+                          multiplex_files [multiplex_files ...]
 
  positional arguments:
-   multiplex_file        Path to a multiplex file
+   multiplex_files       Path(s) to a multiplex file(s)
 
  optional arguments:
    -h, --help            show this help message and exit
@@ -208,7 +208,7 @@ be run::
  Variant 4:    /longest
      sleep_length: 10
 
- $ avocado run --multiplex examples/tests/sleeptest.py.data/sleeptest.yaml sleeptest
+ $ avocado run sleeptest --multiplex-files examples/tests/sleeptest.py.data/sleeptest.yaml
 
 And the output should look like::
 
@@ -235,13 +235,13 @@ the `filter-out` removes one or more paths from being processed.
 From the previous example, if we are interested to use the variants `/medium`
 and `longest`, we do the following command line::
 
- $ avocado run --multiplex examples/tests/sleeptest.py.data/sleeptest.yaml sleeptest \
+ $ avocado run sleeptest --multiplex-files examples/tests/sleeptest.py.data/sleeptest.yaml \
        --filter-only /medium /longest
 
 And if you want to remove `/small` from the variants created,
 we do the following::
 
- $ avocado run --multiplex examples/tests/sleeptest.py.data/sleeptest.yaml sleeptest \
+ $ avocado run sleeptest --multiplex-files examples/tests/sleeptest.py.data/sleeptest.yaml \
        --filter-out /medium
 
 Note that both filters can be arranged in the same command line.