Merge branch 'ldoktor-yaml_testsuite_loader2'

Signed-off-by: NAmador Pahim <apahim@redhat.com>

avocado/core/loader.py

@@ -208,7 +208,6 @@ class TestLoaderProxy(object):
                 extra_params['loader_options'] = loaders[i][1]
             plugin = self.registered_plugins[supported_loaders.index(name)]
             self._initialized_plugins.append(plugin(args, extra_params))
-        self._update_mappings()
 
     def _update_mappings(self):
         """
@@ -217,11 +216,11 @@ class TestLoaderProxy(object):
         # Plugins are initialized, let's update mappings
         self._label_mapping = {MissingTest: "MISSING"}
         for plugin in self._initialized_plugins:
-            self._label_mapping.update(plugin.get_type_label_mapping())
+            self._label_mapping.update(plugin.get_full_type_label_mapping())
         self._decorator_mapping = {MissingTest:
                                    output.TERM_SUPPORT.fail_header_str}
         for plugin in self._initialized_plugins:
-            self._decorator_mapping.update(plugin.get_decorator_mapping())
+            self._decorator_mapping.update(plugin.get_full_decorator_mapping())
 
     def get_extra_listing(self):
         for loader_plugin in self._initialized_plugins:
@@ -234,9 +233,15 @@ class TestLoaderProxy(object):
         return base_path
 
     def get_type_label_mapping(self):
+        if self._label_mapping is None:
+            raise RuntimeError("LoaderProxy.discover has to be called before "
+                               "LoaderProxy.get_type_label_mapping")
         return self._label_mapping
 
     def get_decorator_mapping(self):
+        if self._label_mapping is None:
+            raise RuntimeError("LoaderProxy.discover has to be called before "
+                               "LoaderProxy.get_decorator_mapping")
         return self._decorator_mapping
 
     def discover(self, references, which_tests=DEFAULT, force=None):
@@ -292,6 +297,7 @@ class TestLoaderProxy(object):
                 else:
                     raise LoaderUnhandledReferenceError(unhandled_references,
                                                         self._initialized_plugins)
+        self._update_mappings()
         return tests
 
     def load_test(self, test_factory):
@@ -390,6 +396,12 @@ class TestLoader(object):
         """
         raise NotImplementedError
 
+    def get_full_type_label_mapping(self):     # pylint: disable=R0201
+        """
+        Allows extending the type-label-mapping after the object is initialized
+        """
+        return self.get_type_label_mapping()
+
     @staticmethod
     def get_decorator_mapping():
         """
@@ -399,6 +411,12 @@ class TestLoader(object):
         """
         raise NotImplementedError
 
+    def get_full_decorator_mapping(self):      # pylint: disable=R0201
+        """
+        Allows extending the decorator-mapping after the object is initialized
+        """
+        return self.get_decorator_mapping()
+
     def discover(self, reference, which_tests=DEFAULT):
         """
         Discover (possible) tests from an reference.

avocado/core/mux.py

@@ -22,7 +22,6 @@ a custom Varianter plugin.
 #
 
 import collections
-import hashlib
 import itertools
 import re
 
@@ -167,13 +166,8 @@ class MuxPlugin(object):
         self.variant_ids = self._get_variant_ids()
 
     def _get_variant_ids(self):
-        variant_ids = []
-        for variant in MuxTree(self.root):
-            variant.sort(key=lambda x: x.path)
-            fingerprint = "-".join(_.fingerprint() for _ in variant)
-            variant_ids.append("-".join(node.name for node in variant) + '-' +
-                               hashlib.sha1(fingerprint).hexdigest()[:4])
-        return variant_ids
+        return [varianter.generate_variant_id(variant)
+                for variant in MuxTree(self.root)]
 
     def __iter__(self):
         """

avocado/core/runner.py

@@ -29,6 +29,7 @@ from . import test
 from . import exceptions
 from . import output
 from . import status
+from . import varianter
 from .loader import loader
 from .status import mapping
 from ..utils import wait
@@ -506,10 +507,19 @@ class TestRunner(object):
         params = variant.get("variant"), variant.get("mux_path")
         if params:
             if "params" in template[1]:
-                msg = ("Unable to use test variants %s, params are already"
-                       " present in test factory: %s"
-                       % (template[0], template[1]))
-                raise ValueError(msg)
+                if not varianter.is_empty_variant(params[0]):
+                    msg = ("Specifying test params from test loader and "
+                           "from varianter at the same time is not yet "
+                           "supported. Please remove either variants defined"
+                           "by the varianter (%s) or make the test loader of"
+                           "test %s to not to fill variants."
+                           % (variant, template))
+                    raise NotImplementedError(msg)
+                params = template[1]["params"]
+                variant_id = varianter.generate_variant_id(params[0])
+                return template, {"variant": params[0],
+                                  "variant_id": variant_id,
+                                  "mux_path": params[1]}
             factory = [template[0], template[1].copy()]
             factory[1]["params"] = params
         else:

avocado/core/varianter.py

@@ -19,6 +19,7 @@
 Multiplex and create variants.
 """
 
+import hashlib
 import re
 
 from . import tree
@@ -309,6 +310,30 @@ class AvocadoParam(object):
                 yield (leaf.environment.origin[key].path, key, value)
 
 
+def is_empty_variant(variant):
+    """
+    Reports whether the variant contains any data
+
+    :param variant: Avocado test variant (list of TreeNode-like objects)
+    :return: True when the variant does not contain (any useful) data
+    """
+    return not variant or variant == [tree.TreeNode()] * len(variant)
+
+
+def generate_variant_id(variant):
+    """
+    Basic function to generate variant-id from a variant
+
+    :param variant: Avocado test variant (list of TreeNode-like objects)
+    :return: String compounded of ordered node names and a hash of all
+             values.
+    """
+    variant = sorted(variant, key=lambda x: x.path)
+    fingerprint = "-".join(_.fingerprint() for _ in variant)
+    return ("-".join(node.name for node in variant) + '-' +
+            hashlib.sha1(fingerprint).hexdigest()[:4])
+
+
 def variant_to_str(variant, verbosity, out_args=None, debug=False):
     """
     Reports human readable representation of a variant

docs/source/optional_plugins/varianter_yaml_to_mux.rst

@@ -543,3 +543,29 @@ From this example you can see that querying for ``/env/debug`` works only in
 the first variant, but returns nothing in the second variant. Keep this in mind
 and when you use the ``!mux`` flag always query for the pre-mux path,
 ``/env/*`` in this example.
+
+
+Yaml_to_mux_loader plugin
+=========================
+
+This plugin is part of the `Yaml_to_mux` plugin and it understands the same
+content, only it works on loader-level, rather than on test variants level.
+The result is that this plugin tries to open the test reference as if it was
+a file specifying variants and if it succeeds it iterates through variants
+and looks for `test_reference` entries. On success it attempts to discover
+the reference using either loader defined by `test_reference_resolver_class`
+or it fall-backs to `FileLoader` when not specified. Then it assigns the
+current variant's params to all of the discovered tests. This way one can
+freely assign various variants to different tests.
+
+Keep in mind YAML files (in Avocado) are ordered, therefor variant name won't
+re-arrange the test order. The only exception is when you use the same variant
+name twice, then the second one will get merged into the first one.
+
+Also note that in case of no `test_reference` or just when no tests are
+discovered in the current variant, there is no error, no warning and
+the loader reports the discovered tests (if any) without the variant
+which did not produced any tests.
+
+The simplest way to learn about this plugin is to look at examples in
+``examples/yaml_to_mux_loader/``.

examples/yaml_to_mux_loader/advanced.yaml

+# The purpose of this example is to show you can even override the test
+# timeout and therefor make some variants fail and other pass
+#
+# Also notice the order of params and tests...
+
+timeout: !mux
+    short:
+        timeout: 2
+    longer:
+        timeout: 6
+    no_timeout:
+tests: !mux
+    passtest:
+        test_reference: passtest.py
+    sleeptest:
+        test_reference: sleeptest.py
+        !include : ../tests/sleeptest.py.data/sleeptest.yaml
+    failtest:
+        test_reference: failtest.py
+        some_test_variants: !mux
+            this_fails:
+            this_also_fails:
+            and_this_fails_as_well:

examples/yaml_to_mux_loader/basic.yaml

+# This example shows how to define test as well as params together in one
+# file. To execute it simply install avocado_varianter_yaml_to_mux plugin
+# and run this as if it was a test (avocado run $this_file)
+#
+# test_reference - specifies the test reference to be loaded with the default
+#     (file) loader.
+
+!mux
+passtest:
+    test_reference: passtest.py
+sleeptest:
+    test_reference: sleeptest.py
+failtest:
+    test_reference: failtest.py
+    some_test_variants: !mux
+        this_fails:
+        this_also_fails:
+        and_this_fails_as_well:

examples/yaml_to_mux_loader/loaders.yaml

+# This rather advanced example shows how to specify custom loaders. Note
+# it's also possible to enable loaders that are not enabled by Avocado
+# simply by passing the `test_reference_resolver_class`.
+#
+# test_reference_resolver_class - loadable location of the loader class
+# test_reference_resolver_args  - args to override current Avocado args
+#                                 before being passed to the loader
+#                                 class. (dict)
+# test_reference_resolver_extra - extra_params to be passed to resolver (dict)
+
+tests: !mux
+    instrumented_default:
+        test_reference: passtest.py
+    instrumented_custom:
+        test_reference: passtest.sh
+        # Force-set the FileLoader
+        test_reference_resolver_class: "avocado.core.loader.FileLoader"
+        # Make sure only SIMPLE test types will be detected
+        test_reference_resolver_extra: !!python/dict
+            allowed_test_types: SIMPLE
+    silently_skipped_test:
+        test_reference: passtest.sh
+        # This test will be skipped as it won't be discovered because of type-mismatch
+        test_reference_resolver_class: "avocado.core.loader.FileLoader"
+        test_reference_resolver_extra: !!python/dict
+            allowed_test_types: INSTRUMENTED
+    external_echo:
+        test_reference: "external_echo"
+        # Use ExternalLoader
+        test_reference_resolver_class: "avocado.core.loader.ExternalLoader"
+        # Set the loader_option to "/bin/echo"
+        test_reference_resolver_extra:
+            !!python/dict
+            loader_options: "/bin/echo"
+    external_false:
+        test_reference: "external_false"
+        test_reference_resolver_class: "avocado.core.loader.ExternalLoader"
+        test_reference_resolver_extra: !!python/dict
+            loader_options: "/bin/false"
+    # This demonstrates features which require Avocado-vt installed
+    # avocado-vt-simple:
+    #     test_reference: boot
+    #     test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader"
+    # avocado-vt:
+    #      test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader"
+    #      test_reference_resolver_args:
+    #          !!python/dict
+    #          # Replace this with path to custom --vt-config compatible file
+    #          vt_config: migrate.cfg

optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py

@@ -18,7 +18,7 @@ import os
 import re
 import sys
 
-from avocado.core import tree, exit_codes, mux
+from avocado.core import tree, exit_codes, mux, varianter, loader
 from avocado.core.output import LOG_UI
 from avocado.core.plugin_interfaces import CLI, Varianter
 
@@ -314,10 +314,111 @@ class YamlToMuxCLI(CLI):
                              help="DEPRECATED: Filter out path(s) from "
                              "multiplexing (use --mux-filter-out instead)")
 
+            mux = subparser.add_argument_group("yaml to mux testsuite options")
+            mux.add_argument("--mux-suite-only", nargs="+",
+                             help="Filter only part of the YAML suite file")
+            mux.add_argument("--mux-suite-out", nargs="+",
+                             help="Filter out part of the YAML suite file")
+
     def run(self, args):
         """
         The YamlToMux varianter plugin handles these
         """
+        loader.loader.register_plugin(YamlTestsuiteLoader)
+
+
+class YamlTestsuiteLoader(loader.TestLoader):
+
+    """
+    Gets variants from a YAML file and uses `test_reference` entries
+    to create a test suite.
+    """
+
+    name = "yaml_testsuite"
+    _extra_type_label_mapping = {}
+    _extra_decorator_mapping = {}
+
+    @staticmethod
+    def get_type_label_mapping():
+        """
+        No type is discovered by default, uses "full_*_mappings" to report
+        the actual types after "discover()" is called.
+        """
+        return {}
+
+    def get_full_type_label_mapping(self):
+        return self._extra_type_label_mapping
+
+    @staticmethod
+    def get_decorator_mapping():
+        return {}
+
+    def get_full_decorator_mapping(self):
+        return self._extra_decorator_mapping
+
+    def _get_loader(self, params):
+        """
+        Initializes test loader according to params.
+
+        Uses params.get():
+          test_reference_resolver_class - loadable location of the loader class
+          test_reference_resolver_args - args to override current Avocado args
+                                         before being passed to the loader
+                                         class. (dict)
+          test_reference_resolver_extra - extra_params to be passed to resolver
+                                          (dict)
+        """
+        resolver_class = params.get("test_reference_resolver_class")
+        if not resolver_class:
+            if params.get("test_reference"):
+                resolver_class = "avocado.core.loader.FileLoader"
+            else:
+                # Don't supply the default when no `test_reference` is given
+                # to avoid listing default FileLoader tests
+                return None
+        mod, klass = resolver_class.rsplit(".", 1)
+        try:
+            loader_class = getattr(__import__(mod, fromlist=[klass]), klass)
+        except ImportError:
+            raise RuntimeError("Unable to import class defined by test_"
+                               "reference_resolver_class '%s.%s'"
+                               % (mod, klass))
+        _args = params.get("test_reference_resolver_args")
+        if not _args:
+            args = self.args
+        else:
+            args = copy.deepcopy(self.args)
+            for key, value in _args.iteritems():
+                setattr(args, key, value)
+        extra_params = params.get("test_reference_resolver_extra", default={})
+        return loader_class(args, extra_params)
+
+    def discover(self, reference, which_tests=loader.DEFAULT):
+        tests = []
+        try:
+            root = mux.apply_filters(create_from_yaml([reference], False),
+                                     getattr(self.args, "mux_suite_only", []),
+                                     getattr(self.args, "mux_suite_out", []))
+        except Exception:
+            return []
+        mux_tree = mux.MuxTree(root)
+        for variant in mux_tree:
+            params = varianter.AvocadoParams(variant, "YamlTestsuiteLoader",
+                                             ["/run/*"], {})
+            reference = params.get("test_reference")
+            test_loader = self._get_loader(params)
+            if not test_loader:
+                continue
+            _tests = test_loader.discover(reference, which_tests)
+            self._extra_type_label_mapping.update(
+                test_loader.get_full_type_label_mapping())
+            self._extra_decorator_mapping.update(
+                test_loader.get_full_decorator_mapping())
+            if _tests:
+                for tst in _tests:
+                    tst[1]["params"] = (variant, ["/run/*"])
+                tests.extend(_tests)
+        return tests
 
 
 class YamlToMux(mux.MuxPlugin, Varianter):

selftests/functional/test_loader.py

@@ -261,6 +261,64 @@ class LoaderTestFunctional(unittest.TestCase):
                     % (AVOCADO, self.tmpdir, mytest))
         self._run_with_timeout(cmd_line, 5)
 
+    @unittest.skipUnless(os.path.exists("/bin/true"), "/bin/true not "
+                         "available")
+    @unittest.skipUnless(os.path.exists("/bin/echo"), "/bin/echo not "
+                         "available")
+    def test_yaml_loader_list(self):
+        # Verifies that yaml_loader list won't crash and is able to detect
+        # various test types
+        result = process.run("%s list -V --loaders yaml_testsuite -- "
+                             "examples/yaml_to_mux_loader/loaders.yaml"
+                             % AVOCADO)
+        # This has to be defined like this as pep8 complains about tailing
+        # empty spaces when using """
+        self.assertRegexpMatches(result.stdout, r"Type *Test *Tag\(s\)\n"
+                                 r"INSTRUMENTED *passtest.py:PassTest.test *"
+                                 "fast\n"
+                                 r"SIMPLE.*passtest.sh *\n"
+                                 r"EXTERNAL *external_echo *\n"
+                                 r"EXTERNAL *external_false *\n")
+        # Also check whether list without loaders won't crash
+        result = process.run("%s list -V -- "
+                             "examples/yaml_to_mux_loader/loaders.yaml"
+                             % AVOCADO)
+
+    def test_yaml_loader_run(self):
+        # Checks that yaml_loader supplies correct params and that
+        # --mux-suite-only filters the test suite
+        result = process.run("%s --show test run --dry-run --mux-suite-only "
+                             "/run/tests/sleeptest -- examples/yaml_to_mux_"
+                             "loader/advanced.yaml" % AVOCADO)
+        test = -1
+        exp_timeouts = [2] * 4 + [6] * 4 + [None] * 4
+        exp_timeout = None
+        exp_sleep_lengths = [0.5, 1, 5, 10] * 3
+        exp_sleep_length = None
+        for line in result.stdout.splitlines():
+            if line.startswith("START "):
+                self.assertFalse(exp_timeout, "%s was not found in test %ss "
+                                 "output:\n%s" % (exp_timeout, test, result))
+                self.assertFalse(exp_timeout, "%s was not found in test %ss "
+                                 "output:\n%s" % (exp_sleep_length, test,
+                                                  result))
+                self.assertLess(test, 12, "Number of tests is greater than "
+                                "12:\n%s" % result)
+                test += 1
+                timeout = exp_timeouts[test]
+                if timeout:
+                    exp_timeout = "timeout ==> %s" % timeout
+                else:
+                    exp_timeout = "(key=timeout, path=*, default=None) => None"
+                exp_sleep_length = ("sleep_length ==> %s"
+                                    % exp_sleep_lengths[test])
+            elif exp_timeout and exp_timeout in line:
+                exp_timeout = None
+            elif exp_sleep_length and exp_sleep_length in line:
+                exp_sleep_length = None
+        self.assertEqual(test, 11, "Number of tests is not 12 (%s):\n%s"
+                         % (test, result))
+
     def tearDown(self):
         shutil.rmtree(self.tmpdir)