optional_plugins: Support for any test loader for yaml_to_mux_loader

Use the new ability to report full mappings after the discovery and
allow using `test_reference_resolver_class` to specify the resolver
class intended for test discovery.

To allow sufficient flexibility also support
`test_reference_resolver_args` to override resolver arguments and
`test_reference_resolver_extra` to extend the resolver extra_params.

Note the overridden arguments apply only to the resolver, not to whole
Avocado so it's not possible to override wrappers or such just for a
single test execution. Anyway it is possible to set `vt_config` just for
that single loader, which is more-than useful.
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

docs/source/optional_plugins/varianter_yaml_to_mux.rst

@@ -553,13 +553,19 @@ content, only it works on loader-level, rather than on test variants level.
 The result is that this plugin tries to open the test reference as if it was
 a file specifying variants and if it succeeds it iterates through variants
 and looks for `test_reference` entries. On success it attempts to discover
-the reference using `FileLoader` and then it assigns the current variant's
-params to it. This way one can freely assign various variants to different
-tests.
+the reference using either loader defined by `test_reference_resolver_class`
+or it fall-backs to `FileLoader` when not specified. Then it assigns the
+current variant's params to all of the discovered tests. This way one can
+freely assign various variants to different tests.
 
 Keep in mind YAML files (in Avocado) are ordered, therefor variant name won't
 re-arrange the test order. The only exception is when you use the same variant
 name twice, then the second one will get merged into the first one.
 
+Also note that in case of no `test_reference` or just when no tests are
+discovered in the current variant, there is no error, no warning and
+the loader reports the discovered tests (if any) without the variant
+which did not produced any tests.
+
 The simplest way to learn about this plugin is to look at examples in
 ``examples/yaml_to_mux_loader/``.

examples/yaml_to_mux_loader/loaders.yaml

+# This rather advanced example shows how to specify custom loaders. Note
+# it's also possible to enable loaders that are not enabled by Avocado
+# simply by passing the `test_reference_resolver_class`.
+#
+# test_reference_resolver_class - loadable location of the loader class
+# test_reference_resolver_args  - args to override current Avocado args
+#                                 before being passed to the loader
+#                                 class. (dict)
+# test_reference_resolver_extra - extra_params to be passed to resolver (dict)
+
+tests: !mux
+    instrumented_default:
+        test_reference: passtest.py
+    instrumented_custom:
+        test_reference: passtest.sh
+        # Force-set the FileLoader
+        test_reference_resolver_class: "avocado.core.loader.FileLoader"
+        # Make sure only SIMPLE test types will be detected
+        test_reference_resolver_extra: !!python/dict
+            allowed_test_types: SIMPLE
+    silently_skipped_test:
+        test_reference: passtest.sh
+        # This test will be skipped as it won't be discovered because of type-mismatch
+        test_reference_resolver_class: "avocado.core.loader.FileLoader"
+        test_reference_resolver_extra: !!python/dict
+            allowed_test_types: INSTRUMENTED
+    external_echo:
+        test_reference: "external_echo"
+        # Use ExternalLoader
+        test_reference_resolver_class: "avocado.core.loader.ExternalLoader"
+        # Set the loader_option to "/bin/echo"
+        test_reference_resolver_extra:
+            !!python/dict
+            loader_options: "/bin/echo"
+    external_false:
+        test_reference: "external_false"
+        test_reference_resolver_class: "avocado.core.loader.ExternalLoader"
+        test_reference_resolver_extra: !!python/dict
+            loader_options: "/bin/false"
+    # This demonstrates features which require Avocado-vt installed
+    # avocado-vt-simple:
+    #     test_reference: boot
+    #     test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader"
+    # avocado-vt:
+    #      test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader"
+    #      test_reference_resolver_args:
+    #          !!python/dict
+    #          # Replace this with path to custom --vt-config compatible file
+    #          vt_config: migrate.cfg

optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py

@@ -335,22 +335,63 @@ class YamlTestsuiteLoader(loader.TestLoader):
     """
 
     name = "yaml_testsuite"
+    _extra_type_label_mapping = {}
+    _extra_decorator_mapping = {}
 
     @staticmethod
     def get_type_label_mapping():
         """
-        Currently this plugin uses loader.FileLoader, therefor it can only
-        resolve it's test types.
+        No type is discovered by default, uses "full_*_mappings" to report
+        the actual types after "discover()" is called.
         """
-        return loader.FileLoader.get_type_label_mapping()
+        return {}
+
+    def get_full_type_label_mapping(self):
+        return self._extra_type_label_mapping
 
     @staticmethod
     def get_decorator_mapping():
+        return {}
+
+    def get_full_decorator_mapping(self):
+        return self._extra_decorator_mapping
+
+    def _get_loader(self, params):
         """
-        Currently this plugin uses loader.FileLoader, therefor it can only
-        resolve it's test types.
+        Initializes test loader according to params.
+
+        Uses params.get():
+          test_reference_resolver_class - loadable location of the loader class
+          test_reference_resolver_args - args to override current Avocado args
+                                         before being passed to the loader
+                                         class. (dict)
+          test_reference_resolver_extra - extra_params to be passed to resolver
+                                          (dict)
         """
-        return loader.FileLoader.get_decorator_mapping()
+        resolver_class = params.get("test_reference_resolver_class")
+        if not resolver_class:
+            if params.get("test_reference"):
+                resolver_class = "avocado.core.loader.FileLoader"
+            else:
+                # Don't supply the default when no `test_reference` is given
+                # to avoid listing default FileLoader tests
+                return None
+        mod, klass = resolver_class.rsplit(".", 1)
+        try:
+            loader_class = getattr(__import__(mod, fromlist=[klass]), klass)
+        except ImportError:
+            raise RuntimeError("Unable to import class defined by test_"
+                               "reference_resolver_class '%s.%s'"
+                               % (mod, klass))
+        _args = params.get("test_reference_resolver_args")
+        if not _args:
+            args = self.args
+        else:
+            args = copy.deepcopy(self.args)
+            for key, value in _args.iteritems():
+                setattr(args, key, value)
+        extra_params = params.get("test_reference_resolver_extra", default={})
+        return loader_class(args, extra_params)
 
     def discover(self, reference, which_tests=loader.DEFAULT):
         tests = []
@@ -361,12 +402,19 @@ class YamlTestsuiteLoader(loader.TestLoader):
         except Exception:
             return []
         mux_tree = mux.MuxTree(root)
-        f_loader = loader.FileLoader(self.args, {})
         for variant in mux_tree:
             params = varianter.AvocadoParams(variant, "YamlTestsuiteLoader",
                                              ["/run/*"], {})
             reference = params.get("test_reference")
-            _tests = f_loader.discover(reference)
+            test_loader = self._get_loader(params)
+            if not test_loader:
+                continue
+            _tests = test_loader.discover(reference, which_tests)
+            self._extra_type_label_mapping.update(
+                test_loader.get_full_type_label_mapping())
+            self._extra_decorator_mapping.update(
+                test_loader.get_full_decorator_mapping())
+            if _tests:
                 for tst in _tests:
                     tst[1]["params"] = (variant, ["/run/*"])
                 tests.extend(_tests)

selftests/functional/test_loader.py

@@ -261,6 +261,29 @@ class LoaderTestFunctional(unittest.TestCase):
                     % (AVOCADO, self.tmpdir, mytest))
         self._run_with_timeout(cmd_line, 5)
 
+    @unittest.skipUnless(os.path.exists("/bin/true"), "/bin/true not "
+                         "available")
+    @unittest.skipUnless(os.path.exists("/bin/echo"), "/bin/echo not "
+                         "available")
+    def test_yaml_loader_list(self):
+        # Verifies that yaml_loader list won't crash and is able to detect
+        # various test types
+        result = process.run("%s list -V --loaders yaml_testsuite -- "
+                             "examples/yaml_to_mux_loader/loaders.yaml"
+                             % AVOCADO)
+        # This has to be defined like this as pep8 complains about tailing
+        # empty spaces when using """
+        self.assertRegexpMatches(result.stdout, r"Type *Test *Tag\(s\)\n"
+                                 r"INSTRUMENTED *passtest.py:PassTest.test *"
+                                 "fast\n"
+                                 r"SIMPLE.*passtest.sh *\n"
+                                 r"EXTERNAL *external_echo *\n"
+                                 r"EXTERNAL *external_false *\n")
+        # Also check whether list without loaders won't crash
+        result = process.run("%s list -V -- "
+                             "examples/yaml_to_mux_loader/loaders.yaml"
+                             % AVOCADO)
+
     def test_yaml_loader_run(self):
         # Checks that yaml_loader supplies correct params and that
         # --mux-suite-only filters the test suite