From e99f45e1cbc74d2a8afd702e29760654782742fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Sat, 5 Aug 2017 17:30:05 +0200 Subject: [PATCH] optional_plugins: Support for any test loader for yaml_to_mux_loader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the new ability to report full mappings after the discovery and allow using `test_reference_resolver_class` to specify the resolver class intended for test discovery. To allow sufficient flexibility also support `test_reference_resolver_args` to override resolver arguments and `test_reference_resolver_extra` to extend the resolver extra_params. Note the overridden arguments apply only to the resolver, not to whole Avocado so it's not possible to override wrappers or such just for a single test execution. Anyway it is possible to set `vt_config` just for that single loader, which is more-than useful. Signed-off-by: Lukáš Doktor --- .../varianter_yaml_to_mux.rst | 12 +++- examples/yaml_to_mux_loader/loaders.yaml | 49 +++++++++++++ .../avocado_varianter_yaml_to_mux/__init__.py | 70 ++++++++++++++++--- selftests/functional/test_loader.py | 23 ++++++ 4 files changed, 140 insertions(+), 14 deletions(-) create mode 100644 examples/yaml_to_mux_loader/loaders.yaml diff --git a/docs/source/optional_plugins/varianter_yaml_to_mux.rst b/docs/source/optional_plugins/varianter_yaml_to_mux.rst index 5e1a4175..d019d7fb 100644 --- a/docs/source/optional_plugins/varianter_yaml_to_mux.rst +++ b/docs/source/optional_plugins/varianter_yaml_to_mux.rst @@ -553,13 +553,19 @@ content, only it works on loader-level, rather than on test variants level. The result is that this plugin tries to open the test reference as if it was a file specifying variants and if it succeeds it iterates through variants and looks for `test_reference` entries. On success it attempts to discover -the reference using `FileLoader` and then it assigns the current variant's -params to it. This way one can freely assign various variants to different -tests. +the reference using either loader defined by `test_reference_resolver_class` +or it fall-backs to `FileLoader` when not specified. Then it assigns the +current variant's params to all of the discovered tests. This way one can +freely assign various variants to different tests. Keep in mind YAML files (in Avocado) are ordered, therefor variant name won't re-arrange the test order. The only exception is when you use the same variant name twice, then the second one will get merged into the first one. +Also note that in case of no `test_reference` or just when no tests are +discovered in the current variant, there is no error, no warning and +the loader reports the discovered tests (if any) without the variant +which did not produced any tests. + The simplest way to learn about this plugin is to look at examples in ``examples/yaml_to_mux_loader/``. diff --git a/examples/yaml_to_mux_loader/loaders.yaml b/examples/yaml_to_mux_loader/loaders.yaml new file mode 100644 index 00000000..117767fc --- /dev/null +++ b/examples/yaml_to_mux_loader/loaders.yaml @@ -0,0 +1,49 @@ +# This rather advanced example shows how to specify custom loaders. Note +# it's also possible to enable loaders that are not enabled by Avocado +# simply by passing the `test_reference_resolver_class`. +# +# test_reference_resolver_class - loadable location of the loader class +# test_reference_resolver_args - args to override current Avocado args +# before being passed to the loader +# class. (dict) +# test_reference_resolver_extra - extra_params to be passed to resolver (dict) + +tests: !mux + instrumented_default: + test_reference: passtest.py + instrumented_custom: + test_reference: passtest.sh + # Force-set the FileLoader + test_reference_resolver_class: "avocado.core.loader.FileLoader" + # Make sure only SIMPLE test types will be detected + test_reference_resolver_extra: !!python/dict + allowed_test_types: SIMPLE + silently_skipped_test: + test_reference: passtest.sh + # This test will be skipped as it won't be discovered because of type-mismatch + test_reference_resolver_class: "avocado.core.loader.FileLoader" + test_reference_resolver_extra: !!python/dict + allowed_test_types: INSTRUMENTED + external_echo: + test_reference: "external_echo" + # Use ExternalLoader + test_reference_resolver_class: "avocado.core.loader.ExternalLoader" + # Set the loader_option to "/bin/echo" + test_reference_resolver_extra: + !!python/dict + loader_options: "/bin/echo" + external_false: + test_reference: "external_false" + test_reference_resolver_class: "avocado.core.loader.ExternalLoader" + test_reference_resolver_extra: !!python/dict + loader_options: "/bin/false" + # This demonstrates features which require Avocado-vt installed + # avocado-vt-simple: + # test_reference: boot + # test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader" + # avocado-vt: + # test_reference_resolver_class: "avocado_vt.loader.VirtTestLoader" + # test_reference_resolver_args: + # !!python/dict + # # Replace this with path to custom --vt-config compatible file + # vt_config: migrate.cfg diff --git a/optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py b/optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py index 9f594019..46561126 100644 --- a/optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py +++ b/optional_plugins/varianter_yaml_to_mux/avocado_varianter_yaml_to_mux/__init__.py @@ -335,22 +335,63 @@ class YamlTestsuiteLoader(loader.TestLoader): """ name = "yaml_testsuite" + _extra_type_label_mapping = {} + _extra_decorator_mapping = {} @staticmethod def get_type_label_mapping(): """ - Currently this plugin uses loader.FileLoader, therefor it can only - resolve it's test types. + No type is discovered by default, uses "full_*_mappings" to report + the actual types after "discover()" is called. """ - return loader.FileLoader.get_type_label_mapping() + return {} + + def get_full_type_label_mapping(self): + return self._extra_type_label_mapping @staticmethod def get_decorator_mapping(): + return {} + + def get_full_decorator_mapping(self): + return self._extra_decorator_mapping + + def _get_loader(self, params): """ - Currently this plugin uses loader.FileLoader, therefor it can only - resolve it's test types. + Initializes test loader according to params. + + Uses params.get(): + test_reference_resolver_class - loadable location of the loader class + test_reference_resolver_args - args to override current Avocado args + before being passed to the loader + class. (dict) + test_reference_resolver_extra - extra_params to be passed to resolver + (dict) """ - return loader.FileLoader.get_decorator_mapping() + resolver_class = params.get("test_reference_resolver_class") + if not resolver_class: + if params.get("test_reference"): + resolver_class = "avocado.core.loader.FileLoader" + else: + # Don't supply the default when no `test_reference` is given + # to avoid listing default FileLoader tests + return None + mod, klass = resolver_class.rsplit(".", 1) + try: + loader_class = getattr(__import__(mod, fromlist=[klass]), klass) + except ImportError: + raise RuntimeError("Unable to import class defined by test_" + "reference_resolver_class '%s.%s'" + % (mod, klass)) + _args = params.get("test_reference_resolver_args") + if not _args: + args = self.args + else: + args = copy.deepcopy(self.args) + for key, value in _args.iteritems(): + setattr(args, key, value) + extra_params = params.get("test_reference_resolver_extra", default={}) + return loader_class(args, extra_params) def discover(self, reference, which_tests=loader.DEFAULT): tests = [] @@ -361,15 +402,22 @@ class YamlTestsuiteLoader(loader.TestLoader): except Exception: return [] mux_tree = mux.MuxTree(root) - f_loader = loader.FileLoader(self.args, {}) for variant in mux_tree: params = varianter.AvocadoParams(variant, "YamlTestsuiteLoader", ["/run/*"], {}) reference = params.get("test_reference") - _tests = f_loader.discover(reference) - for tst in _tests: - tst[1]["params"] = (variant, ["/run/*"]) - tests.extend(_tests) + test_loader = self._get_loader(params) + if not test_loader: + continue + _tests = test_loader.discover(reference, which_tests) + self._extra_type_label_mapping.update( + test_loader.get_full_type_label_mapping()) + self._extra_decorator_mapping.update( + test_loader.get_full_decorator_mapping()) + if _tests: + for tst in _tests: + tst[1]["params"] = (variant, ["/run/*"]) + tests.extend(_tests) return tests diff --git a/selftests/functional/test_loader.py b/selftests/functional/test_loader.py index 1b3a3c6d..051024f9 100644 --- a/selftests/functional/test_loader.py +++ b/selftests/functional/test_loader.py @@ -261,6 +261,29 @@ class LoaderTestFunctional(unittest.TestCase): % (AVOCADO, self.tmpdir, mytest)) self._run_with_timeout(cmd_line, 5) + @unittest.skipUnless(os.path.exists("/bin/true"), "/bin/true not " + "available") + @unittest.skipUnless(os.path.exists("/bin/echo"), "/bin/echo not " + "available") + def test_yaml_loader_list(self): + # Verifies that yaml_loader list won't crash and is able to detect + # various test types + result = process.run("%s list -V --loaders yaml_testsuite -- " + "examples/yaml_to_mux_loader/loaders.yaml" + % AVOCADO) + # This has to be defined like this as pep8 complains about tailing + # empty spaces when using """ + self.assertRegexpMatches(result.stdout, r"Type *Test *Tag\(s\)\n" + r"INSTRUMENTED *passtest.py:PassTest.test *" + "fast\n" + r"SIMPLE.*passtest.sh *\n" + r"EXTERNAL *external_echo *\n" + r"EXTERNAL *external_false *\n") + # Also check whether list without loaders won't crash + result = process.run("%s list -V -- " + "examples/yaml_to_mux_loader/loaders.yaml" + % AVOCADO) + def test_yaml_loader_run(self): # Checks that yaml_loader supplies correct params and that # --mux-suite-only filters the test suite -- GitLab