Merge branch 'ldoktor-loader2'

avocado/core/job.py

@@ -48,6 +48,7 @@ from ..utils import archive
 from ..utils import astring
 from ..utils import path
 from ..utils import runtime
+from ..utils import stacktrace
 
 try:
     from .plugins import htmlresult
@@ -446,7 +447,12 @@ class Job(object):
         self.view.start_file_logging(self.logfile,
                                      self.loglevel,
                                      self.unique_id)
-        test_suite = self._make_test_suite(urls)
+        try:
+            test_suite = self._make_test_suite(urls)
+        except loader.LoaderError, details:
+            stacktrace.log_exc_info(sys.exc_info(), 'avocado.app.tracebacks')
+            self._remove_job_results()
+            raise exceptions.OptionValidationError(details)
         if not test_suite:
             self._remove_job_results()
             e_msg = ("No tests found for given urls, try 'avocado list -V %s' "

avocado/core/loader.py

@@ -17,16 +17,19 @@
 Test loader module.
 """
 
+import collections
 import imp
 import inspect
 import os
 import re
 import sys
+import shlex
 import fnmatch
 
 from . import data_dir
 from . import output
 from . import test
+from . import exceptions
 from .settings import settings
 from ..utils import path
 from ..utils import stacktrace
@@ -79,11 +82,20 @@ class TestLoaderProxy(object):
         self._initialized_plugins = []
         self.registered_plugins = []
         self.url_plugin_mapping = {}
+        self._test_types = {}
 
     def register_plugin(self, plugin):
         try:
             if issubclass(plugin, TestLoader):
                 self.registered_plugins.append(plugin)
+                for test_type in plugin.get_type_label_mapping().itervalues():
+                    if (test_type in self._test_types and
+                            self._test_types[test_type] != plugin):
+                        msg = ("Multiple plugins using the same test_type not "
+                               "yet supported (%s, %s)"
+                               % (test_type, self._test_types))
+                        raise NotImplementedError(msg)
+                    self._test_types[test_type] = plugin
             else:
                 raise ValueError
         except ValueError:
@@ -91,23 +103,45 @@ class TestLoaderProxy(object):
                                       "TestLoader" % plugin)
 
     def load_plugins(self, args):
+        def _err_list_loaders():
+            return ("Loaders: %s\nTypes: %s" % (names,
+                                                self._test_types.keys()))
         self._initialized_plugins = []
         # Add (default) file loader if not already registered
         if FileLoader not in self.registered_plugins:
-            self.registered_plugins.append(FileLoader)
+            self.register_plugin(FileLoader)
         # Load plugin by the priority from settings
-        names = [_.name for _ in self.registered_plugins]
-        priority = settings.get_value("plugins", "loader_plugins_priority",
-                                      list, [])
-        sorted_plugins = [self.registered_plugins[names.index(name)]
-                          for name in priority
-                          if name in names]
-        for plugin in sorted_plugins:
-            self._initialized_plugins.append(plugin(args))
-        for plugin in self.registered_plugins:
-            if plugin in sorted_plugins:
-                continue
-            self._initialized_plugins.append(plugin(args))
+        names = ["@" + _.name for _ in self.registered_plugins]
+        loaders = getattr(args, 'loaders', None)
+        if not loaders:
+            loaders = settings.get_value("plugins", "loaders", list, [])
+        if '?' in loaders:
+            raise LoaderError("Loaders: %s\nTypes: %s"
+                              % (names, self._test_types.keys()))
+        if "DEFAULT" in loaders:   # Replace DEFAULT with unused loaders
+            idx = loaders.index("DEFAULT")
+            loaders = (loaders[:idx] + [_ for _ in names if _ not in loaders] +
+                       loaders[idx+1:])
+            while "DEFAULT" in loaders:    # Remove duplicite DEFAULT entries
+                loaders.remove("DEFAULT")
+
+        loaders = [_.split(':', 1) for _ in loaders]
+        priority = [_[0] for _ in loaders]
+        for i, name in enumerate(priority):
+            extra_params = {}
+            if name in names:
+                plugin = self.registered_plugins[names.index(name)]
+            elif name in self._test_types:
+                plugin = self._test_types[name]
+                extra_params['allowed_test_types'] = name
+            else:
+                raise InvalidLoaderPlugin("Loader '%s' not available:\n"
+                                          "Loaders: %s\nTypes: %s"
+                                          % (name, names,
+                                             self._test_types.keys()))
+            if len(loaders[i]) == 2:
+                extra_params['loader_options'] = loaders[i][1]
+            self._initialized_plugins.append(plugin(args, extra_params))
 
     def get_extra_listing(self):
         for loader_plugin in self._initialized_plugins:
@@ -153,8 +187,7 @@ class TestLoaderProxy(object):
         if not urls:
             for loader_plugin in self._initialized_plugins:
                 try:
-                    tests.extend(loader_plugin.discover(None,
-                                                        list_tests))
+                    tests.extend(loader_plugin.discover(None, list_tests))
                 except Exception, details:
                     handle_exception(loader_plugin, details)
         else:
@@ -200,13 +233,14 @@ class TestLoader(object):
     Base for test loader classes
     """
 
-    def __init__(self, args):
+    def __init__(self, args, extra_params):    # pylint: disable=W0613
         self.args = args
 
     def get_extra_listing(self):
         pass
 
-    def get_type_label_mapping(self):
+    @staticmethod
+    def get_type_label_mapping():
         """
         Get label mapping for display in test listing.
 
@@ -214,7 +248,8 @@ class TestLoader(object):
         """
         raise NotImplementedError
 
-    def get_decorator_mapping(self):
+    @staticmethod
+    def get_decorator_mapping():
         """
         Get label mapping for display in test listing.
 
@@ -222,7 +257,7 @@ class TestLoader(object):
         """
         raise NotImplementedError
 
-    def discover(self, url, list_tests=False):
+    def discover(self, url, list_tests=DEFAULT):
         """
         Discover (possible) tests from an url.
 
@@ -254,6 +289,43 @@ class FilteredOut(object):
     pass
 
 
+def add_file_loader_options(parser):
+    loader = parser.add_argument_group('loader options')
+    loader.add_argument('--loaders', nargs='*', help="Overrides the priority "
+                        "of the test loaders. You can specify either "
+                        "@loader_name or TEST_TYPE. By default it tries all "
+                        "available loaders according to priority set in "
+                        "settings->plugins.loaders.")
+    loader.add_argument('--inner-runner', default=None,
+                        metavar='EXECUTABLE',
+                        help=('Path to an specific test runner that '
+                              'allows the use of its own tests. This '
+                              'should be used for running tests that '
+                              'do not conform to Avocado\' SIMPLE test'
+                              'interface and can not run standalone'))
+
+    chdir_help = ('Change directory before executing tests. This option '
+                  'may be necessary because of requirements and/or '
+                  'limitations of the inner test runner. If the inner '
+                  'runner requires to be run from its own base directory,'
+                  'use "runner" here. If the inner runner runs tests based'
+                  ' on files and requires to be run from the directory '
+                  'where those files are located, use "test" here and '
+                  'specify the test directory with the option '
+                  '"--inner-runner-testdir". Defaults to "%(default)s"')
+    loader.add_argument('--inner-runner-chdir', default='off',
+                        choices=('runner', 'test', 'off'),
+                        help=chdir_help)
+
+    loader.add_argument('--inner-runner-testdir', metavar='DIRECTORY',
+                        default=None,
+                        help=('Where test files understood by the inner'
+                              ' test runner are located in the '
+                              'filesystem. Obviously this assumes and '
+                              'only applies to inner test runners that '
+                              'run tests from files'))
+
+
 class FileLoader(TestLoader):
 
     """
@@ -262,8 +334,67 @@ class FileLoader(TestLoader):
 
     name = 'file'
 
-    def get_type_label_mapping(self):
+    def __init__(self, args, extra_params):
+        super(FileLoader, self).__init__(args, extra_params)
+        loader_options = extra_params.get('loader_options')
+        if loader_options == '?':
+            raise LoaderError("File loader accept option to sets the "
+                              "inner-runner executable.")
+        self._inner_runner = self._process_inner_runner(args, loader_options)
+        self.test_type = extra_params.get('allowed_test_types')
+
+    @staticmethod
+    def _process_inner_runner(args, extra_params):
+        """ Enables the inner_runner when asked for """
+        runner = getattr(args, 'inner_runner', None)
+        chdir = getattr(args, 'inner_runner_chdir', 'off')
+        test_dir = getattr(args, 'inner_runner_testdir', None)
+        if extra_params:
+            if runner:
+                msg = ("Inner runner specified via booth: --loaders (%s) and "
+                       "--inner-runner (%s). Please use only one of them"
+                       % (extra_params, runner))
+                raise LoaderError(msg)
+            runner = extra_params
+
+        if runner:
+            inner_runner_and_args = shlex.split(runner)
+            if len(inner_runner_and_args) > 1:
+                executable = inner_runner_and_args[0]
+            else:
+                executable = runner
+            if not os.path.exists(executable):
+                msg = ('Could not find the inner runner executable "%s"'
+                       % executable)
+                raise LoaderError(msg)
+            if chdir == 'test':
+                if not test_dir:
+                    msg = ('Option "--inner-runner-chdir=test" requires '
+                           '"--inner-runner-testdir" to be set.')
+                    raise LoaderError(msg)
+            elif test_dir:
+                msg = ('Option "--inner-runner-testdir" requires '
+                       '"--inner-runner-chdir=test".')
+                raise LoaderError(msg)
+
+            cls_inner_runner = collections.namedtuple('InnerRunner',
+                                                      ['runner', 'chdir',
+                                                       'test_dir'])
+            return cls_inner_runner(runner, chdir, test_dir)
+
+        elif chdir != "off":
+            msg = ('Option "--inner-runner-chdir" requires '
+                   '"--inner-runner" to be set.')
+            raise LoaderError(msg)
+        elif test_dir:
+            msg = ('Option "--inner-runner-test-dir" requires '
+                   '"--inner-runner" to be set.')
+            raise LoaderError(msg)
+
+    @staticmethod
+    def get_type_label_mapping():
         return {test.SimpleTest: 'SIMPLE',
+                test.InnerRunnerTest: 'INNER_RUNNER',
                 test.BuggyTest: 'BUGGY',
                 test.NotATest: 'NOT_A_TEST',
                 test.MissingTest: 'MISSING',
@@ -272,9 +403,11 @@ class FileLoader(TestLoader):
                 test.Test: 'INSTRUMENTED',
                 FilteredOut: 'FILTERED'}
 
-    def get_decorator_mapping(self):
+    @staticmethod
+    def get_decorator_mapping():
         term_support = output.TermSupport()
         return {test.SimpleTest: term_support.healthy_str,
+                test.InnerRunnerTest: term_support.healthy_str,
                 test.BuggyTest: term_support.fail_header_str,
                 test.NotATest: term_support.warn_header_str,
                 test.MissingTest: term_support.fail_header_str,
@@ -290,12 +423,45 @@ class FileLoader(TestLoader):
         Recursively walk in a directory and find tests params.
         The tests are returned in alphabetic order.
 
+        Afterwards when "allowed_test_types" is supplied it verifies if all
+        found tests are of the allowed type. If not return None (even on
+        partial match).
+
+        :param url: the directory path to inspect.
+        :param list_tests: list corrupted/invalid tests too
+        :return: list of matching tests
+        """
+        tests = self._discover(url, list_tests)
+        if self.test_type:
+            mapping = self.get_type_label_mapping()
+            if self.test_type == 'INSTRUMENTED':
+                # Instrumented is parent of all of supported tests, we need to
+                # exclude the rest of the supported tests
+                filtered_clss = tuple(_ for _ in mapping.iterkeys()
+                                      if _ is not test.Test)
+                for tst in tests:
+                    if (not issubclass(tst[0], test.Test) or
+                            issubclass(tst[0], filtered_clss)):
+                        return None
+            else:
+                test_class = (key for key, value in mapping.iteritems()
+                              if value == self.test_type).next()
+                for tst in tests:
+                    if not issubclass(tst[0], test_class):
+                        return None
+        return tests
+
+    def _discover(self, url, list_tests=DEFAULT):
+        """
+        Recursively walk in a directory and find tests params.
+        The tests are returned in alphabetic order.
+
         :param url: the directory path to inspect.
         :param list_tests: list corrupted/invalid tests too
         :return: list of matching tests
         """
-        if test.INNER_RUNNER is not None:
-            return self._make_tests(url, [], [])
+        if self._inner_runner:
+            return self._make_inner_runner_test(url)
 
         if url is None:
             if list_tests is DEFAULT:
@@ -453,6 +619,14 @@ class FileLoader(TestLoader):
         params.setdefault('id', uid)
         return [(klass, {'name': uid, 'params': params})]
 
+    def _make_inner_runner_test(self, test_path):
+        """
+        Creates inner-runner test (adds self._inner_runner as test argument)
+        """
+        tst = self._make_test(test.InnerRunnerTest, test_path)
+        tst[0][1]['inner_runner'] = self._inner_runner
+        return tst
+
     def _make_tests(self, test_path, list_non_tests, subtests_filter=None):
         """
         Create test templates from given path
@@ -464,9 +638,6 @@ class FileLoader(TestLoader):
             """ Always return empty list """
             return []
 
-        if test.INNER_RUNNER is not None:
-            return self._make_test(test.SimpleTest, test_path)
-
         if list_non_tests:   # return broken test with params
             make_broken = self._make_test
         else:               # return empty set instead

avocado/core/plugins/inner_runner.py

-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#
-# See LICENSE for more details.
-#
-# Copyright: Red Hat Inc. 2015
-# Author: Cleber Rosa <cleber@redhat.com>
-
-"""Allows the use of an intermediary inner test runner."""
-
-import os
-import sys
-import shlex
-
-from . import plugin
-from .. import test
-from .. import output
-from .. import exit_codes
-
-
-class InnerRunner(plugin.Plugin):
-
-    """
-    Allows the use of an intermediary inner test runner
-    """
-
-    name = 'inner_runner'
-    enabled = True
-
-    def configure(self, parser):
-        inner_grp = parser.runner.add_argument_group('inner test runner support')
-        inner_grp.add_argument('--inner-runner', default=None,
-                               metavar='EXECUTABLE',
-                               help=('Path to an specific test runner that '
-                                     'allows the use of its own tests. This '
-                                     'should be used for running tests that '
-                                     'do not conform to Avocado\' SIMPLE test'
-                                     'interface and can not run standalone'))
-
-        chdir_help = ('Change directory before executing tests. This option '
-                      'may be necessary because of requirements and/or '
-                      'limitations of the inner test runner. If the inner '
-                      'runner requires to be run from its own base directory,'
-                      'use "runner" here. If the inner runner runs tests based'
-                      ' on files and requires to be run from the directory '
-                      'where those files are located, use "test" here and '
-                      'specify the test directory with the option '
-                      '"--inner-runner-testdir". Defaults to "%(default)s"')
-        inner_grp.add_argument('--inner-runner-chdir', default='off',
-                               choices=('runner', 'test', 'off'),
-                               help=chdir_help)
-
-        inner_grp.add_argument('--inner-runner-testdir', metavar='DIRECTORY',
-                               default=None,
-                               help=('Where test files understood by the inner'
-                                     ' test runner are located in the '
-                                     'filesystem. Obviously this assumes and '
-                                     'only applies to inner test runners that '
-                                     'run tests from files'))
-        self.configured = True
-
-    def activate(self, app_args):
-        self.view = output.View(app_args=app_args)
-
-        if hasattr(app_args, 'inner_runner'):
-            if app_args.inner_runner:
-                inner_runner_and_args = shlex.split(app_args.inner_runner)
-                if len(inner_runner_and_args) > 1:
-                    executable = inner_runner_and_args[0]
-                else:
-                    executable = app_args.inner_runner
-                if not os.path.exists(executable):
-                    msg = 'Could not find the inner runner executable "%s"' % executable
-                    self.view.notify(event='error', msg=msg)
-                    sys.exit(exit_codes.AVOCADO_FAIL)
-                test.INNER_RUNNER = app_args.inner_runner
-
-        if hasattr(app_args, 'inner_runner_testdir'):
-            if app_args.inner_runner_testdir:
-                test.INNER_RUNNER_TESTDIR = app_args.inner_runner_testdir
-
-        if hasattr(app_args, 'inner_runner_chdir'):
-            if app_args.inner_runner_chdir:
-                if app_args.inner_runner_chdir == 'test':
-                    if app_args.inner_runner_testdir is None:
-                        msg = ('Option "--inner-runner-testdir" is mandatory '
-                               'when "--inner-runner-chdir=test" is used.')
-                        self.view.notify(event='error', msg=msg)
-                        sys.exit(exit_codes.AVOCADO_FAIL)
-                test.INNER_RUNNER_CHDIR = app_args.inner_runner_chdir

avocado/core/plugins/runner.py

@@ -22,6 +22,7 @@ from . import plugin
 from .. import exit_codes
 from .. import output
 from .. import job
+from .. import loader
 from .. import multiplexer
 from ..settings import settings
 
@@ -115,6 +116,8 @@ class TestRunner(plugin.Plugin):
                                      'present for the test. '
                                      'Current: on (output check enabled)'))
 
+        loader.add_file_loader_options(self.parser)
+
         if multiplexer.MULTIPLEX_CAPABLE:
             mux = self.parser.add_argument_group('multiplexer use on test execution')
             mux.add_argument('-m', '--multiplex-files', nargs='*',

avocado/core/plugins/test_list.py

@@ -32,7 +32,12 @@ class TestLister(object):
         use_paginator = args.paginator == 'on'
         self.view = output.View(app_args=args, use_paginator=use_paginator)
         self.term_support = output.TermSupport()
-        loader.loader.load_plugins(args)
+        try:
+            loader.loader.load_plugins(args)
+        except loader.LoaderError, details:
+            sys.stderr.write(str(details))
+            sys.stderr.write('\n')
+            sys.exit(exit_codes.AVOCADO_FAIL)
         self.args = args
 
     def _extra_listing(self):
@@ -159,6 +164,7 @@ class TestList(plugin.Plugin):
                                  choices=('on', 'off'), default='on',
                                  help='Turn the paginator on/off. '
                                       'Current: %(default)s')
+        loader.add_file_loader_options(self.parser)
         super(TestList, self).configure(self.parser)
         parser.lister = self.parser
 

avocado/core/test.py

@@ -43,11 +43,6 @@ from ..utils import stacktrace
 from ..utils import data_structures
 
 
-INNER_RUNNER = None
-INNER_RUNNER_TESTDIR = None
-INNER_RUNNER_CHDIR = None
-
-
 class Test(unittest.TestCase):
 
     """
@@ -563,12 +558,9 @@ class SimpleTest(Test):
                                 r' \d\d:\d\d:\d\d WARN \|')
 
     def __init__(self, name, params=None, base_logdir=None, tag=None, job=None):
-        if INNER_RUNNER is None:
-            self.path = os.path.abspath(name)
-        else:
-            self.path = name
         super(SimpleTest, self).__init__(name=name, base_logdir=base_logdir,
                                          params=params, tag=tag, job=job)
+        self.path = name
         basedir = os.path.dirname(self.path)
         basename = os.path.basename(self.path)
         datadirname = basename + '.data'
@@ -587,7 +579,7 @@ class SimpleTest(Test):
         self.log.info("Exit status: %s", result.exit_status)
         self.log.info("Duration: %s", result.duration)
 
-    def test(self):
+    def test(self, override_command=None):
         """
         Run the executable, and log its detailed execution.
         """
@@ -595,34 +587,14 @@ class SimpleTest(Test):
             test_params = dict([(str(key), str(val)) for key, val in
                                 self.params.iteritems()])
 
-            pre_cwd = os.getcwd()
-            new_cwd = None
-            if INNER_RUNNER is not None:
-                self.log.info('Running test with the inner level test '
-                              'runner: "%s"', INNER_RUNNER)
-
-                # Change work directory if needed by the inner runner
-                if INNER_RUNNER_CHDIR == 'runner':
-                    new_cwd = os.path.dirname(INNER_RUNNER)
-                elif INNER_RUNNER_CHDIR == 'test':
-                    new_cwd = INNER_RUNNER_TESTDIR
-                else:
-                    new_cwd = None
-                if new_cwd is not None:
-                    self.log.debug('Changing working directory to "%s" '
-                                   'because of inner runner requirements ',
-                                   new_cwd)
-                    os.chdir(new_cwd)
-
-                command = "%s %s" % (INNER_RUNNER, self.path)
+            if override_command is not None:
+                command = override_command
             else:
                 command = pipes.quote(self.path)
 
             # process.run uses shlex.split(), the self.path needs to be escaped
             result = process.run(command, verbose=True,
                                  env=test_params)
-            if new_cwd is not None:
-                os.chdir(pre_cwd)
 
             self._log_detailed_cmd_info(result)
         except process.CmdError, details:
@@ -638,6 +610,43 @@ class SimpleTest(Test):
                                           "the log for details.")
 
 
+class InnerRunnerTest(SimpleTest):
+    def __init__(self, name, params=None, base_logdir=None, tag=None, job=None,
+                 inner_runner=None):
+        self.assertIsNotNone(inner_runner, "Inner runner test requires "
+                             "inner_runner parameter, got None instead.")
+        self.inner_runner = inner_runner
+        super(InnerRunnerTest, self).__init__(name, params, base_logdir, tag,
+                                              job)
+
+    def test(self):
+        pre_cwd = os.getcwd()
+        new_cwd = None
+        try:
+            self.log.info('Running test with the inner level test '
+                          'runner: "%s"', self.inner_runner.runner)
+
+            # Change work directory if needed by the inner runner
+            if self.inner_runner.chdir == 'runner':
+                new_cwd = os.path.dirname(self.inner_runner.runner)
+            elif self.inner_runner.chdir == 'test':
+                new_cwd = self.inner_runner.test_dir
+            else:
+                new_cwd = None
+            if new_cwd is not None:
+                self.log.debug('Changing working directory to "%s" '
+                               'because of inner runner requirements ',
+                               new_cwd)
+                os.chdir(new_cwd)
+
+            command = "%s %s" % (self.inner_runner.runner, self.path)
+
+            return super(InnerRunnerTest, self).test(command)
+        finally:
+            if new_cwd is not None:
+                os.chdir(pre_cwd)
+
+
 class MissingTest(Test):
 
     """

docs/source/Loaders.rst

+==============
+Test discovery
+==============
+
+In this section you can learn how tests are being discovered and how to affect
+this process.
+
+
+The order of test loaders
+=========================
+
+Avocado supports different types of test starting with `SIMPLE` tests, which
+are simply executable files, then unittest-like tests called `INSTRUMENTED`
+up to some tests like the `avocado-vt` ones, which uses complex
+matrix of tests from config files that don't directl map to existing files.
+Given the number of loaders, the mapping from test names on the command line
+to executed tests might not always be unique. Additionally some people might
+always (or for given run) want to execute only tests of a single type.
+
+To adjust this behavior you can either tweak ``plugins.loaders`` in avocado
+settings (``/etc/avocado/``), or temporarily using ``--loaders``
+(option of ``avocado run``) option.
+
+This option allows you to specify order and some params of the available test
+loaders. You can specify either ``@`` + loader_name (``@file``),
+TEST_TYPE (``SIMPLE``) and for some loaders even additional params passed
+after ``:`` (``@file:/bin/echo -e`` or ``INNER_RUNNER:/bin/echo -e``). You can
+also supply ``DEFAULT``, which injects into that position all the remaining
+unused loaders.
+
+To get help about ``--loaders``::
+
+    $ avocado run --loaders ?
+    $ avocado run --loaders @file:?
+
+Example of how ``--loaders`` affects the produced tests (manually gathered
+as some of them result in error)::
+
+    $ avocado run passtest boot this_does_not_exist /bin/echo
+        > INSTRUMENTED passtest.py:PassTest.test
+        > VT           io-github-autotest-qemu.boot
+        > MISSING      this_does_not_exist
+        > SIMPLE       /bin/echo
+    $ avocado run passtest boot this_does_not_exist /bin/echo --loaders DEFAULT "@file:/bin/echo -e"
+        > INSTRUMENTED passtest.py:PassTest.test
+        > VT           io-github-autotest-qemu.boot
+        > INNER_RUNNER this_does_not_exist
+        > SIMPLE       /bin/echo
+    $ avocado run passtest boot this_does_not_exist /bin/echo --loaders SIMPLE INSTRUMENTED DEFAULT INNER_RUNNER:/bin/echo
+        > INSTRUMENTED passtest.py:PassTest.test
+        > VT           io-github-autotest-qemu.boot
+        > INNER_RUNNER this_does_not_exist
+        > SIMPLE       /bin/echo
+

docs/source/index.rst

@@ -12,6 +12,7 @@ Contents:
    WritingTests
    ResultFormats
    Configuration
+   Loaders
    MultiplexConfig
    RunningTestsRemotely
    DebuggingWithGDB

etc/avocado/avocado.conf

@@ -56,6 +56,10 @@ password =
 # avocado.core.plugins.htmlresult  ImportError No module named pystache
 # add 'avocado.core.plugins.htmlresult' as an element of the list below.
 skip_broken_plugin_notification = []
-# Optionally you can specify the priority of loader plugins. Unspecified
-# plugins will be sorted accordingly to the plugin priorities.
-loader_plugins_priority = ['file']
+# Optionally you can specify the priority of test loaders (@file) or test
+# types (SIMPLE). Some of the plugins even support extra params
+# (INNER_RUNNER:/bin/echo -e). Plugins will be used accordingly to the plugin
+# priorities. It's possible to list plugins multiple times (with different
+# options or just types).
+# The type "DEFAULT" will be replaced with all available unused loaders.
+loaders = ['@file', 'DEFAULT']

selftests/functional/test_basic.py

@@ -443,9 +443,10 @@ class InnerRunnerTest(unittest.TestCase):
                     '--inner-runner-chdir=test %s')
         cmd_line %= (self.tmpdir, self.pass_script.path)
         result = process.run(cmd_line, ignore_status=True)
-        expected_output = 'Option "--inner-runner-testdir" is mandatory'
+        expected_output = ('Option "--inner-runner-chdir=test" requires '
+                           '"--inner-runner-testdir" to be set')
         self.assertIn(expected_output, result.stderr)
-        expected_rc = 3
+        expected_rc = 2
         self.assertEqual(result.exit_status, expected_rc,
                          "Avocado did not return rc %d:\n%s" %
                          (expected_rc, result))

selftests/unit/test_loader.py

@@ -81,7 +81,7 @@ class MultipleMethods(Test):
 class LoaderTest(unittest.TestCase):
 
     def setUp(self):
-        self.loader = loader.FileLoader({})
+        self.loader = loader.FileLoader(None, {})
         self.queue = multiprocessing.Queue()
 
     def test_load_simple(self):