Be lenient with test references

Introducing the --force-execution to create the Job regardless the fact
that some test references might not be resolved to tests.

Reference: https://trello.com/c/bDUJjhq0Signed-off-by: NAmador Pahim <apahim@redhat.com>

avocado/core/job.py

@@ -291,7 +291,8 @@ class Job(object):
         """
         loader.loader.load_plugins(self.args)
         try:
-            suite = loader.loader.discover(references)
+            force = getattr(self.args, 'ignore_missing_references', 'off')
+            suite = loader.loader.discover(references, force=force)
             if getattr(self.args, 'filter_by_tags', False):
                 suite = loader.filter_test_tags(
                     suite,

avocado/core/loader.py

@@ -233,7 +233,7 @@ class TestLoaderProxy(object):
     def get_decorator_mapping(self):
         return self._decorator_mapping
 
-    def discover(self, references, which_tests=DEFAULT):
+    def discover(self, references, which_tests=DEFAULT, force=None):
         """
         Discover (possible) tests from test references.
 
@@ -241,6 +241,8 @@ class TestLoaderProxy(object):
         :type references: builtin.list
         :param which_tests: Limit tests to be displayed (ALL, AVAILABLE or
                             DEFAULT)
+        :param force: don't raise an exception when some test references
+                      are not resolved to tests.
         :return: A list of test factories (tuples (TestClass, test_params))
         """
         def handle_exception(plugin, details):
@@ -278,8 +280,12 @@ class TestLoaderProxy(object):
                 tests.extend([(test.MissingTest, {'name': reference})
                               for reference in unhandled_references])
             else:
-                raise LoaderUnhandledReferenceError(unhandled_references,
-                                                    self._initialized_plugins)
+                if force == 'on':
+                    LOG_UI.error(LoaderUnhandledReferenceError(unhandled_references,
+                                                               self._initialized_plugins))
+                else:
+                    raise LoaderUnhandledReferenceError(unhandled_references,
+                                                        self._initialized_plugins)
         return tests
 
     def load_test(self, test_factory):

avocado/plugins/replay.py

@@ -208,7 +208,8 @@ class Replay(CLI):
                      'external_runner',
                      'external_runner_testdir',
                      'external_runner_chdir',
-                     'failfast']
+                     'failfast',
+                     'ignore_missing_references']
         if replay_args is None:
             LOG_UI.warn('Source job args data not found. These options will '
                         'not be loaded in this replay job: %s',

avocado/plugins/run.py

@@ -85,6 +85,10 @@ class Run(CLICmd):
                             default='off', help='Keep job temporary files '
                             '(useful for avocado debugging). Defaults to off.')
 
+        parser.add_argument('--ignore-missing-references', choices=('on', 'off'),
+                            help="Force the job execution, even if some of "
+                            "the test references are not resolved to tests.")
+
         sysinfo_default = settings.get_value('sysinfo.collect',
                                              'enabled',
                                              key_type='bool',

docs/source/GetStartedGuide.rst

@@ -327,6 +327,34 @@ The ``--failfast`` option accepts the argument ``off``. Since it's disabled
 by default, the ``off`` argument only makes sense in replay jobs, when the
 original job was executed with ``--failfast on``.
 
+Ignoring Missing Test References
+================================
+
+When you provide a list of test references, Avocado will try to resolve
+all of them to tests. If one or more test references can not be resolved
+to tests, the Job will not be created. Example::
+
+    $ avocado run passtest.py badtest.py
+    Unable to resolve reference(s) 'badtest.py' with plugins(s) 'file', 'robot', 'external', try running 'avocado list -V badtest.py' to see the details.
+
+But if you want to execute the Job anyway, with the tests that could be
+resolved, you can use ``--ignore-missing-references on``. The same message
+will appear in the UI, but the Job will be executed::
+
+    $ avocado run passtest.py badtest.py --ignore-missing-references on
+    Unable to resolve reference(s) 'badtest.py' with plugins(s) 'file', 'robot', 'external', try running 'avocado list -V badtest.py' to see the details.
+    JOB ID     : 85927c113074b9defd64ea595d6d1c3fdfc1f58f
+    JOB LOG    : $HOME/avocado/job-results/job-2017-05-17T10.54-85927c1/job.log
+     (1/1) passtest.py:PassTest.test: PASS (0.02 s)
+    RESULTS    : PASS 1 | ERROR 0 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0 | CANCEL 0
+    JOB TIME   : 0.11 s
+    JOB HTML   : $HOME/avocado/job-results/job-2017-05-17T10.54-85927c1/html/results.html
+
+The ``--ignore-missing-references`` option accepts the argument ``off``.
+Since it's disabled by default, the ``off`` argument only makes sense in
+replay jobs, when the original job was executed with
+``--ignore-missing-references on``.
+
 .. _running-external-runner:
 
 Running Tests With An External Runner

selftests/functional/test_basic.py

@@ -190,6 +190,31 @@ class RunnerOperationTest(unittest.TestCase):
         self.assertEqual(result.exit_status, expected_rc,
                          "Avocado did not return rc %d:\n%s" % (expected_rc, result))
 
+    def test_runner_ignore_missing_references_one_missing(self):
+        os.chdir(basedir)
+        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
+                    'passtest.py badtest.py --ignore-missing-references on'
+                    % (AVOCADO, self.tmpdir))
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertIn("Unable to resolve reference(s) 'badtest.py'", result.stderr)
+        self.assertIn('PASS 1 | ERROR 0 | FAIL 0 | SKIP 0', result.stdout)
+        expected_rc = exit_codes.AVOCADO_ALL_OK
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+
+    def test_runner_ignore_missing_references_all_missing(self):
+        os.chdir(basedir)
+        cmd_line = ('%s run --sysinfo=off --job-results-dir %s '
+                    'badtest.py badtest2.py --ignore-missing-references on'
+                    % (AVOCADO, self.tmpdir))
+        result = process.run(cmd_line, ignore_status=True)
+        self.assertIn("Unable to resolve reference(s) 'badtest.py', 'badtest2.py'",
+                      result.stderr)
+        self.assertEqual('', result.stdout)
+        expected_rc = exit_codes.AVOCADO_JOB_FAIL
+        self.assertEqual(result.exit_status, expected_rc,
+                         "Avocado did not return rc %d:\n%s" % (expected_rc, result))
+
     @unittest.skipIf(not CC_BINARY,
                      "C compiler is required by the underlying datadir.py test")
     def test_datadir_alias(self):