Functional tests: prevent the persistence of temporary job results

Some functional tests, most of which execute a full avocado command,
were missing the `--job-results-dir`, pointing to a directory that
would be removed after the tests.  This has led to the pollution of
the standard job results directory (usually `~/avocado/job-results`)
when selftests are run.

This proposal fixes all but one set of functional tests, which has
to be fixed with a different set of changes (see reference).

Besides the functional tests, some unittests are also creating
job results that should be discarded.  It's believed that at least
some of this is caused by the relationship between tests and jobs,
but this is speculation and it's not attepted to be fixed on this
patch.

Reference: https://trello.com/c/I2ieqhR5Signed-off-by: NCleber Rosa <crosa@redhat.com>

selftests/functional/test_basic.py

@@ -385,7 +385,7 @@ class RunnerOperationTest(unittest.TestCase):
 
     def test_empty_test_list(self):
         os.chdir(basedir)
-        cmd_line = './scripts/avocado run --sysinfo=off'
+        cmd_line = './scripts/avocado run --sysinfo=off --job-results-dir %s' % self.tmpdir
         result = process.run(cmd_line, ignore_status=True)
         expected_rc = exit_codes.AVOCADO_JOB_FAIL
         expected_output = 'No urls provided nor any arguments produced'
@@ -394,7 +394,7 @@ class RunnerOperationTest(unittest.TestCase):
 
     def test_not_found(self):
         os.chdir(basedir)
-        cmd_line = './scripts/avocado run --sysinfo=off sbrubles'
+        cmd_line = './scripts/avocado run --sysinfo=off --job-results-dir %s sbrubles' % self.tmpdir
         result = process.run(cmd_line, ignore_status=True)
         expected_rc = exit_codes.AVOCADO_JOB_FAIL
         self.assertEqual(result.exit_status, expected_rc)
@@ -402,8 +402,8 @@ class RunnerOperationTest(unittest.TestCase):
         self.assertNotIn('Unable to discover url', result.stdout)
 
     def test_invalid_unique_id(self):
-        cmd_line = ('./scripts/avocado run --sysinfo=off --force-job-id foobar'
-                    ' passtest.py')
+        cmd_line = ('./scripts/avocado run --sysinfo=off --job-results-dir '
+                    '%s --force-job-id foobar passtest.py' % self.tmpdir)
         result = process.run(cmd_line, ignore_status=True)
         self.assertNotEqual(result.exit_status, exit_codes.AVOCADO_ALL_OK)
         self.assertIn('needs to be a 40 digit hex', result.stderr)
@@ -485,8 +485,8 @@ class RunnerOperationTest(unittest.TestCase):
         os.chdir(basedir)
         test = script.make_script(os.path.join(self.tmpdir, 'test.py'),
                                   INVALID_PYTHON_TEST)
-        cmd_line = './scripts/avocado --show test run --sysinfo=off '\
-                   '--job-results-dir %s %s' % (self.tmpdir, test)
+        cmd_line = ('./scripts/avocado --show test run --sysinfo=off '
+                    '--job-results-dir %s %s') % (self.tmpdir, test)
         result = process.run(cmd_line, ignore_status=True)
         expected_rc = exit_codes.AVOCADO_TESTS_FAIL
         self.assertEqual(result.exit_status, expected_rc,
@@ -498,8 +498,9 @@ class RunnerOperationTest(unittest.TestCase):
     @unittest.skipIf(not READ_BINARY, "read binary not available.")
     def test_read(self):
         os.chdir(basedir)
-        result = process.run("./scripts/avocado run %s" % READ_BINARY,
-                             timeout=10, ignore_status=True)
+        cmd = "./scripts/avocado run --job-results-dir %s %s" % (self.tmpdir,
+                                                                 READ_BINARY)
+        result = process.run(cmd, timeout=10, ignore_status=True)
         self.assertLess(result.duration, 8, "Duration longer than expected."
                         "\n%s" % result)
         self.assertEqual(result.exit_status, 1, "Expected exit status is 1\n%s"
@@ -587,11 +588,17 @@ class RunnerHumanOutputTest(unittest.TestCase):
                          ECHO_BINARY.replace('/', '_'))
 
     def test_replay_skip_skipped(self):
-        result = process.run("./scripts/avocado run skiponsetup.py --json -")
+        cmd = ("./scripts/avocado run --job-results-dir %s --json - "
+               "skiponsetup.py" % self.tmpdir)
+        result = process.run(cmd)
         result = json.loads(result.stdout)
-        jobid = result["job_id"]
-        process.run(str("./scripts/avocado run --replay %s "
-                        "--replay-test-status PASS" % jobid))
+        jobid = str(result["job_id"])
+        replay_data_dir = os.path.dirname(str(result["debuglog"]))
+        cmd = ("./scripts/avocado run --job-results-dir %s --replay-data-dir "
+               "%s --replay %s --replay-test-status PASS") % (self.tmpdir,
+                                                              replay_data_dir,
+                                                              jobid)
+        process.run(cmd)
 
     def tearDown(self):
         shutil.rmtree(self.tmpdir)

selftests/functional/test_multiplex.py

@@ -107,8 +107,8 @@ class MultiplexTests(unittest.TestCase):
         self.run_and_check(cmd_line, expected_rc, (4, 0))
 
     def test_empty_file(self):
-        cmd_line = ("./scripts/avocado run -m selftests/.data/empty_file "
-                    "-- passtest.py")
+        cmd_line = ("./scripts/avocado run --job-results-dir %s -m "
+                    "selftests/.data/empty_file -- passtest.py" % self.tmpdir)
         result = self.run_and_check(cmd_line, exit_codes.AVOCADO_ALL_OK,
                                     (1, 0))
 

selftests/functional/test_plugin_jobscripts.py

@@ -71,8 +71,8 @@ class JobScriptsTest(unittest.TestCase):
                                         SCRIPT_PRE_POST_CFG % (self.pre_dir,
                                                                self.post_dir))
         with config:
-            cmd = './scripts/avocado --config %s run --sysinfo=off %s' % (config,
-                                                                          test_check_touch)
+            cmd = ('./scripts/avocado --config %s run --job-results-dir %s '
+                   '--sysinfo=off %s' % (config, self.tmpdir, test_check_touch))
             result = process.run(cmd)
 
         # Pre/Post scripts failures do not (currently?) alter the exit status
@@ -93,7 +93,8 @@ class JobScriptsTest(unittest.TestCase):
         config = script.TemporaryScript("non_zero.conf",
                                         SCRIPT_NON_ZERO_CFG % self.pre_dir)
         with config:
-            cmd = './scripts/avocado --config %s run --sysinfo=off passtest.py' % config
+            cmd = ('./scripts/avocado --config %s run --job-results-dir %s '
+                   '--sysinfo=off passtest.py' % (config, self.tmpdir))
             result = process.run(cmd)
 
         # Pre/Post scripts failures do not (currently?) alter the exit status
@@ -114,7 +115,8 @@ class JobScriptsTest(unittest.TestCase):
         config = script.TemporaryScript("non_existing_dir.conf",
                                         SCRIPT_NON_EXISTING_DIR_CFG % self.pre_dir)
         with config:
-            cmd = './scripts/avocado --config %s run --sysinfo=off passtest.py' % config
+            cmd = ('./scripts/avocado --config %s run --job-results-dir %s '
+                   '--sysinfo=off passtest.py' % (config, self.tmpdir))
             result = process.run(cmd)
 
         # Pre/Post scripts failures do not (currently?) alter the exit status