avocado.core.runner: Avoid infinite hang on job.abort()

When we abort the test and the test ignores SIGTERM, avocado hangs for
infinity. Let's wait a while and send SIGKILL.
Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

avocado/core/runner.py

@@ -152,18 +152,18 @@ class TestStatus(object):
             else:       # test_status
                 self.status = msg
 
-    def abort(self, test_alive, started, timeout, first, step):
+    def abort(self, proc, started, timeout, first, step):
         """
         Handle job abortion
-        :param test_alive: Whether the test process is still alive
+        :param proc: The test's process
         :param started: Time when the test started
         :param timeout: Timeout for waiting on status
         :param first: Delay before first check
         :param step: Step between checks for the status
         """
-        if test_alive and wait.wait_for(lambda: self.status, timeout,
-                                        first, step):
-            return self.status
+        if proc.is_alive() and wait.wait_for(lambda: self.status, timeout,
+                                             first, step):
+            status = self.status
         else:
             test_state = self.early_status
             test_state['time_elapsed'] = time.time() - started
@@ -178,7 +178,17 @@ class TestStatus(object):
             test_log.error('ERROR %s -> TestAbortedError: '
                            'Test aborted unexpectedly',
                            test_state['name'])
-            return test_state
+            status = test_state
+        if proc.is_alive():
+            for _ in xrange(5):     # I really want to destroy it
+                os.kill(proc.pid, signal.SIGKILL)
+                if not proc.is_alive():
+                    break
+                time.sleep(0.1)
+            else:
+                raise exceptions.TestError("Unable to destroy test's process "
+                                           "(%s)" % proc.pid)
+        return status
 
 
 class TestRunner(object):
@@ -367,8 +377,8 @@ class TestRunner(object):
         if test_status.status:
             test_state = test_status.status
         else:
-            test_state = test_status.abort(proc.is_alive(), time_started,
-                                           cycle_timeout, first, step)
+            test_state = test_status.abort(proc, time_started, cycle_timeout,
+                                           first, step)
 
         # don't process other tests from the list
         if ctrl_c_count > 0:

selftests/functional/test_basic.py

@@ -6,6 +6,8 @@ import sys
 import tempfile
 import xml.dom.minidom
 import glob
+import aexpect
+import signal
 
 if sys.version_info[:2] == (2, 6):
     import unittest2 as unittest
@@ -482,6 +484,36 @@ class RunnerSimpleTest(unittest.TestCase):
                          "Avocado did not return rc %d:\n%s" %
                          (expected_rc, result))
 
+    def test_kill_stopped_sleep(self):
+        sleep = process.run("which sleep", ignore_status=True, shell=True)
+        if sleep.exit_status:
+            self.skipTest("Sleep binary not found in PATH")
+        sleep = "'%s 60'" % sleep.stdout.strip()
+        proc = aexpect.Expect("./scripts/avocado run %s --job-results-dir %s "
+                              "--sysinfo=off --job-timeout 3"
+                              % (sleep, self.tmpdir))
+        proc.read_until_output_matches(["\(1/1\)"], timeout=3,
+                                       internal_timeout=0.01)
+        # We need pid of the avocado, not the shell executing it
+        pid = int(process.get_children_pids(proc.get_pid())[0])
+        os.kill(pid, signal.SIGTSTP)   # This freezes the process
+        deadline = time.time() + 5
+        while time.time() < deadline:
+            if not proc.is_alive():
+                break
+        else:
+            proc.kill(signal.SIGKILL)
+            self.fail("Avocado process still alive 1s after job-timeout:\n%s"
+                      % proc.get_output())
+        output = proc.get_output()
+        self.assertIn("ctrl+z pressed, stopping test", output, "SIGTSTP "
+                      "message not in the output, test was probably not "
+                      "stopped.")
+        self.assertIn("TIME", output, "TIME not in the output, avocado "
+                      "probably died unexpectadly")
+        self.assertEqual(proc.get_status(), 1, "Avocado did not finish with "
+                         "1.")
+
     def tearDown(self):
         self.pass_script.remove()
         self.fail_script.remove()