Merge pull request #3414 from stephanreiter/master

[JENKINS-17116] - When aborting a build, wait up to 2min for process termination

core/pom.xml

@@ -515,7 +515,7 @@ THE SOFTWARE.
     <dependency>
       <groupId>org.jvnet.winp</groupId>
       <artifactId>winp</artifactId>
-      <version>1.26</version>
+      <version>1.27</version>
     </dependency>
     <dependency>
       <groupId>org.jenkins-ci</groupId>

core/src/main/java/hudson/util/ProcessTree.java

@@ -144,6 +144,8 @@ public abstract class ProcessTree implements Iterable<OSProcess>, IProcessTree,
      */
     public abstract void killAll(Map<String, String> modelEnvVars) throws InterruptedException;
 
+    private final long softKillWaitSeconds = Integer.getInteger("SoftKillWaitSeconds", 2 * 60); // by default processes get at most 2 minutes to respond to SIGTERM (JENKINS-17116)
+
     /**
      * Convenience method that does {@link #killAll(Map)} and {@link OSProcess#killRecursively()}.
      * This is necessary to reliably kill the process and its descendants, as some OS
@@ -501,6 +503,8 @@ public abstract class ProcessTree implements Iterable<OSProcess>, IProcessTree,
                 return;
 
             LOGGER.log(FINER, "Killing recursively {0}", getPid());
+            // Firstly try to kill the root process gracefully, then do a forcekill if it does not help (algorithm is described in JENKINS-17116)
+            killSoftly();
             p.killRecursively();
             killByKiller();
         }
@@ -512,10 +516,39 @@ public abstract class ProcessTree implements Iterable<OSProcess>, IProcessTree,
             }
             
             LOGGER.log(FINER, "Killing {0}", getPid());
+            // Firstly try to kill it gracefully, then do a forcekill if it does not help (algorithm is described in JENKINS-17116)
+            killSoftly();
             p.kill();
             killByKiller();
         }
 
+        private void killSoftly() throws InterruptedException {
+            // send Ctrl+C to the process
+            try {
+                if (!p.sendCtrlC()) {
+                    return;
+                }
+            }
+            catch (WinpException e) {
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.log(Level.FINE, "Failed to send CTRL+C to pid=" + getPid(), e);
+                }
+                return;
+            }
+
+            // after that wait for it to cease to exist
+            long deadline = System.nanoTime() + softKillWaitSeconds * 1000000000;
+            int sleepTime = 10; // initially we sleep briefly, then sleep up to 1sec
+            do {
+                if (!p.isRunning()) {
+                    break;
+                }
+
+                Thread.sleep(sleepTime);
+                sleepTime = Math.min(sleepTime * 2, 1000);
+            } while (System.nanoTime() < deadline);
+        }
+
         @Override
         public synchronized List<String> getArguments() {
             if(args==null) {
@@ -718,12 +751,29 @@ public abstract class ProcessTree implements Iterable<OSProcess>, IProcessTree,
          * Tries to kill this process.
          */
         public void kill() throws InterruptedException {
+            // after sending SIGTERM, wait for the process to cease to exist
+            long deadline = System.nanoTime() + softKillWaitSeconds * 1000000000;
+            kill(deadline);
+        }
+
+        private void kill(long deadline) throws InterruptedException {
             if (getVeto() != null) 
                 return;
             try {
                 int pid = getPid();
                 LOGGER.fine("Killing pid="+pid);
                 UnixReflection.destroy(pid);
+                // after sending SIGTERM, wait for the process to cease to exist
+                int sleepTime = 10; // initially we sleep briefly, then sleep up to 1sec
+                File status = getFile("status");
+                do {
+                    if (!status.exists()) {
+                        break; // status is gone, process therefore as well
+                    }
+
+                    Thread.sleep(sleepTime);
+                    sleepTime = Math.min(sleepTime * 2, 1000);
+                } while (System.nanoTime() < deadline);
             } catch (IllegalAccessException e) {
                 // this is impossible
                 IllegalAccessError x = new IllegalAccessError();
@@ -740,11 +790,22 @@ public abstract class ProcessTree implements Iterable<OSProcess>, IProcessTree,
         }
 
         public void killRecursively() throws InterruptedException {
+            // after sending SIGTERM, wait for the processes to cease to exist until the deadline
+            long deadline = System.nanoTime() + softKillWaitSeconds * 1000000000;
+            killRecursively(deadline);
+        }
+
+        private void killRecursively(long deadline) throws InterruptedException {
             // We kill individual processes of a tree, so handling vetoes inside #kill() is enough for UnixProcess es
             LOGGER.fine("Recursively killing pid="+getPid());
-            for (OSProcess p : getChildren())
-                p.killRecursively();
-            kill();
+            for (OSProcess p : getChildren()) {
+                if (p instanceof UnixProcess) {
+                    ((UnixProcess)p).killRecursively(deadline);
+                } else {
+                    p.killRecursively(); // should not happen, fallback to non-deadline version
+                }
+            }
+            kill(deadline);
         }
 
         /**