Allow the administrator to yank out dead executors.

changelog.html

@@ -55,6 +55,8 @@ Upcoming changes</a>
     Build names (e.g., "#123") can be now modified by users/plugins to arbitrary text.
     (<a href="http://issues.hudson-ci.org/browse/HUDSON-53">issue 53</a>,
      <a href="http://issues.hudson-ci.org/browse/HUDSON-4884">issue 4884</a>)
+  <li class=rfe>
+    Allow the administrator to yank out dead executors.
   <li class=bug>
     Fixed an issue preventing to copy data on AIX, HP-UX or Linux for S/390. 
     (<a href="http://issues.hudson-ci.org/browse/HUDSON-8155">issue 8155</a>)

core/src/main/java/hudson/model/Computer.java

@@ -685,10 +685,23 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
      */
     /*package*/ synchronized void removeExecutor(Executor e) {
         executors.remove(e);
-        if(executors.isEmpty())
+        if(!isAlive())
             Hudson.getInstance().removeComputer(this);
     }
 
+    /**
+     * Returns true if any of the executors are functioning.
+     *
+     * Note that if an executor dies, we'll leave it in {@link #executors} until
+     * the administrator yanks it out, so that we can see why it died.
+     */
+    private boolean isAlive() {
+        for (Executor e : executors)
+            if (e.isAlive())
+                return true;
+        return false;
+    }
+
     /**
      * Interrupt all {@link Executor}s.
      */

core/src/main/java/hudson/model/Executor.java

@@ -33,6 +33,8 @@ import hudson.model.queue.WorkUnit;
 import hudson.util.TimeUnit2;
 import hudson.util.InterceptingProxy;
 import hudson.security.ACL;
+import org.kohsuke.stapler.HttpResponse;
+import org.kohsuke.stapler.HttpResponses;
 import org.kohsuke.stapler.StaplerRequest;
 import org.kohsuke.stapler.StaplerResponse;
 import org.kohsuke.stapler.export.ExportedBean;
@@ -77,6 +79,8 @@ public class Executor extends Thread implements ModelObject {
 
     private Throwable causeOfDeath;
 
+    private boolean induceDeath;
+
     public Executor(Computer owner, int n) {
         super("Executor #"+n+" for "+owner.getDisplayName());
         this.owner = owner;
@@ -107,6 +111,7 @@ public class Executor extends Thread implements ModelObject {
                 // sometime an interrupt aborts a build but without clearing the flag.
                 // see issue #1583
                 if (Thread.interrupted())   continue;
+                if (induceDeath)        throw new ThreadDeath();
 
                 SubTask task;
                 try {
@@ -167,6 +172,14 @@ public class Executor extends Thread implements ModelObject {
         }
     }
 
+    /**
+     * For testing only. Simulate a fatal unexpected failure.
+     */
+    public void killHard() {
+        induceDeath = true;
+        interrupt();
+    }
+
     /**
      * Returns true if we should keep going.
      */
@@ -361,6 +374,17 @@ public class Executor extends Thread implements ModelObject {
         rsp.forwardToPreviousPage(req);
     }
 
+    /**
+     * Throws away this executor and get a new one.
+     */
+    public HttpResponse doYank() {
+        Hudson.getInstance().checkPermission(Hudson.ADMINISTER);
+        if (isAlive())
+            throw new Failure("Can't yank a live executor");
+        owner.removeExecutor(this);
+        return HttpResponses.redirectViaContextPath("/");
+    }
+
     /**
      * Checks if the current user has a permission to stop this build.
      */

core/src/main/resources/hudson/model/Executor/causeOfDeath.jelly

@@ -47,6 +47,9 @@ THE SOFTWARE.
           <pre>
             <a href="http://wiki.hudson-ci.org/display/HUDSON/Dead+Executor">${%more info}</a>
           </pre>
+          <form method="post" action="yank" name="yank">
+            <f:submit value="${%Restart this thread}"/>
+          </form>
         </j:otherwise>
       </j:choose>
     </l:main-panel>

test/src/test/java/hudson/model/ExecutorTest.java

+package hudson.model;
+
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import org.jvnet.hudson.test.HudsonTestCase;
+
+/**
+ * @author Kohsuke Kawaguchi
+ */
+public class ExecutorTest extends HudsonTestCase {
+    public void testYank() throws Exception {
+        Computer c = Hudson.getInstance().toComputer();
+        Executor e = c.getExecutors().get(0);
+
+        // kill an executor
+        e.killHard();
+        while (e.isAlive())
+            Thread.sleep(10);
+
+        // make sure it's dead
+        assertTrue(c.getExecutors().contains(e));
+        assertTrue(e.getCauseOfDeath()!=null);
+
+        // test the UI
+        HtmlPage p = createWebClient().goTo("/");
+        p = p.getAnchorByText("Dead (!)").click();
+        assertTrue(p.getWebResponse().getContentAsString().contains(ThreadDeath.class.getName()));
+        submit(p.getFormByName("yank"));
+
+        assertFalse(c.getExecutors().contains(e));
+
+    }
+}