[FIXED JENKINS-27565] Refactor the Queue and Nodes to use a consistent locking strategy

The test system I set up to verify resolution of customer(s)' issues driving this change, required
additional changes in order to fully resolve the issues at hand. As a result I am bundling these
changes:

- Moves nodes to being store in separate config files outside of the main config file (improves performance) [FIXED JENKINS-27562]
- Makes the Jenkins is loading screen not block on the extensions loading lock [FIXED JENKINS-27563]
- Removes race condition rendering the list of executors [FIXED JENKINS-27564] [FIXED JENKINS-15355]
- Tidy up the locks that were causing deadlocks with the once retention strategy in durable tasks [FIXED JENKINS-27476]
- Remove any requirement from Jenkins Core to lock on the Queue when rendering the Jenkins UI [FIXED-JENKINS-27566]

core/src/main/java/hudson/Functions.java

@@ -28,6 +28,7 @@ package hudson;
 import hudson.cli.CLICommand;
 import hudson.console.ConsoleAnnotationDescriptor;
 import hudson.console.ConsoleAnnotatorFactory;
+import hudson.init.InitMilestone;
 import hudson.model.AbstractProject;
 import hudson.model.Action;
 import hudson.model.Describable;
@@ -201,7 +202,12 @@ public class Functions {
     public static String rfc822Date(Calendar cal) {
         return Util.RFC822_DATETIME_FORMATTER.format(cal.getTime());
     }
-    
+
+    public static boolean isExtensionsAvailable() {
+        final Jenkins jenkins = Jenkins.getInstance();
+        return jenkins == null ? false : jenkins.getInitLevel().compareTo(InitMilestone.EXTENSIONS_AUGMENTED) >= 0;
+    }
+
     public static void initPageVariables(JellyContext context) {
         StaplerRequest currentRequest = Stapler.getCurrentRequest();
         String rootURL = currentRequest.getContextPath();

core/src/main/java/hudson/model/AbstractCIBase.java

@@ -48,8 +48,6 @@ public abstract class AbstractCIBase extends Node implements ItemGroup<TopLevelI
 
     private static final Logger LOGGER = Logger.getLogger(AbstractCIBase.class.getName());
 
-    private final transient Object updateComputerLock = new Object();
-
     /**
      * If you are calling this on Hudson something is wrong.
      *
@@ -137,15 +135,20 @@ public abstract class AbstractCIBase extends Node implements ItemGroup<TopLevelI
         used.add(c);
     }
 
-    /*package*/ void removeComputer(Computer computer) {
-        Map<Node,Computer> computers = getComputerMap();
-        for (Map.Entry<Node, Computer> e : computers.entrySet()) {
-            if (e.getValue() == computer) {
-                computers.remove(e.getKey());
-                computer.onRemoved();
-                return;
+    /*package*/ void removeComputer(final Computer computer) {
+        Queue.withLock(new Runnable() {
+            @Override
+            public void run() {
+                Map<Node,Computer> computers = getComputerMap();
+                for (Map.Entry<Node, Computer> e : computers.entrySet()) {
+                    if (e.getValue() == computer) {
+                        computers.remove(e.getKey());
+                        computer.onRemoved();
+                        return;
+                    }
+                }
             }
-        }
+        });
     }
 
     /*package*/ @CheckForNull Computer getComputer(Node n) {
@@ -160,37 +163,40 @@ public abstract class AbstractCIBase extends Node implements ItemGroup<TopLevelI
      * This method tries to reuse existing {@link Computer} objects
      * so that we won't upset {@link Executor}s running in it.
      */
-    protected void updateComputerList(boolean automaticSlaveLaunch) throws IOException {
-        Map<Node,Computer> computers = getComputerMap();
-        synchronized(updateComputerLock) {// just so that we don't have two code updating computer list at the same time
-            Map<String,Computer> byName = new HashMap<String,Computer>();
-            for (Computer c : computers.values()) {
-                Node node = c.getNode();
-                if (node == null)
-                    continue;   // this computer is gone
-                byName.put(node.getNodeName(),c);
-            }
+    protected void updateComputerList(final boolean automaticSlaveLaunch) {
+        final Map<Node,Computer> computers = getComputerMap();
+        Queue.withLock(new Runnable() {
+            @Override
+            public void run() {
+                Map<String,Computer> byName = new HashMap<String,Computer>();
+                for (Computer c : computers.values()) {
+                    Node node = c.getNode();
+                    if (node == null)
+                        continue;   // this computer is gone
+                    byName.put(node.getNodeName(),c);
+                }
 
-            final Set<Computer> old = new HashSet<Computer>(computers.values());
-            Set<Computer> used = new HashSet<Computer>();
+                final Set<Computer> old = new HashSet<Computer>(computers.values());
+                Set<Computer> used = new HashSet<Computer>();
 
-            updateComputer(this, byName, used, automaticSlaveLaunch);
-            for (Node s : getNodes()) {
-                long start = System.currentTimeMillis();
-                updateComputer(s, byName, used, automaticSlaveLaunch);
-                if(LOG_STARTUP_PERFORMANCE)
-                    LOGGER.info(String.format("Took %dms to update node %s",
-                            System.currentTimeMillis()-start, s.getNodeName()));
-            }
+                updateComputer(AbstractCIBase.this, byName, used, automaticSlaveLaunch);
+                for (Node s : getNodes()) {
+                    long start = System.currentTimeMillis();
+                    updateComputer(s, byName, used, automaticSlaveLaunch);
+                    if(LOG_STARTUP_PERFORMANCE)
+                        LOGGER.info(String.format("Took %dms to update node %s",
+                                System.currentTimeMillis()-start, s.getNodeName()));
+                }
 
-            // find out what computers are removed, and kill off all executors.
-            // when all executors exit, it will be removed from the computers map.
-            // so don't remove too quickly
-            old.removeAll(used);
-            for (Computer c : old) {
-                killComputer(c);
+                // find out what computers are removed, and kill off all executors.
+                // when all executors exit, it will be removed from the computers map.
+                // so don't remove too quickly
+                old.removeAll(used);
+                for (Computer c : old) {
+                    killComputer(c);
+                }
             }
-        }
+        });
         getQueue().scheduleMaintenance();
         for (ComputerListener cl : ComputerListener.all())
             cl.onConfigurationChange();

core/src/main/java/hudson/model/Computer.java

@@ -176,6 +176,21 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
 
     protected final Object statusChangeLock = new Object();
 
+    private transient final List<RuntimeException> terminatedBy = Collections.synchronizedList(new ArrayList
+            <RuntimeException>());
+
+    public void recordTermination() {
+        try {
+            throw new RuntimeException(String.format("Termination requested by %s", Thread.currentThread()));
+        } catch (RuntimeException e) {
+            terminatedBy.add(e);
+        }
+    }
+
+    public List<RuntimeException> getTerminatedBy() {
+        return new ArrayList<RuntimeException>(terminatedBy);
+    }
+
     public Computer(Node node) {
         setNode(node);
     }
@@ -404,6 +419,7 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
      * @since 1.320
      */
     public Future<?> disconnect(OfflineCause cause) {
+        recordTermination();
         offlineCause = cause;
         if (Util.isOverridden(Computer.class,getClass(),"disconnect"))
             return disconnect();    // legacy subtypes that extend disconnect().
@@ -419,6 +435,7 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
      *      Use {@link #disconnect(OfflineCause)} and specify the cause.
      */
     public Future<?> disconnect() {
+        recordTermination();
         if (Util.isOverridden(Computer.class,getClass(),"disconnect",OfflineCause.class))
             // if the subtype already derives disconnect(OfflineCause), delegate to it
             return disconnect(null);
@@ -808,6 +825,21 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
         return new ArrayList<OneOffExecutor>(oneOffExecutors);
     }
 
+    public List<DisplayExecutor> getDisplayExecutors() {
+        List<DisplayExecutor> result = new ArrayList<DisplayExecutor>(executors.size()+oneOffExecutors.size());
+        int index = 0;
+        for (Executor e: executors) {
+            result.add(new DisplayExecutor(Integer.toString(index+1), String.format("executors/%d", index), e));
+            index++;
+        }
+        index = 0;
+        for (OneOffExecutor e: oneOffExecutors) {
+            result.add(new DisplayExecutor("", String.format("oneOffExecutors/%d", index), e));
+            index++;
+        }
+        return result;
+    }
+
     /**
      * Returns true if all the executors of this computer are idle.
      */
@@ -867,14 +899,21 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
     /**
      * Called by {@link Executor} to kill excessive executors from this computer.
      */
-    /*package*/ synchronized void removeExecutor(Executor e) {
-        executors.remove(e);
-        addNewExecutorIfNecessary();
-        if(!isAlive())
-        {
-            AbstractCIBase ciBase = Jenkins.getInstance();
-            ciBase.removeComputer(this);
-        }
+    /*package*/ void removeExecutor(final Executor e) {
+        Queue.withLock(new Runnable() {
+            @Override
+            public void run() {
+                synchronized (Computer.this) {
+                    executors.remove(e);
+                    addNewExecutorIfNecessary();
+                    if(!isAlive())
+                    {
+                        AbstractCIBase ciBase = Jenkins.getInstance();
+                        ciBase.removeComputer(Computer.this);
+                    }
+                }
+            }
+        });
     }
 
     /**
@@ -1413,6 +1452,71 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
         }
     }
 
+    public static class DisplayExecutor implements ModelObject {
+
+        @Nonnull
+        private final String displayName;
+        @Nonnull
+        private final String url;
+        @Nonnull
+        private final Executor executor;
+
+        public DisplayExecutor(@Nonnull String displayName, @Nonnull String url, @Nonnull Executor executor) {
+            this.displayName = displayName;
+            this.url = url;
+            this.executor = executor;
+        }
+
+        @Override
+        @Nonnull
+        public String getDisplayName() {
+            return displayName;
+        }
+
+        @Nonnull
+        public String getUrl() {
+            return url;
+        }
+
+        @Nonnull
+        public Executor getExecutor() {
+            return executor;
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder sb = new StringBuilder("DisplayExecutor{");
+            sb.append("displayName='").append(displayName).append('\'');
+            sb.append(", url='").append(url).append('\'');
+            sb.append(", executor=").append(executor);
+            sb.append('}');
+            return sb.toString();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+
+            DisplayExecutor that = (DisplayExecutor) o;
+
+            if (!executor.equals(that.executor)) {
+                return false;
+            }
+
+            return true;
+        }
+
+        @Override
+        public int hashCode() {
+            return executor.hashCode();
+        }
+    }
+
     public static final PermissionGroup PERMISSIONS = new PermissionGroup(Computer.class,Messages._Computer_Permissions_Title());
     public static final Permission CONFIGURE = new Permission(PERMISSIONS,"Configure", Messages._Computer_ConfigurePermission_Description(), Permission.CONFIGURE, PermissionScope.COMPUTER);
     /**

core/src/main/java/hudson/model/Hudson.java

@@ -121,7 +121,7 @@ public class Hudson extends Jenkins {
      *      Use {@link #getNodes()}. Since 1.252.
      */
     public List<Slave> getSlaves() {
-        return (List)slaves;
+        return (List)getNodes();
     }
 
     /**

core/src/main/java/hudson/model/Node.java

@@ -373,6 +373,10 @@ public abstract class Node extends AbstractModelObject implements Reconfigurable
             if (c!=null)    return c;
         }
 
+        if (!isAcceptingTasks()) {
+            return CauseOfBlockage.fromMessage(Messages._Node_BecauseNodeIsNotAcceptingTasks(getNodeName()));
+        }
+
         // Looks like we can take the task
         return null;
     }

core/src/main/java/hudson/model/ResourceController.java

@@ -29,6 +29,7 @@ import java.util.Set;
 import java.util.Collection;
 import java.util.AbstractCollection;
 import java.util.Iterator;
+import java.util.concurrent.Callable;
 import java.util.concurrent.CopyOnWriteArraySet;
 import javax.annotation.Nonnull;
 
@@ -74,25 +75,36 @@ public class ResourceController {
      * @throws InterruptedException
      *      the thread can be interrupted while waiting for the available resources.
      */
-    public void execute(@Nonnull Runnable task, ResourceActivity activity ) throws InterruptedException {
-        ResourceList resources = activity.getResourceList();
-        synchronized(this) {
-            while(inUse.isCollidingWith(resources))
-                wait();
-
-            // we have a go
-            inProgress.add(activity);
-            inUse = ResourceList.union(inUse,resources);
-        }
+    public void execute(@Nonnull Runnable task, final ResourceActivity activity ) throws InterruptedException {
+        final ResourceList resources = activity.getResourceList();
+        _withLock(new Runnable() {
+            @Override
+            public void run() {
+                while(inUse.isCollidingWith(resources))
+                    try {
+                        // TODO revalidate the resource list after re-acquiring lock, for now we just let the build fail
+                        _await();
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException(e);
+                    }
+
+                // we have a go
+                inProgress.add(activity);
+                inUse = ResourceList.union(inUse,resources);
+            }
+        });
 
         try {
             task.run();
         } finally {
-            synchronized(this) {
-                inProgress.remove(activity);
-                inUse = ResourceList.union(resourceView);
-                notifyAll();
-            }
+            _withLock(new Runnable() {
+                @Override
+                public void run() {
+                    inProgress.remove(activity);
+                    inUse = ResourceList.union(resourceView);
+                    _signalAll();
+                }
+            });
         }
     }
 
@@ -105,8 +117,17 @@ public class ResourceController {
      * another activity might acquire resources before the caller
      * gets to call {@link #execute(Runnable, ResourceActivity)}.
      */
-    public synchronized boolean canRun(ResourceList resources) {
-        return !inUse.isCollidingWith(resources);
+    public boolean canRun(final ResourceList resources) {
+        try {
+            return _withLock(new Callable<Boolean>() {
+                @Override
+                public Boolean call() {
+                    return !inUse.isCollidingWith(resources);
+                }
+            });
+        } catch (Exception e) {
+            throw new IllegalStateException("Inner callable does not throw exception");
+        }
     }
 
     /**
@@ -117,8 +138,17 @@ public class ResourceController {
      * If more than one such resource exists, one is chosen and returned.
      * This method is used for reporting what's causing the blockage.
      */
-    public synchronized Resource getMissingResource(ResourceList resources) {
-        return resources.getConflict(inUse);
+    public Resource getMissingResource(final ResourceList resources) {
+        try {
+            return _withLock(new Callable<Resource>() {
+                @Override
+                public Resource call() {
+                    return resources.getConflict(inUse);
+                }
+            });
+        } catch (Exception e) {
+            throw new IllegalStateException("Inner callable does not throw exception");
+        }
     }
 
     /**
@@ -133,5 +163,25 @@ public class ResourceController {
                 return a;
         return null;
     }
+
+    protected void _await() throws InterruptedException {
+        wait();
+    }
+
+    protected void _signalAll() {
+        notifyAll();
+    }
+
+    protected void _withLock(Runnable runnable) {
+        synchronized (this) {
+            runnable.run();
+        }
+    }
+
+    protected <V> V _withLock(java.util.concurrent.Callable<V> callable) throws Exception {
+        synchronized (this) {
+            return callable.call();
+        }
+    }
 }
 

core/src/main/java/hudson/slaves/AbstractCloudSlave.java

@@ -23,6 +23,7 @@
  */
 package hudson.slaves;
 
+import hudson.model.Computer;
 import hudson.model.Descriptor.FormException;
 import jenkins.model.Jenkins;
 import hudson.model.Slave;
@@ -57,6 +58,10 @@ public abstract class AbstractCloudSlave extends Slave {
      * Releases and removes this slave.
      */
     public void terminate() throws InterruptedException, IOException {
+        final Computer computer = toComputer();
+        if (computer != null) {
+            computer.recordTermination();
+        }
         try {
             // TODO: send the output to somewhere real
             _terminate(new StreamTaskListener(System.out, Charset.defaultCharset()));

core/src/main/java/hudson/slaves/ComputerRetentionWork.java

 /*
  * The MIT License
- * 
+ *
  * Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi, Stephen Connolly
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
  * in the Software without restriction, including without limitation the rights
  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  * copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice shall be included in
  * all copies or substantial portions of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.WeakHashMap;
 
 import hudson.model.Computer;
+import hudson.model.Queue;
 import jenkins.model.Jenkins;
 import hudson.model.Node;
 import hudson.model.PeriodicWork;
@@ -56,16 +57,21 @@ public class ComputerRetentionWork extends PeriodicWork {
     @SuppressWarnings("unchecked")
     protected void doRun() {
         final long startRun = System.currentTimeMillis();
-        for (Computer c : Jenkins.getInstance().getComputers()) {
-            Node n = c.getNode();
-            if (n!=null && n.isHoldOffLaunchUntilSave())
-                continue;
-            if (!nextCheck.containsKey(c) || startRun > nextCheck.get(c)) {
-                // at the moment I don't trust strategies to wait more than 60 minutes
-                // strategies need to wait at least one minute
-                final long waitInMins = Math.max(1, Math.min(60, c.getRetentionStrategy().check(c)));
-                nextCheck.put(c, startRun + waitInMins*1000*60 /*MINS->MILLIS*/);
-            }
+        for (final Computer c : Jenkins.getInstance().getComputers()) {
+            Queue.withLock(new Runnable() {
+                @Override
+                public void run() {
+                    Node n = c.getNode();
+                    if (n!=null && n.isHoldOffLaunchUntilSave())
+                        return;
+                    if (!nextCheck.containsKey(c) || startRun > nextCheck.get(c)) {
+                        // at the moment I don't trust strategies to wait more than 60 minutes
+                        // strategies need to wait at least one minute
+                        final long waitInMins = Math.max(1, Math.min(60, c.getRetentionStrategy().check(c)));
+                        nextCheck.put(c, startRun + waitInMins*1000*60 /*MINS->MILLIS*/);
+                    }
+                }
+            });
         }
     }
 }

core/src/main/java/hudson/slaves/NodeProvisioner.java

@@ -35,6 +35,7 @@ import net.jcip.annotations.GuardedBy;
 import javax.annotation.Nonnull;
 import java.awt.Color;
 import java.util.Arrays;
+import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 import java.util.concurrent.ExecutionException;
 import java.util.List;
@@ -42,6 +43,10 @@ import java.util.Collection;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Logger;
 import java.util.logging.Level;
 import java.io.IOException;
@@ -120,9 +125,14 @@ public class NodeProvisioner {
      */
     private final Label label;
 
-    @GuardedBy("self")
+    @GuardedBy("#provisioningLock")
     private final List<PlannedNode> pendingLaunches = new ArrayList<PlannedNode>();
 
+    private final Lock provisioningLock = new ReentrantLock();
+
+    @GuardedBy("#provisioningLock")
+    private StrategyState provisioningState = null;
+
     private transient volatile long lastSuggestedReview;
 
     /**
@@ -148,8 +158,11 @@ public class NodeProvisioner {
      * @since 1.401
      */
     public List<PlannedNode> getPendingLaunches() {
-        synchronized (pendingLaunches) {
+        provisioningLock.lock();
+        try {
             return new ArrayList<PlannedNode>(pendingLaunches);
+        } finally {
+            provisioningLock.unlock();
         }
     }
 
@@ -174,80 +187,96 @@ public class NodeProvisioner {
      * Periodically invoked to keep track of the load.
      * Launches additional nodes if necessary.
      */
-    private synchronized void update() {
-        Jenkins jenkins = Jenkins.getInstance();
-        lastSuggestedReview = System.currentTimeMillis();
+    private void update() {
+        provisioningLock.lock();
+        try {
+            lastSuggestedReview = System.currentTimeMillis();
+            Queue.withLock(new Runnable() {
+                @Override
+                public void run() {
+                    Jenkins jenkins = Jenkins.getInstance();
+                    // clean up the cancelled launch activity, then count the # of executors that we are about to
+                    // bring up.
+
+                    int plannedCapacitySnapshot = 0;
+                    List<PlannedNode> completedLaunches = new ArrayList<PlannedNode>();
+
+                    for (Iterator<PlannedNode> itr = pendingLaunches.iterator(); itr.hasNext(); ) {
+                        PlannedNode f = itr.next();
+                        if (f.future.isDone()) {
+                            completedLaunches.add(f);
+                            itr.remove();
+                        } else {
+                            plannedCapacitySnapshot += f.numExecutors;
+                        }
+                    }
 
-        // clean up the cancelled launch activity, then count the # of executors that we are about to bring up.
-        int plannedCapacitySnapshot = 0;
-        List<PlannedNode> completedLaunches = new ArrayList<PlannedNode>();
+                    for (PlannedNode f : completedLaunches) {
+                        try {
+                            Node node = f.future.get();
+                            for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
+                                cl.onComplete(f, node);
+                            }
 
-        synchronized (pendingLaunches) {
-            for (Iterator<PlannedNode> itr = pendingLaunches.iterator(); itr.hasNext(); ) {
-                PlannedNode f = itr.next();
-                if (f.future.isDone()) {
-                    completedLaunches.add(f);
-                    itr.remove();
-                } else {
-                    plannedCapacitySnapshot += f.numExecutors;
-                }
-            }
-        }
+                            jenkins.addNode(node);
+                            LOGGER.log(Level.INFO,
+                                    "{0} provisioning successfully completed. We have now {1,number,integer} computer"
+                                            + "(s)",
+                                    new Object[]{f.displayName, jenkins.getComputers().length});
+                        } catch (InterruptedException e) {
+                            throw new AssertionError(e); // since we confirmed that the future is already done
+                        } catch (ExecutionException e) {
+                            LOGGER.log(Level.WARNING, "Provisioned slave " + f.displayName + " failed to launch",
+                                    e.getCause());
+                            for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
+                                cl.onFailure(f, e.getCause());
+                            }
+                        } catch (IOException e) {
+                            LOGGER.log(Level.WARNING, "Provisioned slave " + f.displayName + " failed to launch", e);
+                            for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
+                                cl.onFailure(f, e);
+                            }
+                        }
 
-        for (PlannedNode f : completedLaunches) {
-            try {
-                Node node = f.future.get();
-                for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
-                    cl.onComplete(f, node);
-                }
+                        f.spent();
+                    }
 
-                jenkins.addNode(node);
-                LOGGER.log(Level.INFO,
-                        "{0} provisioning successfully completed. We have now {1,number,integer} computer(s)",
-                        new Object[]{f.displayName, jenkins.getComputers().length});
-            } catch (InterruptedException e) {
-                throw new AssertionError(e); // since we confirmed that the future is already done
-            } catch (ExecutionException e) {
-                LOGGER.log(Level.WARNING, "Provisioned slave " + f.displayName + " failed to launch", e.getCause());
-                for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
-                    cl.onFailure(f, e.getCause());
-                }
-            } catch (IOException e) {
-                LOGGER.log(Level.WARNING, "Provisioned slave " + f.displayName + " failed to launch", e);
-                for (CloudProvisioningListener cl : CloudProvisioningListener.all()) {
-                    cl.onFailure(f, e);
+                    float plannedCapacity = plannedCapacitySnapshot;
+                    plannedCapacitiesEMA.update(plannedCapacity);
+
+                    int idleSnapshot = stat.computeIdleExecutors();
+                    int queueLengthSnapshot = stat.computeQueueLength();
+
+                    if (queueLengthSnapshot <= idleSnapshot) {
+                        LOGGER.log(Level.FINE,
+                                "Queue length {0} is less than the idle capacity {1}. No provisioning strategy "
+                                        + "required",
+                                new Object[]{queueLengthSnapshot, idleSnapshot});
+                        provisioningState = null;
+                    } else {
+                        provisioningState = new StrategyState(queueLengthSnapshot, label, idleSnapshot,
+                                stat.computeTotalExecutors(),
+                                plannedCapacitySnapshot);
+                    }
                 }
-            }
-
-            f.spent();
-        }
+            });
 
-        float plannedCapacity = plannedCapacitySnapshot;
-        plannedCapacitiesEMA.update(plannedCapacity);
-
-        int idleSnapshot = stat.computeIdleExecutors();
-        int queueLengthSnapshot = stat.computeQueueLength();
-
-        if (queueLengthSnapshot <= idleSnapshot) {
-            LOGGER.log(Level.FINE,
-                    "Queue length {0} is less than the idle capacity {1}. No provisioning strategy required",
-                    new Object[]{queueLengthSnapshot, idleSnapshot});
-        } else {
-            StrategyState state =
-                    new StrategyState(queueLengthSnapshot, label, idleSnapshot, stat.computeTotalExecutors(),
-                            plannedCapacitySnapshot);
-            List<Strategy> strategies = Jenkins.getInstance().getExtensionList(Strategy.class);
-            for (Strategy strategy : strategies.isEmpty()
-                    ? Arrays.<Strategy>asList(new StandardStrategyImpl())
-                    : strategies) {
-                LOGGER.log(Level.FINER, "Consulting {0} provisioning strategy with state {1}",
-                        new Object[]{strategy, state});
-                if (StrategyDecision.PROVISIONING_COMPLETED == strategy.apply(state)) {
-                    LOGGER.log(Level.FINER, "Provisioning strategy {0} declared provisioning complete",
-                            strategy);
-                    break;
+            if (provisioningState != null) {
+                List<Strategy> strategies = Jenkins.getInstance().getExtensionList(Strategy.class);
+                for (Strategy strategy : strategies.isEmpty()
+                        ? Arrays.<Strategy>asList(new StandardStrategyImpl())
+                        : strategies) {
+                    LOGGER.log(Level.FINER, "Consulting {0} provisioning strategy with state {1}",
+                            new Object[]{strategy, provisioningState});
+                    if (StrategyDecision.PROVISIONING_COMPLETED == strategy.apply(provisioningState)) {
+                        LOGGER.log(Level.FINER, "Provisioning strategy {0} declared provisioning complete",
+                                strategy);
+                        break;
+                    }
                 }
             }
+        } finally {
+            provisioningLock.unlock();
         }
     }
 
@@ -384,8 +413,13 @@ public class NodeProvisioner {
          * The additional planned capacity for this {@link #getLabel()} and provisioned by previous strategies during
          * the current updating of the {@link NodeProvisioner}.
          */
-        public synchronized int getAdditionalPlannedCapacity() {
-            return additionalPlannedCapacity;
+        public int getAdditionalPlannedCapacity() {
+            provisioningLock.lock();
+            try {
+                return additionalPlannedCapacity;
+            } finally {
+                provisioningLock.unlock();
+            }
         }
 
         /**
@@ -451,13 +485,14 @@ public class NodeProvisioner {
                     additionalPlannedCapacity += f.numExecutors;
                 }
             }
-            synchronized (pendingLaunches) {
+            provisioningLock.lock();
+            try {
                 pendingLaunches.addAll(plannedNodes);
-            }
-            if (additionalPlannedCapacity > 0) {
-                synchronized (this) {
-                    this.additionalPlannedCapacity += additionalPlannedCapacity;
+                if (additionalPlannedCapacity > 0) {
+                        this.additionalPlannedCapacity += additionalPlannedCapacity;
                 }
+            } finally {
+                provisioningLock.unlock();
             }
         }
 

core/src/main/java/hudson/slaves/RetentionStrategy.java

@@ -33,8 +33,10 @@ import hudson.util.DescriptorList;
 import java.util.Collections;
 import java.util.HashMap;
 import jenkins.model.Jenkins;
+import net.jcip.annotations.GuardedBy;
 import org.kohsuke.stapler.DataBoundConstructor;
 
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -54,6 +56,7 @@ public abstract class RetentionStrategy<T extends Computer> extends AbstractDesc
      * @return The number of minutes after which the strategy would like to be checked again. The strategy may be
      *         rechecked earlier or later that this!
      */
+    @GuardedBy("hudson.model.Queue#lock")
     public abstract long check(T c);
 
     /**
@@ -91,8 +94,13 @@ public abstract class RetentionStrategy<T extends Computer> extends AbstractDesc
      *
      * @since 1.275
      */
-    public void start(T c) {
-        check(c);
+    public void start(final T c) {
+        Queue.withLock(new Runnable() {
+            @Override
+            public void run() {
+                check(c);
+            }
+        });
     }
 
     /**
@@ -257,21 +265,13 @@ public abstract class RetentionStrategy<T extends Computer> extends AbstractDesc
             } else if (c.isIdle()) {
                 final long idleMilliseconds = System.currentTimeMillis() - c.getIdleStartMilliseconds();
                 if (idleMilliseconds > idleDelay * 1000 * 60 /*MINS->MILLIS*/) {
-                    Queue.withLock(new Runnable() {
-                        @Override
-                        public void run() {
-                            // re-check idle now that we are within the Queue lock
-                            if (c.isIdle()) {
-                                final long idleMilliseconds = System.currentTimeMillis() - c.getIdleStartMilliseconds();
-                                if (idleMilliseconds > idleDelay * 1000 * 60 /*MINS->MILLIS*/) {
-                                    // we've been idle for long enough
-                                    logger.log(Level.INFO, "Disconnecting computer {0} as it has been idle for {1}",
-                                            new Object[]{c.getName(), Util.getTimeSpanString(idleMilliseconds)});
-                                    c.disconnect(OfflineCause.create(Messages._RetentionStrategy_Demand_OfflineIdle()));
-                                }
-                            }
-                        }
-                    });
+                    // we've been idle for long enough
+                    logger.log(Level.INFO, "Disconnecting computer {0} as it has been idle for {1}",
+                            new Object[]{c.getName(), Util.getTimeSpanString(idleMilliseconds)});
+                    c.disconnect(OfflineCause.create(Messages._RetentionStrategy_Demand_OfflineIdle()));
+                } else {
+                    // no point revisiting until we can be confident we will be idle
+                    return TimeUnit.MILLISECONDS.toMinutes(TimeUnit.MINUTES.toMillis(idleDelay) - idleMilliseconds);
                 }
             }
             return 1;

core/src/main/java/hudson/slaves/SlaveComputer.java

@@ -649,7 +649,7 @@ public class SlaveComputer extends Computer {
     }
 
     @Override
-    protected void setNode(Node node) {
+    protected void setNode(final Node node) {
         super.setNode(node);
         launcher = grabLauncher(node);
 
@@ -657,10 +657,16 @@ public class SlaveComputer extends Computer {
         // "constructed==null" test is an ugly hack to avoid launching before the object is fully
         // constructed.
         if(constructed!=null) {
-            if (node instanceof Slave)
-                ((Slave)node).getRetentionStrategy().check(this);
-            else
+            if (node instanceof Slave) {
+                Queue.withLock(new Runnable() {
+                    @Override
+                    public void run() {
+                        ((Slave)node).getRetentionStrategy().check(SlaveComputer.this);
+                    }
+                });
+            } else {
                 connect(false);
+            }
         }
     }
 

core/src/main/java/jenkins/model/Jenkins.java

@@ -289,6 +289,7 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.logging.Level;
 import java.util.logging.LogRecord;
 import java.util.logging.Logger;
@@ -498,17 +499,18 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
     }
 
     /**
-     * Set of installed cluster nodes.
-     * <p>
-     * We use this field with copy-on-write semantics.
-     * This field has mutable list (to keep the serialization look clean),
-     * but it shall never be modified. Only new completely populated slave
-     * list can be set here.
-     * <p>
-     * The field name should be really {@code nodes}, but again the backward compatibility
-     * prevents us from renaming.
+     * Legacy store of the set of installed cluster nodes.
+     * @deprecated in favour of {@link Nodes}
+     */
+    @Deprecated
+    protected transient volatile NodeList slaves;
+
+    /**
+     * The holder of the set of installed cluster nodes.
+     *
+     * @since 1.FIXME
      */
-    protected volatile NodeList slaves;
+    private transient final Nodes nodes = new Nodes(this);
 
     /**
      * Quiet period.
@@ -1214,7 +1216,7 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
         return null;
     }
 
-    protected void updateComputerList() throws IOException {
+    protected void updateComputerList() {
         updateComputerList(AUTOMATIC_SLAVE_LAUNCH);
     }
 
@@ -1663,7 +1665,7 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
      * Gets the slave node of the give name, hooked under this Hudson.
      */
     public @CheckForNull Node getNode(String name) {
-        return slaves.getNode(name);
+        return nodes.getNode(name);
     }
 
     /**
@@ -1682,38 +1684,25 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
      * represents the master.
      */
     public List<Node> getNodes() {
-        return slaves;
+        return nodes.getNodes();
     }
 
     /**
      * Adds one more {@link Node} to Hudson.
      */
-    public synchronized void addNode(Node n) throws IOException {
-        if(n==null)     throw new IllegalArgumentException();
-        ArrayList<Node> nl = new ArrayList<Node>(this.slaves);
-        if(!nl.contains(n)) // defensive check
-            nl.add(n);
-        setNodes(nl);
+    public void addNode(Node n) throws IOException {
+        nodes.addNode(n);
     }
 
     /**
      * Removes a {@link Node} from Hudson.
      */
-    public synchronized void removeNode(@Nonnull Node n) throws IOException {
-        Computer c = n.toComputer();
-        if (c!=null)
-            c.disconnect(OfflineCause.create(Messages._Hudson_NodeBeingRemoved()));
-
-        ArrayList<Node> nl = new ArrayList<Node>(this.slaves);
-        nl.remove(n);
-        setNodes(nl);
+    public void removeNode(@Nonnull Node n) throws IOException {
+        nodes.removeNode(n);
     }
 
-    public void setNodes(final List<? extends Node> nodes) throws IOException {
-        Jenkins.this.slaves = new NodeList(nodes);
-        updateComputerList();
-        trimLabels();
-        save();
+    public void setNodes(final List<? extends Node> n) throws IOException {
+        nodes.setNodes(n);
     }
 
     public DescribableList<NodeProperty<?>, NodePropertyDescriptor> getNodeProperties() {
@@ -1730,7 +1719,7 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
      * This should be called when the assumptions behind label cache computation changes,
      * but we also call this periodically to self-heal any data out-of-sync issue.
      */
-    private void trimLabels() {
+    /*package*/ void trimLabels() {
         for (Iterator<Label> itr = labels.values().iterator(); itr.hasNext();) {
             Label l = itr.next();
             resetLabel(l);
@@ -2627,10 +2616,6 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
         TaskGraphBuilder g = new TaskGraphBuilder();
         Handle loadHudson = g.requires(EXTENSIONS_AUGMENTED).attains(JOB_LOADED).add("Loading global config", new Executable() {
             public void run(Reactor session) throws Exception {
-                // JENKINS-8043: some slaves (eg. swarm slaves) are not saved into the config file
-                // and will get overwritten when reloading. Make a backup copy now, and re-add them later
-                NodeList oldSlaves = slaves;
-
                 XmlFile cfg = getConfigFile();
                 if (cfg.exists()) {
                     // reset some data that may not exist in the disk file
@@ -2643,23 +2628,14 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
                 }
 
                 // if we are loading old data that doesn't have this field
-                if (slaves == null) slaves = new NodeList();
+                if (slaves != null && !slaves.isEmpty() && nodes.isLegacy()) {
+                    nodes.setNodes(slaves);
+                    slaves = null;
+                } else {
+                    nodes.load();
+                }
 
                 clouds.setOwner(Jenkins.this);
-
-                // JENKINS-8043: re-add the slaves which were not saved into the config file
-                // and are now missing, but still connected.
-                if (oldSlaves != null) {
-                    ArrayList<Node> newSlaves = new ArrayList<Node>(slaves);
-                    for (Node n: oldSlaves) {
-                        if (n instanceof EphemeralNode) {
-                            if(!newSlaves.contains(n)) {
-                                newSlaves.add(n);
-                            }
-                        }
-                    }
-                    setNodes(newSlaves);
-                }
             }
         });
 
@@ -2697,7 +2673,7 @@ public class Jenkins extends AbstractCIBase implements DirectlyModifiableTopLeve
                 rebuildDependencyGraph();
 
                 {// recompute label objects - populates the labels mapping.
-                    for (Node slave : slaves)
+                    for (Node slave : nodes.getNodes())
                         // Note that not all labels are visible until the slaves have connected.
                         slave.getAssignedLabels();
                     getAssignedLabels();

core/src/main/java/jenkins/model/Nodes.java

+package jenkins.model;
+
+import hudson.BulkChange;
+import hudson.Util;
+import hudson.XmlFile;
+import hudson.model.Computer;
+import hudson.model.ItemGroupMixIn;
+import hudson.model.Node;
+import hudson.model.Queue;
+import hudson.model.Saveable;
+import hudson.model.listeners.SaveableListener;
+import hudson.slaves.EphemeralNode;
+import hudson.slaves.OfflineCause;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Manages all the nodes for Jenkins.
+ *
+ * @since 1.FIXME
+ */
+public class Nodes implements Saveable {
+
+    /**
+     * The {@link Jenkins} instance that we are tracking nodes for.
+     */
+    @Nonnull
+    private final Jenkins jenkins;
+
+    /**
+     * The map of nodes.
+     */
+    private final ConcurrentMap<String, Node> nodes = new ConcurrentSkipListMap<String, Node>();
+
+    /**
+     * Constructor, intended to be called only from {@link Jenkins}.
+     *
+     * @param jenkins A reference to the {@link Jenkins} that this instance is tracking nodes for, beware not to
+     *                let this reference escape from a partially constructed {@link Nodes} as when we are passed the
+     *                reference the {@link Jenkins} instance has not completed instantiation.
+     */
+    /*package*/ Nodes(@Nonnull Jenkins jenkins) {
+        this.jenkins = jenkins;
+    }
+
+    /**
+     * Returns the list of nodes.
+     *
+     * @return the list of nodes.
+     */
+    @Nonnull
+    public List<Node> getNodes() {
+        return new ArrayList<Node>(nodes.values());
+    }
+
+    /**
+     * Sets the list of nodes.
+     *
+     * @param nodes the new list of nodes.
+     * @throws IOException if the new list of nodes could not be persisted.
+     */
+    public void setNodes(@Nonnull Collection<? extends Node> nodes) throws IOException {
+        this.nodes.clear();
+        for (Node n : nodes) {
+            this.nodes.put(n.getNodeName(), n);
+        }
+        save();
+    }
+
+    /**
+     * Adds a node. If a node of the same name already exists then that node will be replaced.
+     *
+     * @param node the new node.
+     * @throws IOException if the list of nodes could not be persisted.
+     */
+    public void addNode(final @Nonnull Node node) throws IOException {
+        if (node != nodes.get(node.getNodeName())) {
+            // TODO we should not need to lock the queue for adding nodes but until we have a way to update the
+            // computer list for just the new node
+            Queue.withLock(new Runnable() {
+                @Override
+                public void run() {
+                    nodes.put(node.getNodeName(), node);
+                    jenkins.updateComputerList();
+                    jenkins.trimLabels();
+                }
+            });
+            if (node instanceof EphemeralNode) {
+                Util.deleteRecursive(new File(getNodesDir(), node.getNodeName()));
+            } else {
+                XmlFile xmlFile = new XmlFile(Jenkins.XSTREAM,
+                        new File(new File(getNodesDir(), node.getNodeName()), "config.xml"));
+                xmlFile.write(node);
+            }
+        }
+    }
+
+    /**
+     * Removes a node. If the node instance is not in the list of nodes, then this will be a no-op, even if
+     * there is another instance with the same {@link Node#getNodeName()}.
+     *
+     * @param node the node instance to remove.
+     * @throws IOException if the list of nodes could not be persisted.
+     */
+    public void removeNode(final @Nonnull Node node) throws IOException {
+        if (node == nodes.get(node.getNodeName())) {
+            Queue.withLock(new Runnable() {
+                @Override
+                public void run() {
+                    Computer c = node.toComputer();
+                    if (c != null) {
+                        c.recordTermination();
+                        c.disconnect(OfflineCause.create(hudson.model.Messages._Hudson_NodeBeingRemoved()));
+                    }
+                    if (node == nodes.remove(node.getNodeName())) {
+                        jenkins.updateComputerList();
+                        jenkins.trimLabels();
+                    }
+                }
+            });
+            Util.deleteRecursive(new File(getNodesDir(), node.getNodeName()));
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void save() throws IOException {
+        if (BulkChange.contains(this)) {
+            return;
+        }
+        final File nodesDir = getNodesDir();
+        final Set<String> existing = new HashSet<String>();
+        for (Node n : nodes.values()) {
+            if (n instanceof EphemeralNode) {
+                continue;
+            }
+            existing.add(n.getNodeName());
+            XmlFile xmlFile = new XmlFile(Jenkins.XSTREAM, new File(new File(nodesDir, n.getNodeName()), "config.xml"));
+            xmlFile.write(n);
+            SaveableListener.fireOnChange(this, xmlFile);
+        }
+        for (File forDeletion : nodesDir.listFiles(new FileFilter() {
+            @Override
+            public boolean accept(File pathname) {
+                return pathname.isDirectory() && !existing.contains(pathname.getName());
+            }
+        })) {
+            Util.deleteRecursive(forDeletion);
+        }
+    }
+
+    /**
+     * Returns the named node.
+     *
+     * @param name the {@link Node#getNodeName()} of the node to retrieve.
+     * @return the {@link Node} or {@code null} if the node could not be found.
+     */
+    @CheckForNull
+    public Node getNode(String name) {
+        return nodes.get(name);
+    }
+
+    /**
+     * Loads the nodes from disk.
+     *
+     * @throws IOException if the nodes could not be deserialized.
+     */
+    public void load() throws IOException {
+        final File nodesDir = getNodesDir();
+        final File[] subdirs = nodesDir.listFiles(new FileFilter() {
+            public boolean accept(File child) {
+                return child.isDirectory();
+            }
+        });
+        final Map<String, Node> newNodes = new TreeMap<String, Node>();
+        if (subdirs != null) {
+            for (File subdir : subdirs) {
+                try {
+                    XmlFile xmlFile = new XmlFile(Jenkins.XSTREAM, new File(subdir, "config.xml"));
+                    if (xmlFile.exists()) {
+                        Node node = (Node) xmlFile.read();
+                        newNodes.put(node.getNodeName(), node);
+                    }
+                } catch (IOException e) {
+                    Logger.getLogger(ItemGroupMixIn.class.getName()).log(Level.WARNING, "could not load " + subdir, e);
+                }
+            }
+        }
+        Queue.withLock(new Runnable() {
+            @Override
+            public void run() {
+                for (Iterator<Map.Entry<String, Node>> i = nodes.entrySet().iterator(); i.hasNext(); ) {
+                    if (!(i.next().getValue() instanceof EphemeralNode)) {
+                        i.remove();
+                    }
+                }
+                nodes.putAll(newNodes);
+                jenkins.updateComputerList();
+                jenkins.trimLabels();
+            }
+        });
+    }
+
+    /**
+     * Returns the directory that the nodes are stored in.
+     *
+     * @return the directory that the nodes are stored in.
+     * @throws IOException
+     */
+    private File getNodesDir() throws IOException {
+        final File nodesDir = new File(jenkins.getRootDir(), "nodes");
+        if (!nodesDir.isDirectory() && !nodesDir.mkdirs()) {
+            throw new IOException(String.format("Could not mkdirs %s", nodesDir));
+        }
+        return nodesDir;
+    }
+
+    /**
+     * Returns {@code true} if and only if the list of nodes is stored in the legacy location.
+     *
+     * @return {@code true} if and only if the list of nodes is stored in the legacy location.
+     */
+    public boolean isLegacy() {
+        return !new File(jenkins.getRootDir(), "nodes").isDirectory();
+    }
+}

core/src/main/java/jenkins/util/AtmostOneTaskExecutor.java

@@ -2,6 +2,8 @@ package jenkins.util;
 
 import com.google.common.util.concurrent.SettableFuture;
 import hudson.remoting.AtmostOneThreadExecutor;
+import hudson.util.DaemonThreadFactory;
+import hudson.util.NamingThreadFactory;
 
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executor;
@@ -63,7 +65,12 @@ public class AtmostOneTaskExecutor<V> {
     }
 
     public AtmostOneTaskExecutor(Callable<V> task) {
-        this(new AtmostOneThreadExecutor(),task);
+        this(new AtmostOneThreadExecutor(new NamingThreadFactory(
+                        new DaemonThreadFactory(),
+                        String.format("AtmostOneTaskExecutor[%s]", task)
+                )),
+                task
+        );
     }
 
     public synchronized Future<V> submit() {

core/src/main/resources/hudson/model/Messages.properties

@@ -313,6 +313,7 @@ RunParameterDefinition.DisplayName=Run Parameter
 PasswordParameterDefinition.DisplayName=Password Parameter
 
 Node.BecauseNodeIsReserved={0} is reserved for jobs with matching label expression
+Node.BecauseNodeIsNotAcceptingTasks={0} is not accepting tasks
 Node.LabelMissing={0} doesn\u2019t have label {1}
 Node.LackingBuildPermission={0} doesn\u2019t have a permission to run on {1}
 Node.Mode.NORMAL=Utilize this node as much as possible

core/src/main/resources/lib/hudson/executors.jelly

@@ -39,102 +39,121 @@ THE SOFTWARE.
       <j:if test="${!c.acceptingTasks}"> <st:nbsp/> (${%suspended})</j:if>
     </d:tag>
     <d:tag name="executor">
-      <j:choose>
-        <j:when test="${!c.offline or (c.offline and !e.idle)}">
-          <tr>
-            <td class="pane" align="right" style="vertical-align: top">
-              ${name}
-            </td>
-            <j:choose>
-              <j:when test="${!e.active}">
-                <td class="pane">
-                  <a href="${rootURL}/${c.url}${url}/causeOfDeath">
-                    <div class="error">${%Dead} (!)</div>
-                  </a>
-                </td>
-                <td class="pane"/>
-                <td class="pane"/>
-              </j:when>
-              <j:when test="${e.idle}">
-                <td class="pane">
-                  <j:choose>
-                    <j:when test="${c.offline}">
-                      <a href="${rootURL}/${c.url}" class="model-link inside">${%Offline}</a>
-                    </j:when>
-                    <j:otherwise>
-                      ${%Idle}
-                    </j:otherwise>
-                  </j:choose>
-                </td>
-                <td class="pane"/>
-                <td class="pane"/>
-              </j:when>
-              <j:otherwise>
-                <!-- not actually optional, but it helps with backward compatibility -->
-                <j:set var="executor" value="${e}" />
-                <st:include it="${e.currentExecutable}" page="executorCell.jelly" optional="true">
-                  <td class="pane">
-                    <div style="white-space: normal">
-                      <j:set var="exe" value="${e.currentExecutable}" />
-                      <j:invokeStatic var="exeparent"
-                         className="hudson.model.queue.Executables" method="getParentOf">
-                        <j:arg type="hudson.model.Queue$Executable" value="${exe}" />
-                      </j:invokeStatic>
-                      <j:choose>
-                        <j:when test="${h.hasPermission(exeparent,exeparent.READ)}">
-                          <a href="${rootURL}/${exeparent.url}"><l:breakable value="${exeparent.fullDisplayName}"/></a>
-                          <t:buildProgressBar build="${exe}" executor="${executor}"/>
-                        </j:when>
-                        <j:otherwise>
-                          <span>${%Unknown Task}</span>
-                        </j:otherwise>
-                      </j:choose>
-                    </div>
-                  </td>
+      <j:if test="${!c.offline or (c.offline and !e.idle)}">
+        <tr>
+          <td class="pane" align="right" style="vertical-align: top">
+            ${name}
+          </td>
+          <j:choose>
+            <j:when test="${!e.active}">
+              <td class="pane">
+                <a href="${rootURL}/${c.url}${url}/causeOfDeath">
+                  <div class="error">${%Dead} (!)</div>
+                </a>
+              </td>
+              <td class="pane"/>
+              <td class="pane"/>
+            </j:when>
+            <j:when test="${e.idle}">
+              <td class="pane">
+                <j:choose>
+                  <j:when test="${c.offline}">
+                    <a href="${rootURL}/${c.url}" class="model-link inside">${%Offline}</a>
+                  </j:when>
+                  <j:otherwise>
+                    ${%Idle}
+                  </j:otherwise>
+                </j:choose>
+              </td>
+              <td class="pane"/>
+              <td class="pane"/>
+            </j:when>
+            <j:otherwise>
+              <!-- not actually optional, but it helps with backward compatibility -->
+              <j:set var="executor" value="${e}" />
+              <st:include it="${e.currentExecutable}" page="executorCell.jelly" optional="true">
                 <td class="pane">
-                  <j:if test="${h.hasPermission(exeparent,exeparent.READ)}">
-                      <a href="${rootURL}/${exe.url}" class="model-link inside"><l:breakable value="${exe.displayName}"/></a>
-                  </j:if>
+                  <div style="white-space: normal">
+                    <j:set var="exe" value="${e.currentExecutable}" />
+                    <j:set var="wu" value="${e.currentWorkUnit}" />
+                    <j:choose>
+                      <j:when test="${exe == null and wu != null}">
+                        <j:set var="exeparent" value="${wu.work}"/>
+                        <j:choose>
+                          <j:when test="${h.hasPermission(exeparent,exeparent.READ)}">
+                            <a href="${rootURL}/${exeparent.url}"><l:breakable value="${exeparent.fullDisplayName}"/></a>
+                            <t:progressBar tooltip="${%Pending}" pos="-1" href="${rootURL}/${exeparent.url}"/>
+                          </j:when>
+                          <j:otherwise>
+                            <span>${%Unknown Task}</span>
+                          </j:otherwise>
+                        </j:choose>
+                      </j:when>
+                      <j:when test="${exe == null and wu == null}">
+                        <!-- went idle concurrent with testing for idle -->
+                        ${%Idle}
+                      </j:when>
+                      <j:otherwise>
+                        <j:invokeStatic var="exeparent"
+                           className="hudson.model.queue.Executables" method="getParentOf">
+                          <j:arg type="hudson.model.Queue$Executable" value="${exe}" />
+                        </j:invokeStatic>
+                        <j:choose>
+                          <j:when test="${h.hasPermission(exeparent,exeparent.READ)}">
+                            <a href="${rootURL}/${exeparent.url}"><l:breakable value="${exeparent.fullDisplayName}"/></a>
+                            <t:buildProgressBar build="${exe}" executor="${executor}"/>
+                          </j:when>
+                          <j:otherwise>
+                            <span>${%Unknown Task}</span>
+                          </j:otherwise>
+                        </j:choose>
+                      </j:otherwise>
+                    </j:choose>
+                  </div>
                 </td>
-                </st:include>
-                <td class="pane" align="center" valign="middle">
-                  <j:if test="${e.hasStopPermission()}">
-                    <l:stopButton href="${rootURL}/${c.url}${url}/stop" alt="${%terminate this build}"/>
-                  </j:if>
-                </td>
-              </j:otherwise>
-            </j:choose>
-          </tr>
-        </j:when>
-      </j:choose>
+              <td class="pane">
+                <j:if test="${h.hasPermission(exeparent,exeparent.READ)}">
+                    <a href="${rootURL}/${exe.url}" class="model-link inside"><l:breakable value="${exe.displayName}"/></a>
+                </j:if>
+              </td>
+              </st:include>
+              <td class="pane" align="center" valign="middle">
+                <j:if test="${e.hasStopPermission()}">
+                  <l:stopButton href="${rootURL}/${c.url}${url}/stop" alt="${%terminate this build}"/>
+                </j:if>
+              </td>
+            </j:otherwise>
+          </j:choose>
+        </tr>
+      </j:if>
     </d:tag>
   </d:taglib>
   <j:set var="computers" value="${attrs.computers?:app.computers}" />
-    <l:pane width="3" id="executors"
-        title="&lt;a href='${rootURL}/computer/'>${%Build Executor Status}&lt;/a>"
-        collapsedText="${%Computers(computers.size() - 1, app.unlabeledLoad.computeTotalExecutors() - app.unlabeledLoad.computeIdleExecutors(), app.unlabeledLoad.computeTotalExecutors())}">
-      <colgroup><col width="30"/><col width="200*"/><col width="24"/></colgroup>
+  <j:set var="computersSize" value="${computers.size()}"/>
+  <l:pane width="3" id="executors"
+      title="&lt;a href='${rootURL}/computer/'>${%Build Executor Status}&lt;/a>"
+      collapsedText="${%Computers(computersSize - 1, app.unlabeledLoad.computeTotalExecutors() - app.unlabeledLoad.computeIdleExecutors(), app.unlabeledLoad.computeTotalExecutors())}">
+    <colgroup><col width="30"/><col width="200*"/><col width="24"/></colgroup>
 
-      <j:forEach var="c" items="${computers}">
-        <tr>
-          <j:if test="${computers.size() gt 1 and (c.executors.size()!=0 or c.oneOffExecutors.size()!=0)}">
-            <th class="pane" colspan="3">
-              <local:computerCaption title="${c.displayName}" />
-            </th>
-          </j:if>
-        </tr>
-        <j:forEach var="e" items="${c.executors}" varStatus="eloop">
-          <local:executor name="${eloop.index+1}" url="executors/${eloop.index}" />
-        </j:forEach>
-        <j:forEach var="e" items="${c.oneOffExecutors}" varStatus="eloop">
-          <local:executor name="" url="oneOffExecutors/${eloop.index}" />
-        </j:forEach>
+    <j:forEach var="c" items="${computers}">
+      <j:set var="cDisplayExecutors" value="${c.displayExecutors}"/>
+      <tr>
+        <j:if test="${computersSize gt 1 and !cDisplayExecutors.isEmpty()}">
+          <th class="pane" colspan="3">
+            <local:computerCaption title="${c.displayName}" />
+          </th>
+        </j:if>
+      </tr>
+      <j:forEach var="de" items="${cDisplayExecutors}" varStatus="eloop">
+        <j:set var="e" value="${de.executor}"/>
+        <local:executor name="${de.displayName}" url="${de.url}" />
       </j:forEach>
-    </l:pane>
-    <!-- schedule updates only for the full page reload -->
-    <j:if test="${ajax==null and !h.isAutoRefresh(request) and h.hasPermission(app.READ)}">
-      <script defer="defer">
-        refreshPart('executors',"${rootURL}/${h.hasView(it,'ajaxExecutors')?it.url:''}ajaxExecutors");
-      </script>
-    </j:if>
+    </j:forEach>
+  </l:pane>
+  <!-- schedule updates only for the full page reload -->
+  <j:if test="${ajax==null and !h.isAutoRefresh(request) and h.hasPermission(app.READ)}">
+    <script defer="defer">
+      refreshPart('executors',"${rootURL}/${h.hasView(it,'ajaxExecutors')?it.url:''}ajaxExecutors");
+    </script>
+  </j:if>
 </j:jelly>

core/src/main/resources/lib/layout/layout.jelly

@@ -69,11 +69,14 @@ ${h.initPageVariables(context)}
 <j:set var="isMSIE" value="${userAgent.contains('MSIE')}"/>
 <j:set var="_" value="${request.getSession()}"/>
 <j:set var="_" value="${h.configureAutoRefresh(request, response, attrs.norefresh!=null and !attrs.norefresh.equals(false))}"/>
+  <j:set var="extensionsAvailable" value="${h.extensionsAvailable}"/>
   <j:if test="${request.servletPath=='/' || request.servletPath==''}">
     ${h.advertiseHeaders(response)}
-    <j:forEach var="pd" items="${h.pageDecorators}">
-      <st:include it="${pd}" page="httpHeaders.jelly" optional="true"/>
-    </j:forEach>
+    <j:if test="${extensionsAvailable}">
+      <j:forEach var="pd" items="${h.pageDecorators}">
+        <st:include it="${pd}" page="httpHeaders.jelly" optional="true"/>
+      </j:forEach>
+    </j:if>
   </j:if>
 <x:doctype name="html" />
 <html>
@@ -126,9 +129,11 @@ ${h.initPageVariables(context)}
     <script src="${resURL}/scripts/hudson-behavior.js" type="text/javascript"></script>
     <script src="${resURL}/scripts/sortable.js" type="text/javascript"/>
 
-    <script>
-        crumb.init("${h.getCrumbRequestField()}", "${h.getCrumb(request)}");
-    </script>
+    <j:if test="${extensionsAvailable}">
+      <script>
+          crumb.init("${h.getCrumbRequestField()}", "${h.getCrumb(request)}");
+      </script>
+    </j:if>
 
     <link rel="stylesheet" href="${resURL}/scripts/yui/container/assets/container.css" type="text/css"/>
     <link rel="stylesheet" href="${resURL}/scripts/yui/assets/skins/sam/skin.css" type="text/css" />
@@ -143,9 +148,11 @@ ${h.initPageVariables(context)}
     <meta name="ROBOTS" content="INDEX,NOFOLLOW" />
     <j:set var="mode" value="header" />
     <d:invokeBody />
-    <j:forEach var="pd" items="${h.pageDecorators}">
-      <st:include it="${pd}" page="header.jelly" optional="true" />
-    </j:forEach>
+    <j:if test="${extensionsAvailable}">
+      <j:forEach var="pd" items="${h.pageDecorators}">
+        <st:include it="${pd}" page="header.jelly" optional="true" />
+      </j:forEach>
+    </j:if>
 
     <j:if test="${isMSIE}">
       <script src="${resURL}/scripts/msie.js" type="text/javascript"/>
@@ -259,9 +266,11 @@ ${h.initPageVariables(context)}
         <span class="jenkins_ver">
           <a href="${h.getFooterURL()}">Jenkins ver. ${h.version}</a>
         </span>
+        <j:if test="${extensionsAvailable}">
         <j:forEach var="pd" items="${h.pageDecorators}">
           <st:include it="${pd}" page="footer.jelly" optional="true" />
         </j:forEach>
+        </j:if>
       </div>
     </div>
   </body>