added a mechanism to take future load of the system into account.

git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@35013 71c3de6d-444a-0410-be80-ed276b4c234a

core/src/main/java/hudson/model/queue/FutureLoad.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2010, InfraDNA, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package hudson.model.queue;
+
+/**
+ * Estimated future load to Hudson.
+ *
+ * @author Kohsuke Kawaguchi
+ * @see LoadPredictor
+ */
+public final class FutureLoad {
+    /**
+     * When is this load expected to start?
+     */
+    public final long startTime;
+    /**
+     * How many executors is this going to consume?
+     */
+    public final int numExecutors;
+    /**
+     * How long is task expected to continue, in milliseconds?
+     */
+    public final long duration;
+
+    public FutureLoad(long startTime, long duration, int numExecutors) {
+        this.startTime = startTime;
+        this.numExecutors = numExecutors;
+        this.duration = duration;
+    }
+
+    public String toString() {
+        return "startTime="+startTime+",#executors="+numExecutors+",duration="+duration;
+    }
+}

core/src/main/java/hudson/model/queue/LoadPredictor.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2010, InfraDNA, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package hudson.model.queue;
+
+import hudson.Extension;
+import hudson.ExtensionList;
+import hudson.ExtensionPoint;
+import hudson.model.Computer;
+import hudson.model.Executor;
+import hudson.model.Hudson;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Predicts future load to the system, to assist the scheduling decisions
+ *
+ * <p>
+ * When Hudson makes a scheduling decision, Hudson considers predicted future load
+ * &mdash; e.g., "We do currently have one available executor, but we know we need this for something else in 30 minutes,
+ * so we can't currently schedule a build that takes 1 hour."
+ *
+ * <p>
+ * This extension point plugs in such estimation of future load.
+ *
+ * @author Kohsuke Kawaguchi
+ */
+public abstract class LoadPredictor implements ExtensionPoint {
+    /**
+     * Estimates load starting from the 'start' timestamp, up to the 'end' timestamp.
+     *
+     * @param start
+     *      Where to start enumeration. Always bigger or equal to the current time of the execution.
+     */
+    public abstract Iterable<FutureLoad> predict(Computer computer, long start, long end);
+
+    /**
+     * All the registered instances.
+     */
+    public static ExtensionList<LoadPredictor> all() {
+        return Hudson.getInstance().getExtensionList(LoadPredictor.class);
+    }
+
+    /**
+     * Considers currently running tasks and their completion.
+     */
+    @Extension
+    public static class CurrentlyRunningTasks extends LoadPredictor {
+        @Override
+        public Iterable<FutureLoad> predict(final Computer computer, long start, long eternity) {
+            long now = System.currentTimeMillis();
+            List<FutureLoad> fl = new ArrayList<FutureLoad>();
+            for (Executor e : computer.getExecutors()) {
+                if (e.isIdle())     continue;
+
+                long eta = e.getEstimatedRemainingTimeMillis();
+                long end = eta<0 ? eternity : now + eta; // when does this task end?
+                if (end < start)    continue;   // should be over by the 'start' time.
+                fl.add(new FutureLoad(start, end-start, 1));
+            }
+            return fl;
+        }
+    }
+}

core/src/main/java/hudson/model/queue/MappingWorksheet.java

@@ -24,6 +24,7 @@
 package hudson.model.queue;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import hudson.model.Computer;
 import hudson.model.Executor;
 import hudson.model.Label;
@@ -40,6 +41,9 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
+
+import static java.lang.Math.*;
 
 /**
  * Defines a mapping problem for answering "where do we execute this task?"
@@ -273,7 +277,7 @@ public class MappingWorksheet {
 
 
     public MappingWorksheet(Task task, List<JobOffer> offers) {
-        // executors
+        // group executors by their computers
         Map<Computer,List<JobOffer>> j = new HashMap<Computer, List<JobOffer>>();
         for (JobOffer o : offers) {
             Computer c = o.executor.getOwner();
@@ -283,9 +287,36 @@ public class MappingWorksheet {
             l.add(o);
         }
 
+        {// take load prediction into account and reduce the available executor pool size accordingly
+            long duration = task.getEstimatedDuration();
+            if (duration > 0) {
+                long now = System.currentTimeMillis();
+                for (Entry<Computer, List<JobOffer>> e : j.entrySet()) {
+                    final List<JobOffer> list = e.getValue();
+                    final int max = e.getKey().countExecutors();
+
+                    // build up the prediction model. cut the chase if we hit the max.
+                    Timeline timeline = new Timeline();
+                    int peak = 0;
+                    OUTER:
+                    for (LoadPredictor lp : LoadPredictor.all()) {
+                        for (FutureLoad fl : Iterables.limit(lp.predict(e.getKey(), now, now + duration),100)) {
+                            peak = max(peak,timeline.insert(fl.startTime, fl.startTime+fl.duration, fl.numExecutors));
+                            if (peak>=max)  break OUTER;
+                        }
+                    }
+
+                    int minIdle = max-peak; // minimum number of idle nodes during this time period
+                    if (minIdle<list.size())
+                        e.setValue(list.subList(0,minIdle));
+                }
+            }
+        }
+
         // build into the final shape
         List<ExecutorChunk> executors = new ArrayList<ExecutorChunk>();
         for (List<JobOffer> group : j.values()) {
+            if (group.isEmpty())    continue;   // evict empty group
             ExecutorChunk ec = new ExecutorChunk(group, executors.size());
             if (ec.node==null)  continue;   // evict out of sync node
             executors.add(ec);

core/src/main/java/hudson/model/queue/Timeline.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2010, InfraDNA, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package hudson.model.queue;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static java.lang.Math.*;
+
+/**
+* Represents a mutable q(t), a discrete value that changes over the time.
+*
+* <p>
+* Internally represented by a set of ranges and the value of q(t) in that range,
+* as a map from "starting time of a range" to "value of q(t)".
+*/
+final class Timeline {
+    private final TreeMap<Long, int[]> data = new TreeMap<Long, int[]>();
+
+    /**
+     * Obtains q(t) for the given t.
+     */
+    private int at(long t) {
+        SortedMap<Long, int[]> head = data.headMap(t);
+        if (head.isEmpty()) return 0;
+        return data.get(head.lastKey())[0];
+    }
+
+    /**
+     * Splits the range set at the given timestamp (if it hasn't been split yet)
+     */
+    private void splitAt(long t) {
+        if (data.containsKey(t)) return; // already split at this timestamp
+
+        SortedMap<Long, int[]> head = data.headMap(t);
+
+        int v = head.isEmpty() ? 0 : data.get(head.lastKey())[0];
+        data.put(t, new int[]{v});
+    }
+
+    /**
+     * increases q(t) by n for t in [start,end).
+     *
+     * @return peak value of q(t) in this range as a result of addition.
+     */
+    int insert(long start, long end, int n) {
+        splitAt(start);
+        splitAt(end);
+
+        int peak = 0;
+        for (Map.Entry<Long, int[]> e : data.tailMap(start).headMap(end).entrySet()) {
+            peak = max(peak, e.getValue()[0] += n);
+        }
+        return peak;
+    }
+}

test/pom.xml

@@ -185,6 +185,12 @@ THE SOFTWARE.
       <artifactId>easymock</artifactId>
       <version>2.4</version>
     </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <version>1.8.5</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <profiles>

test/src/test/java/hudson/model/queue/LoadPredictorTest.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2010, InfraDNA, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package hudson.model.queue;
+
+import hudson.model.Computer;
+import hudson.model.Executor;
+import hudson.model.Node;
+import hudson.model.Queue.JobOffer;
+import hudson.model.Queue.Task;
+import org.jvnet.hudson.test.HudsonTestCase;
+import org.jvnet.hudson.test.TestExtension;
+
+import java.lang.reflect.Field;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import static java.util.Arrays.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * @author Kohsuke Kawaguchi
+ */
+public class LoadPredictorTest extends HudsonTestCase {
+    @TestExtension
+    public static class LoadPredictorImpl extends LoadPredictor {
+        @Override
+        public Iterable<FutureLoad> predict(Computer computer, long start, long end) {
+            return asList(new FutureLoad(start+5000, end-(start+5000), 1));
+        }
+    }
+
+    /**
+     * Makes sure that {@link LoadPredictor} is taken into account when building {@link MappingWorksheet}.
+     * The scenario is:
+     *
+     * - a computer with 1 executor, idle.
+     * - a future load of size 1 is predicted
+     * - hence the consideration of the current task at hand shall fail, as it'll collide with the estimated future load.
+     */
+    public void test1() throws Exception {
+        Task t = mock(Task.class);
+        when(t.getEstimatedDuration()).thenReturn(10000L);
+        when(t.getSubTasks()).thenReturn((Collection) asList(t));
+
+
+        Computer c = createMockComputer(1);
+
+        JobOffer o = createMockOffer(c.getExecutors().get(0));
+
+        MappingWorksheet mw = new MappingWorksheet(t, asList(o));
+
+        // the test load predictor should have pushed down the executor count to 0
+        assertTrue(mw.executors.isEmpty());
+        assertEquals(1,mw.works.size());
+    }
+
+    /**
+     * Test scenario is:
+     *
+     * - a computer with two executors, one is building something now
+     * - a future load of size 1 is predicted but it'll start after the currently building something is completed.
+     * - hence the currently available executor should be considered available (unlike in test1)
+     */
+    public void test2() throws Exception {
+        Task t = mock(Task.class);
+        when(t.getEstimatedDuration()).thenReturn(10000L);
+        when(t.getSubTasks()).thenReturn((Collection) asList(t));
+
+
+        Computer c = createMockComputer(2);
+        Executor e = c.getExecutors().get(0);
+
+        when(e.isIdle()).thenReturn(false);
+        when(e.getEstimatedRemainingTimeMillis()).thenReturn(300L);
+
+        JobOffer o = createMockOffer(c.getExecutors().get(1));
+
+        MappingWorksheet mw = new MappingWorksheet(t, asList(o));
+
+        // since the currently busy executor will free up before a future predicted load starts,
+        // we should have a valid executor remain in the queue
+        assertEquals(1,mw.executors.size());
+        assertEquals(1,mw.works.size());
+    }
+
+    private JobOffer createMockOffer(Executor e) throws NoSuchFieldException, IllegalAccessException {
+        JobOffer o = mock(JobOffer.class);
+        setExecutor(o, e);
+        return o;
+    }
+
+    private void setExecutor(JobOffer o, Executor e) throws NoSuchFieldException, IllegalAccessException {
+        Field f = o.getClass().getField("executor");
+        f.setAccessible(true);
+        f.set(o, e);
+    }
+
+    private Computer createMockComputer(int nExecutors) throws Exception {
+        Node n = mock(Node.class);
+        Computer c = mock(Computer.class);
+        when(c.getNode()).thenReturn(n);
+
+        List executors = new CopyOnWriteArrayList();
+
+        for (int i=0; i<nExecutors; i++) {
+            Executor e = mock(Executor.class);
+            when(e.isIdle()).thenReturn(true);
+            when(e.getOwner()).thenReturn(c);
+            executors.add(e);
+        }
+
+        Field f = Computer.class.getDeclaredField("executors");
+        f.setAccessible(true);
+        f.set(c, executors);
+
+        when(c.getExecutors()).thenReturn(executors);
+        
+        return c;
+    }
+}