diff --git a/core/src/main/java/hudson/model/queue/BackFiller.java b/core/src/main/java/hudson/model/queue/BackFiller.java new file mode 100644 index 0000000000000000000000000000000000000000..c568885f1cfd2ba9de31dbe867fe4b4c5d90aa39 --- /dev/null +++ b/core/src/main/java/hudson/model/queue/BackFiller.java @@ -0,0 +1,211 @@ +package hudson.model.queue; + +import com.google.common.collect.Iterables; +import hudson.Extension; +import hudson.model.Computer; +import hudson.model.Executor; +import hudson.model.Hudson; +import hudson.model.InvisibleAction; +import hudson.model.Queue.BuildableItem; +import hudson.model.queue.MappingWorksheet.ExecutorChunk; +import hudson.model.queue.MappingWorksheet.ExecutorSlot; +import hudson.model.queue.MappingWorksheet.Mapping; +import hudson.model.queue.MappingWorksheet.WorkChunk; +import hudson.util.TimeUnit2; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Experimental. + * + * @author Kohsuke Kawaguchi + */ +public class BackFiller extends LoadPredictor { + private boolean recursion = false; + + @Override + public Iterable predict(MappingWorksheet plan, Computer computer, long start, long end) { + TimeRange timeRange = new TimeRange(start, end - start); + List loads = new ArrayList(); + + for (BuildableItem bi : Hudson.getInstance().getQueue().getBuildableItems()) { + TentativePlan tp = bi.getAction(TentativePlan.class); + if (tp==null) {// do this even for bi==plan.item ensures that we have FIFO semantics in tentative plans. + tp = makeTentativePlan(bi); + if (tp==null) continue; // no viable plan. + } + + if (tp.isStale()) { + // if the tentative plan is stale, just keep on pushing it to the current time + // (if we recreate the plan, it'll be put at the end of the queue, whereas this job + // should actually get priority over others) + tp.range.shiftTo(System.currentTimeMillis()); + } + + // don't let its own tentative plan count when considering a scheduling for a job + if (plan.item==bi) continue; + + + // no overlap in the time span, meaning this plan is for a distant future + if (!timeRange.overlapsWith(tp.range)) continue; + + // if this tentative plan has no baring on this computer, that's ignorable + Integer i = tp.footprint.get(computer); + if (i==null) continue; + + return Collections.singleton(tp.range.toFutureLoad(i)); + } + + return loads; + } + + private static final class PseudoExecutorSlot extends ExecutorSlot { + private Executor executor; + + private PseudoExecutorSlot(Executor executor) { + this.executor = executor; + } + + @Override + public Executor getExecutor() { + return executor; + } + + @Override + public boolean isAvailable() { + return true; + } + + // this slot isn't executable + @Override + protected void set(WorkUnit p) { + throw new UnsupportedOperationException(); + } + } + + private TentativePlan makeTentativePlan(BuildableItem bi) { + if (recursion) return null; + recursion = true; + try { + // pretend for now that all executors are available and decide some assignment that's executable. + List slots = new ArrayList(); + for (Computer c : Hudson.getInstance().getComputers()) { + if (c.isOffline()) continue; + for (Executor e : c.getExecutors()) { + slots.add(new PseudoExecutorSlot(e)); + } + } + + // also ignore all load predictions as we just want to figure out some executable assignment + // and we are not trying to figure out if this task is executable right now. + MappingWorksheet worksheet = new MappingWorksheet(bi, slots, Collections.emptyList()); + Mapping m = Hudson.getInstance().getQueue().getLoadBalancer().map(bi.task, worksheet); + if (m==null) return null; + + // figure out how many executors we need on each computer? + Map footprint = new HashMap(); + for (Entry e : m.toMap().entrySet()) { + Computer c = e.getValue().computer; + Integer v = footprint.get(c); + if (v==null) v = 0; + v += e.getKey().size(); + footprint.put(c,v); + } + + // the point of a tentative plan is to displace other jobs to create a point in time + // where this task can start executing. An incorrectly estimated duration is not + // a problem in this regard, as we just need enough idle executors in the right moment. + // The downside of guessing the duration wrong is that we can end up creating tentative plans + // afterward that may be incorrect, but those plans will be rebuilt. + long d = bi.task.getEstimatedDuration(); + if (d<=0) d = TimeUnit2.MINUTES.toMillis(5); + + TimeRange slot = new TimeRange(System.currentTimeMillis(), d); + + // now, based on the real predicted loads, figure out the approximation of when we can + // start executing this guy. + for (Entry e : footprint.entrySet()) { + Computer computer = e.getKey(); + Timeline timeline = new Timeline(); + for (LoadPredictor lp : LoadPredictor.all()) { + for (FutureLoad fl : Iterables.limit(lp.predict(worksheet, computer, slot.start, slot.end),100)) { + timeline.insert(fl.startTime, fl.startTime+fl.duration, fl.numExecutors); + } + } + + Long x = timeline.fit(slot.start, slot.duration, computer.countExecutors()-e.getValue()); + // if no suitable range was found in [slot.start,slot.end), slot.end would be a good approximation + if (x==null) x = slot.end; + slot = slot.shiftTo(x); + } + + TentativePlan tp = new TentativePlan(footprint, slot); + bi.addAction(tp); + return tp; + } finally { + recursion = false; + } + } + + /** + * Represents a duration in time. + */ + private static final class TimeRange { + public final long start; + public final long duration; + public final long end; + + private TimeRange(long start, long duration) { + this.start = start; + this.duration = duration; + this.end = start+duration; + } + + public boolean overlapsWith(TimeRange that) { + return (this.start <= that.start && that.start <=this.end) + || (that.start <= this.start && this.start <=that.end); + } + + public FutureLoad toFutureLoad(int size) { + return new FutureLoad(start,duration,size); + } + + public TimeRange shiftTo(long newStart) { + if (newStart==start) return this; + return new TimeRange(newStart,duration); + } + } + + public static final class TentativePlan extends InvisibleAction { + private final Map footprint; + public final TimeRange range; + + public TentativePlan(Map footprint, TimeRange range) { + this.footprint = footprint; + this.range = range; + } + + public Object writeReplace() {// don't persist + return null; + } + + public boolean isStale() { + return range.end < System.currentTimeMillis(); + } + } + + /** + * Once this feature stabilizes, move it to the heavyjob plugin + */ + @Extension + public static BackFiller newInstance() { + if (Boolean.getBoolean(BackFiller.class.getName())) + return new BackFiller(); + return null; + } +}