From 70db23bf56bf327b86333acc81bc423a4b28de9a Mon Sep 17 00:00:00 2001 From: kohsuke Date: Tue, 21 Jul 2009 17:02:03 +0000 Subject: [PATCH] Merged revisions 19919,19998,20022 via svnmerge from https://www.dev.java.net/svn/hudson/branches/matrix-parent ........ r19919 | kohsuke | 2009-07-18 18:45:59 -0700 (Sat, 18 Jul 2009) | 1 line committing the work in progress. ........ r19998 | kohsuke | 2009-07-20 18:58:15 -0700 (Mon, 20 Jul 2009) | 1 line [HUDSON-936] I believe this should do. ........ r20022 | kohsuke | 2009-07-21 09:48:03 -0700 (Tue, 21 Jul 2009) | 1 line adding a switch to enable this selectively, so that we can give this feature a longer soak time. ........ git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@20025 71c3de6d-444a-0410-be80-ed276b4c234a --- .../java/hudson/matrix/MatrixProject.java | 3 +- core/src/main/java/hudson/model/Computer.java | 23 +++- core/src/main/java/hudson/model/Executor.java | 27 +++-- core/src/main/java/hudson/model/Hudson.java | 1 + .../java/hudson/model/OneOffExecutor.java | 65 +++++++++++ core/src/main/java/hudson/model/Queue.java | 41 ++++++- .../main/resources/lib/hudson/executors.jelly | 102 +++++++++--------- 7 files changed, 199 insertions(+), 63 deletions(-) create mode 100644 core/src/main/java/hudson/model/OneOffExecutor.java diff --git a/core/src/main/java/hudson/matrix/MatrixProject.java b/core/src/main/java/hudson/matrix/MatrixProject.java index 97cffe130c..4bb975a442 100644 --- a/core/src/main/java/hudson/matrix/MatrixProject.java +++ b/core/src/main/java/hudson/matrix/MatrixProject.java @@ -44,6 +44,7 @@ import hudson.model.SCMedItem; import hudson.model.Saveable; import hudson.model.TopLevelItem; import hudson.model.TopLevelItemDescriptor; +import hudson.model.Queue.FlyweightTask; import hudson.model.Descriptor.FormException; import hudson.tasks.BuildStep; import hudson.tasks.BuildStepDescriptor; @@ -86,7 +87,7 @@ import org.kohsuke.stapler.HttpResponse; * * @author Kohsuke Kawaguchi */ -public class MatrixProject extends AbstractProject implements TopLevelItem, SCMedItem, ItemGroup, Saveable { +public class MatrixProject extends AbstractProject implements TopLevelItem, SCMedItem, ItemGroup, Saveable, FlyweightTask { /** * Other configuration axes. * diff --git a/core/src/main/java/hudson/model/Computer.java b/core/src/main/java/hudson/model/Computer.java index 9f5cb2b055..0c8a69c0e7 100644 --- a/core/src/main/java/hudson/model/Computer.java +++ b/core/src/main/java/hudson/model/Computer.java @@ -98,6 +98,8 @@ import java.net.Inet4Address; public /*transient*/ abstract class Computer extends Actionable implements AccessControlled, ExecutorListener { private final CopyOnWriteArrayList executors = new CopyOnWriteArrayList(); + // TODO: + private final CopyOnWriteArrayList oneOffExecutors = new CopyOnWriteArrayList(); private int numExecutors; @@ -451,7 +453,7 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces } else { // if the number is increased, add new ones while(executors.size()(executors); } + /** + * Gets the read-only snapshot view of all {@link OneOffExecutor}s. + */ + @Exported + public List getOneOffExecutors() { + return new ArrayList(oneOffExecutors); + } + /** * Returns true if all the executors of this computer is idle. */ @@ -656,6 +666,17 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces return null; } + /** + * Starts executing a fly-weight task. + */ + /*package*/ final void startFlyWeightTask(Queue.BuildableItem p) { + oneOffExecutors.add(new OneOffExecutor(this, p)); + } + + /*package*/ final void remove(OneOffExecutor e) { + oneOffExecutors.remove(e); + } + private static class ListPossibleNames implements Callable,IOException> { public List call() throws IOException { List names = new ArrayList(); diff --git a/core/src/main/java/hudson/model/Executor.java b/core/src/main/java/hudson/model/Executor.java index d6f8b6f65e..656d802797 100644 --- a/core/src/main/java/hudson/model/Executor.java +++ b/core/src/main/java/hudson/model/Executor.java @@ -24,6 +24,7 @@ package hudson.model; import hudson.Util; +import hudson.model.Queue.*; import hudson.util.TimeUnit2; import hudson.security.ACL; import org.kohsuke.stapler.StaplerRequest; @@ -45,7 +46,7 @@ import java.util.logging.Level; */ @ExportedBean public class Executor extends Thread implements ModelObject { - private final Computer owner; + protected final Computer owner; private final Queue queue; private long startTime; @@ -65,11 +66,11 @@ public class Executor extends Thread implements ModelObject { private Throwable causeOfDeath; - public Executor(Computer owner) { - super("Executor #"+owner.getExecutors().size()+" for "+owner.getDisplayName()); + public Executor(Computer owner, int n) { + super("Executor #"+n+" for "+owner.getDisplayName()); this.owner = owner; this.queue = Hudson.getInstance().getQueue(); - this.number = owner.getExecutors().size(); + this.number = n; start(); } @@ -79,10 +80,7 @@ public class Executor extends Thread implements ModelObject { try { finishTime = System.currentTimeMillis(); - while(true) { - if(Hudson.getInstance() == null || Hudson.getInstance().isTerminating()) - return; - + while(shouldRun()) { synchronized(owner) { if(owner.getNumExecutors(), Stapl public static boolean KILL_AFTER_LOAD = Boolean.getBoolean(Hudson.class.getName()+".killAfterLoad"); public static boolean LOG_STARTUP_PERFORMANCE = Boolean.getBoolean(Hudson.class.getName()+".logStartupPerformance"); private static final boolean CONSISTENT_HASH = true; // Boolean.getBoolean(Hudson.class.getName()+".consistentHash"); + public static boolean FLYWEIGHT_SUPPORT = Boolean.getBoolean(Hudson.class.getName()+".flyweightSupport"); private static final Logger LOGGER = Logger.getLogger(Hudson.class.getName()); diff --git a/core/src/main/java/hudson/model/OneOffExecutor.java b/core/src/main/java/hudson/model/OneOffExecutor.java new file mode 100644 index 0000000000..3ba78ccf09 --- /dev/null +++ b/core/src/main/java/hudson/model/OneOffExecutor.java @@ -0,0 +1,65 @@ +/* + * The MIT License + * + * Copyright (c) 2004-2009, Sun Microsystems, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package hudson.model; + +import hudson.model.Queue.FlyweightTask; + +/** + * {@link Executor} that's temporarily added to carry out tasks that doesn't consume + * regular executors, like a matrix project parent build. + * + * @author Kohsuke Kawaguchi + * @see FlyweightTask + */ +public class OneOffExecutor extends Executor { + private Queue.Item item; + + public OneOffExecutor(Computer owner, Queue.Item item) { + super(owner,-1); + this.item = item; + } + + @Override + protected boolean shouldRun() { + // TODO: consulting super.shouldRun() here means we'll lose the item if it gets scheduled + // when super.shouldRun() returns false. + return super.shouldRun() && item!=null; + } + + @Override + protected Queue.Item grabJob() throws InterruptedException { + Queue.Item r = item; + item = null; + return r; + } + + @Override + public void run() { + try { + super.run(); + } finally { + owner.remove(this); + } + } +} diff --git a/core/src/main/java/hudson/model/Queue.java b/core/src/main/java/hudson/model/Queue.java index 88aefa5589..1f1afd8282 100644 --- a/core/src/main/java/hudson/model/Queue.java +++ b/core/src/main/java/hudson/model/Queue.java @@ -35,6 +35,8 @@ import hudson.triggers.Trigger; import hudson.util.OneShotEvent; import hudson.util.TimeUnit2; import hudson.util.XStream2; +import hudson.util.ConsistentHash; +import hudson.util.ConsistentHash.Hash; import java.io.BufferedReader; import java.io.File; @@ -852,7 +854,7 @@ public class Queue extends ResourceController implements Saveable { // ready to be executed LOGGER.fine(p.task.getFullDisplayName() + " no longer blocked"); itr.remove(); - buildables.put(p.task,new BuildableItem(p)); + makeBuildable(new BuildableItem(p)); } } @@ -862,16 +864,15 @@ public class Queue extends ResourceController implements Saveable { if (!top.timestamp.before(new GregorianCalendar())) return; // finished moving all ready items from queue + waitingList.remove(top); Task p = top.task; if (!isBuildBlocked(p)) { // ready to be executed immediately - waitingList.remove(top); LOGGER.fine(p.getFullDisplayName() + " ready to build"); - buildables.put(p,new BuildableItem(top)); + makeBuildable(new BuildableItem(top)); } else { // this can't be built now because another build is in progress // set this project aside. - waitingList.remove(top); LOGGER.fine(p.getFullDisplayName() + " is blocked"); blockedProjects.put(p,new BlockedItem(top)); } @@ -881,6 +882,31 @@ public class Queue extends ResourceController implements Saveable { sortingHandler.sortBuildableItems(buildables); } + private void makeBuildable(BuildableItem p) { + if(Hudson.FLYWEIGHT_SUPPORT && p.task instanceof FlyweightTask) { + ConsistentHash hash = new ConsistentHash(new Hash() { + public String hash(Node node) { + return node.getNodeName(); + } + }); + Hudson h = Hudson.getInstance(); + hash.add(h, h.getNumExecutors()*100); + for (Node n : h.getNodes()) + hash.add(n,n.getNumExecutors()*100); + + for (Node n : hash.list(p.task.getFullDisplayName())) { + Computer c = n.toComputer(); + if (c==null) continue; + c.startFlyWeightTask(p); + return; + } + // if the execution get here, it means we couldn't schedule it anywhere. + // so do the scheduling like other normal jobs. + } + + buildables.put(p.task,p); + } + public Api getApi() { return new Api(this); } @@ -891,6 +917,13 @@ public class Queue extends ResourceController implements Saveable { */ public interface TransientTask extends Task {} + /** + * Marks {@link Task}s that do not consume {@link Executor}. + * @see OneOffExecutor + * @since 1.318 + */ + public interface FlyweightTask extends Task {} + /** * Task whose execution is controlled by the queue. * diff --git a/core/src/main/resources/lib/hudson/executors.jelly b/core/src/main/resources/lib/hudson/executors.jelly index f39ce4d2e3..7db6121699 100644 --- a/core/src/main/resources/lib/hudson/executors.jelly +++ b/core/src/main/resources/lib/hudson/executors.jelly @@ -36,6 +36,56 @@ THE SOFTWARE. (${%offline}) (${%suspended}) + + + + ${name} + + + + + + ${%Dead} (!) + + + + + + + + + ${%Offline} + + + ${%Idle} + + + + + + + +
${%Building} + + + ${e.currentExecutable} + + + + ${%Unknown Task} + + +
+ + + + ${%terminate this build} + + + + + + @@ -67,55 +117,11 @@ THE SOFTWARE. - - - ${eloop.index+1} - - - - - - ${%Dead} (!) - - - - - - - - - ${%Offline} - - - ${%Idle} - - - - - - - -
${%Building} - - - ${e.currentExecutable} - - - - ${%Unknown Task} - - -
- - - - ${%terminate this build} - - - - - + + + + -- GitLab