Merged revisions 19919,19998,20022 via svnmerge from

https://www.dev.java.net/svn/hudson/branches/matrix-parent

........
  r19919 | kohsuke | 2009-07-18 18:45:59 -0700 (Sat, 18 Jul 2009) | 1 line
  
  committing the work in progress.
........
  r19998 | kohsuke | 2009-07-20 18:58:15 -0700 (Mon, 20 Jul 2009) | 1 line
  
  [HUDSON-936] I believe this should do.
........
  r20022 | kohsuke | 2009-07-21 09:48:03 -0700 (Tue, 21 Jul 2009) | 1 line
  
  adding a switch to enable this selectively, so that we can give this feature a longer soak time.
........


git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@20025 71c3de6d-444a-0410-be80-ed276b4c234a

core/src/main/java/hudson/matrix/MatrixProject.java

@@ -44,6 +44,7 @@ import hudson.model.SCMedItem;
 import hudson.model.Saveable;
 import hudson.model.TopLevelItem;
 import hudson.model.TopLevelItemDescriptor;
+import hudson.model.Queue.FlyweightTask;
 import hudson.model.Descriptor.FormException;
 import hudson.tasks.BuildStep;
 import hudson.tasks.BuildStepDescriptor;
@@ -86,7 +87,7 @@ import org.kohsuke.stapler.HttpResponse;
  *
  * @author Kohsuke Kawaguchi
  */
-public class MatrixProject extends AbstractProject<MatrixProject,MatrixBuild> implements TopLevelItem, SCMedItem, ItemGroup<MatrixConfiguration>, Saveable {
+public class MatrixProject extends AbstractProject<MatrixProject,MatrixBuild> implements TopLevelItem, SCMedItem, ItemGroup<MatrixConfiguration>, Saveable, FlyweightTask  {
     /**
      * Other configuration axes.
      *

core/src/main/java/hudson/model/Computer.java

@@ -98,6 +98,8 @@ import java.net.Inet4Address;
 public /*transient*/ abstract class Computer extends Actionable implements AccessControlled, ExecutorListener {
 
     private final CopyOnWriteArrayList<Executor> executors = new CopyOnWriteArrayList<Executor>();
+    // TODO: 
+    private final CopyOnWriteArrayList<OneOffExecutor> oneOffExecutors = new CopyOnWriteArrayList<OneOffExecutor>();
 
     private int numExecutors;
     
@@ -451,7 +453,7 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
         } else {
             // if the number is increased, add new ones
             while(executors.size()<numExecutors)
-                executors.add(new Executor(this));
+                executors.add(new Executor(this,executors.size()));
         }
     }
 
@@ -486,6 +488,14 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
         return new ArrayList<Executor>(executors);
     }
 
+    /**
+     * Gets the read-only snapshot view of all {@link OneOffExecutor}s.
+     */
+    @Exported
+    public List<OneOffExecutor> getOneOffExecutors() {
+        return new ArrayList<OneOffExecutor>(oneOffExecutors);
+    }
+
     /**
      * Returns true if all the executors of this computer is idle.
      */
@@ -656,6 +666,17 @@ public /*transient*/ abstract class Computer extends Actionable implements Acces
         return null;
     }
 
+    /**
+     * Starts executing a fly-weight task.
+     */
+    /*package*/ final void startFlyWeightTask(Queue.BuildableItem p) {
+        oneOffExecutors.add(new OneOffExecutor(this, p));
+    }
+
+    /*package*/ final void remove(OneOffExecutor e) {
+        oneOffExecutors.remove(e);
+    }
+
     private static class ListPossibleNames implements Callable<List<String>,IOException> {
         public List<String> call() throws IOException {
             List<String> names = new ArrayList<String>();

core/src/main/java/hudson/model/Executor.java

@@ -24,6 +24,7 @@
 package hudson.model;
 
 import hudson.Util;
+import hudson.model.Queue.*;
 import hudson.util.TimeUnit2;
 import hudson.security.ACL;
 import org.kohsuke.stapler.StaplerRequest;
@@ -45,7 +46,7 @@ import java.util.logging.Level;
  */
 @ExportedBean
 public class Executor extends Thread implements ModelObject {
-    private final Computer owner;
+    protected final Computer owner;
     private final Queue queue;
 
     private long startTime;
@@ -65,11 +66,11 @@ public class Executor extends Thread implements ModelObject {
 
     private Throwable causeOfDeath;
 
-    public Executor(Computer owner) {
-        super("Executor #"+owner.getExecutors().size()+" for "+owner.getDisplayName());
+    public Executor(Computer owner, int n) {
+        super("Executor #"+n+" for "+owner.getDisplayName());
         this.owner = owner;
         this.queue = Hudson.getInstance().getQueue();
-        this.number = owner.getExecutors().size();
+        this.number = n;
         start();
     }
 
@@ -79,10 +80,7 @@ public class Executor extends Thread implements ModelObject {
 
         try {
             finishTime = System.currentTimeMillis();
-            while(true) {
-                if(Hudson.getInstance() == null || Hudson.getInstance().isTerminating())
-                    return;
-
+            while(shouldRun()) {
                 synchronized(owner) {
                     if(owner.getNumExecutors()<owner.getExecutors().size()) {
                         // we've got too many executors.
@@ -98,7 +96,7 @@ public class Executor extends Thread implements ModelObject {
 
                 Queue.Item queueItem;
                 try {
-                	queueItem = queue.pop();
+                	queueItem = grabJob();
                 } catch (InterruptedException e) {
                     continue;
                 }
@@ -144,6 +142,17 @@ public class Executor extends Thread implements ModelObject {
         }
     }
 
+    /**
+     * Returns true if we should keep going.
+     */
+    protected boolean shouldRun() {
+        return Hudson.getInstance() != null && !Hudson.getInstance().isTerminating();
+    }
+
+    protected Queue.Item grabJob() throws InterruptedException {
+        return queue.pop();
+    }
+
     /**
      * Returns the current {@link Queue.Task} this executor is running.
      *

core/src/main/java/hudson/model/Hudson.java

@@ -3369,6 +3369,7 @@ public final class Hudson extends Node implements ItemGroup<TopLevelItem>, Stapl
     public static boolean KILL_AFTER_LOAD = Boolean.getBoolean(Hudson.class.getName()+".killAfterLoad");
     public static boolean LOG_STARTUP_PERFORMANCE = Boolean.getBoolean(Hudson.class.getName()+".logStartupPerformance");
     private static final boolean CONSISTENT_HASH = true; // Boolean.getBoolean(Hudson.class.getName()+".consistentHash");
+    public static boolean FLYWEIGHT_SUPPORT = Boolean.getBoolean(Hudson.class.getName()+".flyweightSupport");
 
     private static final Logger LOGGER = Logger.getLogger(Hudson.class.getName());
 

core/src/main/java/hudson/model/OneOffExecutor.java

+/*
+ * The MIT License
+ *
+ * Copyright (c) 2004-2009, Sun Microsystems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package hudson.model;
+
+import hudson.model.Queue.FlyweightTask;
+
+/**
+ * {@link Executor} that's temporarily added to carry out tasks that doesn't consume
+ * regular executors, like a matrix project parent build.
+ *
+ * @author Kohsuke Kawaguchi
+ * @see FlyweightTask
+ */
+public class OneOffExecutor extends Executor {
+    private Queue.Item item;
+
+    public OneOffExecutor(Computer owner, Queue.Item item) {
+        super(owner,-1);
+        this.item = item;
+    }
+
+    @Override
+    protected boolean shouldRun() {
+        // TODO: consulting super.shouldRun() here means we'll lose the item if it gets scheduled
+        // when super.shouldRun() returns false.
+        return super.shouldRun() && item!=null;
+    }
+
+    @Override
+    protected Queue.Item grabJob() throws InterruptedException {
+        Queue.Item r = item;
+        item = null;
+        return r;
+    }
+
+    @Override
+    public void run() {
+        try {
+            super.run();
+        } finally {
+            owner.remove(this);
+        }
+    }
+}

core/src/main/java/hudson/model/Queue.java

@@ -35,6 +35,8 @@ import hudson.triggers.Trigger;
 import hudson.util.OneShotEvent;
 import hudson.util.TimeUnit2;
 import hudson.util.XStream2;
+import hudson.util.ConsistentHash;
+import hudson.util.ConsistentHash.Hash;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -852,7 +854,7 @@ public class Queue extends ResourceController implements Saveable {
                 // ready to be executed
                 LOGGER.fine(p.task.getFullDisplayName() + " no longer blocked");
                 itr.remove();
-                buildables.put(p.task,new BuildableItem(p));
+                makeBuildable(new BuildableItem(p));
             }
         }
 
@@ -862,16 +864,15 @@ public class Queue extends ResourceController implements Saveable {
             if (!top.timestamp.before(new GregorianCalendar()))
                 return; // finished moving all ready items from queue
 
+            waitingList.remove(top);
             Task p = top.task;
             if (!isBuildBlocked(p)) {
                 // ready to be executed immediately
-                waitingList.remove(top);
                 LOGGER.fine(p.getFullDisplayName() + " ready to build");
-                buildables.put(p,new BuildableItem(top));
+                makeBuildable(new BuildableItem(top));
             } else {
                 // this can't be built now because another build is in progress
                 // set this project aside.
-                waitingList.remove(top);
                 LOGGER.fine(p.getFullDisplayName() + " is blocked");
                 blockedProjects.put(p,new BlockedItem(top));
             }
@@ -881,6 +882,31 @@ public class Queue extends ResourceController implements Saveable {
         	sortingHandler.sortBuildableItems(buildables);
     }
 
+    private void makeBuildable(BuildableItem p) {
+        if(Hudson.FLYWEIGHT_SUPPORT && p.task instanceof FlyweightTask) {
+            ConsistentHash<Node> hash = new ConsistentHash<Node>(new Hash<Node>() {
+                public String hash(Node node) {
+                    return node.getNodeName();
+                }
+            });
+            Hudson h = Hudson.getInstance();
+            hash.add(h, h.getNumExecutors()*100);
+            for (Node n : h.getNodes())
+                hash.add(n,n.getNumExecutors()*100);
+            
+            for (Node n : hash.list(p.task.getFullDisplayName())) {
+                Computer c = n.toComputer();
+                if (c==null)    continue;
+                c.startFlyWeightTask(p);
+                return;
+            }
+            // if the execution get here, it means we couldn't schedule it anywhere.
+            // so do the scheduling like other normal jobs.
+        }
+        
+        buildables.put(p.task,p);
+    }
+
     public Api getApi() {
         return new Api(this);
     }
@@ -891,6 +917,13 @@ public class Queue extends ResourceController implements Saveable {
      */
     public interface TransientTask extends Task {}
 
+    /**
+     * Marks {@link Task}s that do not consume {@link Executor}.
+     * @see OneOffExecutor
+     * @since 1.318
+     */
+    public interface FlyweightTask extends Task {}
+
     /**
      * Task whose execution is controlled by the queue.
      *

core/src/main/resources/lib/hudson/executors.jelly

@@ -36,6 +36,56 @@ THE SOFTWARE.
       <j:if test="${c.offline}"> (${%offline})</j:if>
       <j:if test="${!c.acceptingTasks}"> (${%suspended})</j:if>
     </d:tag>
+    <d:tag name="executor">
+      <tr>
+        <td class="pane" align="right" style="vertical-align: top">
+          ${name}
+        </td>
+        <j:choose>
+          <j:when test="${!e.alive}">
+            <td class="pane">
+              <a href="${rootURL}/computers/${cloop.index}/${url}/causeOfDeath">
+                <span class="error">${%Dead} (!)</span>
+              </a>
+            </td>
+            <td class="pane"/>
+          </j:when>
+          <j:when test="${e.idle}">
+            <td class="pane">
+              <j:choose>
+                <j:when test="${c.offline}">
+                  <a href="${rootURL}/${c.url}">${%Offline}</a>
+                </j:when>
+                <j:otherwise>
+                  ${%Idle}
+                </j:otherwise>
+              </j:choose>
+            </td>
+            <td class="pane"/>
+          </j:when>
+          <j:otherwise>
+            <td class="pane">
+              <div nowrap="true">${%Building}
+                <j:choose>
+                  <j:when test="${h.hasPermission(e.currentExecutable.parent,e.currentExecutable.parent.READ)}">
+                    <a href="${rootURL}/${e.currentExecutable.url}">${e.currentExecutable}</a>
+                    <t:buildProgressBar build="${e.currentExecutable}" executor="${e}"/>
+                  </j:when>
+                  <j:otherwise>
+                    <span>${%Unknown Task}</span>
+                  </j:otherwise>
+                </j:choose>
+              </div>
+            </td>
+            <td class="pane" align="center" valign="middle">
+              <j:if test="${e.hasStopPermission()}">
+                <a href="${rootURL}/computers/${cloop.index}/${url}/stop"><img src="${imagesURL}/16x16/stop.gif" alt="${%terminate this build}" /></a>
+              </j:if>
+            </td>
+          </j:otherwise>
+        </j:choose>
+      </tr>
+    </d:tag>
   </d:taglib>
   <j:set var="computers" value="${attrs.computers?:app.computers}" />
 	<l:pane title="&lt;a href='${rootURL}/computer/'>${%Build Executor Status}&lt;/a>" width="3" id="executors">
@@ -67,55 +117,11 @@ THE SOFTWARE.
 	    </j:choose>
 
 	    <j:forEach var="e" items="${c.executors}" varStatus="eloop">
-	      <tr>
-	        <td class="pane" align="right" style="vertical-align: top">
-	          ${eloop.index+1}
-	        </td>
-	        <j:choose>
-            <j:when test="${!e.alive}">
-              <td class="pane">
-                <a href="${rootURL}/computers/${cloop.index}/executors/${eloop.index}/causeOfDeath">
-                  <span class="error">${%Dead} (!)</span>
-                </a>
-              </td>
-              <td class="pane"/>
-            </j:when>
-            <j:when test="${e.idle}">
-	            <td class="pane">
-                <j:choose>
-                  <j:when test="${c.offline}">
-                    <a href="${rootURL}/${c.url}">${%Offline}</a>
-                  </j:when>
-                  <j:otherwise>
-                    ${%Idle}
-                  </j:otherwise>
-                </j:choose>
-	            </td>
-	            <td class="pane"/>
-	          </j:when>
-	          <j:otherwise>
-	            <td class="pane">
-                <div nowrap="true">${%Building}
-                  <j:choose>
-                    <j:when test="${h.hasPermission(e.currentExecutable.parent,e.currentExecutable.parent.READ)}">
-                      <a href="${rootURL}/${e.currentExecutable.url}">${e.currentExecutable}</a>
-                      <t:buildProgressBar build="${e.currentExecutable}" executor="${e}"/>
-                    </j:when>
-                    <j:otherwise>
-                      <span>${%Unknown Task}</span>
-                    </j:otherwise>
-                  </j:choose>
-                </div>
-              </td>
-	            <td class="pane" align="center" valign="middle">
-                <j:if test="${e.hasStopPermission()}">
-                  <a href="${rootURL}/computers/${cloop.index}/executors/${eloop.index}/stop"><img src="${imagesURL}/16x16/stop.gif" alt="${%terminate this build}" /></a>
-                </j:if>
-	            </td>
-	          </j:otherwise>
-	        </j:choose>
-	      </tr>
+        <local:executor name="${eloop.index+1}" url="executors/${eloop.index}" />
 	    </j:forEach>
+      <j:forEach var="e" items="${c.oneOffExecutors}" varStatus="eloop">
+        <local:executor name="" url="oneOffExecutors/${eloop.index}" />
+      </j:forEach>
 	  </j:forEach>
 	</l:pane>
     <!-- schedule updates only for the full page reload -->