Improved the tracking of queued jobs and their eventual builds in the REST API.

This came from Alpha CSP guys during JUC Israel. The remote API needs to expose a better way of tracking a submitted task.

changelog.html

@@ -61,6 +61,8 @@ Upcoming changes</a>
   <li class='major bug'>
     Fingerprint action deserialization problem fixed.
     (<a href="https://issues.jenkins-ci.org/browse/JENKINS-17125">issue 17125</a>)
+  <li class='rfe'>
+    Improved the tracking of queued jobs and their eventual builds in the REST API.
 </ul>
 </div><!--=TRUNK-END=-->
 

core/src/main/java/hudson/model/AbstractProject.java

@@ -1798,8 +1798,11 @@ public abstract class AbstractProject<P extends AbstractProject<P,R>,R extends A
         if (!isBuildable())
             throw HttpResponses.error(SC_INTERNAL_SERVER_ERROR,new IOException(getFullName()+" is not buildable"));
 
-        Jenkins.getInstance().getQueue().schedule(this, (int)delay.getTime(), getBuildCause(req));
-        rsp.sendRedirect(".");
+        WaitingItem item = Jenkins.getInstance().getQueue().schedule(this, (int) delay.getTime(), getBuildCause(req));
+        if (item!=null) {
+            rsp.sendRedirect(SC_CREATED,req.getContextPath()+'/'+item.getUrl());
+        } else
+            rsp.sendRedirect(".");
     }
 
     /**

core/src/main/java/hudson/model/Executor.java

@@ -214,6 +214,7 @@ public class Executor extends Thread implements ModelObject {
                         task = workUnit.work;
                         startTime = System.currentTimeMillis();
                         executable = task.createExecutable();
+                        workUnit.setExecutable(executable);
                     }
                     if (LOGGER.isLoggable(FINE))
                         LOGGER.log(FINE, getName()+" is going to execute "+executable);

core/src/main/java/hudson/model/ParametersDefinitionProperty.java

@@ -34,6 +34,7 @@ import java.util.AbstractList;
 
 import javax.servlet.ServletException;
 
+import hudson.model.Queue.WaitingItem;
 import jenkins.model.Jenkins;
 import jenkins.util.TimeDuration;
 import net.sf.json.JSONArray;
@@ -48,6 +49,8 @@ import org.kohsuke.stapler.export.ExportedBean;
 import hudson.Extension;
 import org.kohsuke.stapler.export.Flavor;
 
+import static javax.servlet.http.HttpServletResponse.SC_CREATED;
+
 /**
  * Keeps a list of the parameters defined for a project.
  *
@@ -132,11 +135,13 @@ public class ParametersDefinitionProperty extends JobProperty<AbstractProject<?,
             values.add(parameterValue);
         }
 
-    	Jenkins.getInstance().getQueue().schedule(
+    	WaitingItem item = Jenkins.getInstance().getQueue().schedule(
                 owner, delay.getTime(), new ParametersAction(values), new CauseAction(new Cause.UserIdCause()));
-
-        // send the user back to the job top page.
-        rsp.sendRedirect(".");
+        if (item!=null)
+            rsp.sendRedirect(SC_CREATED,req.getContextPath()+'/'+item.getUrl());
+        else
+            // send the user back to the job top page.
+            rsp.sendRedirect(".");
     }
 
     public void buildWithParameters(StaplerRequest req, StaplerResponse rsp) throws IOException, ServletException {

core/src/main/java/hudson/model/Queue.java

@@ -24,6 +24,8 @@
  */
 package hudson.model;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
 import com.google.common.collect.ImmutableList;
 import com.infradna.tool.bridge_method_injector.WithBridgeMethods;
 import hudson.AbortException;
@@ -90,6 +92,11 @@ import java.util.Set;
 import java.util.Timer;
 import java.util.TreeSet;
 import java.util.Map.Entry;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicLong;
@@ -116,7 +123,7 @@ import org.kohsuke.stapler.interceptor.RequirePOST;
  * <p>
  * This class implements the core scheduling logic. {@link Task} represents the executable
  * task that are placed in the queue. While in the queue, it's wrapped into {@link Item}
- * so that we can keep track of additional data used for deciding what to exeucte when.
+ * so that we can keep track of additional data used for deciding what to execute when.
  *
  * <p>
  * Items in queue goes through several stages, as depicted below:
@@ -125,7 +132,7 @@ import org.kohsuke.stapler.interceptor.RequirePOST;
  *                           |        ^
  *                           |        |
  *                           |        v
- *                           +--> buildables ---> pending ---> (executed)
+ *                           +--> buildables ---> pending ---> left
  * </pre>
  *
  * <p>
@@ -167,8 +174,36 @@ public class Queue extends ResourceController implements Saveable {
      */
     private final ItemList<BuildableItem> pendings = new ItemList<BuildableItem>();
 
+    /**
+     * Items that left queue would stay here for a while to enable tracking via {@link Item#id}.
+     *
+     * This map is forgetful, since we can't remember everything that executed in the past.
+     */
+//    private final FifoMap<Integer,LeftItem> leftItems = new FifoMap<Integer,LeftItem>(256);
+    private final Cache<Integer,LeftItem> leftItems = CacheBuilder.newBuilder().expireAfterWrite(5*60, TimeUnit.SECONDS).build();
+
     private final CachedItemList itemsView = new CachedItemList();
 
+//    class FifoMap<K,V> {
+//        private final ConcurrentMap<K,V> store = new ConcurrentHashMap<K,V>();
+//        private final BlockingQueue<K> order = new LinkedBlockingQueue<K>();
+//        private final int size;
+//
+//        FifoMap(int size) {
+//            this.size = size;
+//        }
+//
+//        private void add(K k, V v) {
+//            store.put(k,v);
+//            order.add(k);
+//            while (order.size()>size)
+//                store.remove(order.remove());
+//        }
+//
+//        public V get(K k) {
+//            return store.get(k);
+//        }
+//    }
     /**
      * Maintains a copy of {@link Queue#getItems()}
      *
@@ -612,6 +647,10 @@ public class Queue extends ResourceController implements Saveable {
         LOGGER.log(Level.FINE, "Cancelling {0} item#{1}", new Object[] {item.task, item.id});
         // use bitwise-OR to make sure that all the branches get evaluated all the time
         boolean r = (item instanceof WaitingItem && waitingList.remove(item)) | blockedProjects.remove(item) | buildables.remove(item);
+
+        LeftItem li = new LeftItem(item);
+        leftItems.put(li.id,li);
+
         if(r)
             item.onCancelled();
         return r;
@@ -683,7 +722,8 @@ public class Queue extends ResourceController implements Saveable {
     	for (Item item: blockedProjects) if (item.id == id) return item;
     	for (Item item: buildables) if (item.id == id) return item;
         for (Item item: pendings) if (item.id == id) return item;
-    	return null;
+
+        return leftItems.getIfPresent(id);
     }
 
     /**
@@ -720,6 +760,13 @@ public class Queue extends ResourceController implements Saveable {
         return new ArrayList<BuildableItem>(pendings.values());
     }
 
+    /**
+     * Returns the snapshot of all {@link LeftItem}s.
+     */
+    public Collection<LeftItem> getLeftItems() {
+        return Collections.unmodifiableCollection(leftItems.asMap().values());
+    }
+
     /**
      * Gets all items that are in the queue but not blocked
      *
@@ -852,6 +899,9 @@ public class Queue extends ResourceController implements Saveable {
             OneOffExecutor ooe = (OneOffExecutor) exec;
             final WorkUnit wu = ooe.getAssignedWorkUnit();
             pendings.remove(wu.context.item);
+
+            LeftItem li = new LeftItem(wu.context);
+            leftItems.put(li.id,li);
             return wu;
         }
 
@@ -895,8 +945,11 @@ public class Queue extends ResourceController implements Saveable {
                     LOGGER.log(Level.FINE, "Pop returning {0} for {1}", new Object[] {offer.workUnit, exec.getName()});
 
                     // TODO: I think this has to be done by the last executor that leaves the pop(), not by main executor
-                    if (offer.workUnit.isMainWork())
+                    if (offer.workUnit.isMainWork()) {
                         pendings.remove(offer.workUnit.context.item);
+                        LeftItem li = new LeftItem(offer.workUnit.context);
+                        leftItems.put(li.id,li);
+                    }
 
                     return offer.workUnit;
                 }
@@ -1407,6 +1460,17 @@ public class Queue extends ResourceController implements Saveable {
         	this(item.task, item.getActions(), item.id, item.future, item.inQueueSince);
         }
 
+        /**
+         * Returns the URL of this {@link Item} relative to the context path of Jenkins
+         *
+         * @return
+         *      URL that ends with '/'.
+         */
+        @Exported
+        public String getUrl() {
+            return "queue/item/"+id+'/';
+        }
+
         /**
          * Gets a human-readable status message describing why it's in the queue.
          */
@@ -1468,6 +1532,10 @@ public class Queue extends ResourceController implements Saveable {
             future.setAsCancelled();
         }
 
+        public Api getApi() {
+            return new Api(this);
+        }
+
         private Object readResolve() {
             this.future = new FutureImpl(task);
             return this;
@@ -1691,6 +1759,52 @@ public class Queue extends ResourceController implements Saveable {
         }
     }
 
+    /**
+     * {@link Item} in the {@link Queue#leftItems} stage. These are items that had left the queue
+     * by either began executing or by getting cancelled.
+     */
+    public final static class LeftItem extends Item {
+        public final WorkUnitContext outcome;
+
+        /**
+         * When item has left the queue and begin executing.
+         */
+        public LeftItem(WorkUnitContext wuc) {
+            super(wuc.item);
+            this.outcome = wuc;
+        }
+
+        /**
+         * When item is cancelled.
+         */
+        public LeftItem(Item cancelled) {
+            super(cancelled);
+            this.outcome = null;
+        }
+
+        @Override
+        public CauseOfBlockage getCauseOfBlockage() {
+            return null;
+        }
+
+        /**
+         * If this is representing an item that started executing, this property returns
+         * the primary executable (such as {@link AbstractBuild}) that created out of it.
+         */
+        @Exported
+        public Executable getExecutable() {
+            return outcome!=null ? outcome.getPrimaryWorkUnit().getExecutable() : null;
+        }
+
+        /**
+         * Is this representing a cancelled item?
+         */
+        @Exported
+        public boolean isCancelled() {
+            return outcome==null;
+        }
+    }
+
     private static final Logger LOGGER = Logger.getLogger(Queue.class.getName());
 
     /**

core/src/main/java/hudson/model/queue/WorkUnit.java

@@ -48,6 +48,7 @@ public final class WorkUnit {
     public final WorkUnitContext context;
 
     private volatile Executor executor;
+    private Executable executable;
 
     WorkUnit(WorkUnitContext context, SubTask work) {
         this.context = context;
@@ -69,10 +70,14 @@ public final class WorkUnit {
     }
 
     /**
-     * If the execution has already started, return the current executable.
+     * If the execution has already started, return the executable that was created.
      */
     public Executable getExecutable() {
-        return executor!=null ? executor.getCurrentExecutable() : null;
+        return executable;
+    }
+
+    public void setExecutable(Executable executable) {
+        this.executable = executable;
     }
 
     /**

core/src/main/resources/hudson/model/Job/_api.jelly

@@ -37,6 +37,11 @@ THE SOFTWARE.
   <p>
     To programmatically schedule a new build, post to <a href="../build">this URL</a>. 
     If the build has parameters, post to <a href="../buildWithParameters">this URL</a> and provide the parameters as form data.
+    Either way, the successful queueing will result in 201 status code with <tt>Location</tt> HTTP header
+    pointing the URL of the item in the queue. By polling the <tt>api/xml</tt> sub-URL of the queue item,
+    you can track the status of the queued task. Generally, the task will go through some state transitions,
+    then eventually it becomes either cancelled (look for the "cancelled" boolean property), or gets executed
+    (look for the "executable" property that typically points to the <tt>AbstractBuild</tt> object.)
   </p>
   <p>
     To programmatically schedule SCM polling, post to <a href="../polling">this URL</a>.