From 173acc3e2d0ddd3df41877d0dd6e43a212cf8b83 Mon Sep 17 00:00:00 2001 From: kohsuke Date: Sun, 22 Jul 2007 22:59:06 +0000 Subject: [PATCH] implemented the resource-based mutual exclusion to Queue, so that tasks can do mutual exclusion without the built-in knowledge of each other. git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@3784 71c3de6d-444a-0410-be80-ed276b4c234a --- .../main/java/hudson/maven/MavenModule.java | 9 ++ .../java/hudson/model/AbstractProject.java | 14 +++ core/src/main/java/hudson/model/Executor.java | 2 +- core/src/main/java/hudson/model/Queue.java | 38 ++++-- core/src/main/java/hudson/model/Resource.java | 64 ++++++++++ .../java/hudson/model/ResourceController.java | 72 ++++++++++++ .../main/java/hudson/model/ResourceList.java | 111 ++++++++++++++++++ 7 files changed, 302 insertions(+), 8 deletions(-) create mode 100644 core/src/main/java/hudson/model/Resource.java create mode 100644 core/src/main/java/hudson/model/ResourceController.java create mode 100644 core/src/main/java/hudson/model/ResourceList.java diff --git a/core/src/main/java/hudson/maven/MavenModule.java b/core/src/main/java/hudson/maven/MavenModule.java index 4ef8e2a591..a17b8e1447 100644 --- a/core/src/main/java/hudson/maven/MavenModule.java +++ b/core/src/main/java/hudson/maven/MavenModule.java @@ -15,6 +15,7 @@ import hudson.model.JDK; import hudson.model.Job; import hudson.model.Label; import hudson.model.Node; +import hudson.model.Resource; import hudson.tasks.LogRotator; import hudson.util.DescribableList; import org.apache.maven.project.MavenProject; @@ -227,6 +228,14 @@ public final class MavenModule extends AbstractMavenProject,R extends A return newBuild(); } + /** + * Gets the {@link Resource} that represents the workspace of this project. + */ + public Resource getWorkspaceResource() { + return new Resource(getFullDisplayName()+" workspace"); + } + + /** + * List of necessary resources to perform the build of this project. + */ + public ResourceList getResourceList() { + return new ResourceList().w(getWorkspaceResource()); + } + public boolean checkout(AbstractBuild build, Launcher launcher, BuildListener listener, File changelogFile) throws IOException { SCM scm = getScm(); if(scm==null) diff --git a/core/src/main/java/hudson/model/Executor.java b/core/src/main/java/hudson/model/Executor.java index 51ceff2807..0bcacedc6e 100644 --- a/core/src/main/java/hudson/model/Executor.java +++ b/core/src/main/java/hudson/model/Executor.java @@ -56,7 +56,7 @@ public class Executor extends Thread { try { startTime = System.currentTimeMillis(); executable = task.createExecutable(); - executable.run(); + queue.execute(executable,task.getResourceList()); } catch (Throwable e) { // for some reason the executor died. this is really // a bug in the code, but we don't want the executor to die, diff --git a/core/src/main/java/hudson/model/Queue.java b/core/src/main/java/hudson/model/Queue.java index ef5d3a75bf..63cebcddb8 100644 --- a/core/src/main/java/hudson/model/Queue.java +++ b/core/src/main/java/hudson/model/Queue.java @@ -37,7 +37,7 @@ import java.util.logging.Logger; * * @author Kohsuke Kawaguchi */ -public class Queue { +public class Queue extends ResourceController { /** * Items in the queue ordered by {@link Item#timestamp}. * @@ -49,7 +49,9 @@ public class Queue { /** * {@link Project}s that can be built immediately - * but blocked because another build is in progress. + * but blocked because another build is in progress, + * required {@link Resource}s are not available, or otherwise blocked + * by {@link Task#isBuildBlocked()}. */ private final Set blockedProjects = new HashSet(); @@ -302,7 +304,7 @@ public class Queue { Task p = itr.next(); // one last check to make sure this build is not blocked. - if(p.isBuildBlocked()) { + if(isBuildBlocked(p)) { itr.remove(); blockedProjects.add(p); continue; @@ -451,6 +453,13 @@ public class Queue { } } + /** + * Checks if the given task is blocked. + */ + private boolean isBuildBlocked(Task t) { + return t.isBuildBlocked() || !canRun(t.getResourceList()); + } + /** * Queue maintenance. @@ -465,7 +474,7 @@ public class Queue { Iterator itr = blockedProjects.iterator(); while(itr.hasNext()) { Task p = itr.next(); - if(!p.isBuildBlocked()) { + if(!isBuildBlocked(p)) { // ready to be executed LOGGER.fine(p.getName()+" no longer blocked"); itr.remove(); @@ -480,7 +489,7 @@ public class Queue { return; // finished moving all ready items from queue Task p = top.task; - if(!p.isBuildBlocked()) { + if(!isBuildBlocked(p)) { // ready to be executed immediately queue.remove(top); LOGGER.fine(p.getName()+" ready to build"); @@ -553,9 +562,18 @@ public class Queue { long getEstimatedDuration(); /** - * Creates {@link Executable}, which performs the actual execution of the task. + * Creates {@link Executable}, which performs the actual execution of the task. */ Executable createExecutable() throws IOException; + + /** + * Gets the list of {@link Resource}s that this task requires. + * Used to make sure no two conflicting tasks run concurrently. + *

+ * This method must always return the {@link ResourceList} + * that contains the exact same set of {@link Resource}s. + */ + ResourceList getResourceList(); } public interface Executable extends Runnable { @@ -593,7 +611,10 @@ public class Queue { public final int id; /** - * Build is blocked because another build is in progress. + * Build is blocked because another build is in progress, + * required {@link Resource}s are not available, or otherwise blocked + * by {@link Task#isBuildBlocked()}. + * * This flag is only used in {@link Queue#getItems()} for * 'pseudo' items that are actually not really in the queue. */ @@ -642,6 +663,9 @@ public class Queue { } if(isBlocked) { + Resource r = getMissingResource(task.getResourceList()); + if(r!=null) + return "Waiting for "+r.displayName; return task.getWhyBlocked(); } diff --git a/core/src/main/java/hudson/model/Resource.java b/core/src/main/java/hudson/model/Resource.java new file mode 100644 index 0000000000..ef313f7568 --- /dev/null +++ b/core/src/main/java/hudson/model/Resource.java @@ -0,0 +1,64 @@ +package hudson.model; + +/** + * Represents things that {@link Queue.Executable} uses while running. + * + *

+ * This is used in {@link Queue} to support basic mutual exclusion/locks. If two + * {@link Queue.Task}s require the same {@link Resource}, they will not + * be run at the same time. + * + *

+ * Resources are compared by using their names. + * + * @since 1.121 + */ +public final class Resource { + /** + * Human-readable name of this resource. + * Used for rendering HTML. + */ + public final String displayName; + + /** + * Parent resource. + * + *

+ * A child resource is considered a part of the parent resource, + * so acquiring the parent resource always imply acquiring all + * the child resources. + */ + public final Resource parent; + + public Resource(Resource parent, String displayName) { + this.parent = parent; + this.displayName = displayName; + } + + public Resource(String displayName) { + this(null,displayName); + } + + public boolean isCollidingWith(Resource that) { + assert that!=null; + for(Resource r=that; r!=null; r=r.parent) + if(this.equals(r)) + return true; + for(Resource r=this; r!=null; r=r.parent) + if(that.equals(r)) + return true; + return false; + } + + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Resource that = (Resource) o; + + return displayName.equals(that.displayName) && parent.equals(that.parent); + } + + public int hashCode() { + return displayName.hashCode(); + } +} diff --git a/core/src/main/java/hudson/model/ResourceController.java b/core/src/main/java/hudson/model/ResourceController.java new file mode 100644 index 0000000000..9250ce8cf0 --- /dev/null +++ b/core/src/main/java/hudson/model/ResourceController.java @@ -0,0 +1,72 @@ +package hudson.model; + +import java.util.Set; +import java.util.HashSet; + +/** + * Controls mutual exclusion of {@link ResourceList}. + * @author Kohsuke Kawaguchi + */ +public class ResourceController { + /** + * {@link ResourceList}s that are used by activities that are in progress. + */ + private final Set inProgress = new HashSet(); + + private ResourceList inUse = ResourceList.EMPTY; + + /** + * Performs the task that requires the given list of resources. + * + *

+ * The execution is blocked until the resource is available. + * + * @throws InterruptedException + * the thread can be interrupted while waiting for the available resources. + */ + public void execute( Runnable task, ResourceList resources ) throws InterruptedException { + synchronized(this) { + while(inUse.isCollidingWith(resources)) + wait(); + + // we have a go + inProgress.add(resources); + inUse = ResourceList.union(inUse,resources); + } + + try { + task.run(); + } finally { + synchronized(this) { + inProgress.remove(resources); + inUse = ResourceList.union(inProgress); + notifyAll(); + } + } + } + + /** + * Checks if an activity that requires the given resource list + * can run immediately. + * + *

+ * This method is really only useful as a hint, since + * another activity might acquire resources before the caller + * gets to call {@link #execute(Runnable, ResourceList)}. + */ + public synchronized boolean canRun(ResourceList resources) { + return !inUse.isCollidingWith(resources); + } + + /** + * Of the resource in the given resource list, return the one that's + * currently in use. + * + *

+ * If more than one such resource exists, one is chosen and returned. + * This method is used for reporting what's causing the blockage. + */ + public synchronized Resource getMissingResource(ResourceList resources) { + return resources.getConflict(inUse); + } +} diff --git a/core/src/main/java/hudson/model/ResourceList.java b/core/src/main/java/hudson/model/ResourceList.java new file mode 100644 index 0000000000..143558cf0b --- /dev/null +++ b/core/src/main/java/hudson/model/ResourceList.java @@ -0,0 +1,111 @@ +package hudson.model; + +import java.util.Set; +import java.util.HashSet; +import java.util.Map; +import java.util.HashMap; +import java.util.Arrays; +import java.util.Collection; + +/** + * List of {@link Resource}s that an activity needs. + * + *

+ * There are two ways to access resources. Read and write. + * As with usual reader/writer pattern, multiple read accesses can + * co-exist concurrently, but write access requires exclusive access. + * + * @author Kohsuke Kawaguchi + * @since 1.121 + */ +public final class ResourceList { + /** + * All resources (R/W.) + */ + private final Set all = new HashSet(); + + /** + * Write accesses. + */ + private final Set write = new HashSet(); + + /** + * Creates union of all resources. + */ + public static ResourceList union(ResourceList... lists) { + return union(Arrays.asList(lists)); + } + + /** + * Creates union of all resources. + */ + public static ResourceList union(Collection lists) { + switch(lists.size()) { + case 0: + return EMPTY; + case 1: + return lists.iterator().next(); + default: + ResourceList r = new ResourceList(); + for (ResourceList l : lists) { + r.all.addAll(l.all); + r.write.addAll(l.write); + } + return r; + } + } + + /** + * Adds a resource for read access. + */ + public ResourceList r(Resource r) { + all.add(r); + return this; + } + + /** + * Adds a resource for write access. + */ + public ResourceList w(Resource r) { + all.add(r); + write.add(r); + return this; + } + + /** + * Checks if this resource list and that resource list has any conflicting + * resource access. + */ + public boolean isCollidingWith(ResourceList that) { + return getConflict(that)!=null; + } + + /** + * Returns the resource in this list that's colliding with the given resource list. + */ + public Resource getConflict(ResourceList that) { + for (Resource r : this.write) + for (Resource l : that.all) + if(r.isCollidingWith(l)) + return r; + for (Resource r : that.write) + for (Resource l : this.all) + if(r.isCollidingWith(l)) + return l; + return null; + } + + public String toString() { + Map m = new HashMap(); + for (Resource r : all) + m.put(r,"R"); + for (Resource r : write) + m.put(r,"W"); + return m.toString(); + } + + /** + * Empty resource list. + */ + public static final ResourceList EMPTY = new ResourceList(); +} -- GitLab