From dd5865980e861a8f8658d28ba442a308abc5174b Mon Sep 17 00:00:00 2001 From: kohsuke Date: Tue, 13 Mar 2007 14:17:35 +0000 Subject: [PATCH] extracted the generally useful portion of FingerprintMap as KeyedDataStorage. git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@2500 71c3de6d-444a-0410-be80-ed276b4c234a --- .../java/hudson/model/FingerprintMap.java | 122 +++--------- .../java/hudson/util/KeyedDataStorage.java | 185 ++++++++++++++++++ 2 files changed, 212 insertions(+), 95 deletions(-) create mode 100644 core/src/main/java/hudson/util/KeyedDataStorage.java diff --git a/core/src/main/java/hudson/model/FingerprintMap.java b/core/src/main/java/hudson/model/FingerprintMap.java index d0df9c06d9..a20b5c8f55 100644 --- a/core/src/main/java/hudson/model/FingerprintMap.java +++ b/core/src/main/java/hudson/model/FingerprintMap.java @@ -1,11 +1,10 @@ package hudson.model; import hudson.Util; +import hudson.util.KeyedDataStorage; import java.io.File; import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.concurrent.ConcurrentHashMap; /** * Cache of {@link Fingerprint}s. @@ -17,47 +16,7 @@ import java.util.concurrent.ConcurrentHashMap; * * @author Kohsuke Kawaguchi */ -public final class FingerprintMap { - /** - * The value is either {@code WeakReference} or {@link Loading}. - * - * If it's {@link WeakReference}, that represents the currently available value. - * If it's {@link Loading}, then that indicates the fingerprint is being loaded. - * The thread can wait on this object to be notified when the loading completes. - */ - private final ConcurrentHashMap core = new ConcurrentHashMap(); - - /** - * Used in {@link FingerprintMap#core} to indicate that the loading of a fingerprint - * is in progress, so that we can avoid creating two {@link Fingerprint}s for the same hash code, - * but do so without having a single lock. - */ - private static class Loading { - private Fingerprint value; - private boolean set; - - public synchronized void set(Fingerprint value) { - this.set = true; - this.value = value; - notifyAll(); - } - - /** - * Blocks until the value is {@link #set(Fingerprint)} by another thread - * and returns the value. - */ - public synchronized Fingerprint get() { - try { - while(!set) - wait(); - return value; - } catch (InterruptedException e) { - // assume the loading failed, but make sure we process interruption properly later - Thread.currentThread().interrupt(); - return null; - } - } - } +public final class FingerprintMap extends KeyedDataStorage { /** * Returns true if there's some data in the fingerprint database. @@ -77,64 +36,16 @@ public final class FingerprintMap { } public Fingerprint getOrCreate(AbstractBuild build, String fileName, String md5sum) throws IOException { - assert build!=null; - assert fileName!=null; - Fingerprint fp = get(md5sum); - if(fp!=null) - return fp; // found it. - - // not found. need to create one. - // creates a new one. - // since it's nearly impossible for two different files to have the same md5 sum, - // this part is not synchronized. - fp = new Fingerprint(build,fileName,toByteArray(md5sum)); - - core.put(md5sum,new WeakReference(fp)); - - return fp; + return super.getOrCreate(md5sum, new FingerprintParams(build,fileName)); } - public synchronized Fingerprint get(String md5sum) throws IOException { + protected Fingerprint get(String md5sum, boolean createIfNotExist, FingerprintParams createParams) throws IOException { + // sanity check if(md5sum.length()!=32) return null; // illegal input md5sum = md5sum.toLowerCase(); - while(true) { - Object value = core.get(md5sum); - - if(value instanceof WeakReference) { - WeakReference wfp = (WeakReference) value; - Fingerprint fp = wfp.get(); - if(fp!=null) - return fp; // found it - } - if(value instanceof Loading) { - // another thread is loading it. get the value from there. - return ((Loading)value).get(); - } - - // the fingerprint doesn't seem to be loaded thus far, so let's load it now. - // the care needs to be taken that other threads might be trying to do the same. - Loading l = new Loading(); - if(value==null ? core.putIfAbsent(md5sum,l)!=null : !core.replace(md5sum,value,l)) { - // the value has changed since then. another thread is attempting to do the same. - // go back to square 1 and try it again. - continue; - } - - Fingerprint fp = Fingerprint.load(toByteArray(md5sum)); - // let other threads know that the value is available now - l.set(fp); - - // the map needs to be updated to reflect the result of loading - if(fp!=null) - core.put(md5sum,new WeakReference(fp)); - else - core.remove(md5sum); - - return fp; - } - + return super.get(md5sum,createIfNotExist,createParams); } private byte[] toByteArray(String md5sum) { @@ -144,4 +55,25 @@ public final class FingerprintMap { return data; } + protected Fingerprint create(String md5sum, FingerprintParams createParams) throws IOException { + return new Fingerprint(createParams.build, createParams.fileName, toByteArray(md5sum)); + } + + protected Fingerprint load(String key) throws IOException { + return Fingerprint.load(toByteArray(key)); + } } + +class FingerprintParams { + final AbstractBuild build; + final String fileName; + + + public FingerprintParams(AbstractBuild build, String fileName) { + this.build = build; + this.fileName = fileName; + + assert build!=null; + assert fileName!=null; + } +} \ No newline at end of file diff --git a/core/src/main/java/hudson/util/KeyedDataStorage.java b/core/src/main/java/hudson/util/KeyedDataStorage.java new file mode 100644 index 0000000000..4b02ba1f17 --- /dev/null +++ b/core/src/main/java/hudson/util/KeyedDataStorage.java @@ -0,0 +1,185 @@ +package hudson.util; + +import hudson.model.Fingerprint; +import hudson.model.FingerprintMap; + +import java.io.IOException; +import java.lang.ref.WeakReference; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Convenient base class for implementing data storage. + * + *

+ * One typical pattern of data storage in Hudson is the one that {@link Fingerprint} + * uses, where each data is keyed by an unique key (MD5 sum), and that key is used + * to determine the file system location of the data. + * + * On memory, each data is represented by one object ({@link Fingerprint}), and + * write access to the same data is coordinated by using synchronization. + * + *

+ * With such storage, care has to be taken to ensure that there's only one data + * object in memory for any given key. That means load and create operation + * needs to be synchronized. This class implements this logic in a fairly efficient + * way, and thus intends to help plugins that want to use such data storage. + * + * @since 1.87 + * @author Kohsuke Kawaguchi + * @see FingerprintMap + */ +public abstract class KeyedDataStorage { + /** + * The value is either {@code WeakReference} or {@link Loading}. + * + * If it's {@link WeakReference}, that represents the currently available value. + * If it's {@link Loading}, then that indicates the fingerprint is being loaded. + * The thread can wait on this object to be notified when the loading completes. + */ + private final ConcurrentHashMap core = new ConcurrentHashMap(); + + /** + * Used in {@link KeyedDataStorage#core} to indicate that the loading of a fingerprint + * is in progress, so that we can avoid creating two {@link Fingerprint}s for the same hash code, + * but do so without having a single lock. + */ + private static class Loading { + private T value; + private boolean set; + + public synchronized void set(T value) { + this.set = true; + this.value = value; + notifyAll(); + } + + /** + * Blocks until the value is {@link #set(Object)} by another thread + * and returns the value. + */ + public synchronized T get() { + try { + while(!set) + wait(); + return value; + } catch (InterruptedException e) { + // assume the loading failed, but make sure we process interruption properly later + Thread.currentThread().interrupt(); + return null; + } + } + } + + /** + * Atomically gets the existing data object if any, or if it doesn't exist + * {@link #create(String,Object) create} it and return it. + * + * @return + * Never null. + * @param createParams + * Additional parameters needed to create a new data object. Can be null. + */ + public T getOrCreate(String key, P createParams) throws IOException { + return get(key,true,createParams); + } + + /** + * Finds the data object that matches the given key if available, or null + * if not found. + */ + public T get(String key) throws IOException { + return get(key,false,null); + } + + /** + * Implementation of get/getOrCreate. + */ + protected T get(String key, boolean createIfNotExist, P createParams) throws IOException { + while(true) { + Object value = core.get(key); + + if(value instanceof WeakReference) { + WeakReference wfp = (WeakReference) value; + T t = wfp.get(); + if(t!=null) + return t; // found it + } + if(value instanceof Loading) { + // another thread is loading it. get the value from there. + T t = ((Loading)value).get(); + if(t!=null || !createIfNotExist) + return t; // found it (t!=null) or we are just 'get' (!createIfNotExist) + } + + // the fingerprint doesn't seem to be loaded thus far, so let's load it now. + // the care needs to be taken that other threads might be trying to do the same. + Loading l = new Loading(); + if(value==null ? core.putIfAbsent(key,l)!=null : !core.replace(key,value,l)) { + // the value has changed since then. another thread is attempting to do the same. + // go back to square 1 and try it again. + continue; + } + + T t = null; + try { + t = load(key); + if(t==null && createIfNotExist) { + t = create(key,createParams); // create the new data + if(t==null) + throw new IllegalStateException(); // bug in the derived classes + } + } finally { + // let other threads know that the value is available now. + // when the original thread failed to load, this should set it to null. + l.set(t); + } + + // the map needs to be updated to reflect the result of loading + if(t!=null) + core.put(key,new WeakReference(t)); + else + core.remove(key); + + return t; + } + + } + + /** + * Attempts to load an existing data object from disk. + * + *

+ * {@link KeyedDataStorage} class serializes the requests so that + * no two threads call the {@link #load(String)} method with the + * same parameter concurrently. This ensures that there's only + * up to one data object for any key. + * + * @return + * null if no such data exists. + * @throws IOException + * if load operation fails. This exception will be + * propagated to the caller. + */ + protected abstract T load(String key) throws IOException; + + /** + * Creates a new data object. + * + *

+ * This method is called by {@link #getOrCreate(String,Object)} + * if the data that matches the specified key does not exist. + *

+ * Because of concurrency, another thread might call {@link #get(String)} + * as soon as a new data object is created, so it's important that + * this method returns a properly initialized "valid" object. + * + * @return + * never null. If construction fails, abort with an exception. + * @throws IOException + * if the method fails to create a new data object, it can throw + * {@link IOException} (or any other exception) and that will be + * propagated to the caller. + */ + protected abstract T create(String key, P createParams) throws IOException; + +} -- GitLab