[FIXED JENKINS-18417]

FingerprintCleanupThread can now partially clean up a fingerprint record
by removing portions of it that's not referencing existing stuff.

changelog.html

@@ -55,7 +55,9 @@ Upcoming changes</a>
 <!-- Record your changes in the trunk here. -->
 <div id="trunk" style="display:none"><!--=TRUNK-BEGIN=-->
 <ul class=image>
-  <li class=>
+  <li class=bug>
+    Clean up fingerprint records that correspond to the deleted build recods
+    (<a href="https://issues.jenkins-ci.org/browse/JENKINS-18417">issue 18417</a>)
 </ul>
 </div><!--=TRUNK-END=-->
 

core/src/main/java/hudson/model/Fingerprint.java

@@ -249,6 +249,13 @@ public class Fingerprint implements ModelObject, Saveable {
             return this.end<that.start ||that.end<this.start;
         }
 
+        /**
+         * Returns true if two {@link Range}s do not share any common integer.
+         */
+        public boolean isDisjoint(Range that) {
+            return this.end<=that.start || that.end<=this.start;
+        }
+
         /**
          * Returns true if this range only represents a single number.
          */
@@ -256,6 +263,13 @@ public class Fingerprint implements ModelObject, Saveable {
             return end-1==start;
         }
 
+        /**
+         * If this range contains every int that's in the other range, return true
+         */
+        public boolean contains(Range that) {
+            return this.start<=that.start && that.end<=this.end;
+        }
+
         /**
          * Returns the {@link Range} that combines two ranges.
          */
@@ -265,10 +279,35 @@ public class Fingerprint implements ModelObject, Saveable {
                 Math.min(this.start,that.start),
                 Math.max(this.end  ,that.end  ));
         }
+
+        /**
+         * Returns the {@link Range} that represents the intersection of the two.
+         */
+        public Range intersect(Range that) {
+            assert !isDisjoint(that);
+            return new Range(
+                Math.max(this.start, that.start),
+                Math.min(this.end, that.end));
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            Range that = (Range) o;
+            return start == that.start && end == that.end;
+
+        }
+
+        @Override
+        public int hashCode() {
+            return 31 * start + end;
+        }
     }
 
     /**
-     * Set of {@link Range}s.
+     * Set of {@link Range}s. Mutable.
      */
     @ExportedBean(defaultVisibility=3)
     public static final class RangeSet {
@@ -283,6 +322,11 @@ public class Fingerprint implements ModelObject, Saveable {
             this.ranges = data;
         }
 
+        private RangeSet(Range initial) {
+            this();
+            ranges.add(initial);
+        }
+
         /**
          * List all numbers in this range set, in the ascending order.
          */
@@ -370,6 +414,12 @@ public class Fingerprint implements ModelObject, Saveable {
             ranges.add(new Range(n,n+1));
         }
 
+        public synchronized void addAll(int... n) {
+            for (int i : n)
+                add(i);
+        }
+
+
         private void checkCollapse(int i) {
             if(i<0 || i==ranges.size()-1)     return;
             Range lhs = ranges.get(i);
@@ -425,6 +475,121 @@ public class Fingerprint implements ModelObject, Saveable {
             this.ranges.addAll(that.ranges.subList(rhs,that.ranges.size()));
         }
 
+        /**
+         * Updates this range set by the intersection of this range and the given range.
+         *
+         * @return true if this range set was modified as a result.
+         */
+        public synchronized boolean retainAll(RangeSet that) {
+            List<Range> intersection = new ArrayList<Range>();
+
+            int lhs=0,rhs=0;
+            while(lhs<this.ranges.size() && rhs<that.ranges.size()) {
+                Range lr = this.ranges.get(lhs);
+                Range rr = that.ranges.get(rhs);
+
+                if(lr.end<=rr.start) {// lr has no overlap with that.ranges
+                    lhs++;
+                    continue;
+                }
+                if(rr.end<=lr.start) {// rr has no overlap with this.ranges
+                    rhs++;
+                    continue;
+                }
+
+                // overlap. figure out the intersection
+                Range v = lr.intersect(rr);
+                intersection.add(v);
+
+                // move on to the next pair
+                if (lr.end<rr.end) {
+                    lhs++;
+                } else {
+                    rhs++;
+                }
+            }
+
+            boolean same = this.ranges.equals(intersection);
+
+            if (!same) {
+                this.ranges.clear();
+                this.ranges.addAll(intersection);
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+        /**
+         * Updates this range set by removing all the values in the given range set.
+         *
+         * @return true if this range set was modified as a result.
+         */
+        public synchronized boolean removeAll(RangeSet that) {
+            boolean modified = false;
+            List<Range> sub = new ArrayList<Range>();
+
+            int lhs=0,rhs=0;
+            while(lhs<this.ranges.size() && rhs<that.ranges.size()) {
+                Range lr = this.ranges.get(lhs);
+                Range rr = that.ranges.get(rhs);
+
+                if(lr.end<=rr.start) {// lr has no overlap with that.ranges. lr stays
+                    sub.add(lr);
+                    lhs++;
+                    continue;
+                }
+                if(rr.end<=lr.start) {// rr has no overlap with this.ranges
+                    rhs++;
+                    continue;
+                }
+
+                // some overlap between lr and rr
+                assert !lr.isDisjoint(rr);
+                modified = true;
+
+                if (rr.contains(lr)) {
+                    // lr completely removed by rr
+                    lhs++;
+                    continue;
+                }
+
+                // we want to look at A and B below, if they are non-null.
+                // |------------| lr
+                //     |-----|    rr
+                //   A         B
+                //
+                // note that lr and rr could be something like or the other way around
+                // |------------| lr
+                //         |------------| rr
+                //     A             (no B)
+
+                if (lr.start<rr.start) {// if A is non-empty, that will stay
+                    Range a = new Range(lr.start, rr.start);
+                    sub.add(a);
+                }
+
+                if (rr.end<lr.end) {// if B is non-empty
+                    // we still need to check that with that.ranges, so keep it in the place of lr.
+                    // how much of them will eventually stay is up to the remainder of that.ranges
+                    this.ranges.set(lhs,new Range(rr.end,lr.end));
+                    rhs++;
+                } else {
+                    // if B is empty, we are done considering lr
+                    lhs++;
+                }
+            }
+
+            if (!modified)  return false;   // no changes
+
+            // whatever that remains in lhs will survive
+            sub.addAll(this.ranges.subList(lhs,this.ranges.size()));
+
+            this.ranges.clear();
+            this.ranges.addAll(sub);
+            return true;
+        }
+
         @Override
         public synchronized String toString() {
             StringBuilder buf = new StringBuilder();
@@ -435,6 +600,20 @@ public class Fingerprint implements ModelObject, Saveable {
             return buf.toString();
         }
 
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            return ranges.equals(((RangeSet) o).ranges);
+
+        }
+
+        @Override
+        public int hashCode() {
+            return ranges.hashCode();
+        }
+
         public synchronized boolean isEmpty() {
             return ranges.isEmpty();
         }
@@ -712,7 +891,7 @@ public class Fingerprint implements ModelObject, Saveable {
     }
 
     public synchronized void add(AbstractBuild b) throws IOException {
-        add(b.getParent().getFullName(),b.getNumber());
+        add(b.getParent().getFullName(), b.getNumber());
     }
 
     /**
@@ -762,6 +941,65 @@ public class Fingerprint implements ModelObject, Saveable {
         return false;
     }
 
+    /**
+     * Trim off references to non-existent builds and jobs, thereby making the fingerprint smaller.
+     *
+     * @return true
+     *      if this record was modified.
+     */
+    public synchronized boolean trim() throws IOException {
+        boolean modified = false;
+
+        for (Entry<String,RangeSet> e : new Hashtable<String,RangeSet>(usages).entrySet()) {// copy because we mutate
+            Job j = Jenkins.getInstance().getItemByFullName(e.getKey(),Job.class);
+            if(j==null) {// no such job any more. recycle the record
+                modified = true;
+                usages.remove(e.getKey());
+                continue;
+            }
+
+            Run firstBuild = j.getFirstBuild();
+            if(firstBuild==null) {// no builds. recycle the whole record
+                modified = true;
+                usages.remove(e.getKey());
+                continue;
+            }
+
+            RangeSet cur = e.getValue();
+
+            // builds that are around without the keepLog flag on are normally clustered together (in terms of build #)
+            // so our basic strategy is to discard everything up to the first ephemeral build, except those builds
+            // that are marked as kept
+            RangeSet kept = new RangeSet();
+            Run r = firstBuild;
+            while (r!=null && r.isKeepLog()) {
+                kept.add(r.getNumber());
+                r = r.getNextBuild();
+            }
+
+            if (r==null) {
+                // all the build records are permanently kept ones, so we'll just have to keep 'kept' out of whatever currently in 'cur'
+                modified |= cur.retainAll(kept);
+            } else {
+                // otherwise we are ready to discard [0,r.number) except those marked as 'kept'
+                RangeSet discarding =  new RangeSet(new Range(-1,r.getNumber()));
+                discarding.removeAll(kept);
+                modified |= cur.removeAll(discarding);
+            }
+
+            if (cur.isEmpty()) {
+                usages.remove(e.getKey());
+                modified = true;
+            }
+        }
+
+        if (modified)
+            save();
+
+        return modified;
+    }
+
+
     /**
      * Gets the associated {@link FingerprintFacet}s.
      *

core/src/main/java/hudson/model/FingerprintCleanupThread.java

@@ -61,7 +61,7 @@ public final class FingerprintCleanupThread extends AsyncPeriodicWork {
         return Jenkins.getInstance().getExtensionList(AsyncPeriodicWork.class).get(FingerprintCleanupThread.class);
     }
 
-    protected void execute(TaskListener listener) {
+    public void execute(TaskListener listener) {
         int numFiles = 0;
 
         File root = new File(Jenkins.getInstance().getRootDir(),"fingerprints");
@@ -103,11 +103,16 @@ public final class FingerprintCleanupThread extends AsyncPeriodicWork {
             if(!fp.isAlive()) {
                 fingerprintFile.delete();
                 return true;
+            } else {
+                // get the fingerprint in the official map so have the changes visible to Jenkins
+                // otherwise the mutation made in FingerprintMap can override our trimming.
+                fp = Jenkins.getInstance()._getFingerprint(fp.getHashString());
+                return fp.trim();
             }
         } catch (IOException e) {
             logger.log(Level.WARNING, "Failed to process "+fingerprintFile, e);
+            return false;
         }
-        return false;
     }
 
     private static final FileFilter LENGTH2DIR_FILTER = new FileFilter() {

core/src/main/java/hudson/tasks/Fingerprinter.java

@@ -158,10 +158,8 @@ public class Fingerprinter extends Recorder implements Serializable, DependencyD
 
             for ( ListIterator iter = builds.listIterator(); iter.hasNext(); ) {
                 Run build = (Run) iter.next();
-                List<FingerprintAction> fingerprints = build.getActions(FingerprintAction.class);
-                for (FingerprintAction action : fingerprints) {
-                    Map<AbstractProject,Integer> deps = action.getDependencies();
-                    for (AbstractProject key : deps.keySet()) {
+                for (FingerprintAction action : build.getActions(FingerprintAction.class)) {
+                    for (AbstractProject key : action.getDependencies().keySet()) {
                         if (key == owner) {
                             continue;   // Avoid self references
                         }

core/src/test/java/hudson/model/FingerprintTest.java

@@ -123,6 +123,91 @@ public class FingerprintTest {
         assertEquals("[1,2),[3,4),[5,6),[7,8)",x.toString());
     }
 
+    @Test
+    public void retainAll1() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3, 10,11,       20);
+        y.addAll(  2,      11,12, 19,20,21);
+
+        assertTrue(x.retainAll(y));
+
+        RangeSet z = new RangeSet();
+        z.addAll(2,11,20);
+
+        assertEquals(x,z);
+    }
+
+    @Test
+    public void retainAll2() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3,4,5,6,7,8,9,10,      13,14,15,16,17,18,19,20);
+        y.addAll(  2,3,  5,6,    9,10,11,12,13,   15,16,   18,19);
+
+        assertTrue(x.retainAll(y));
+
+        RangeSet z = new RangeSet();
+        z.addAll(2,3,5,6,9,10,13,15,16,18,19);
+
+        assertEquals(x,z);
+    }
+
+    @Test
+    public void retainAll3() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3,4,5);
+
+        assertTrue(x.retainAll(y));
+        assertTrue(x.isEmpty());
+    }
+
+    @Test
+    public void removeAll1() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3, 10,11,       20);
+        y.addAll(  2,      11,12, 19,20,21);
+
+        assertTrue(x.removeAll(y));
+
+        RangeSet z = new RangeSet();
+        z.addAll(1,3,10);
+
+        assertEquals(x,z);
+    }
+
+    @Test
+    public void removeAll2() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3,4,5,6,7,8,9,10,      13,14,15,16,17,18,19,20);
+        y.addAll(  2,3,  5,6,    9,10,11,12,13,   15,16,   18,19);
+
+        assertTrue(x.removeAll(y));
+
+        RangeSet z = new RangeSet();
+        z.addAll(1,4,7,8,14,17,20);
+
+        assertEquals(x,z);
+    }
+
+    @Test
+    public void removeAll3() {
+        RangeSet x = new RangeSet();
+        RangeSet y = new RangeSet();
+
+        x.addAll(1,2,3,4,5);
+
+        assertFalse(x.removeAll(y));
+    }
+
     @Test public void deserialize() throws Exception {
         assertEquals("Fingerprint["
                 + "original=stapler/org.kohsuke.stapler:stapler-jelly #123,"

test/src/test/java/hudson/tasks/FingerprinterTest.java

@@ -24,12 +24,14 @@
 
 package hudson.tasks;
 
+import hudson.Util;
 import hudson.XmlFile;
 import hudson.matrix.Axis;
 import hudson.matrix.AxisList;
 import hudson.matrix.MatrixProject;
 import hudson.model.AbstractProject;
 import hudson.model.Fingerprint;
+import hudson.model.FingerprintCleanupThread;
 import hudson.model.FreeStyleBuild;
 import hudson.model.FreeStyleProject;
 import hudson.model.Hudson;
@@ -38,9 +40,13 @@ import hudson.util.RunList;
 import java.io.File;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import static org.junit.Assert.*;
+
+import hudson.util.StreamTaskListener;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
@@ -275,6 +281,50 @@ public class FingerprinterTest {
         assertEquals(build, action.getBuild());
         assertEquals("{a=f31efcf9afe30617d6c46b919e702822}", action.getRecords().toString());
     }
+
+    @SuppressWarnings("unchecked")
+    @Bug(18417)
+    @Test
+    public void fingerprintCleanup() throws Exception {
+        // file names shouldn't matter
+        FreeStyleProject p1 = createFreeStyleProjectWithFingerprints(singleContents, singleFiles);
+        FreeStyleProject p2 = createFreeStyleProjectWithFingerprints(singleContents, singleFiles2);
+        FreeStyleProject p3 = createFreeStyleProjectWithFingerprints(singleContents, singleFiles);
+
+        j.assertBuildStatusSuccess(p1.scheduleBuild2(0));
+        j.assertBuildStatusSuccess(p2.scheduleBuild2(0));
+        j.assertBuildStatusSuccess(p3.scheduleBuild2(0));
+
+        Fingerprint f = j.jenkins._getFingerprint(Util.getDigestOf(singleContents[0]+"\n"));
+        assertEquals(3,f.getUsages().size());
+
+        assertEquals(Arrays.asList(p1), p2.getUpstreamProjects());
+        assertEquals(Arrays.asList(p1), p3.getUpstreamProjects());
+        assertEquals(new HashSet(Arrays.asList(p2,p3)), new HashSet(p1.getDownstreamProjects()));
+
+        // discard the p3 records
+        p3.delete();
+        new FingerprintCleanupThread().execute(StreamTaskListener.fromStdout());
+
+        // records for p3 should have been deleted now
+        assertEquals(2,f.getUsages().size());
+        assertEquals(Arrays.asList(p1), p2.getUpstreamProjects());
+        assertEquals(Arrays.asList(p2), p1.getDownstreamProjects());
+
+
+        // do a new build in p2 #2 that points to a separate fingerprints
+        p2.getBuildersList().clear();
+        p2.getPublishersList().clear();
+        addFingerprinterToProject(p2,singleContents2,singleFiles2);
+        j.assertBuildStatusSuccess(p2.scheduleBuild2(0));
+
+        // another garbage collection that gets rid of p2 records from the fingerprint
+        p2.getBuildByNumber(1).delete();
+        new FingerprintCleanupThread().execute(StreamTaskListener.fromStdout());
+
+        assertEquals(1,f.getUsages().size());
+    }
+
     
     private FreeStyleProject createFreeStyleProjectWithFingerprints(String[] contents, String[] files) throws IOException, Exception {
         FreeStyleProject project = j.createFreeStyleProject();