Verifying that the heap size of a SuiteResult (from Surefire) is linear in the...

Verifying that the heap size of a SuiteResult (from Surefire) is linear in the sizes of TEST-*.xml + *-output.txt even when there is lots of stdio.

test/pom.xml

@@ -140,6 +140,16 @@ THE SOFTWARE.
       <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.netbeans.modules</groupId>
+      <artifactId>org-netbeans-insane</artifactId>
+      <version>RELEASE72</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.stephenc.findbugs</groupId>
+      <artifactId>findbugs-annotations</artifactId>
+      <version>1.3.9-1</version>
+    </dependency>
   </dependencies>
 
   <build>

test/src/main/java/org/jvnet/hudson/test/MemoryAssert.java

+/*
+ * The MIT License
+ *
+ * Copyright 2012 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package org.jvnet.hudson.test;
+
+import java.lang.ref.WeakReference;
+import java.util.Collections;
+import static org.junit.Assert.*;
+import org.netbeans.insane.scanner.CountingVisitor;
+import org.netbeans.insane.scanner.ScannerUtils;
+
+/**
+ * Static utility methods for verifying heap memory usage.
+ * Uses the <a href="http://performance.netbeans.org/insane/">INSANE library</a>
+ * to traverse the heap from within your test.
+ * <p>Object sizes are in an idealized JVM in which pointers are 4 bytes
+ * (realistic even for modern 64-bit JVMs in which {@code -XX:+UseCompressedOops} is the default)
+ * but objects are aligned on 8-byte boundaries (so dropping an {@code int} field does not always save memory).
+ * <p>{@code import static org.jvnet.hudson.test.MemoryAssert.*;} to use.
+ */
+public class MemoryAssert {
+
+    private MemoryAssert() {}
+
+    /**
+     * Verifies that an object and its transitive reference graph occupy at most a predetermined amount of memory.
+     * The referents of {@link WeakReference} and the like are ignored.
+     * <p>To use, run your test for the first time with {@code max} of {@code 0};
+     * when it fails, use the reported actual size as your assertion maximum.
+     * When improving memory usage, run again with {@code 0} and tighten the test to both demonstrate
+     * your improvement quantitatively and prevent regressions.
+     * @param o the object to measure
+     * @param max the maximum desired memory usage (in bytes)
+     */
+    public static void assertHeapUsage(Object o, int max) throws Exception {
+        CountingVisitor v = new CountingVisitor();
+        ScannerUtils.scan(ScannerUtils.skipNonStrongReferencesFilter(), v, Collections.singleton(o), false);
+        int memoryUsage = v.getTotalSize();
+        assertTrue(o + " consumes " + memoryUsage + " bytes of heap, " + (memoryUsage - max) + " over the limit of " + max, memoryUsage <= max);
+    }
+
+}

test/src/test/java/hudson/tasks/junit/SuiteResultTest.java

+/*
+ * The MIT License
+ *
+ * Copyright 2012 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package hudson.tasks.junit;
+
+import edu.umd.cs.findbugs.annotations.SuppressWarnings;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.util.List;
+import static org.jvnet.hudson.test.MemoryAssert.*;
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+public class SuiteResultTest {
+
+    @SuppressWarnings({"DM_DEFAULT_ENCODING", "OS_OPEN_STREAM", "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE"})
+    @Test public void sizeSurefire() throws Exception {
+        File data = File.createTempFile("TEST-", ".xml");
+        try {
+            Writer w = new FileWriter(data);
+            try {
+                PrintWriter pw = new PrintWriter(w);
+                pw.println("<testsuites name='x'>");
+                pw.println("<testsuite failures='10' errors='0' tests='20' name='x'>");
+                // Simulating Surefire 2.12.4 with redirectTestOutputToFile=true:
+                for (int i = 0; i < 10; i++) { // these pass and SF omits stdio
+                    pw.println("<testcase name='t" + i + "' classname='x'/>");
+                }
+                for (int i = 10; i < 20; i++) { // these fail and SF includes per-test stdio
+                    pw.println("<testcase name='t" + i + "' classname='x'>");
+                    pw.println("<failure type='java.lang.AssertionError'>some stack trace</failure>");
+                    pw.print("<system-out>");
+                    for (int j = 0; j < 1000; j++) {
+                        pw.println("t" + i + " out #" + j);
+                    }
+                    pw.println("</system-out>");
+                    pw.print("<system-err>");
+                    for (int j = 0; j < 1000; j++) {
+                        pw.println("t" + i + " err #" + j);
+                    }
+                    pw.println("</system-err>");
+                    pw.println("</testcase>");
+                }
+                pw.println("</testsuite>");
+                pw.println("</testsuites>");
+                pw.flush();
+            } finally {
+                w.close();
+            }
+            File data2 = new File(data.getParentFile(), data.getName().replaceFirst("^TEST-(.+)[.]xml$", "$1-output.txt"));
+            try {
+                w = new FileWriter(data2);
+                try {
+                    PrintWriter pw = new PrintWriter(w);
+                    for (int i = 0; i < 20; i++) { // stdio for complete suite (incl. passing tests)
+                        for (int j = 0; j < 1000; j++) {
+                            pw.println("t" + i + " out #" + j);
+                            pw.println("t" + i + " err #" + j);
+                        }
+                    }
+                    pw.flush();
+                } finally {
+                    w.close();
+                }
+                SuiteResult sr = parseOne(data);
+                assertHeapUsage(sr, 242 + /* Unicode overhead */2 * (int) (/*259946*/data.length() + /*495600*/data2.length() + /* SuiteResult.file */data.getAbsolutePath().length()));
+                // XXX serialize using TestResultAction.XSTREAM and verify that round-tripped object has same size
+            } finally {
+                data2.delete();
+            }
+        } finally {
+            data.delete();
+        }
+    }
+
+    private SuiteResult parseOne(File file) throws Exception {
+        List<SuiteResult> results = SuiteResult.parse(file, false);
+        assertEquals(1,results.size());
+        return results.get(0);
+    }
+
+}

test/src/test/java/org/jvnet/hudson/test/MemoryAssertTest.java

+/*
+ * The MIT License
+ *
+ * Copyright 2012 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package org.jvnet.hudson.test;
+
+import java.lang.ref.WeakReference;
+import static org.jvnet.hudson.test.MemoryAssert.*;
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+public class MemoryAssertTest {
+
+    @Test public void heapUsage() throws Exception {
+        Object[] biggie = new Object[1000];
+        assertHeapUsage(biggie, 4016);
+        assertHeapUsage(new WeakReference<Object>(biggie), 56);
+        AssertionError e = null;
+        try {
+            assertHeapUsage(biggie, 1016);
+        } catch (AssertionError _e) {
+            e = _e;
+        }
+        assertNotNull(e);
+        assertTrue(e.toString(), e.getMessage().contains("3000"));
+    }
+
+}