8044140: Create NMT (Native Memory Tracking) tests for NMT2

Summary: Create new/modify existing tests for NMT2, which is an internal redesign to address scalability issues in the first implementation.
Reviewed-by: ctornqvi, zgu
Contributed-by: NGeorge Triantafillou <george.triantafillou@oracle.com>

test/TEST.ROOT

 # 
-# Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,6 @@
 # It also contains test-suite configuration information.
 
 # The list of keywords supported in this test suite
-keys=cte_test jcmd nmt regression gc
+keys=cte_test jcmd nmt regression gc stress
 
 groups=TEST.groups [closed/TEST.groups]

test/TEST.groups

@@ -73,15 +73,23 @@ needs_jdk = \
   runtime/jsig/Test8017498.sh \
   runtime/Metaspace/FragmentMetaspace.java \
   runtime/NMT/BaselineWithParameter.java \
+  runtime/NMT/JcmdBaselineDetail.java \
+  runtime/NMT/JcmdDetailDiff.java \
+  runtime/NMT/JcmdScaleDetail.java \
   runtime/NMT/JcmdScale.java \
   runtime/NMT/JcmdWithNMTDisabled.java \
+  runtime/NMT/MallocRoundingReportTest.java \
+  runtime/NMT/MallocSiteHashOverflow.java \
+  runtime/NMT/MallocStressTest.java \
   runtime/NMT/MallocTestType.java \
   runtime/NMT/ReleaseCommittedMemory.java \
+  runtime/NMT/ReleaseNoCommit.java \
   runtime/NMT/ShutdownTwice.java \
   runtime/NMT/SummaryAfterShutdown.java \
   runtime/NMT/SummarySanityCheck.java \
   runtime/NMT/ThreadedMallocTestType.java \
   runtime/NMT/ThreadedVirtualAllocTestType.java \
+  runtime/NMT/VirtualAllocCommitUncommitRecommit.java \
   runtime/NMT/VirtualAllocTestType.java \
   runtime/RedefineObject/TestRedefineObject.java \
   runtime/Thread/TestThreadDumpMonitorContention.java \

test/runtime/NMT/BaselineWithParameter.java

@@ -27,6 +27,7 @@
  * @key nmt jcmd regression
  * @summary Regression test for invoking a jcmd with baseline=false, result was that the target VM crashed
  * @library /testlibrary
+ * @ignore
  * @run main/othervm -XX:NativeMemoryTracking=detail BaselineWithParameter
  */
 

test/runtime/NMT/CommandLineDetail.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Running with NMT detail should not result in an error
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/CommandLineEmptyArgument.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Empty argument to NMT should result in an informative error message
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/CommandLineInvalidArgument.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Invalid argument to NMT should result in an informative error message
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/CommandLineSummary.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Running with NMT summary should not result in an error
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/CommandLineTurnOffNMT.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Turning off NMT should not result in an error
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/JcmdScale.java

@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @summary Test the NMT scale parameter
  * @library /testlibrary
+ * @ignore
  * @run main/othervm -XX:NativeMemoryTracking=summary JcmdScale
  */
 

test/runtime/NMT/JcmdWithNMTDisabled.java

@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @summary Verify that jcmd correctly reports that NMT is not enabled
  * @library /testlibrary
+ * @ignore
  * First run without enabling NMT
  * @run main/othervm JcmdWithNMTDisabled
  * Then run with explicitly disabling NMT, should not be any difference

test/runtime/NMT/MallocTestType.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
  * @key nmt jcmd
  * @library /testlibrary /testlibrary/whitebox
  * @build MallocTestType
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail MallocTestType
  */
@@ -51,11 +52,6 @@ public class MallocTestType {
     long memAlloc1 = wb.NMTMalloc(512 * 1024);
     wb.NMTFree(memAlloc2);
 
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
-
     // Run 'jcmd <pid> VM.native_memory summary'
     pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "summary"});
     output = new OutputAnalyzer(pb.start());
@@ -64,10 +60,6 @@ public class MallocTestType {
     // Free the memory allocated by NMTAllocTest
     wb.NMTFree(memAlloc1);
 
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
     output = new OutputAnalyzer(pb.start());
     output.shouldNotContain("Test (reserved=");
   }

test/runtime/NMT/PrintNMTStatistics.java

@@ -28,6 +28,7 @@
  * @summary Make sure PrintNMTStatistics works on normal JVM exit
  * @library /testlibrary /testlibrary/whitebox
  * @build PrintNMTStatistics
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main PrintNMTStatistics
  */
@@ -45,10 +46,6 @@ public class PrintNMTStatistics {
     // We start a new java process running with an argument and use WB API to ensure
     // we have data for NMT on VM exit
     if (args.length > 0) {
-      // Use WB API to ensure that all data has been merged before we continue
-      if (!WhiteBox.getWhiteBox().NMTWaitForDataMerge()) {
-        throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-      }
       return;
     }
 

test/runtime/NMT/PrintNMTStatisticsWithNMTDisabled.java

@@ -26,6 +26,7 @@
  * @key nmt
  * @summary Trying to enable PrintNMTStatistics should result in a warning
  * @library /testlibrary
+ * @ignore
  */
 
 import com.oracle.java.testlibrary.*;

test/runtime/NMT/ReleaseCommittedMemory.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
  * @key nmt regression
  * @library /testlibrary /testlibrary/whitebox
  * @build ReleaseCommittedMemory
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail ReleaseCommittedMemory
  */
@@ -44,7 +45,6 @@ public class ReleaseCommittedMemory {
     addr = wb.NMTReserveMemory(reserveSize);
     wb.NMTCommitMemory(addr, 128*1024);
     wb.NMTReleaseMemory(addr, reserveSize);
-    wb.NMTWaitForDataMerge();
   }
 }
 

test/runtime/NMT/ShutdownTwice.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @summary Run shutdown twice
  * @library /testlibrary
+ * @ignore
  * @run main/othervm -XX:NativeMemoryTracking=detail ShutdownTwice
  */
 
@@ -45,12 +46,12 @@ public class ShutdownTwice {
     output = new OutputAnalyzer(pb.start());
 
     // Verify that jcmd reports that NMT is shutting down
-    output.shouldContain("Shutdown is in progress, it will take a few moments to completely shutdown");
+    output.shouldContain("Native memory tracking has been turned off");
 
     // Run shutdown again
     output = new OutputAnalyzer(pb.start());
 
     // Verify that jcmd reports that NMT has been shutdown already
-    output.shouldContain("Native memory tracking has been shutdown by user");
+    output.shouldContain("Native memory tracking has been shutdown");
   }
 }

test/runtime/NMT/SummaryAfterShutdown.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @summary Verify that jcmd correctly reports that NMT is not enabled after a shutdown
  * @library /testlibrary
+ * @ignore
  * @run main/othervm -XX:NativeMemoryTracking=detail SummaryAfterShutdown
  */
 
@@ -44,13 +45,13 @@ public class SummaryAfterShutdown {
     output = new OutputAnalyzer(pb.start());
 
     // Verify that jcmd reports that NMT is shutting down
-    output.shouldContain("Shutdown is in progress, it will take a few moments to completely shutdown");
+    output.shouldContain("Native memory tracking has been turned off");
 
     // Run 'jcmd <pid> VM.native_memory summary'
     pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "summary"});
     output = new OutputAnalyzer(pb.start());
 
     // Verify that jcmd reports that NMT has been shutdown
-    output.shouldContain("Native memory tracking has been shutdown by user");
+    output.shouldContain("Native memory tracking has been shutdown");
   }
 }

test/runtime/NMT/SummarySanityCheck.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
  * @summary Sanity check the output of NMT
  * @library /testlibrary /testlibrary/whitebox
  * @build SummarySanityCheck
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:NativeMemoryTracking=summary -XX:+WhiteBoxAPI SummarySanityCheck
  */
@@ -44,11 +45,6 @@ public class SummarySanityCheck {
     // Grab my own PID
     String pid = Integer.toString(ProcessTools.getProcessId());
 
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!WhiteBox.getWhiteBox().NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
-
     ProcessBuilder pb = new ProcessBuilder();
 
     // Run  'jcmd <pid> VM.native_memory summary scale=KB'
@@ -69,13 +65,13 @@ public class SummarySanityCheck {
     // Match '- <mtType> (reserved=<reserved>KB, committed=<committed>KB)
     Pattern mtTypePattern = Pattern.compile("-\\s+(?<typename>[\\w\\s]+)\\(reserved=(?<reserved>\\d+)KB,\\scommitted=(?<committed>\\d+)KB\\)");
     // Match 'Total: reserved=<reserved>KB, committed=<committed>KB'
-    Pattern totalMemoryPattern = Pattern.compile("Total\\:\\s\\sreserved=(?<reserved>\\d+)KB,\\s\\scommitted=(?<committed>\\d+)KB");
+    Pattern totalMemoryPattern = Pattern.compile("Total\\:\\sreserved=(?<reserved>\\d+)KB,\\scommitted=(?<committed>\\d+)KB");
 
     for (int i = 0; i < lines.length; i++) {
       if (lines[i].startsWith("Total")) {
         Matcher totalMemoryMatcher = totalMemoryPattern.matcher(lines[i]);
 
-        if (totalMemoryMatcher.matches() && totalMemoryMatcher.groupCount() == 2) {
+        if (totalMemoryMatcher.matches()) {
           totalCommitted = Integer.parseInt(totalMemoryMatcher.group("committed"));
           totalReserved = Integer.parseInt(totalMemoryMatcher.group("reserved"));
         } else {

test/runtime/NMT/ThreadedMallocTestType.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @library /testlibrary /testlibrary/whitebox
  * @build ThreadedMallocTestType
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail ThreadedMallocTestType
  */
@@ -58,11 +59,6 @@ public class ThreadedMallocTestType {
     allocThread.start();
     allocThread.join();
 
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
-
     // Run 'jcmd <pid> VM.native_memory summary'
     pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "summary"});
     output = new OutputAnalyzer(pb.start());
@@ -80,11 +76,6 @@ public class ThreadedMallocTestType {
     freeThread.start();
     freeThread.join();
 
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
-
     output = new OutputAnalyzer(pb.start());
     output.shouldNotContain("Test (reserved=");
   }

test/runtime/NMT/ThreadedVirtualAllocTestType.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
  * @key nmt jcmd
  * @library /testlibrary /testlibrary/whitebox
  * @build ThreadedVirtualAllocTestType
+ * @ignore
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail ThreadedVirtualAllocTestType
  */
@@ -60,8 +61,6 @@ public class ThreadedVirtualAllocTestType {
     reserveThread.start();
     reserveThread.join();
 
-    mergeData();
-
     pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "detail"});
     output = new OutputAnalyzer(pb.start());
     output.shouldContain("Test (reserved=512KB, committed=0KB)");
@@ -77,8 +76,6 @@ public class ThreadedVirtualAllocTestType {
     commitThread.start();
     commitThread.join();
 
-    mergeData();
-
     output = new OutputAnalyzer(pb.start());
     output.shouldContain("Test (reserved=512KB, committed=128KB)");
     if (has_nmt_detail) {
@@ -93,8 +90,6 @@ public class ThreadedVirtualAllocTestType {
     uncommitThread.start();
     uncommitThread.join();
 
-    mergeData();
-
     output = new OutputAnalyzer(pb.start());
     output.shouldContain("Test (reserved=512KB, committed=0KB)");
     output.shouldNotMatch("\\[0x[0]*" + Long.toHexString(addr) + " - 0x[0]*" + Long.toHexString(addr + commitSize) + "\\] committed");
@@ -107,17 +102,9 @@ public class ThreadedVirtualAllocTestType {
     releaseThread.start();
     releaseThread.join();
 
-    mergeData();
-
     output = new OutputAnalyzer(pb.start());
     output.shouldNotContain("Test (reserved=");
     output.shouldNotContain("\\[0x[0]*" + Long.toHexString(addr) + " - 0x[0]*" + Long.toHexString(addr + reserveSize) + "\\] reserved");
   }
 
-  public static void mergeData() throws Exception {
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
     }
-}

test/runtime/NMT/VirtualAllocTestType.java

 /*
- * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
  * @summary Test Reserve/Commit/Uncommit/Release of virtual memory and that we track it correctly
  * @key nmt jcmd
  * @library /testlibrary /testlibrary/whitebox
+ * @ignore
  * @build VirtualAllocTestType
  * @run main ClassFileInstaller sun.hotspot.WhiteBox
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail VirtualAllocTestType
@@ -54,7 +55,6 @@ public class VirtualAllocTestType {
     }
 
     addr = wb.NMTReserveMemory(reserveSize);
-    mergeData();
     pb.command(new String[] { JDKToolFinder.getJDKTool("jcmd"), pid, "VM.native_memory", "detail"});
 
     output = new OutputAnalyzer(pb.start());
@@ -65,7 +65,6 @@ public class VirtualAllocTestType {
 
     wb.NMTCommitMemory(addr, commitSize);
 
-    mergeData();
 
     output = new OutputAnalyzer(pb.start());
     output.shouldContain("Test (reserved=256KB, committed=128KB)");
@@ -75,24 +74,15 @@ public class VirtualAllocTestType {
 
     wb.NMTUncommitMemory(addr, commitSize);
 
-    mergeData();
 
     output = new OutputAnalyzer(pb.start());
     output.shouldContain("Test (reserved=256KB, committed=0KB)");
     output.shouldNotMatch("\\[0x[0]*" + Long.toHexString(addr) + " - 0x[0]*" + Long.toHexString(addr + commitSize) + "\\] committed");
 
     wb.NMTReleaseMemory(addr, reserveSize);
-    mergeData();
 
     output = new OutputAnalyzer(pb.start());
     output.shouldNotContain("Test (reserved=");
     output.shouldNotMatch("\\[0x[0]*" + Long.toHexString(addr) + " - 0x[0]*" + Long.toHexString(addr + reserveSize) + "\\] reserved");
   }
-
-  public static void mergeData() throws Exception {
-    // Use WB API to ensure that all data has been merged before we continue
-    if (!wb.NMTWaitForDataMerge()) {
-      throw new Exception("Call to WB API NMTWaitForDataMerge() failed");
-    }
     }
-}

test/testlibrary/whitebox/sun/hotspot/WhiteBox.java

@@ -145,8 +145,6 @@ public class WhiteBox {
 
   // Memory
   public native void readReservedMemory();
-  public native long allocateMetaspace(ClassLoader classLoader, long size);
-  public native void freeMetaspace(ClassLoader classLoader, long addr, long size);
 
   // force Young GC
   public native void youngGC();
@@ -162,17 +160,4 @@ public class WhiteBox {
   // CPU features
   public native String getCPUFeatures();
 
-  // VM flags
-  public native void    setBooleanVMFlag(String name, boolean value);
-  public native void    setIntxVMFlag(String name, long value);
-  public native void    setUintxVMFlag(String name, long value);
-  public native void    setUint64VMFlag(String name, long value);
-  public native void    setStringVMFlag(String name, String value);
-  public native void    setDoubleVMFlag(String name, double value);
-  public native Boolean getBooleanVMFlag(String name);
-  public native Long    getIntxVMFlag(String name);
-  public native Long    getUintxVMFlag(String name);
-  public native Long    getUint64VMFlag(String name);
-  public native String  getStringVMFlag(String name);
-  public native Double  getDoubleVMFlag(String name);
 }