8013120: NMT: Kitchensink crashes with assert(next_region == NULL ||...

8013120: NMT: Kitchensink crashes with assert(next_region == NULL || !next_region->is_committed_region()) failed: Sanity check
Summary: Fixed NMT to deal with releasing virtual memory region when there are still committed regions within it
Reviewed-by: acorn, coleenp

src/share/vm/memory/allocation.inline.hpp

@@ -132,7 +132,7 @@ E* ArrayAllocator<E, F>::allocate(size_t length) {
   int alignment = os::vm_allocation_granularity();
   _size = align_size_up(_size, alignment);
 
-  _addr = os::reserve_memory(_size, NULL, alignment);
+  _addr = os::reserve_memory(_size, NULL, alignment, F);
   if (_addr == NULL) {
     vm_exit_out_of_memory(_size, OOM_MMAP_ERROR, "Allocator (reserve)");
   }

src/share/vm/runtime/os.cpp

@@ -1457,6 +1457,18 @@ char* os::reserve_memory(size_t bytes, char* addr, size_t alignment_hint) {
 
   return result;
 }
+
+char* os::reserve_memory(size_t bytes, char* addr, size_t alignment_hint,
+   MEMFLAGS flags) {
+  char* result = pd_reserve_memory(bytes, addr, alignment_hint);
+  if (result != NULL) {
+    MemTracker::record_virtual_memory_reserve((address)result, bytes, CALLER_PC);
+    MemTracker::record_virtual_memory_type((address)result, flags);
+  }
+
+  return result;
+}
+
 char* os::attempt_reserve_memory_at(size_t bytes, char* addr) {
   char* result = pd_attempt_reserve_memory_at(bytes, addr);
   if (result != NULL) {

src/share/vm/runtime/os.hpp

@@ -255,6 +255,8 @@ class os: AllStatic {
   static int    vm_allocation_granularity();
   static char*  reserve_memory(size_t bytes, char* addr = 0,
                                size_t alignment_hint = 0);
+  static char*  reserve_memory(size_t bytes, char* addr,
+                               size_t alignment_hint, MEMFLAGS flags);
   static char*  reserve_memory_aligned(size_t size, size_t alignment);
   static char*  attempt_reserve_memory_at(size_t bytes, char* addr);
   static void   split_reserved_memory(char *base, size_t size,

src/share/vm/services/memSnapshot.cpp

 /*
- * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -262,13 +262,28 @@ bool VMMemPointerIterator::remove_released_region(MemPointerRecord* rec) {
   assert(cur->is_reserved_region() && cur->contains_region(rec),
     "Sanity check");
   if (rec->is_same_region(cur)) {
-    // release whole reserved region
+
+    // In snapshot, the virtual memory records are sorted in following orders:
+    // 1. virtual memory's base address
+    // 2. virtual memory reservation record, followed by commit records within this reservation.
+    //    The commit records are also in base address order.
+    // When a reserved region is released, we want to remove the reservation record and all
+    // commit records following it.
 #ifdef ASSERT
-    VMMemRegion* next_region = (VMMemRegion*)peek_next();
-    // should not have any committed memory in this reserved region
-    assert(next_region == NULL || !next_region->is_committed_region(), "Sanity check");
+    address low_addr = cur->addr();
+    address high_addr = low_addr + cur->size();
 #endif
+    // remove virtual memory reservation record
     remove();
+    // remove committed regions within above reservation
+    VMMemRegion* next_region = (VMMemRegion*)current();
+    while (next_region != NULL && next_region->is_committed_region()) {
+      assert(next_region->addr() >= low_addr &&
+             next_region->addr() + next_region->size() <= high_addr,
+            "Range check");
+      remove();
+      next_region = (VMMemRegion*)current();
+    }
   } else if (rec->addr() == cur->addr() ||
     rec->addr() + rec->size() == cur->addr() + cur->size()) {
     // released region is at either end of this region

test/runtime/NMT/ReleaseCommittedMemory.java

+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8013120
+ * @summary Release committed memory and make sure NMT handles it correctly
+ * @key nmt regression
+ * @library /testlibrary /testlibrary/whitebox
+ * @build ReleaseCommittedMemory
+ * @run main ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:NativeMemoryTracking=detail ReleaseCommittedMemory
+ */
+
+import sun.hotspot.WhiteBox;
+
+public class ReleaseCommittedMemory {
+
+  public static void main(String args[]) throws Exception {
+    WhiteBox wb = WhiteBox.getWhiteBox();
+    long reserveSize = 256 * 1024;
+    long addr;
+
+    addr = wb.NMTReserveMemory(reserveSize);
+    wb.NMTCommitMemory(addr, 128*1024);
+    wb.NMTReleaseMemory(addr, reserveSize);
+    wb.NMTWaitForDataMerge();
+  }
+}
+