7185614: NMT ON: "check by caller" assertion failed on nsk ThreadMXBean test

7187429: NMT ON: Merge failure should cause NMT to shutdown
Summary: Fixed NMT assertion failures
Reviewed-by: acorn, kvn

src/share/vm/services/memSnapshot.cpp

@@ -344,7 +344,7 @@ void MemSnapshot::promote() {
           // due to platform dependent behaviors. On some platforms, we see uncommit/release
           // native thread stack, but some, we don't.
           if (!cur_vm->is_uncommit_record() && !cur_vm->is_deallocation_record()) {
-            ShouldNotReachHere();
+            fatal(err_msg("Should not reach here, pointer flags = [%x]", cur_vm->flags()));
           }
 #endif
         }

src/share/vm/services/memSnapshot.hpp

@@ -63,13 +63,13 @@ class MemPointerIterator : public MemPointerArrayIteratorImpl {
       MemPointer* p1 = (MemPointer*)ptr;
       MemPointer* p2 = (MemPointer*)_array->at(_pos - 1);
       assert(!is_dup_pointer(p1, p2),
-        "dup pointer");
+        err_msg("duplicated pointer, flag = [%x]", (unsigned int)((MemPointerRecord*)p1)->flags()));
     }
      if (_pos < _array->length() -1) {
       MemPointer* p1 = (MemPointer*)ptr;
       MemPointer* p2 = (MemPointer*)_array->at(_pos + 1);
       assert(!is_dup_pointer(p1, p2),
-        "dup pointer");
+        err_msg("duplicated pointer, flag = [%x]", (unsigned int)((MemPointerRecord*)p1)->flags()));
      }
     return _array->insert_at(ptr, _pos);
   }
@@ -79,14 +79,14 @@ class MemPointerIterator : public MemPointerArrayIteratorImpl {
       MemPointer* p1 = (MemPointer*)ptr;
       MemPointer* p2 = (MemPointer*)_array->at(_pos - 1);
       assert(!is_dup_pointer(p1, p2),
-        "dup pointer");
+        err_msg("duplicated pointer, flag = [%x]", (unsigned int)((MemPointerRecord*)p1)->flags()));
     }
     if (_pos < _array->length() - 1) {
       MemPointer* p1 = (MemPointer*)ptr;
       MemPointer* p2 = (MemPointer*)_array->at(_pos + 1);
 
       assert(!is_dup_pointer(p1, p2),
-        "dup pointer");
+        err_msg("duplicated pointer, flag = [%x]", (unsigned int)((MemPointerRecord*)p1)->flags()));
      }
     if (_array->insert_at(ptr, _pos + 1)) {
       _pos ++;

src/share/vm/services/memTrackWorker.cpp

@@ -99,9 +99,11 @@ void MemTrackWorker::run() {
     }
     if (rec != NULL) {
       // merge the recorder into staging area
-      bool result = snapshot->merge(rec);
-      assert(result, "merge failed");
-      debug_only(_merge_count ++;)
+      if (!snapshot->merge(rec)) {
+        MemTracker::shutdown(MemTracker::NMT_out_of_memory);
+      } else {
+        NOT_PRODUCT(_merge_count ++;)
+      }
       MemTracker::release_thread_recorder(rec);
     } else {
       // no more recorder to merge, promote staging area
@@ -129,7 +131,7 @@ void MemTrackWorker::run() {
   }
   assert(MemTracker::shutdown_in_progress(), "just check");
 
-  // transites to final shutdown
+  // transits to final shutdown
   MemTracker::final_shutdown();
 }
 

src/share/vm/services/memTracker.hpp

@@ -184,7 +184,6 @@ class MemTracker : AllStatic {
   // record a 'malloc' call
   static inline void record_malloc(address addr, size_t size, MEMFLAGS flags,
                             address pc = 0, Thread* thread = NULL) {
-    assert(is_on(), "check by caller");
     if (NMT_CAN_TRACK(flags)) {
       create_memory_record(addr, (flags|MemPointerRecord::malloc_tag()), size, pc, thread);
     }
@@ -285,7 +284,6 @@ class MemTracker : AllStatic {
 
   // retrieve global snapshot
   static MemSnapshot* get_snapshot() {
-    assert(is_on(), "native memory tracking is off");
     if (shutdown_in_progress()) {
       return NULL;
     }