7181986: NMT ON: Assertion failure when running jdi ExpiredRequestDeletionTest

Summary: Changed _query_lock to heap object from static object. Also fixed _query_lock and snapshot lock ranks, so they can participate deadlock detection.
Reviewed-by: coleenp, dholmes, kvn

src/share/vm/services/memSnapshot.cpp

@@ -173,7 +173,7 @@ MemSnapshot::MemSnapshot() {
     _staging_area = new (std::nothrow)MemPointerArrayImpl<SeqMemPointerRecord>();
   }
 
-  _lock = new (std::nothrow) Mutex(Monitor::native, "memSnapshotLock");
+  _lock = new (std::nothrow) Mutex(Monitor::max_nonleaf - 1, "memSnapshotLock");
   NOT_PRODUCT(_untracked_count = 0;)
 }
 

src/share/vm/services/memTracker.cpp

@@ -54,7 +54,7 @@ void SyncThreadRecorderClosure::do_thread(Thread* thread) {
 MemRecorder*                    MemTracker::_global_recorder = NULL;
 MemSnapshot*                    MemTracker::_snapshot = NULL;
 MemBaseline                     MemTracker::_baseline;
-Mutex                           MemTracker::_query_lock(Monitor::native, "NMT_queryLock");
+Mutex*                          MemTracker::_query_lock = NULL;
 volatile MemRecorder*           MemTracker::_merge_pending_queue = NULL;
 volatile MemRecorder*           MemTracker::_pooled_recorders = NULL;
 MemTrackWorker*                 MemTracker::_worker_thread = NULL;
@@ -89,6 +89,12 @@ void MemTracker::bootstrap_single_thread() {
       return;
     }
 
+    _query_lock = new (std::nothrow) Mutex(Monitor::max_nonleaf, "NMT_queryLock");
+    if (_query_lock == NULL) {
+      shutdown(NMT_out_of_memory);
+      return;
+    }
+
     debug_only(_main_thread_tid = os::current_thread_id();)
     _state = NMT_bootstrapping_single_thread;
     NMT_track_callsite = (_tracking_level == NMT_detail && can_walk_stack());
@@ -164,7 +170,7 @@ void MemTracker::final_shutdown() {
   {
     // shared baseline and snapshot are the only objects needed to
     // create query results
-    MutexLockerEx locker(&_query_lock, true);
+    MutexLockerEx locker(_query_lock, true);
     // cleanup baseline data and snapshot
     _baseline.clear();
     delete _snapshot;
@@ -534,7 +540,7 @@ void MemTracker::thread_exiting(JavaThread* thread) {
 
 // baseline current memory snapshot
 bool MemTracker::baseline() {
-  MutexLockerEx lock(&_query_lock, true);
+  MutexLockerEx lock(_query_lock, true);
   MemSnapshot* snapshot = get_snapshot();
   if (snapshot != NULL) {
     return _baseline.baseline(*snapshot, false);
@@ -545,7 +551,7 @@ bool MemTracker::baseline() {
 // print memory usage from current snapshot
 bool MemTracker::print_memory_usage(BaselineOutputer& out, size_t unit, bool summary_only) {
   MemBaseline  baseline;
-  MutexLockerEx lock(&_query_lock, true);
+  MutexLockerEx lock(_query_lock, true);
   MemSnapshot* snapshot = get_snapshot();
   if (snapshot != NULL && baseline.baseline(*snapshot, summary_only)) {
     BaselineReporter reporter(out, unit);
@@ -557,7 +563,7 @@ bool MemTracker::print_memory_usage(BaselineOutputer& out, size_t unit, bool sum
 
 // compare memory usage between current snapshot and baseline
 bool MemTracker::compare_memory_usage(BaselineOutputer& out, size_t unit, bool summary_only) {
-  MutexLockerEx lock(&_query_lock, true);
+  MutexLockerEx lock(_query_lock, true);
   if (_baseline.baselined()) {
     MemBaseline baseline;
     MemSnapshot* snapshot = get_snapshot();

src/share/vm/services/memTracker.hpp

@@ -126,6 +126,8 @@ class MemTracker : AllStatic {
         return "Native memory tracking has been shutdown by user";
       case NMT_normal:
         return "Native memory tracking has been shutdown due to process exiting";
+      case NMT_out_of_memory:
+        return "Native memory tracking has been shutdown due to out of native memory";
       case NMT_initialization:
         return "Native memory tracking failed to initialize";
       case NMT_error_reporting:
@@ -336,7 +338,7 @@ class MemTracker : AllStatic {
   static MemBaseline      _baseline;
 
   // query lock
-  static Mutex            _query_lock;
+  static Mutex*           _query_lock;
 
   // a thread can start to allocate memory before it is attached
   // to VM 'Thread', those memory activities are recorded here.