7036236: VM crashes assert((!inside_attrs()) || is_error_reported()) failed ...

Summary: Eliminate the race condition.
Reviewed-by: kvn

src/share/vm/code/codeCache.cpp

@@ -971,8 +971,6 @@ size_t CodeCache::largest_free_block() {
   if (CodeCache_lock->owned_by_self()) {
     return _heap->largest_free_block();
   } else {
-    // Avoid lock ordering problems with ttyLock.
-    ttyUnlocker ttyul;
     MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
     return _heap->largest_free_block();
   }

src/share/vm/compiler/compileBroker.cpp

@@ -1736,8 +1736,14 @@ void CompileBroker::handle_full_code_cache() {
   UseInterpreter = true;
   if (UseCompiler || AlwaysCompileLoopMethods ) {
     if (xtty != NULL) {
+      stringStream s;
+      // Dump code cache state into a buffer before locking the tty,
+      // because log_state() will use locks causing lock conflicts.
+      CodeCache::log_state(&s);
+      // Lock to prevent tearing
+      ttyLocker ttyl;
       xtty->begin_elem("code_cache_full");
-      CodeCache::log_state(xtty);
+      xtty->print(s.as_string());
       xtty->stamp();
       xtty->end_elem();
     }

src/share/vm/runtime/sweeper.cpp

@@ -418,6 +418,11 @@ void NMethodSweeper::speculative_disconnect_nmethods(bool is_full) {
 // state of the code cache if it's requested.
 void NMethodSweeper::log_sweep(const char* msg, const char* format, ...) {
   if (PrintMethodFlushing) {
+    stringStream s;
+    // Dump code cache state into a buffer before locking the tty,
+    // because log_state() will use locks causing lock conflicts.
+    CodeCache::log_state(&s);
+
     ttyLocker ttyl;
     tty->print("### sweeper: %s ", msg);
     if (format != NULL) {
@@ -426,10 +431,15 @@ void NMethodSweeper::log_sweep(const char* msg, const char* format, ...) {
       tty->vprint(format, ap);
       va_end(ap);
     }
-    CodeCache::log_state(tty); tty->cr();
+    tty->print_cr(s.as_string());
   }
 
   if (LogCompilation && (xtty != NULL)) {
+    stringStream s;
+    // Dump code cache state into a buffer before locking the tty,
+    // because log_state() will use locks causing lock conflicts.
+    CodeCache::log_state(&s);
+
     ttyLocker ttyl;
     xtty->begin_elem("sweeper state='%s' traversals='" INTX_FORMAT "' ", msg, (intx)traversal_count());
     if (format != NULL) {
@@ -438,7 +448,7 @@ void NMethodSweeper::log_sweep(const char* msg, const char* format, ...) {
       xtty->vprint(format, ap);
       va_end(ap);
     }
-    CodeCache::log_state(xtty);
+    xtty->print(s.as_string());
     xtty->stamp();
     xtty->end_elem();
   }