6650373: Assert in methodOopDesc::make_adapters()

Summary: AdapterHandlerLibrary::get_create_adapter_index() returns incorrect value (-2) when CodeCache is full.
Reviewed-by: sgoldman

src/share/vm/opto/output.cpp

@@ -921,11 +921,8 @@ static void turn_off_compiler(Compile* C) {
     // blown the code cache size.
     C->record_failure("excessive request to CodeCache");
   } else {
-    UseInterpreter            = true;
-    UseCompiler               = false;
-    AlwaysCompileLoopMethods  = false;
+    // Let CompilerBroker disable further compilations.
     C->record_failure("CodeCache is full");
-    warning("CodeCache is full. Compiling has been disabled");
   }
 }
 

src/share/vm/runtime/sharedRuntime.cpp

@@ -1839,7 +1839,25 @@ int AdapterHandlerLibrary::get_create_adapter_index(methodHandle method) {
                                                                         regs);
 
     B = BufferBlob::create(AdapterHandlerEntry::name, &buffer);
-    if (B == NULL)  return -2;          // Out of CodeCache space
+    if (B == NULL) {
+      // CodeCache is full, disable compilation
+      // Ought to log this but compile log is only per compile thread
+      // and we're some non descript Java thread.
+      UseInterpreter = true;
+      if (UseCompiler || AlwaysCompileLoopMethods ) {
+#ifndef PRODUCT
+        warning("CodeCache is full. Compiler has been disabled");
+        if (CompileTheWorld || ExitOnFullCodeCache) {
+          before_exit(JavaThread::current());
+          exit_globals(); // will delete tty
+          vm_direct_exit(CompileTheWorld ? 0 : 1);
+        }
+#endif
+        UseCompiler               = false;
+        AlwaysCompileLoopMethods  = false;
+      }
+      return 0; // Out of CodeCache space (_handlers[0] == NULL)
+    }
     entry->relocate(B->instructions_begin());
 #ifndef PRODUCT
     // debugging suppport