8130309: Need to bailout cleanly if creation of stubs fails when codecache is out of space

Summary: Check for failed expansion of stub section in code buffer and bailout.
Reviewed-by: kvn, thartmann

src/cpu/ppc/vm/compiledIC_ppc.cpp

@@ -94,7 +94,7 @@ bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site) {
 
 const int IC_pos_in_java_to_interp_stub = 8;
 #define __ _masm.
-void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
+address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 #ifdef COMPILER2
   // Get the mark within main instrs section which is set to the address of the call.
   address call_addr = cbuf.insts_mark();
@@ -106,8 +106,7 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
   // Start the stub.
   address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
   if (stub == NULL) {
-    Compile::current()->env()->record_out_of_memory_failure();
-    return;
+    return NULL; // CodeCache is full
   }
 
   // For java_to_interp stubs we use R11_scratch1 as scratch register
@@ -149,6 +148,7 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 
  // End the stub.
   __ end_a_stub();
+  return stub;
 #else
   ShouldNotReachHere();
 #endif

src/cpu/ppc/vm/ppc.ad

@@ -1171,7 +1171,7 @@ void CallStubImpl::emit_trampoline_stub(MacroAssembler &_masm, int destination_t
   // Start the stub.
   address stub = __ start_a_stub(Compile::MAX_stubs_size/2);
   if (stub == NULL) {
-    Compile::current()->env()->record_out_of_memory_failure();
+    ciEnv::current()->record_failure("CodeCache is full");
     return;
   }
 
@@ -1249,7 +1249,7 @@ EmitCallOffsets emit_call_with_trampoline_stub(MacroAssembler &_masm, address en
 
     // Emit the trampoline stub which will be related to the branch-and-link below.
     CallStubImpl::emit_trampoline_stub(_masm, entry_point_toc_offset, offsets.insts_call_instruction_offset);
-    if (Compile::current()->env()->failing()) { return offsets; } // Code cache may be full.
+    if (ciEnv::current()->failing()) { return offsets; } // Code cache may be full.
     __ relocate(rtype);
   }
 
@@ -3488,7 +3488,7 @@ encode %{
 
         // Emit the trampoline stub which will be related to the branch-and-link below.
         CallStubImpl::emit_trampoline_stub(_masm, entry_point_toc_offset, start_offset);
-        if (Compile::current()->env()->failing()) { return; } // Code cache may be full.
+        if (ciEnv::current()->failing()) { return; } // Code cache may be full.
         __ relocate(_optimized_virtual ?
                     relocInfo::opt_virtual_call_type : relocInfo::static_call_type);
       }
@@ -3501,7 +3501,11 @@ encode %{
       __ bl(__ pc());  // Emits a relocation.
 
       // The stub for call to interpreter.
-      CompiledStaticCall::emit_to_interp_stub(cbuf);
+      address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+      if (stub == NULL) {
+        ciEnv::current()->record_failure("CodeCache is full"); 
+        return;
+      }
     }
   %}
 
@@ -3546,7 +3550,11 @@ encode %{
 
     assert(_method, "execute next statement conditionally");
     // The stub for call to interpreter.
-    CompiledStaticCall::emit_to_interp_stub(cbuf);
+    address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+    if (stub == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full"); 
+      return;
+    }
 
     // Restore original sp.
     __ ld(R11_scratch1, 0, R1_SP); // Load caller sp.

src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp

@@ -431,6 +431,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   __ mov(length()->as_register(),  O4);
 
   ce->emit_static_call_stub();
+  if (ce->compilation()->bailed_out()) {
+    return; // CodeCache is full
+  }
 
   __ call(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type);
   __ delayed()->nop();

src/cpu/sparc/vm/compiledIC_sparc.cpp

@@ -53,7 +53,7 @@ bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site) {
 // ----------------------------------------------------------------------------
 
 #define __ _masm.
-void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
+address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 #ifdef COMPILER2
   // Stub is fixed up when the corresponding call is converted from calling
   // compiled code to calling interpreted code.
@@ -64,9 +64,10 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 
   MacroAssembler _masm(&cbuf);
 
-  address base =
-  __ start_a_stub(to_interp_stub_size()*2);
-  if (base == NULL) return;  // CodeBuffer::expand failed.
+  address base = __ start_a_stub(to_interp_stub_size());
+  if (base == NULL) {
+    return NULL;  // CodeBuffer::expand failed.
+  }
 
   // Static stub relocation stores the instruction address of the call.
   __ relocate(static_stub_Relocation::spec(mark));
@@ -81,6 +82,7 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 
   // Update current stubs pointer and restore code_end.
   __ end_a_stub();
+  return base;
 #else
   ShouldNotReachHere();
 #endif

src/cpu/sparc/vm/sparc.ad

@@ -1775,9 +1775,11 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
   AddressLiteral exception_blob(OptoRuntime::exception_blob()->entry_point());
   MacroAssembler _masm(&cbuf);
 
-  address base =
-  __ start_a_stub(size_exception_handler());
-  if (base == NULL)  return 0;  // CodeBuffer::expand failed
+  address base = __ start_a_stub(size_exception_handler());
+  if (base == NULL) {
+    ciEnv::current()->record_failure("CodeCache is full");
+    return 0;  // CodeBuffer::expand failed
+  }
 
   int offset = __ offset();
 
@@ -1798,9 +1800,11 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
   AddressLiteral deopt_blob(SharedRuntime::deopt_blob()->unpack());
   MacroAssembler _masm(&cbuf);
 
-  address base =
-  __ start_a_stub(size_deopt_handler());
-  if (base == NULL)  return 0;  // CodeBuffer::expand failed
+  address base = __ start_a_stub(size_deopt_handler());
+  if (base == NULL) {
+    ciEnv::current()->record_failure("CodeCache is full");
+    return 0;  // CodeBuffer::expand failed
+  }
 
   int offset = __ offset();
   __ save_frame(0);
@@ -2601,7 +2605,12 @@ encode %{
       emit_call_reloc(cbuf, $meth$$method, relocInfo::static_call_type);
     }
     if (_method) {  // Emit stub for static call.
-      CompiledStaticCall::emit_to_interp_stub(cbuf);
+      address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+      // Stub does not fit into scratch buffer if TraceJumps is enabled
+      if (stub == NULL && !(TraceJumps && Compile::current()->in_scratch_emit_size())) {
+        ciEnv::current()->record_failure("CodeCache is full");
+        return;
+      } 
     }
   %}
 

src/cpu/x86/vm/c1_CodeStubs_x86.cpp

@@ -502,6 +502,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->align_call(lir_static_call);
 
   ce->emit_static_call_stub();
+  if (ce->compilation()->bailed_out()) {
+    return; // CodeCache is full
+  }
   AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(),
                          relocInfo::static_call_type);
   __ call(resolve);

src/cpu/x86/vm/compiledIC_x86.cpp

@@ -50,7 +50,7 @@ bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site) {
 // ----------------------------------------------------------------------------
 
 #define __ _masm.
-void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
+address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
   // Stub is fixed up when the corresponding call is converted from
   // calling compiled code to calling interpreted code.
   // movq rbx, 0
@@ -62,9 +62,10 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
   // That's why we must use the macroassembler to generate a stub.
   MacroAssembler _masm(&cbuf);
 
-  address base =
-  __ start_a_stub(to_interp_stub_size()*2);
-  if (base == NULL) return;  // CodeBuffer::expand failed.
+  address base = __ start_a_stub(to_interp_stub_size());
+  if (base == NULL) {
+    return NULL;  // CodeBuffer::expand failed.
+  }
   // Static stub relocation stores the instruction address of the call.
   __ relocate(static_stub_Relocation::spec(mark), Assembler::imm_operand);
   // Static stub relocation also tags the Method* in the code-stream.
@@ -74,6 +75,7 @@ void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
 
   // Update current stubs pointer and restore insts_end.
   __ end_a_stub();
+  return base;
 }
 #undef __
 

src/cpu/x86/vm/x86.ad

@@ -550,7 +550,10 @@ int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf) {
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
   address base = __ start_a_stub(size_exception_handler());
-  if (base == NULL)  return 0;  // CodeBuffer::expand failed
+  if (base == NULL) {
+    ciEnv::current()->record_failure("CodeCache is full");
+    return 0;  // CodeBuffer::expand failed
+  }
   int offset = __ offset();
   __ jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
   assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
@@ -565,7 +568,10 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
   // That's why we must use the macroassembler to generate a handler.
   MacroAssembler _masm(&cbuf);
   address base = __ start_a_stub(size_deopt_handler());
-  if (base == NULL)  return 0;  // CodeBuffer::expand failed
+  if (base == NULL) {
+    ciEnv::current()->record_failure("CodeCache is full");
+    return 0;  // CodeBuffer::expand failed
+  }
   int offset = __ offset();
 
 #ifdef _LP64

src/cpu/x86/vm/x86_32.ad

@@ -1870,7 +1870,11 @@ encode %{
                      static_call_Relocation::spec(), RELOC_IMM32 );
     }
     if (_method) {  // Emit stub for static call.
-      CompiledStaticCall::emit_to_interp_stub(cbuf);
+      address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+      if (stub == NULL) {
+        ciEnv::current()->record_failure("CodeCache is full");
+        return;
+      } 
     }
   %}
 

src/cpu/x86/vm/x86_64.ad

@@ -2125,7 +2125,11 @@ encode %{
     }
     if (_method) {
       // Emit stub for static call.
-      CompiledStaticCall::emit_to_interp_stub(cbuf);
+      address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+      if (stub == NULL) {
+        ciEnv::current()->record_failure("CodeCache is full");
+        return;
+      } 
     }
   %}
 

src/cpu/zero/vm/compiledIC_zero.cpp

@@ -60,8 +60,9 @@ bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site) {
 
 // ----------------------------------------------------------------------------
 
-void CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
+address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf) {
   ShouldNotReachHere(); // Only needed for COMPILER2.
+  return NULL;
 }
 
 int CompiledStaticCall::to_interp_stub_size() {

src/share/vm/c1/c1_LIRAssembler.cpp

@@ -464,6 +464,7 @@ void LIR_Assembler::emit_call(LIR_OpJavaCall* op) {
 
   // emit the static call stub stuff out of line
   emit_static_call_stub();
+  CHECK_BAILOUT();
 
   switch (op->code()) {
   case lir_static_call:

src/share/vm/code/compiledIC.hpp

@@ -320,7 +320,7 @@ class CompiledStaticCall: public NativeCall {
   friend CompiledStaticCall* compiledStaticCall_at(Relocation* call_site);
 
   // Code
-  static void emit_to_interp_stub(CodeBuffer &cbuf);
+  static address emit_to_interp_stub(CodeBuffer &cbuf);
   static int to_interp_stub_size();
   static int reloc_to_interp_stub();
 

src/share/vm/opto/compile.cpp

@@ -608,6 +608,10 @@ uint Compile::scratch_emit_size(const Node* n) {
     n->as_MachBranch()->label_set(&fakeL, 0);
   }
   n->emit(buf, this->regalloc());
+
+  // Emitting into the scratch buffer should not fail
+  assert (!failing(), err_msg_res("Must not have pending failure. Reason is: %s", failure_reason()));
+
   if (is_branch) // Restore label.
     n->as_MachBranch()->label_set(saveL, save_bnum);
 

src/share/vm/opto/output.cpp

@@ -1502,6 +1502,13 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
       n->emit(*cb, _regalloc);
       current_offset  = cb->insts_size();
 
+      // Above we only verified that there is enough space in the instruction section.
+      // However, the instruction may emit stubs that cause code buffer expansion.
+      // Bail out here if expansion failed due to a lack of code cache space.
+      if (failing()) {
+        return;
+      }
+
 #ifdef ASSERT
       if (n->size(_regalloc) < (current_offset-instr_offset)) {
         n->dump();
@@ -1630,11 +1637,14 @@ void Compile::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
   if (_method) {
     // Emit the exception handler code.
     _code_offsets.set_value(CodeOffsets::Exceptions, HandlerImpl::emit_exception_handler(*cb));
+    if (failing()) {
+      return; // CodeBuffer::expand failed
+    }
     // Emit the deopt handler code.
     _code_offsets.set_value(CodeOffsets::Deopt, HandlerImpl::emit_deopt_handler(*cb));
 
     // Emit the MethodHandle deopt handler code (if required).
-    if (has_method_handle_invokes()) {
+    if (has_method_handle_invokes() && !failing()) {
       // We can use the same code as for the normal deopt handler, we
       // just need a different entry point address.
       _code_offsets.set_value(CodeOffsets::DeoptMH, HandlerImpl::emit_deopt_handler(*cb));