7057120: Tiered: Allow C1 to inline methods with loops

Summary: Recompile the enclosing methods without inlining of the method that has OSRed to level 4 or recompile the enclosing method at level 4.
Reviewed-by: kvn, never

src/share/vm/c1/c1_GraphBuilder.cpp

@@ -33,6 +33,7 @@
 #include "compiler/compileBroker.hpp"
 #include "interpreter/bytecode.hpp"
 #include "runtime/sharedRuntime.hpp"
+#include "runtime/compilationPolicy.hpp"
 #include "utilities/bitMap.inline.hpp"
 
 class BlockListBuilder VALUE_OBJ_CLASS_SPEC {
@@ -3395,8 +3396,8 @@ void GraphBuilder::fill_sync_handler(Value lock, BlockBegin* sync_handler, bool
 
 bool GraphBuilder::try_inline_full(ciMethod* callee, bool holder_known) {
   assert(!callee->is_native(), "callee must not be native");
-  if (count_backedges() && callee->has_loops()) {
-    INLINE_BAILOUT("too complex for tiered");
+  if (CompilationPolicy::policy()->should_not_inline(compilation()->env(), callee)) {
+    INLINE_BAILOUT("inlining prohibited by policy");
   }
   // first perform tests of things it's not possible to inline
   if (callee->has_exception_handlers() &&

src/share/vm/c1/c1_Runtime1.cpp

@@ -383,8 +383,10 @@ JRT_ENTRY(void, Runtime1::post_jvmti_exception_throw(JavaThread* thread))
   }
 JRT_END
 
-// This is a helper to allow us to safepoint but allow the outer entry
-// to be safepoint free if we need to do an osr
+// counter_overflow() is called from within C1-compiled methods. The enclosing method is the method
+// associated with the top activation record. The inlinee (that is possibly included in the enclosing
+// method) method oop is passed as an argument. In order to do that it is embedded in the code as
+// a constant.
 static nmethod* counter_overflow_helper(JavaThread* THREAD, int branch_bci, methodOopDesc* m) {
   nmethod* osr_nm = NULL;
   methodHandle method(THREAD, m);
@@ -420,7 +422,7 @@ static nmethod* counter_overflow_helper(JavaThread* THREAD, int branch_bci, meth
     bci = branch_bci + offset;
   }
 
-  osr_nm = CompilationPolicy::policy()->event(enclosing_method, method, branch_bci, bci, level, THREAD);
+  osr_nm = CompilationPolicy::policy()->event(enclosing_method, method, branch_bci, bci, level, nm, THREAD);
   return osr_nm;
 }
 

src/share/vm/ci/ciMethod.cpp

@@ -1010,6 +1010,12 @@ int ciMethod::comp_level() {
   return 0;
 }
 
+int ciMethod::highest_osr_comp_level() {
+  check_is_loaded();
+  VM_ENTRY_MARK;
+  return get_methodOop()->highest_osr_comp_level();
+}
+
 // ------------------------------------------------------------------
 // ciMethod::instructions_size
 //

src/share/vm/ci/ciMethod.hpp

@@ -158,6 +158,7 @@ class ciMethod : public ciObject {
   int interpreter_throwout_count() const         { check_is_loaded(); return _interpreter_throwout_count; }
 
   int comp_level();
+  int highest_osr_comp_level();
 
   Bytecodes::Code java_code_at_bci(int bci) {
     address bcp = code() + bci;

src/share/vm/interpreter/interpreterRuntime.cpp

@@ -844,7 +844,7 @@ IRT_ENTRY(nmethod*,
   const int branch_bci = branch_bcp != NULL ? method->bci_from(branch_bcp) : InvocationEntryBci;
   const int bci = branch_bcp != NULL ? method->bci_from(fr.interpreter_frame_bcp()) : InvocationEntryBci;
 
-  nmethod* osr_nm = CompilationPolicy::policy()->event(method, method, branch_bci, bci, CompLevel_none, thread);
+  nmethod* osr_nm = CompilationPolicy::policy()->event(method, method, branch_bci, bci, CompLevel_none, NULL, thread);
 
   if (osr_nm != NULL) {
     // We may need to do on-stack replacement which requires that no

src/share/vm/runtime/advancedThresholdPolicy.cpp

@@ -171,7 +171,7 @@ CompileTask* AdvancedThresholdPolicy::select_task(CompileQueue* compile_queue) {
       // If a method has been stale for some time, remove it from the queue.
       if (is_stale(t, TieredCompileTaskTimeout, method) && !is_old(method)) {
         if (PrintTieredEvents) {
-          print_event(KILL, method, method, task->osr_bci(), (CompLevel)task->comp_level());
+          print_event(REMOVE_FROM_QUEUE, method, method, task->osr_bci(), (CompLevel)task->comp_level());
         }
         CompileTaskWrapper ctw(task); // Frees the task
         compile_queue->remove(task);
@@ -192,7 +192,7 @@ CompileTask* AdvancedThresholdPolicy::select_task(CompileQueue* compile_queue) {
   if (max_task->comp_level() == CompLevel_full_profile && is_method_profiled(max_method)) {
     max_task->set_comp_level(CompLevel_limited_profile);
     if (PrintTieredEvents) {
-      print_event(UPDATE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
+      print_event(UPDATE_IN_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
     }
   }
 
@@ -259,6 +259,17 @@ bool AdvancedThresholdPolicy::should_create_mdo(methodOop method, CompLevel cur_
   return false;
 }
 
+// Inlining control: if we're compiling a profiled method with C1 and the callee
+// is known to have OSRed in a C2 version, don't inline it.
+bool AdvancedThresholdPolicy::should_not_inline(ciEnv* env, ciMethod* callee) {
+  CompLevel comp_level = (CompLevel)env->comp_level();
+  if (comp_level == CompLevel_full_profile ||
+      comp_level == CompLevel_limited_profile) {
+    return callee->highest_osr_comp_level() == CompLevel_full_optimization;
+  }
+  return false;
+}
+
 // Create MDO if necessary.
 void AdvancedThresholdPolicy::create_mdo(methodHandle mh, TRAPS) {
   if (mh->is_native() || mh->is_abstract() || mh->is_accessor()) return;
@@ -420,10 +431,9 @@ void AdvancedThresholdPolicy::submit_compile(methodHandle mh, int bci, CompLevel
   CompileBroker::compile_method(mh, bci, level, mh, hot_count, "tiered", THREAD);
 }
 
-
 // Handle the invocation event.
 void AdvancedThresholdPolicy::method_invocation_event(methodHandle mh, methodHandle imh,
-                                                      CompLevel level, TRAPS) {
+                                                      CompLevel level, nmethod* nm, TRAPS) {
   if (should_create_mdo(mh(), level)) {
     create_mdo(mh, THREAD);
   }
@@ -438,32 +448,81 @@ void AdvancedThresholdPolicy::method_invocation_event(methodHandle mh, methodHan
 // Handle the back branch event. Notice that we can compile the method
 // with a regular entry from here.
 void AdvancedThresholdPolicy::method_back_branch_event(methodHandle mh, methodHandle imh,
-                                                       int bci, CompLevel level, TRAPS) {
+                                                       int bci, CompLevel level, nmethod* nm, TRAPS) {
   if (should_create_mdo(mh(), level)) {
     create_mdo(mh, THREAD);
   }
+  // Check if MDO should be created for the inlined method
+  if (should_create_mdo(imh(), level)) {
+    create_mdo(imh, THREAD);
+  }
 
-  // If the method is already compiling, quickly bail out.
-  if (is_compilation_enabled() && !CompileBroker::compilation_is_in_queue(mh, bci)) {
-    // Use loop event as an opportinity to also check there's been
-    // enough calls.
-    CompLevel cur_level = comp_level(mh());
-    CompLevel next_level = call_event(mh(), cur_level);
-    CompLevel next_osr_level = loop_event(mh(), level);
+  if (is_compilation_enabled()) {
+    CompLevel next_osr_level = loop_event(imh(), level);
+    CompLevel max_osr_level = (CompLevel)imh->highest_osr_comp_level();
     if (next_osr_level  == CompLevel_limited_profile) {
       next_osr_level = CompLevel_full_profile; // OSRs are supposed to be for very hot methods.
     }
-    next_level = MAX2(next_level,
-                      next_osr_level < CompLevel_full_optimization ? next_osr_level : cur_level);
-    bool is_compiling = false;
-    if (next_level != cur_level) {
-      compile(mh, InvocationEntryBci, next_level, THREAD);
-      is_compiling = true;
+
+    // At the very least compile the OSR version
+    if (!CompileBroker::compilation_is_in_queue(imh, bci)) {
+      // Check if there's a method like that already
+      nmethod* osr_nm = NULL;
+      if (max_osr_level >= next_osr_level) {
+        // There is an osr method already with the same
+        // or greater level, check if it has the bci we need
+        osr_nm = imh->lookup_osr_nmethod_for(bci, next_osr_level, false);
+      }
+      if (osr_nm == NULL) {
+        compile(imh, bci, next_osr_level, THREAD);
+      }
     }
 
-    // Do the OSR version
-    if (!is_compiling && next_osr_level != level) {
-      compile(mh, bci, next_osr_level, THREAD);
+    // Use loop event as an opportunity to also check if there's been
+    // enough calls.
+    CompLevel cur_level, next_level;
+    if (mh() != imh()) { // If there is an enclosing method
+      guarantee(nm != NULL, "Should have nmethod here");
+      cur_level = comp_level(mh());
+      next_level = call_event(mh(), cur_level);
+
+      if (max_osr_level == CompLevel_full_optimization) {
+        // The inlinee OSRed to full opt, we need to modify the enclosing method to avoid deopts
+        bool make_not_entrant = false;
+        if (nm->is_osr_method()) {
+          // This is an osr method, just make it not entrant and recompile later if needed
+          make_not_entrant = true;
+        } else {
+          if (next_level != CompLevel_full_optimization) {
+            // next_level is not full opt, so we need to recompile the
+            // enclosing method without the inlinee
+            cur_level = CompLevel_none;
+            make_not_entrant = true;
+          }
+        }
+        if (make_not_entrant) {
+          if (PrintTieredEvents) {
+            int osr_bci = nm->is_osr_method() ? nm->osr_entry_bci() : InvocationEntryBci;
+            print_event(MAKE_NOT_ENTRANT, mh(), mh(), osr_bci, level);
+          }
+          nm->make_not_entrant();
+        }
+      }
+      if (!CompileBroker::compilation_is_in_queue(mh, InvocationEntryBci)) {
+        // Fix up next_level if necessary to avoid deopts
+        if (next_level == CompLevel_limited_profile && max_osr_level == CompLevel_full_profile) {
+          next_level = CompLevel_full_profile;
+        }
+        if (cur_level != next_level) {
+          compile(mh, InvocationEntryBci, next_level, THREAD);
+        }
+      }
+    } else {
+      cur_level = comp_level(imh());
+      next_level = call_event(imh(), cur_level);
+      if (!CompileBroker::compilation_is_in_queue(imh, bci) && next_level != cur_level) {
+        compile(imh, InvocationEntryBci, next_level, THREAD);
+      }
     }
   }
 }

src/share/vm/runtime/advancedThresholdPolicy.hpp

@@ -211,14 +211,16 @@ protected:
   virtual void submit_compile(methodHandle mh, int bci, CompLevel level, TRAPS);
   // event() from SimpleThresholdPolicy would call these.
   virtual void method_invocation_event(methodHandle method, methodHandle inlinee,
-                                       CompLevel level, TRAPS);
+                                       CompLevel level, nmethod* nm, TRAPS);
   virtual void method_back_branch_event(methodHandle method, methodHandle inlinee,
-                                        int bci, CompLevel level, TRAPS);
+                                        int bci, CompLevel level, nmethod* nm, TRAPS);
 public:
   AdvancedThresholdPolicy() : _start_time(0) { }
   // Select task is called by CompileBroker. We should return a task or NULL.
   virtual CompileTask* select_task(CompileQueue* compile_queue);
   virtual void initialize();
+  virtual bool should_not_inline(ciEnv* env, ciMethod* callee);
+
 };
 
 #endif // TIERED

src/share/vm/runtime/compilationPolicy.cpp

@@ -306,7 +306,7 @@ bool NonTieredCompPolicy::is_mature(methodOop method) {
   return (current >= initial + target);
 }
 
-nmethod* NonTieredCompPolicy::event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, TRAPS) {
+nmethod* NonTieredCompPolicy::event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS) {
   assert(comp_level == CompLevel_none, "This should be only called from the interpreter");
   NOT_PRODUCT(trace_frequency_counter_overflow(method, branch_bci, bci));
   if (JvmtiExport::can_post_interpreter_events()) {

src/share/vm/runtime/compilationPolicy.hpp

@@ -62,7 +62,7 @@ public:
   virtual int compiler_count(CompLevel comp_level) = 0;
   // main notification entry, return a pointer to an nmethod if the OSR is required,
   // returns NULL otherwise.
-  virtual nmethod* event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, TRAPS) = 0;
+  virtual nmethod* event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS) = 0;
   // safepoint() is called at the end of the safepoint
   virtual void do_safepoint_work() = 0;
   // reprofile request
@@ -80,6 +80,7 @@ public:
   virtual bool is_mature(methodOop method) = 0;
   // Do policy initialization
   virtual void initialize() = 0;
+  virtual bool should_not_inline(ciEnv* env, ciMethod* method) { return false; }
 };
 
 // A base class for baseline policies.
@@ -101,7 +102,7 @@ public:
   virtual bool is_mature(methodOop method);
   virtual void initialize();
   virtual CompileTask* select_task(CompileQueue* compile_queue);
-  virtual nmethod* event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, TRAPS);
+  virtual nmethod* event(methodHandle method, methodHandle inlinee, int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS);
   virtual void method_invocation_event(methodHandle m, TRAPS) = 0;
   virtual void method_back_branch_event(methodHandle m, int bci, TRAPS) = 0;
 };

src/share/vm/runtime/simpleThresholdPolicy.cpp

@@ -50,15 +50,18 @@ void SimpleThresholdPolicy::print_event(EventType type, methodHandle mh, methodH
   case COMPILE:
     tty->print("compile");
     break;
-  case KILL:
-    tty->print("kill");
+  case REMOVE_FROM_QUEUE:
+    tty->print("remove-from-queue");
     break;
-  case UPDATE:
-    tty->print("update");
+  case UPDATE_IN_QUEUE:
+    tty->print("update-in-queue");
     break;
   case REPROFILE:
     tty->print("reprofile");
     break;
+  case MAKE_NOT_ENTRANT:
+    tty->print("make-not-entrant");
+    break;
   default:
     tty->print("unknown");
   }
@@ -68,7 +71,6 @@ void SimpleThresholdPolicy::print_event(EventType type, methodHandle mh, methodH
   ResourceMark rm;
   char *method_name = mh->name_and_sig_as_C_string();
   tty->print("[%s", method_name);
-  // We can have an inlinee, although currently we don't generate any notifications for the inlined methods.
   if (inlinee_event) {
     char *inlinee_name = imh->name_and_sig_as_C_string();
     tty->print(" [%s]] ", inlinee_name);
@@ -170,7 +172,7 @@ void SimpleThresholdPolicy::reprofile(ScopeDesc* trap_scope, bool is_osr) {
 }
 
 nmethod* SimpleThresholdPolicy::event(methodHandle method, methodHandle inlinee,
-                                      int branch_bci, int bci, CompLevel comp_level, TRAPS) {
+                                      int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS) {
   if (comp_level == CompLevel_none &&
       JvmtiExport::can_post_interpreter_events()) {
     assert(THREAD->is_Java_thread(), "Should be java thread");
@@ -190,12 +192,13 @@ nmethod* SimpleThresholdPolicy::event(methodHandle method, methodHandle inlinee,
   }
 
   if (bci == InvocationEntryBci) {
-    method_invocation_event(method, inlinee, comp_level, THREAD);
+    method_invocation_event(method, inlinee, comp_level, nm, THREAD);
   } else {
-    method_back_branch_event(method, inlinee, bci, comp_level, THREAD);
-    int highest_level = method->highest_osr_comp_level();
+    method_back_branch_event(method, inlinee, bci, comp_level, nm, THREAD);
+    // method == inlinee if the event originated in the main method
+    int highest_level = inlinee->highest_osr_comp_level();
     if (highest_level > comp_level) {
-      osr_nm = method->lookup_osr_nmethod_for(bci, highest_level, false);
+      osr_nm = inlinee->lookup_osr_nmethod_for(bci, highest_level, false);
     }
   }
   return osr_nm;
@@ -360,7 +363,7 @@ CompLevel SimpleThresholdPolicy::loop_event(methodOop method, CompLevel cur_leve
 
 // Handle the invocation event.
 void SimpleThresholdPolicy::method_invocation_event(methodHandle mh, methodHandle imh,
-                                              CompLevel level, TRAPS) {
+                                              CompLevel level, nmethod* nm, TRAPS) {
   if (is_compilation_enabled() && !CompileBroker::compilation_is_in_queue(mh, InvocationEntryBci)) {
     CompLevel next_level = call_event(mh(), level);
     if (next_level != level) {
@@ -372,7 +375,7 @@ void SimpleThresholdPolicy::method_invocation_event(methodHandle mh, methodHandl
 // Handle the back branch event. Notice that we can compile the method
 // with a regular entry from here.
 void SimpleThresholdPolicy::method_back_branch_event(methodHandle mh, methodHandle imh,
-                                               int bci, CompLevel level, TRAPS) {
+                                                     int bci, CompLevel level, nmethod* nm, TRAPS) {
   // If the method is already compiling, quickly bail out.
   if (is_compilation_enabled() && !CompileBroker::compilation_is_in_queue(mh, bci)) {
     // Use loop event as an opportinity to also check there's been

src/share/vm/runtime/simpleThresholdPolicy.hpp

@@ -62,7 +62,7 @@ protected:
   void set_c1_count(int x) { _c1_count = x;    }
   void set_c2_count(int x) { _c2_count = x;    }
 
-  enum EventType { CALL, LOOP, COMPILE, KILL, UPDATE, REPROFILE };
+  enum EventType { CALL, LOOP, COMPILE, REMOVE_FROM_QUEUE, UPDATE_IN_QUEUE, REPROFILE, MAKE_NOT_ENTRANT };
   void print_event(EventType type, methodHandle mh, methodHandle imh, int bci, CompLevel level);
   // Print policy-specific information if necessary
   virtual void print_specific(EventType type, methodHandle mh, methodHandle imh, int bci, CompLevel level) { }
@@ -88,9 +88,9 @@ protected:
     return CompLevel_none;
   }
   virtual void method_invocation_event(methodHandle method, methodHandle inlinee,
-                                       CompLevel level, TRAPS);
+                                       CompLevel level, nmethod* nm, TRAPS);
   virtual void method_back_branch_event(methodHandle method, methodHandle inlinee,
-                                        int bci, CompLevel level, TRAPS);
+                                        int bci, CompLevel level, nmethod* nm, TRAPS);
 public:
   SimpleThresholdPolicy() : _c1_count(0), _c2_count(0) { }
   virtual int compiler_count(CompLevel comp_level) {
@@ -101,17 +101,20 @@ public:
   virtual void do_safepoint_work() { }
   virtual void delay_compilation(methodOop method) { }
   virtual void disable_compilation(methodOop method) { }
-  // TODO: we should honour reprofiling requests in the future. Currently reprofiling
-  // would happen but not to the extent we would ideally like.
   virtual void reprofile(ScopeDesc* trap_scope, bool is_osr);
   virtual nmethod* event(methodHandle method, methodHandle inlinee,
-                         int branch_bci, int bci, CompLevel comp_level, TRAPS);
+                         int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS);
   // Select task is called by CompileBroker. We should return a task or NULL.
   virtual CompileTask* select_task(CompileQueue* compile_queue);
   // Tell the runtime if we think a given method is adequately profiled.
   virtual bool is_mature(methodOop method);
   // Initialize: set compiler thread count
   virtual void initialize();
+  virtual bool should_not_inline(ciEnv* env, ciMethod* callee) {
+    return (env->comp_level() == CompLevel_limited_profile ||
+            env->comp_level() == CompLevel_full_profile) &&
+            callee->has_loops();
+  }
 };
 
 #endif // SHARE_VM_RUNTIME_SIMPLETHRESHOLDPOLICY_HPP