8028159: C2: compiler stack overflow during inlining of @ForceInline methods

Reviewed-by: roland, kvn

src/share/vm/c1/c1_globals.hpp

@@ -341,9 +341,6 @@
   diagnostic(bool, C1PatchInvokeDynamic, true,                              \
              "Patch invokedynamic appendix not known at compile time")      \
                                                                             \
-  develop(intx, MaxForceInlineLevel, 100,                                   \
-          "maximum number of nested @ForceInline calls that are inlined")   \
-                                                                            \
 
 
 // Read default values for c1 globals

src/share/vm/opto/bytecodeInfo.cpp

@@ -389,6 +389,10 @@ bool InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_method,
     return false;
   }
   if (inline_level() > _max_inline_level) {
+    if (callee_method->force_inline() && inline_level() > MaxForceInlineLevel) {
+      set_msg("MaxForceInlineLevel");
+      return false;
+    }
     if (!callee_method->force_inline() || !IncrementalInline) {
       set_msg("inlining too deep");
       return false;

src/share/vm/opto/callGenerator.cpp

@@ -776,7 +776,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod*
         guarantee(!target->is_method_handle_intrinsic(), "should not happen");  // XXX remove
         const int vtable_index = Method::invalid_vtable_index;
         CallGenerator* cg = C->call_generator(target, vtable_index, false, jvms, true, PROB_ALWAYS, NULL, true, true);
-        assert(!cg->is_late_inline() || cg->is_mh_late_inline(), "no late inline here");
+        assert(cg == NULL || !cg->is_late_inline() || cg->is_mh_late_inline(), "no late inline here");
         if (cg != NULL && cg->is_inline())
           return cg;
       }
@@ -846,7 +846,7 @@ CallGenerator* CallGenerator::for_method_handle_inline(JVMState* jvms, ciMethod*
         }
 
         CallGenerator* cg = C->call_generator(target, vtable_index, call_does_dispatch, jvms, true, PROB_ALWAYS, speculative_receiver_type, true, true);
-        assert(!cg->is_late_inline() || cg->is_mh_late_inline(), "no late inline here");
+        assert(cg == NULL || !cg->is_late_inline() || cg->is_mh_late_inline(), "no late inline here");
         if (cg != NULL && cg->is_inline())
           return cg;
       }

src/share/vm/runtime/globals.hpp

@@ -2954,6 +2954,9 @@ class CommandLineFlags {
   product(intx, MaxRecursiveInlineLevel, 1,                                 \
           "maximum number of nested recursive calls that are inlined")      \
                                                                             \
+  develop(intx, MaxForceInlineLevel, 100,                                   \
+          "maximum number of nested @ForceInline calls that are inlined")   \
+                                                                            \
   product_pd(intx, InlineSmallCode,                                         \
           "Only inline already compiled methods if their code size is "     \
           "less than this")                                                 \