6552561: MaxRecursiveInlineLevel flag doesn't operate correctly

Reviewed-by: kvn, never

src/share/vm/opto/bytecodeInfo.cpp

@@ -310,13 +310,14 @@ const char* InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_
     return "inlining too deep";
   }
 
-  // We need to detect recursive inlining of method handle targets: if
-  // the current method is a method handle adapter and one of the
-  // callers is the same method as the callee, we bail out if
-  // MaxRecursiveInlineLevel is hit.
-  if (method()->is_method_handle_adapter()) {
+  // detect direct and indirect recursive inlining
+  {
+    // count the current method and the callee
+    int inline_level = (method() == callee_method) ? 1 : 0;
+    if (inline_level > MaxRecursiveInlineLevel)
+      return "recursively inlining too deep";
+    // count callers of current method and callee
     JVMState* jvms = caller_jvms();
-    int inline_level = 0;
     while (jvms != NULL && jvms->has_method()) {
       if (jvms->method() == callee_method) {
         inline_level++;
@@ -327,10 +328,6 @@ const char* InlineTree::try_to_inline(ciMethod* callee_method, ciMethod* caller_
     }
   }
 
-  if (method() == callee_method && inline_depth() > MaxRecursiveInlineLevel) {
-    return "recursively inlining too deep";
-  }
-
   int size = callee_method->code_size();
 
   if (UseOldInlining && ClipInlining