7071307: MethodHandle bimorphic inlining should consider the frequency

Reviewed-by: twisti, roland, kvn, iveresov

src/cpu/sparc/vm/methodHandles_sparc.cpp

@@ -1262,6 +1262,15 @@ void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHan
     }
     break;
 
+  case _adapter_opt_profiling:
+    if (java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes() != 0) {
+      Address G3_mh_vmcount(G3_method_handle, java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes());
+      __ ld(G3_mh_vmcount, O1_scratch);
+      __ add(O1_scratch, 1, O1_scratch);
+      __ st(O1_scratch, G3_mh_vmcount);
+    }
+    // fall through
+
   case _adapter_retype_only:
   case _adapter_retype_raw:
     // Immediately jump to the next MH layer:

src/cpu/x86/vm/methodHandles_x86.cpp

@@ -1343,6 +1343,13 @@ void MethodHandles::generate_method_handle_stub(MacroAssembler* _masm, MethodHan
     }
     break;
 
+  case _adapter_opt_profiling:
+    if (java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes() != 0) {
+      Address rcx_mh_vmcount(rcx_recv, java_lang_invoke_CountingMethodHandle::vmcount_offset_in_bytes());
+      __ incrementl(rcx_mh_vmcount);
+    }
+    // fall through
+
   case _adapter_retype_only:
   case _adapter_retype_raw:
     // immediately jump to the next MH layer:

src/share/vm/ci/ciCallProfile.hpp

@@ -79,6 +79,17 @@ public:
     assert(i < _limit, "out of Call Profile MorphismLimit");
     return _receiver[i];
   }
+
+  // Rescale the current profile based on the incoming scale
+  ciCallProfile rescale(double scale) {
+    assert(scale >= 0 && scale <= 1.0, "out of range");
+    ciCallProfile call = *this;
+    call._count = (int)(call._count * scale);
+    for (int i = 0; i < _morphism; i++) {
+      call._receiver_count[i] = (int)(call._receiver_count[i] * scale);
+    }
+    return call;
+  }
 };
 
 #endif // SHARE_VM_CI_CICALLPROFILE_HPP

src/share/vm/ci/ciMethodHandle.cpp

@@ -37,7 +37,7 @@
 // ciMethodHandle::get_adapter
 //
 // Return an adapter for this MethodHandle.
-ciMethod* ciMethodHandle::get_adapter_impl(bool is_invokedynamic) const {
+ciMethod* ciMethodHandle::get_adapter_impl(bool is_invokedynamic) {
   VM_ENTRY_MARK;
   Handle h(get_oop());
   methodHandle callee(_callee->get_methodOop());
@@ -73,7 +73,7 @@ ciMethod* ciMethodHandle::get_adapter_impl(bool is_invokedynamic) const {
 // ciMethodHandle::get_adapter
 //
 // Return an adapter for this MethodHandle.
-ciMethod* ciMethodHandle::get_adapter(bool is_invokedynamic) const {
+ciMethod* ciMethodHandle::get_adapter(bool is_invokedynamic) {
   ciMethod* result = get_adapter_impl(is_invokedynamic);
   if (result) {
     // Fake up the MDO maturity.
@@ -86,11 +86,22 @@ ciMethod* ciMethodHandle::get_adapter(bool is_invokedynamic) const {
 }
 
 
+#ifndef PRODUCT
 // ------------------------------------------------------------------
-// ciMethodHandle::print_impl
+// ciMethodHandle::print_chain_impl
 //
 // Implementation of the print method.
-void ciMethodHandle::print_impl(outputStream* st) {
-  st->print(" type=");
-  get_oop()->print();
+void ciMethodHandle::print_chain_impl(outputStream* st) {
+  ASSERT_IN_VM;
+  MethodHandleChain::print(get_oop());
 }
+
+
+// ------------------------------------------------------------------
+// ciMethodHandle::print_chain
+//
+// Implementation of the print_chain method.
+void ciMethodHandle::print_chain(outputStream* st) {
+  GUARDED_VM_ENTRY(print_chain_impl(st););
+}
+#endif

src/share/vm/ci/ciMethodHandle.hpp

@@ -37,19 +37,23 @@ private:
   ciMethod*      _callee;
   ciMethod*      _caller;
   ciCallProfile  _profile;
+  ciMethod*      _method_handle_adapter;
+  ciMethod*      _invokedynamic_adapter;
 
   // Return an adapter for this MethodHandle.
-  ciMethod* get_adapter_impl(bool is_invokedynamic) const;
-  ciMethod* get_adapter(     bool is_invokedynamic) const;
+  ciMethod* get_adapter_impl(bool is_invokedynamic);
+  ciMethod* get_adapter(     bool is_invokedynamic);
 
 protected:
-  void print_impl(outputStream* st);
+  void print_chain_impl(outputStream* st) PRODUCT_RETURN;
 
 public:
   ciMethodHandle(instanceHandle h_i) :
     ciInstance(h_i),
     _callee(NULL),
-    _caller(NULL)
+    _caller(NULL),
+    _method_handle_adapter(NULL),
+    _invokedynamic_adapter(NULL)
   {}
 
   // What kind of ciObject is this?
@@ -60,10 +64,22 @@ public:
   void set_call_profile(ciCallProfile profile)  { _profile = profile; }
 
   // Return an adapter for a MethodHandle call.
-  ciMethod* get_method_handle_adapter() const { return get_adapter(false); }
+  ciMethod* get_method_handle_adapter() {
+    if (_method_handle_adapter == NULL) {
+      _method_handle_adapter = get_adapter(false);
+    }
+    return _method_handle_adapter;
+  }
 
   // Return an adapter for an invokedynamic call.
-  ciMethod* get_invokedynamic_adapter() const { return get_adapter(true);  }
+  ciMethod* get_invokedynamic_adapter() {
+    if (_invokedynamic_adapter == NULL) {
+      _invokedynamic_adapter = get_adapter(true);
+    }
+    return _invokedynamic_adapter;
+  }
+
+  void print_chain(outputStream* st = tty) PRODUCT_RETURN;
 };
 
 #endif // SHARE_VM_CI_CIMETHODHANDLE_HPP

src/share/vm/ci/ciObject.cpp

@@ -194,16 +194,26 @@ bool ciObject::can_be_constant() {
 // ciObject::should_be_constant()
 bool ciObject::should_be_constant() {
   if (ScavengeRootsInCode >= 2)  return true;  // force everybody to be a constant
-  if (!JavaObjectsInPerm && !is_null_object()) {
+  if (is_null_object()) return true;
+
+  ciEnv* env = CURRENT_ENV;
+  if (!JavaObjectsInPerm) {
     // We want Strings and Classes to be embeddable by default since
     // they used to be in the perm world.  Not all Strings used to be
     // embeddable but there's no easy way to distinguish the interned
     // from the regulars ones so just treat them all that way.
-    ciEnv* env = CURRENT_ENV;
     if (klass() == env->String_klass() || klass() == env->Class_klass()) {
       return true;
     }
   }
+  if (EnableInvokeDynamic &&
+      (klass()->is_subclass_of(env->MethodHandle_klass()) ||
+       klass()->is_subclass_of(env->CallSite_klass()))) {
+    assert(ScavengeRootsInCode >= 1, "must be");
+    // We want to treat these aggressively.
+    return true;
+  }
+
   return handle() == NULL || is_perm();
 }
 

src/share/vm/classfile/javaClasses.cpp

@@ -2324,6 +2324,8 @@ int java_lang_invoke_BoundMethodHandle::_vmargslot_offset;
 
 int java_lang_invoke_AdapterMethodHandle::_conversion_offset;
 
+int java_lang_invoke_CountingMethodHandle::_vmcount_offset;
+
 void java_lang_invoke_MethodHandle::compute_offsets() {
   klassOop k = SystemDictionary::MethodHandle_klass();
   if (k != NULL && EnableInvokeDynamic) {
@@ -2372,6 +2374,23 @@ void java_lang_invoke_AdapterMethodHandle::compute_offsets() {
   }
 }
 
+void java_lang_invoke_CountingMethodHandle::compute_offsets() {
+  klassOop k = SystemDictionary::CountingMethodHandle_klass();
+  if (k != NULL && EnableInvokeDynamic) {
+    compute_offset(_vmcount_offset, k, vmSymbols::vmcount_name(), vmSymbols::int_signature(), true);
+  }
+}
+
+int java_lang_invoke_CountingMethodHandle::vmcount(oop mh) {
+  assert(is_instance(mh), "CMH only");
+  return mh->int_field(_vmcount_offset);
+}
+
+void java_lang_invoke_CountingMethodHandle::set_vmcount(oop mh, int count) {
+  assert(is_instance(mh), "CMH only");
+  mh->int_field_put(_vmcount_offset, count);
+}
+
 oop java_lang_invoke_MethodHandle::type(oop mh) {
   return mh->obj_field(_type_offset);
 }
@@ -3043,6 +3062,7 @@ void JavaClasses::compute_offsets() {
     java_lang_invoke_MethodType::compute_offsets();
     java_lang_invoke_MethodTypeForm::compute_offsets();
     java_lang_invoke_CallSite::compute_offsets();
+    java_lang_invoke_CountingMethodHandle::compute_offsets();
   }
   java_security_AccessControlContext::compute_offsets();
   // Initialize reflection classes. The layouts of these classes

src/share/vm/classfile/javaClasses.hpp

@@ -981,6 +981,34 @@ class java_lang_invoke_AdapterMethodHandle: public java_lang_invoke_BoundMethodH
 };
 
 
+// A simple class that maintains an invocation count
+class java_lang_invoke_CountingMethodHandle: public java_lang_invoke_MethodHandle {
+  friend class JavaClasses;
+
+ private:
+  static int _vmcount_offset;
+  static void compute_offsets();
+
+ public:
+  // Accessors
+  static int            vmcount(oop mh);
+  static void       set_vmcount(oop mh, int count);
+
+  // Testers
+  static bool is_subclass(klassOop klass) {
+    return SystemDictionary::CountingMethodHandle_klass() != NULL &&
+      Klass::cast(klass)->is_subclass_of(SystemDictionary::CountingMethodHandle_klass());
+  }
+  static bool is_instance(oop obj) {
+    return obj != NULL && is_subclass(obj->klass());
+  }
+
+  // Accessors for code generation:
+  static int vmcount_offset_in_bytes()          { return _vmcount_offset; }
+};
+
+
+
 // Interface to java.lang.invoke.MemberName objects
 // (These are a private interface for Java code to query the class hierarchy.)
 

src/share/vm/classfile/systemDictionary.hpp

@@ -133,14 +133,14 @@ class SymbolPropertyTable;
   template(reflect_Method_klass,         java_lang_reflect_Method,       Pre) \
   template(reflect_Constructor_klass,    java_lang_reflect_Constructor,  Pre) \
                                                                               \
-  /* NOTE: needed too early in bootstrapping process to have checks based on JDK version */ \
-  /* Universe::is_gte_jdk14x_version() is not set up by this point. */        \
-  /* It's okay if this turns out to be NULL in non-1.4 JDKs. */               \
-  template(reflect_MagicAccessorImpl_klass,          sun_reflect_MagicAccessorImpl,  Opt) \
-  template(reflect_MethodAccessorImpl_klass, sun_reflect_MethodAccessorImpl, Opt_Only_JDK14NewRef) \
-  template(reflect_ConstructorAccessorImpl_klass, sun_reflect_ConstructorAccessorImpl, Opt_Only_JDK14NewRef) \
-  template(reflect_DelegatingClassLoader_klass, sun_reflect_DelegatingClassLoader, Opt) \
-  template(reflect_ConstantPool_klass,  sun_reflect_ConstantPool,       Opt_Only_JDK15) \
+  /* NOTE: needed too early in bootstrapping process to have checks based on JDK version */                        \
+  /* Universe::is_gte_jdk14x_version() is not set up by this point. */                                             \
+  /* It's okay if this turns out to be NULL in non-1.4 JDKs. */                                                    \
+  template(reflect_MagicAccessorImpl_klass,          sun_reflect_MagicAccessorImpl,  Opt)                          \
+  template(reflect_MethodAccessorImpl_klass, sun_reflect_MethodAccessorImpl, Opt_Only_JDK14NewRef)                 \
+  template(reflect_ConstructorAccessorImpl_klass, sun_reflect_ConstructorAccessorImpl, Opt_Only_JDK14NewRef)       \
+  template(reflect_DelegatingClassLoader_klass, sun_reflect_DelegatingClassLoader, Opt)                            \
+  template(reflect_ConstantPool_klass,  sun_reflect_ConstantPool,       Opt_Only_JDK15)                            \
   template(reflect_UnsafeStaticFieldAccessorImpl_klass, sun_reflect_UnsafeStaticFieldAccessorImpl, Opt_Only_JDK15) \
                                                                               \
   /* support for dynamic typing; it's OK if these are NULL in earlier JDKs */ \
@@ -155,6 +155,7 @@ class SymbolPropertyTable;
   template(BootstrapMethodError_klass,     java_lang_BootstrapMethodError,            Pre_JSR292) \
   template(WrongMethodTypeException_klass, java_lang_invoke_WrongMethodTypeException, Pre_JSR292) \
   template(CallSite_klass,                 java_lang_invoke_CallSite,                 Pre_JSR292) \
+  template(CountingMethodHandle_klass,     java_lang_invoke_CountingMethodHandle,     Opt)        \
   template(ConstantCallSite_klass,         java_lang_invoke_ConstantCallSite,         Pre_JSR292) \
   template(MutableCallSite_klass,          java_lang_invoke_MutableCallSite,          Pre_JSR292) \
   template(VolatileCallSite_klass,         java_lang_invoke_VolatileCallSite,         Pre_JSR292) \

src/share/vm/classfile/vmSymbols.hpp

@@ -218,6 +218,7 @@
   template(returnType_name,                           "returnType")                               \
   template(signature_name,                            "signature")                                \
   template(slot_name,                                 "slot")                                     \
+  template(selectAlternative_name,                    "selectAlternative")                        \
                                                                                                   \
   /* Support for annotations (JDK 1.5 and above) */                                               \
                                                                                                   \
@@ -246,9 +247,11 @@
   template(java_lang_invoke_MethodTypeForm_signature, "Ljava/lang/invoke/MethodTypeForm;")        \
   template(java_lang_invoke_MemberName,               "java/lang/invoke/MemberName")              \
   template(java_lang_invoke_MethodHandleNatives,      "java/lang/invoke/MethodHandleNatives")     \
+  template(java_lang_invoke_MethodHandleImpl,         "java/lang/invoke/MethodHandleImpl")        \
   template(java_lang_invoke_AdapterMethodHandle,      "java/lang/invoke/AdapterMethodHandle")     \
   template(java_lang_invoke_BoundMethodHandle,        "java/lang/invoke/BoundMethodHandle")       \
   template(java_lang_invoke_DirectMethodHandle,       "java/lang/invoke/DirectMethodHandle")      \
+  template(java_lang_invoke_CountingMethodHandle,     "java/lang/invoke/CountingMethodHandle")    \
   /* internal up-calls made only by the JVM, via class sun.invoke.MethodHandleNatives: */         \
   template(findMethodHandleType_name,                 "findMethodHandleType")                     \
   template(findMethodHandleType_signature,       "(Ljava/lang/Class;[Ljava/lang/Class;)Ljava/lang/invoke/MethodType;") \
@@ -263,6 +266,7 @@
   template(setTarget_signature,                       "(Ljava/lang/invoke/MethodHandle;)V")       \
   NOT_LP64(  do_alias(machine_word_signature,         int_signature)  )                           \
   LP64_ONLY( do_alias(machine_word_signature,         long_signature) )                           \
+  template(selectAlternative_signature, "(ZLjava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/MethodHandle;") \
                                                                                                   \
   /* common method and field names */                                                             \
   template(object_initializer_name,                   "<init>")                                   \
@@ -347,6 +351,7 @@
   template(vmmethod_name,                             "vmmethod")                                 \
   template(vmtarget_name,                             "vmtarget")                                 \
   template(vmentry_name,                              "vmentry")                                  \
+  template(vmcount_name,                              "vmcount")                                  \
   template(vmslots_name,                              "vmslots")                                  \
   template(vmlayout_name,                             "vmlayout")                                 \
   template(vmindex_name,                              "vmindex")                                  \
@@ -910,6 +915,8 @@
   do_intrinsic(_invokeVarargs,            java_lang_invoke_MethodHandle, invokeVarargs_name, object_array_object_signature, F_R)  \
   do_intrinsic(_invokeDynamic,            java_lang_invoke_InvokeDynamic, star_name,         object_array_object_signature, F_SN) \
                                                                                                                         \
+  do_intrinsic(_selectAlternative,        java_lang_invoke_MethodHandleImpl, selectAlternative_name, selectAlternative_signature, F_S)  \
+                                                                                                                        \
   /* unboxing methods: */                                                                                               \
   do_intrinsic(_booleanValue,             java_lang_Boolean,      booleanValue_name, void_boolean_signature, F_R)       \
    do_name(     booleanValue_name,       "booleanValue")                                                                \

src/share/vm/oops/methodDataOop.hpp

@@ -600,6 +600,11 @@ public:
   uint taken() {
     return uint_at(taken_off_set);
   }
+
+  void set_taken(uint cnt) {
+    set_uint_at(taken_off_set, cnt);
+  }
+
   // Saturating counter
   uint inc_taken() {
     uint cnt = taken() + 1;
@@ -926,6 +931,10 @@ public:
     return uint_at(not_taken_off_set);
   }
 
+  void set_not_taken(uint cnt) {
+    set_uint_at(not_taken_off_set, cnt);
+  }
+
   uint inc_not_taken() {
     uint cnt = not_taken() + 1;
     // Did we wrap? Will compiler screw us??

src/share/vm/opto/bytecodeInfo.cpp

@@ -141,7 +141,21 @@ const char* InlineTree::should_inline(ciMethod* callee_method, ciMethod* caller_
     assert(mha_profile, "must exist");
     CounterData* cd = mha_profile->as_CounterData();
     invoke_count = cd->count();
-    call_site_count = invoke_count;  // use the same value
+    if (invoke_count == 0) {
+      return "method handle not reached";
+    }
+
+    if (_caller_jvms != NULL && _caller_jvms->method() != NULL &&
+        _caller_jvms->method()->method_data() != NULL &&
+        !_caller_jvms->method()->method_data()->is_empty()) {
+      ciMethodData* mdo = _caller_jvms->method()->method_data();
+      ciProfileData* mha_profile = mdo->bci_to_data(_caller_jvms->bci());
+      assert(mha_profile, "must exist");
+      CounterData* cd = mha_profile->as_CounterData();
+      call_site_count = cd->count();
+    } else {
+      call_site_count = invoke_count;  // use the same value
+    }
   }
 
   assert(invoke_count != 0, "require invocation count greater than zero");

src/share/vm/opto/callGenerator.cpp

@@ -149,7 +149,6 @@ JVMState* DirectCallGenerator::generate(JVMState* jvms) {
     call->set_optimized_virtual(true);
     if (method()->is_method_handle_invoke()) {
       call->set_method_handle_invoke(true);
-      kit.C->set_has_method_handle_invokes(true);
     }
   }
   kit.set_arguments_for_java_call(call);
@@ -207,7 +206,6 @@ JVMState* DynamicCallGenerator::generate(JVMState* jvms) {
   call->set_optimized_virtual(true);
   // Take extra care (in the presence of argument motion) not to trash the SP:
   call->set_method_handle_invoke(true);
-  kit.C->set_has_method_handle_invokes(true);
 
   // Pass the target MethodHandle as first argument and shift the
   // other arguments.
@@ -706,18 +704,30 @@ CallGenerator* CallGenerator::for_method_handle_inline(Node* method_handle, JVMS
     }
   } else if (method_handle->Opcode() == Op_Phi && method_handle->req() == 3 &&
              method_handle->in(1)->Opcode() == Op_ConP && method_handle->in(2)->Opcode() == Op_ConP) {
+    float prob = PROB_FAIR;
+    Node* meth_region = method_handle->in(0);
+    if (meth_region->is_Region() &&
+        meth_region->in(1)->is_Proj() && meth_region->in(2)->is_Proj() &&
+        meth_region->in(1)->in(0) == meth_region->in(2)->in(0) &&
+        meth_region->in(1)->in(0)->is_If()) {
+      // If diamond, so grab the probability of the test to drive the inlining below
+      prob = meth_region->in(1)->in(0)->as_If()->_prob;
+      if (meth_region->in(1)->is_IfTrue()) {
+        prob = 1 - prob;
+      }
+    }
+
     // selectAlternative idiom merging two constant MethodHandles.
     // Generate a guard so that each can be inlined.  We might want to
     // do more inputs at later point but this gets the most common
     // case.
-    const TypeOopPtr* oop_ptr = method_handle->in(1)->bottom_type()->is_oopptr();
-    ciObject* const_oop = oop_ptr->const_oop();
-    ciMethodHandle* mh = const_oop->as_method_handle();
-
-    CallGenerator* cg1 = for_method_handle_inline(method_handle->in(1), jvms, caller, callee, profile);
-    CallGenerator* cg2 = for_method_handle_inline(method_handle->in(2), jvms, caller, callee, profile);
+    CallGenerator* cg1 = for_method_handle_inline(method_handle->in(1), jvms, caller, callee, profile.rescale(1.0 - prob));
+    CallGenerator* cg2 = for_method_handle_inline(method_handle->in(2), jvms, caller, callee, profile.rescale(prob));
     if (cg1 != NULL && cg2 != NULL) {
-      return new PredictedDynamicCallGenerator(mh, cg2, cg1, PROB_FAIR);
+      const TypeOopPtr* oop_ptr = method_handle->in(1)->bottom_type()->is_oopptr();
+      ciObject* const_oop = oop_ptr->const_oop();
+      ciMethodHandle* mh = const_oop->as_method_handle();
+      return new PredictedDynamicCallGenerator(mh, cg2, cg1, prob);
     }
   }
   return NULL;

src/share/vm/opto/idealGraphPrinter.cpp

@@ -375,9 +375,9 @@ intptr_t IdealGraphPrinter::get_node_id(Node *n) {
   return (intptr_t)(n);
 }
 
-void IdealGraphPrinter::visit_node(Node *n, void *param) {
+void IdealGraphPrinter::visit_node(Node *n, bool edges, VectorSet* temp_set) {
 
-  if(param) {
+  if (edges) {
 
     // Output edge
     intptr_t dest_id = get_node_id(n);
@@ -599,16 +599,11 @@ void IdealGraphPrinter::visit_node(Node *n, void *param) {
 
 #ifdef ASSERT
     if (node->debug_orig() != NULL) {
+      temp_set->Clear();
       stringStream dorigStream;
       Node* dorig = node->debug_orig();
-      if (dorig) {
+      while (dorig && temp_set->test_set(dorig->_idx)) {
         dorigStream.print("%d ", dorig->_idx);
-        Node* first = dorig;
-        dorig = first->debug_orig();
-        while (dorig && dorig != first) {
-          dorigStream.print("%d ", dorig->_idx);
-          dorig = dorig->debug_orig();
-        }
       }
       print_prop("debug_orig", dorigStream.as_string());
     }
@@ -629,7 +624,7 @@ void IdealGraphPrinter::visit_node(Node *n, void *param) {
   }
 }
 
-void IdealGraphPrinter::walk_nodes(Node *start, void *param) {
+void IdealGraphPrinter::walk_nodes(Node *start, bool edges, VectorSet* temp_set) {
 
 
   VectorSet visited(Thread::current()->resource_area());
@@ -650,7 +645,7 @@ void IdealGraphPrinter::walk_nodes(Node *start, void *param) {
   while(nodeStack.length() > 0) {
 
     Node *n = nodeStack.pop();
-    visit_node(n, param);
+    visit_node(n, edges, temp_set);
 
     if (_traverse_outs) {
       for (DUIterator i = n->outs(); n->has_out(i); i++) {
@@ -689,12 +684,14 @@ void IdealGraphPrinter::print(Compile* compile, const char *name, Node *node, in
   print_attr(GRAPH_NAME_PROPERTY, (const char *)name);
   end_head();
 
+  VectorSet temp_set(Thread::current()->resource_area());
+
   head(NODES_ELEMENT);
-  walk_nodes(node, NULL);
+  walk_nodes(node, false, &temp_set);
   tail(NODES_ELEMENT);
 
   head(EDGES_ELEMENT);
-  walk_nodes(node, (void *)1);
+  walk_nodes(node, true, &temp_set);
   tail(EDGES_ELEMENT);
   if (C->cfg() != NULL) {
     head(CONTROL_FLOW_ELEMENT);

src/share/vm/opto/idealGraphPrinter.hpp

@@ -104,8 +104,8 @@ private:
   void print_indent();
   void print_method(ciMethod *method, int bci, InlineTree *tree);
   void print_inline_tree(InlineTree *tree);
-  void visit_node(Node *n, void *param);
-  void walk_nodes(Node *start, void *param);
+  void visit_node(Node *n, bool edges, VectorSet* temp_set);
+  void walk_nodes(Node *start, bool edges, VectorSet* temp_set);
   void begin_elem(const char *s);
   void end_elem();
   void begin_head(const char *s);

src/share/vm/opto/matcher.cpp

@@ -1106,6 +1106,9 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
       mcall_java->_optimized_virtual = call_java->is_optimized_virtual();
       is_method_handle_invoke = call_java->is_method_handle_invoke();
       mcall_java->_method_handle_invoke = is_method_handle_invoke;
+      if (is_method_handle_invoke) {
+        C->set_has_method_handle_invokes(true);
+      }
       if( mcall_java->is_MachCallStaticJava() )
         mcall_java->as_MachCallStaticJava()->_name =
          call_java->as_CallStaticJava()->_name;

src/share/vm/prims/methodHandleWalk.cpp

@@ -182,10 +182,6 @@ void MethodHandleChain::print(oopDesc* m) {
   HandleMark hm;
   ResourceMark rm;
   Handle mh(m);
-  print(mh);
-}
-
-void MethodHandleChain::print(Handle mh) {
   EXCEPTION_MARK;
   MethodHandleChain mhc(mh, THREAD);
   if (HAS_PENDING_EXCEPTION) {
@@ -222,16 +218,33 @@ void MethodHandleChain::print_impl(TRAPS) {
       if (o != NULL) {
         if (o->is_instance()) {
           tty->print(" instance %s", o->klass()->klass_part()->internal_name());
+          if (java_lang_invoke_CountingMethodHandle::is_instance(o)) {
+            tty->print(" vmcount: %d", java_lang_invoke_CountingMethodHandle::vmcount(o));
+          }
         } else {
           o->print();
         }
       }
+      oop vmt = chain.vmtarget_oop();
+      if (vmt != NULL) {
+        if (vmt->is_method()) {
+          tty->print(" ");
+          methodOop(vmt)->print_short_name(tty);
+        } else if (java_lang_invoke_MethodHandle::is_instance(vmt)) {
+          tty->print(" method handle " INTPTR_FORMAT, vmt);
+        } else {
+          ShouldNotReachHere();
+        }
+      }
     } else if (chain.is_adapter()) {
       tty->print("adapter: arg_slot %d conversion op %s",
                  chain.adapter_arg_slot(),
                  adapter_op_to_string(chain.adapter_conversion_op()));
       switch (chain.adapter_conversion_op()) {
         case java_lang_invoke_AdapterMethodHandle::OP_RETYPE_ONLY:
+          if (java_lang_invoke_CountingMethodHandle::is_instance(chain.method_handle_oop())) {
+            tty->print(" vmcount: %d", java_lang_invoke_CountingMethodHandle::vmcount(chain.method_handle_oop()));
+          }
         case java_lang_invoke_AdapterMethodHandle::OP_RETYPE_RAW:
         case java_lang_invoke_AdapterMethodHandle::OP_CHECK_CAST:
         case java_lang_invoke_AdapterMethodHandle::OP_PRIM_TO_PRIM:
@@ -907,7 +920,10 @@ MethodHandleCompiler::MethodHandleCompiler(Handle root, Symbol* name, Symbol* si
     _non_bcp_klasses(THREAD, 5),
     _cur_stack(0),
     _max_stack(0),
-    _rtype(T_ILLEGAL)
+    _rtype(T_ILLEGAL),
+    _selectAlternative_bci(-1),
+    _taken_count(0),
+    _not_taken_count(0)
 {
 
   // Element zero is always the null constant.
@@ -1115,11 +1131,50 @@ void MethodHandleCompiler::emit_bc(Bytecodes::Code op, int index, int args_size)
     _bytecode.push(0);
     break;
 
+  case Bytecodes::_ifeq:
+    assert((unsigned short) index == index, "index does not fit in 16-bit");
+    _bytecode.push(op);
+    _bytecode.push(index >> 8);
+    _bytecode.push(index);
+    break;
+
   default:
     ShouldNotReachHere();
   }
 }
 
+void MethodHandleCompiler::update_branch_dest(int src, int dst) {
+  switch (_bytecode.at(src)) {
+    case Bytecodes::_ifeq:
+      dst -= src; // compute the offset
+      assert((unsigned short) dst == dst, "index does not fit in 16-bit");
+      _bytecode.at_put(src + 1, dst >> 8);
+      _bytecode.at_put(src + 2, dst);
+      break;
+    default:
+      ShouldNotReachHere();
+  }
+}
+
+void MethodHandleCompiler::emit_load(ArgToken arg) {
+  TokenType tt = arg.token_type();
+  BasicType bt = arg.basic_type();
+
+  switch (tt) {
+    case tt_parameter:
+    case tt_temporary:
+      emit_load(bt, arg.index());
+      break;
+    case tt_constant:
+      emit_load_constant(arg);
+      break;
+    case tt_illegal:
+    case tt_void:
+    default:
+      ShouldNotReachHere();
+  }
+}
+
 
 void MethodHandleCompiler::emit_load(BasicType bt, int index) {
   if (index <= 3) {
@@ -1318,6 +1373,29 @@ MethodHandleCompiler::make_conversion(BasicType type, klassOop tk, Bytecodes::Co
 jvalue MethodHandleCompiler::zero_jvalue = { 0 };
 jvalue MethodHandleCompiler::one_jvalue  = { 1 };
 
+// Fetch any values from CountingMethodHandles and capture them for profiles
+bool MethodHandleCompiler::fetch_counts(ArgToken arg1, ArgToken arg2) {
+  int count1 = -1, count2 = -1;
+  if (arg1.token_type() == tt_constant && arg1.basic_type() == T_OBJECT &&
+      java_lang_invoke_CountingMethodHandle::is_instance(arg1.object()())) {
+    count1 = java_lang_invoke_CountingMethodHandle::vmcount(arg1.object()());
+  }
+  if (arg2.token_type() == tt_constant && arg2.basic_type() == T_OBJECT &&
+      java_lang_invoke_CountingMethodHandle::is_instance(arg2.object()())) {
+    count2 = java_lang_invoke_CountingMethodHandle::vmcount(arg2.object()());
+  }
+  int total = count1 + count2;
+  if (count1 != -1 && count2 != -1 && total != 0) {
+    // Normalize the collect counts to the invoke_count
+    tty->print("counts %d %d scaled by %d = ", count2, count1, _invoke_count);
+    if (count1 != 0) _not_taken_count = (int)(_invoke_count * count1 / (double)total);
+    if (count2 != 0) _taken_count = (int)(_invoke_count * count2 / (double)total);
+    tty->print_cr("%d %d", _taken_count, _not_taken_count);
+    return true;
+  }
+  return false;
+}
+
 // Emit bytecodes for the given invoke instruction.
 MethodHandleWalker::ArgToken
 MethodHandleCompiler::make_invoke(methodHandle m, vmIntrinsics::ID iid,
@@ -1367,6 +1445,29 @@ MethodHandleCompiler::make_invoke(methodHandle m, vmIntrinsics::ID iid,
     }
   }
 
+  if (m->intrinsic_id() == vmIntrinsics::_selectAlternative &&
+      fetch_counts(argv[1], argv[2])) {
+    assert(argc == 3, "three arguments");
+    assert(tailcall, "only");
+
+    // do inline bytecodes so we can drop profile data into it,
+    //   0:   iload_0
+    emit_load(argv[0]);
+    //   1:   ifeq    8
+    _selectAlternative_bci = _bytecode.length();
+    emit_bc(Bytecodes::_ifeq, 0); // emit placeholder offset
+    //   4:   aload_1
+    emit_load(argv[1]);
+    //   5:   areturn;
+    emit_bc(Bytecodes::_areturn);
+    //   8:   aload_2
+    update_branch_dest(_selectAlternative_bci, cur_bci());
+    emit_load(argv[2]);
+    //   9:   areturn
+    emit_bc(Bytecodes::_areturn);
+    return ArgToken();  // Dummy return value.
+  }
+
   check_non_bcp_klass(klass, CHECK_(zero));
   if (m->is_method_handle_invoke()) {
     check_non_bcp_klasses(m->method_handle_type(), CHECK_(zero));
@@ -1377,10 +1478,6 @@ MethodHandleCompiler::make_invoke(methodHandle m, vmIntrinsics::ID iid,
   assert(argc == asc.size() + ((op == Bytecodes::_invokestatic || op == Bytecodes::_invokedynamic) ? 0 : 1),
          "argc mismatch");
 
-  // Inline the method.
-  InvocationCounter* ic = m->invocation_counter();
-  ic->set_carry_flag();
-
   for (int i = 0; i < argc; i++) {
     ArgToken arg = argv[i];
     TokenType tt = arg.token_type();
@@ -1686,7 +1783,7 @@ constantPoolHandle MethodHandleCompiler::get_constant_pool(TRAPS) const {
 }
 
 
-methodHandle MethodHandleCompiler::get_method_oop(TRAPS) const {
+methodHandle MethodHandleCompiler::get_method_oop(TRAPS) {
   methodHandle empty;
   // Create a method that holds the generated bytecode.  invokedynamic
   // has no receiver, normal MH calls do.
@@ -1765,6 +1862,7 @@ methodHandle MethodHandleCompiler::get_method_oop(TRAPS) const {
     assert(m->method_data() == NULL, "there should not be an MDO yet");
     m->set_method_data(mdo);
 
+    bool found_selectAlternative = false;
     // Iterate over all profile data and set the count of the counter
     // data entries to the original call site counter.
     for (ProfileData* profile_data = mdo->first_data();
@@ -1774,7 +1872,15 @@ methodHandle MethodHandleCompiler::get_method_oop(TRAPS) const {
         CounterData* counter_data = profile_data->as_CounterData();
         counter_data->set_count(_invoke_count);
       }
+      if (profile_data->is_BranchData() &&
+          profile_data->bci() == _selectAlternative_bci) {
+        BranchData* bd = profile_data->as_BranchData();
+        bd->set_taken(_taken_count);
+        bd->set_not_taken(_not_taken_count);
+        found_selectAlternative = true;
+      }
     }
+    assert(_selectAlternative_bci == -1 || found_selectAlternative, "must have found profile entry");
   }
 
 #ifndef PRODUCT

src/share/vm/prims/methodHandleWalk.hpp

@@ -74,6 +74,7 @@ public:
     set_method_handle(MethodHandle_vmtarget_oop(), THREAD);
   }
 
+  Handle root()                 { return _root; }
   Handle method_handle()        { return _method_handle; }
   oop    method_handle_oop()    { return _method_handle(); }
   oop    method_type_oop()      { return MethodHandle_type_oop(); }
@@ -110,7 +111,6 @@ public:
   // the signature for each method.  The signatures are printed in
   // slot order to make it easier to understand.
   void print();
-  static void print(Handle mh);
   static void print(oopDesc* mh);
 #endif
 };
@@ -277,6 +277,10 @@ private:
   KlassHandle  _target_klass;
   Thread*      _thread;
 
+  int          _selectAlternative_bci; // These are used for capturing profiles from GWTs
+  int          _taken_count;
+  int          _not_taken_count;
+
   // Values used by the compiler.
   static jvalue zero_jvalue;
   static jvalue one_jvalue;
@@ -372,6 +376,7 @@ private:
 
   unsigned char* bytecode()        const { return _bytecode.adr_at(0); }
   int            bytecode_length() const { return _bytecode.length(); }
+  int            cur_bci()         const { return _bytecode.length(); }
 
   // Fake constant pool.
   int cpool_oop_put(int tag, Handle con) {
@@ -436,6 +441,8 @@ private:
   }
 
   void emit_bc(Bytecodes::Code op, int index = 0, int args_size = -1);
+  void update_branch_dest(int src, int dst);
+  void emit_load(ArgToken arg);
   void emit_load(BasicType bt, int index);
   void emit_store(BasicType bt, int index);
   void emit_load_constant(ArgToken arg);
@@ -455,11 +462,14 @@ private:
   virtual ArgToken make_fetch(BasicType type, klassOop tk, Bytecodes::Code op, const ArgToken& base, const ArgToken& offset, TRAPS);
   virtual ArgToken make_invoke(methodHandle m, vmIntrinsics::ID iid, Bytecodes::Code op, bool tailcall, int argc, ArgToken* argv, TRAPS);
 
+  // Check for profiling information on a GWT and return true if it's found
+  bool fetch_counts(ArgToken a1, ArgToken a2);
+
   // Get a real constant pool.
   constantPoolHandle get_constant_pool(TRAPS) const;
 
   // Get a real methodOop.
-  methodHandle get_method_oop(TRAPS) const;
+  methodHandle get_method_oop(TRAPS);
 
 public:
   MethodHandleCompiler(Handle root, Symbol* name, Symbol* signature, int invoke_count, bool for_invokedynamic, TRAPS);

src/share/vm/prims/methodHandles.cpp

@@ -158,6 +158,8 @@ const char*        MethodHandles::_entry_names[_EK_LIMIT+1] = {
   "adapter_fold/4/ref",
   "adapter_fold/5/ref",
 
+  "adapter_opt_profiling",
+
   NULL
 };
 
@@ -2653,6 +2655,11 @@ void MethodHandles::init_AdapterMethodHandle(Handle mh, Handle target, int argnu
   // Finalize the conversion field.  (Note that it is final to Java code.)
   java_lang_invoke_AdapterMethodHandle::set_conversion(mh(), new_conversion);
 
+  if (java_lang_invoke_CountingMethodHandle::is_instance(mh())) {
+    assert(ek_orig == _adapter_retype_only, "only one handled");
+    ek_opt = _adapter_opt_profiling;
+  }
+
   // Done!
   java_lang_invoke_MethodHandle::set_vmentry(mh(), entry(ek_opt));
 
@@ -2905,8 +2912,12 @@ JVM_ENTRY(jint, MHN_getConstant(JNIEnv *env, jobject igcls, jint which)) {
     return MethodHandles::stack_move_unit();
   case MethodHandles::GC_CONV_OP_IMPLEMENTED_MASK:
     return MethodHandles::adapter_conversion_ops_supported_mask();
-  case MethodHandles::GC_OP_ROT_ARGS_DOWN_LIMIT_BIAS:
-    return MethodHandles::OP_ROT_ARGS_DOWN_LIMIT_BIAS;
+  case MethodHandles::GC_COUNT_GWT:
+#ifdef COMPILER2
+    return true;
+#else
+    return false;
+#endif
   }
   return 0;
 }

src/share/vm/prims/methodHandles.hpp

@@ -187,6 +187,8 @@ class MethodHandles: AllStatic {
     _adapter_opt_fold_FIRST = _adapter_opt_fold_ref,
     _adapter_opt_fold_LAST  = _adapter_opt_fold_5_ref,
 
+    _adapter_opt_profiling,
+
     _EK_LIMIT,
     _EK_FIRST = 0
   };
@@ -266,6 +268,8 @@ class MethodHandles: AllStatic {
       return _adapter_fold_args;
     if (ek >= _adapter_opt_return_FIRST && ek <= _adapter_opt_return_LAST)
       return _adapter_opt_return_any;
+    if (ek == _adapter_opt_profiling)
+      return _adapter_retype_only;
     assert(false, "oob");
     return _EK_LIMIT;
   }
@@ -582,6 +586,7 @@ class MethodHandles: AllStatic {
     GC_JVM_STACK_MOVE_UNIT = 1,
     GC_CONV_OP_IMPLEMENTED_MASK = 2,
     GC_OP_ROT_ARGS_DOWN_LIMIT_BIAS = 3,
+    GC_COUNT_GWT = 4,
 
     // format of result from getTarget / encode_target:
     ETF_HANDLE_OR_METHOD_NAME = 0, // all available data (immediate MH or method)