8063137: Never-taken branches should be pruned when GWT LambdaForms are shared

Reviewed-by: jrose, kvn

src/share/vm/ci/ciMethod.cpp

@@ -70,7 +70,8 @@
 // Loaded method.
 ciMethod::ciMethod(methodHandle h_m, ciInstanceKlass* holder) :
   ciMetadata(h_m()),
-  _holder(holder)
+  _holder(holder),
+  _has_injected_profile(false)
 {
   assert(h_m() != NULL, "no null method");
 
@@ -167,7 +168,8 @@ ciMethod::ciMethod(ciInstanceKlass* holder,
   _liveness(               NULL),
   _can_be_statically_bound(false),
   _method_blocks(          NULL),
-  _method_data(            NULL)
+  _method_data(            NULL),
+  _has_injected_profile(   false)
 #if defined(COMPILER2) || defined(SHARK)
   ,
   _flow(                   NULL),

src/share/vm/ci/ciMethod.hpp

@@ -78,6 +78,7 @@ class ciMethod : public ciMetadata {
   bool _is_c1_compilable;
   bool _is_c2_compilable;
   bool _can_be_statically_bound;
+  bool _has_injected_profile;
 
   // Lazy fields, filled in on demand
   address              _code;
@@ -281,6 +282,9 @@ class ciMethod : public ciMetadata {
   int instructions_size();
   int scale_count(int count, float prof_factor = 1.);  // make MDO count commensurate with IIC
 
+  bool has_injected_profile() const { return _has_injected_profile;     }
+  void set_injected_profile(bool x) {        _has_injected_profile = x; }
+
   // Stack walking support
   bool is_ignored_by_security_stack_walk() const;
 

src/share/vm/classfile/vmSymbols.hpp

@@ -244,7 +244,6 @@
   template(returnType_name,                           "returnType")                               \
   template(signature_name,                            "signature")                                \
   template(slot_name,                                 "slot")                                     \
-  template(selectAlternative_name,                    "selectAlternative")                        \
                                                                                                   \
   /* Support for annotations (JDK 1.5 and above) */                                               \
                                                                                                   \
@@ -296,8 +295,7 @@
   template(setTarget_signature,                       "(Ljava/lang/invoke/MethodHandle;)V")       \
   NOT_LP64(  do_alias(intptr_signature,               int_signature)  )                           \
   LP64_ONLY( do_alias(intptr_signature,               long_signature) )                           \
-  template(selectAlternative_signature, "(ZLjava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/MethodHandle;") \
-                                                                      \
+                                                                                                  \
   /* common method and field names */                                                             \
   template(object_initializer_name,                   "<init>")                                   \
   template(class_initializer_name,                    "<clinit>")                                 \
@@ -868,6 +866,12 @@
    do_name(     fullFence_name,                                  "fullFence")                                           \
    do_alias(    fullFence_signature,                              void_method_signature)                                \
                                                                                                                         \
+  /* Custom branch frequencies profiling support for JSR292 */                                                          \
+  do_class(java_lang_invoke_MethodHandleImpl,               "java/lang/invoke/MethodHandleImpl")                        \
+  do_intrinsic(_profileBoolean, java_lang_invoke_MethodHandleImpl, profileBoolean_name, profileBoolean_signature,    F_S)  \
+   do_name(     profileBoolean_name,                               "profileBoolean")                                     \
+   do_signature(profileBoolean_signature,                           "(Z[I)Z")                                            \
+                                                                                                                        \
   /* unsafe memory references (there are a lot of them...) */                                                           \
   do_signature(getObject_signature,       "(Ljava/lang/Object;J)Ljava/lang/Object;")                                    \
   do_signature(putObject_signature,       "(Ljava/lang/Object;JLjava/lang/Object;)V")                                   \

src/share/vm/opto/classes.hpp

@@ -199,6 +199,7 @@ macro(NeverBranch)
 macro(Opaque1)
 macro(Opaque2)
 macro(Opaque3)
+macro(ProfileBoolean)
 macro(OrI)
 macro(OrL)
 macro(OverflowAddI)

src/share/vm/opto/compile.cpp

@@ -3075,6 +3075,7 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) {
   default:
     assert( !n->is_Call(), "" );
     assert( !n->is_Mem(), "" );
+    assert( nop != Op_ProfileBoolean, "should be eliminated during IGVN");
     break;
   }
 
@@ -3291,6 +3292,9 @@ bool Compile::final_graph_reshaping() {
 bool Compile::too_many_traps(ciMethod* method,
                              int bci,
                              Deoptimization::DeoptReason reason) {
+  if (method->has_injected_profile()) {
+    return false;
+  }
   ciMethodData* md = method->method_data();
   if (md->is_empty()) {
     // Assume the trap has not occurred, or that it occurred only
@@ -3340,6 +3344,9 @@ bool Compile::too_many_traps(Deoptimization::DeoptReason reason,
 bool Compile::too_many_recompiles(ciMethod* method,
                                   int bci,
                                   Deoptimization::DeoptReason reason) {
+  if (method->has_injected_profile()) {
+    return false;
+  }
   ciMethodData* md = method->method_data();
   if (md->is_empty()) {
     // Assume the trap has not occurred, or that it occurred only

src/share/vm/opto/connode.cpp

@@ -1338,6 +1338,30 @@ uint Opaque2Node::cmp( const Node &n ) const {
   return (&n == this);          // Always fail except on self
 }
 
+//=============================================================================
+
+uint ProfileBooleanNode::hash() const { return NO_HASH; }
+uint ProfileBooleanNode::cmp( const Node &n ) const {
+  return (&n == this);
+}
+
+Node *ProfileBooleanNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  if (can_reshape && _delay_removal) {
+    _delay_removal = false;
+    return this;
+  } else {
+    return NULL;
+  }
+}
+
+Node *ProfileBooleanNode::Identity( PhaseTransform *phase ) {
+  if (_delay_removal) {
+    return this;
+  } else {
+    assert(_consumed, "profile should be consumed before elimination");
+    return in(1);
+  }
+}
 
 //------------------------------Value------------------------------------------
 const Type *MoveL2DNode::Value( PhaseTransform *phase ) const {

src/share/vm/opto/connode.hpp

@@ -669,6 +669,31 @@ public:
   bool rtm_opt() const { return (_opt == RTM_OPT); }
 };
 
+//------------------------------ProfileBooleanNode-------------------------------
+// A node represents value profile for a boolean during parsing.
+// Once parsing is over, the node goes away (during IGVN).
+// It is used to override branch frequencies from MDO (see has_injected_profile in parse2.cpp).
+class ProfileBooleanNode : public Node {
+  uint _false_cnt;
+  uint _true_cnt;
+  bool _consumed;
+  bool _delay_removal;
+  virtual uint hash() const ;                  // { return NO_HASH; }
+  virtual uint cmp( const Node &n ) const;
+  public:
+  ProfileBooleanNode(Node *n, uint false_cnt, uint true_cnt) : Node(0, n),
+          _false_cnt(false_cnt), _true_cnt(true_cnt), _delay_removal(true), _consumed(false) {}
+
+  uint false_count() const { return _false_cnt; }
+  uint  true_count() const { return  _true_cnt; }
+
+  void consume() { _consumed = true;  }
+
+  virtual int Opcode() const;
+  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
+  virtual Node *Identity(PhaseTransform *phase);
+  virtual const Type *bottom_type() const { return TypeInt::BOOL; }
+};
 
 //----------------------PartialSubtypeCheckNode--------------------------------
 // The 2nd slow-half of a subtype check.  Scan the subklass's 2ndary superklass

src/share/vm/opto/graphKit.hpp

@@ -708,6 +708,15 @@ class GraphKit : public Phase {
                   klass, reason_string, must_throw, keep_exact_action);
   }
 
+  // Bail out to the interpreter and keep exact action (avoid switching to Action_none).
+  void uncommon_trap_exact(Deoptimization::DeoptReason reason,
+                           Deoptimization::DeoptAction action,
+                           ciKlass* klass = NULL, const char* reason_string = NULL,
+                           bool must_throw = false) {
+    uncommon_trap(Deoptimization::make_trap_request(reason, action),
+                  klass, reason_string, must_throw, /*keep_exact_action=*/true);
+  }
+
   // SP when bytecode needs to be reexecuted.
   virtual int reexecute_sp() { return sp(); }
 

src/share/vm/opto/library_call.cpp

@@ -31,6 +31,7 @@
 #include "opto/addnode.hpp"
 #include "opto/callGenerator.hpp"
 #include "opto/cfgnode.hpp"
+#include "opto/connode.hpp"
 #include "opto/idealKit.hpp"
 #include "opto/mathexactnode.hpp"
 #include "opto/mulnode.hpp"
@@ -323,6 +324,8 @@ class LibraryCallKit : public GraphKit {
   bool inline_updateBytesCRC32();
   bool inline_updateByteBufferCRC32();
   bool inline_multiplyToLen();
+
+  bool inline_profileBoolean();
 };
 
 
@@ -934,6 +937,9 @@ bool LibraryCallKit::try_to_inline(int predicate) {
   case vmIntrinsics::_updateByteBufferCRC32:
     return inline_updateByteBufferCRC32();
 
+  case vmIntrinsics::_profileBoolean:
+    return inline_profileBoolean();
+
   default:
     // If you get here, it may be that someone has added a new intrinsic
     // to the list in vmSymbols.hpp without implementing it here.
@@ -6544,3 +6550,47 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate)
 
   return instof_false;  // even if it is NULL
 }
+
+bool LibraryCallKit::inline_profileBoolean() {
+  Node* counts = argument(1);
+  const TypeAryPtr* ary = NULL;
+  ciArray* aobj = NULL;
+  if (counts->is_Con()
+      && (ary = counts->bottom_type()->isa_aryptr()) != NULL
+      && (aobj = ary->const_oop()->as_array()) != NULL
+      && (aobj->length() == 2)) {
+    // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively.
+    jint false_cnt = aobj->element_value(0).as_int();
+    jint  true_cnt = aobj->element_value(1).as_int();
+
+    method()->set_injected_profile(true);
+
+    if (C->log() != NULL) {
+      C->log()->elem("observe source='profileBoolean' false='%d' true='%d'",
+                     false_cnt, true_cnt);
+    }
+
+    if (false_cnt + true_cnt == 0) {
+      // According to profile, never executed.
+      uncommon_trap_exact(Deoptimization::Reason_intrinsic,
+                          Deoptimization::Action_reinterpret);
+      return true;
+    }
+    // Stop profiling.
+    // MethodHandleImpl::profileBoolean() has profiling logic in it's bytecode.
+    // By replacing method's body with profile data (represented as ProfileBooleanNode
+    // on IR level) we effectively disable profiling.
+    // It enables full speed execution once optimized code is generated.
+    Node* profile = _gvn.transform(new (C) ProfileBooleanNode(argument(0), false_cnt, true_cnt));
+    C->record_for_igvn(profile);
+    set_result(profile);
+    return true;
+  } else {
+    // Continue profiling.
+    // Profile data isn't available at the moment. So, execute method's bytecode version.
+    // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod
+    // is compiled and counters aren't available since corresponding MethodHandle
+    // isn't a compile-time constant.
+    return false;
+  }
+}

src/share/vm/opto/parse.hpp

@@ -549,8 +549,8 @@ class Parse : public GraphKit {
   void do_jsr();
   void do_ret();
 
-  float   dynamic_branch_prediction(float &cnt);
-  float   branch_prediction(float &cnt, BoolTest::mask btest, int target_bci);
+  float   dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test);
+  float   branch_prediction(float &cnt, BoolTest::mask btest, int target_bci, Node* test);
   bool    seems_never_taken(float prob) const;
   bool    path_is_suitable_for_uncommon_trap(float prob) const;
   bool    seems_stable_comparison() const;

src/share/vm/opto/parse2.cpp

@@ -761,35 +761,64 @@ void Parse::do_ret() {
   merge_common(target, pnum);
 }
 
+static bool has_injected_profile(BoolTest::mask btest, Node* test, int& taken, int& not_taken) {
+  if (btest != BoolTest::eq && btest != BoolTest::ne) {
+    // Only ::eq and ::ne are supported for profile injection.
+    return false;
+  }
+  if (test->is_Cmp() &&
+      test->in(1)->Opcode() == Op_ProfileBoolean) {
+    ProfileBooleanNode* profile = (ProfileBooleanNode*)test->in(1);
+    int false_cnt = profile->false_count();
+    int  true_cnt = profile->true_count();
+
+    // Counts matching depends on the actual test operation (::eq or ::ne).
+    // No need to scale the counts because profile injection was designed
+    // to feed exact counts into VM.
+    taken     = (btest == BoolTest::eq) ? false_cnt :  true_cnt;
+    not_taken = (btest == BoolTest::eq) ?  true_cnt : false_cnt;
+
+    profile->consume();
+    return true;
+  }
+  return false;
+}
 //--------------------------dynamic_branch_prediction--------------------------
 // Try to gather dynamic branch prediction behavior.  Return a probability
 // of the branch being taken and set the "cnt" field.  Returns a -1.0
 // if we need to use static prediction for some reason.
-float Parse::dynamic_branch_prediction(float &cnt) {
+float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test) {
   ResourceMark rm;
 
   cnt  = COUNT_UNKNOWN;
 
-  // Use MethodData information if it is available
-  // FIXME: free the ProfileData structure
-  ciMethodData* methodData = method()->method_data();
-  if (!methodData->is_mature())  return PROB_UNKNOWN;
-  ciProfileData* data = methodData->bci_to_data(bci());
-  if (!data->is_JumpData())  return PROB_UNKNOWN;
-
-  // get taken and not taken values
-  int     taken = data->as_JumpData()->taken();
+  int     taken = 0;
   int not_taken = 0;
-  if (data->is_BranchData()) {
-    not_taken = data->as_BranchData()->not_taken();
-  }
 
-  // scale the counts to be commensurate with invocation counts:
-  taken = method()->scale_count(taken);
-  not_taken = method()->scale_count(not_taken);
+  bool use_mdo = !has_injected_profile(btest, test, taken, not_taken);
+
+  if (use_mdo) {
+    // Use MethodData information if it is available
+    // FIXME: free the ProfileData structure
+    ciMethodData* methodData = method()->method_data();
+    if (!methodData->is_mature())  return PROB_UNKNOWN;
+    ciProfileData* data = methodData->bci_to_data(bci());
+    if (!data->is_JumpData())  return PROB_UNKNOWN;
+
+    // get taken and not taken values
+    taken = data->as_JumpData()->taken();
+    not_taken = 0;
+    if (data->is_BranchData()) {
+      not_taken = data->as_BranchData()->not_taken();
+    }
+
+    // scale the counts to be commensurate with invocation counts:
+    taken = method()->scale_count(taken);
+    not_taken = method()->scale_count(not_taken);
+  }
 
   // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful.
-  // We also check that individual counters are positive first, overwise the sum can become positive.
+  // We also check that individual counters are positive first, otherwise the sum can become positive.
   if (taken < 0 || not_taken < 0 || taken + not_taken < 40) {
     if (C->log() != NULL) {
       C->log()->elem("branch target_bci='%d' taken='%d' not_taken='%d'", iter().get_dest(), taken, not_taken);
@@ -839,8 +868,9 @@ float Parse::dynamic_branch_prediction(float &cnt) {
 //-----------------------------branch_prediction-------------------------------
 float Parse::branch_prediction(float& cnt,
                                BoolTest::mask btest,
-                               int target_bci) {
-  float prob = dynamic_branch_prediction(cnt);
+                               int target_bci,
+                               Node* test) {
+  float prob = dynamic_branch_prediction(cnt, btest, test);
   // If prob is unknown, switch to static prediction
   if (prob != PROB_UNKNOWN)  return prob;
 
@@ -930,7 +960,7 @@ void Parse::do_ifnull(BoolTest::mask btest, Node *c) {
   Block* next_block   = successor_for_bci(iter().next_bci());
 
   float cnt;
-  float prob = branch_prediction(cnt, btest, target_bci);
+  float prob = branch_prediction(cnt, btest, target_bci, c);
   if (prob == PROB_UNKNOWN) {
     // (An earlier version of do_ifnull omitted this trap for OSR methods.)
 #ifndef PRODUCT
@@ -1011,7 +1041,7 @@ void Parse::do_if(BoolTest::mask btest, Node* c) {
   Block* next_block   = successor_for_bci(iter().next_bci());
 
   float cnt;
-  float prob = branch_prediction(cnt, btest, target_bci);
+  float prob = branch_prediction(cnt, btest, target_bci, c);
   float untaken_prob = 1.0 - prob;
 
   if (prob == PROB_UNKNOWN) {