diff --git a/src/share/vm/ci/ciMethod.cpp b/src/share/vm/ci/ciMethod.cpp index ae5dd763a320a8e2b4c2f80b75c36695e0d29486..8fc52c02beed6c0db8360ca2d8574235a726ea61 100644 --- a/src/share/vm/ci/ciMethod.cpp +++ b/src/share/vm/ci/ciMethod.cpp @@ -70,7 +70,8 @@ // Loaded method. ciMethod::ciMethod(methodHandle h_m, ciInstanceKlass* holder) : ciMetadata(h_m()), - _holder(holder) + _holder(holder), + _has_injected_profile(false) { assert(h_m() != NULL, "no null method"); @@ -167,7 +168,8 @@ ciMethod::ciMethod(ciInstanceKlass* holder, _liveness( NULL), _can_be_statically_bound(false), _method_blocks( NULL), - _method_data( NULL) + _method_data( NULL), + _has_injected_profile( false) #if defined(COMPILER2) || defined(SHARK) , _flow( NULL), diff --git a/src/share/vm/ci/ciMethod.hpp b/src/share/vm/ci/ciMethod.hpp index 59d2cdff1e6d246b71d6e90ab12e8542754f98b8..a253306cfb6550db6dd7d36e5d22c5dbea3879b3 100644 --- a/src/share/vm/ci/ciMethod.hpp +++ b/src/share/vm/ci/ciMethod.hpp @@ -78,6 +78,7 @@ class ciMethod : public ciMetadata { bool _is_c1_compilable; bool _is_c2_compilable; bool _can_be_statically_bound; + bool _has_injected_profile; // Lazy fields, filled in on demand address _code; @@ -281,6 +282,9 @@ class ciMethod : public ciMetadata { int instructions_size(); int scale_count(int count, float prof_factor = 1.); // make MDO count commensurate with IIC + bool has_injected_profile() const { return _has_injected_profile; } + void set_injected_profile(bool x) { _has_injected_profile = x; } + // Stack walking support bool is_ignored_by_security_stack_walk() const; diff --git a/src/share/vm/classfile/vmSymbols.hpp b/src/share/vm/classfile/vmSymbols.hpp index 2a438dccdca4cb73640edab30898283ed8e4fd50..3fdb90095df5c8fecc8d4927c2365aed007804e1 100644 --- a/src/share/vm/classfile/vmSymbols.hpp +++ b/src/share/vm/classfile/vmSymbols.hpp @@ -244,7 +244,6 @@ template(returnType_name, "returnType") \ template(signature_name, "signature") \ template(slot_name, "slot") \ - template(selectAlternative_name, "selectAlternative") \ \ /* Support for annotations (JDK 1.5 and above) */ \ \ @@ -296,8 +295,7 @@ template(setTarget_signature, "(Ljava/lang/invoke/MethodHandle;)V") \ NOT_LP64( do_alias(intptr_signature, int_signature) ) \ LP64_ONLY( do_alias(intptr_signature, long_signature) ) \ - template(selectAlternative_signature, "(ZLjava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/MethodHandle;") \ - \ + \ /* common method and field names */ \ template(object_initializer_name, "") \ template(class_initializer_name, "") \ @@ -868,6 +866,12 @@ do_name( fullFence_name, "fullFence") \ do_alias( fullFence_signature, void_method_signature) \ \ + /* Custom branch frequencies profiling support for JSR292 */ \ + do_class(java_lang_invoke_MethodHandleImpl, "java/lang/invoke/MethodHandleImpl") \ + do_intrinsic(_profileBoolean, java_lang_invoke_MethodHandleImpl, profileBoolean_name, profileBoolean_signature, F_S) \ + do_name( profileBoolean_name, "profileBoolean") \ + do_signature(profileBoolean_signature, "(Z[I)Z") \ + \ /* unsafe memory references (there are a lot of them...) */ \ do_signature(getObject_signature, "(Ljava/lang/Object;J)Ljava/lang/Object;") \ do_signature(putObject_signature, "(Ljava/lang/Object;JLjava/lang/Object;)V") \ diff --git a/src/share/vm/opto/classes.hpp b/src/share/vm/opto/classes.hpp index 54a63db1a98451783db34ba90f64fbb405a91b4c..b4fbbacca35913a1bddc3be4520c93f5022cc7fa 100644 --- a/src/share/vm/opto/classes.hpp +++ b/src/share/vm/opto/classes.hpp @@ -199,6 +199,7 @@ macro(NeverBranch) macro(Opaque1) macro(Opaque2) macro(Opaque3) +macro(ProfileBoolean) macro(OrI) macro(OrL) macro(OverflowAddI) diff --git a/src/share/vm/opto/compile.cpp b/src/share/vm/opto/compile.cpp index 1edca8a35073bce70fbff04daf5a8253a90cbcb5..e15a666e26bb01a5f01fee42e9a06221c7253b40 100644 --- a/src/share/vm/opto/compile.cpp +++ b/src/share/vm/opto/compile.cpp @@ -3075,6 +3075,7 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) { default: assert( !n->is_Call(), "" ); assert( !n->is_Mem(), "" ); + assert( nop != Op_ProfileBoolean, "should be eliminated during IGVN"); break; } @@ -3291,6 +3292,9 @@ bool Compile::final_graph_reshaping() { bool Compile::too_many_traps(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->has_injected_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only @@ -3340,6 +3344,9 @@ bool Compile::too_many_traps(Deoptimization::DeoptReason reason, bool Compile::too_many_recompiles(ciMethod* method, int bci, Deoptimization::DeoptReason reason) { + if (method->has_injected_profile()) { + return false; + } ciMethodData* md = method->method_data(); if (md->is_empty()) { // Assume the trap has not occurred, or that it occurred only diff --git a/src/share/vm/opto/connode.cpp b/src/share/vm/opto/connode.cpp index fe12b4f49e701696d503d60150603b7d795128ca..bfebd1c5b79b4c6e4dc7c6394ae5a7fc15c6e49f 100644 --- a/src/share/vm/opto/connode.cpp +++ b/src/share/vm/opto/connode.cpp @@ -1338,6 +1338,30 @@ uint Opaque2Node::cmp( const Node &n ) const { return (&n == this); // Always fail except on self } +//============================================================================= + +uint ProfileBooleanNode::hash() const { return NO_HASH; } +uint ProfileBooleanNode::cmp( const Node &n ) const { + return (&n == this); +} + +Node *ProfileBooleanNode::Ideal(PhaseGVN *phase, bool can_reshape) { + if (can_reshape && _delay_removal) { + _delay_removal = false; + return this; + } else { + return NULL; + } +} + +Node *ProfileBooleanNode::Identity( PhaseTransform *phase ) { + if (_delay_removal) { + return this; + } else { + assert(_consumed, "profile should be consumed before elimination"); + return in(1); + } +} //------------------------------Value------------------------------------------ const Type *MoveL2DNode::Value( PhaseTransform *phase ) const { diff --git a/src/share/vm/opto/connode.hpp b/src/share/vm/opto/connode.hpp index 30692763d5b042f6a648c1677f2227f8a19cb3cb..55580283516684b033dbcf409775bb5daedfaf1b 100644 --- a/src/share/vm/opto/connode.hpp +++ b/src/share/vm/opto/connode.hpp @@ -669,6 +669,31 @@ public: bool rtm_opt() const { return (_opt == RTM_OPT); } }; +//------------------------------ProfileBooleanNode------------------------------- +// A node represents value profile for a boolean during parsing. +// Once parsing is over, the node goes away (during IGVN). +// It is used to override branch frequencies from MDO (see has_injected_profile in parse2.cpp). +class ProfileBooleanNode : public Node { + uint _false_cnt; + uint _true_cnt; + bool _consumed; + bool _delay_removal; + virtual uint hash() const ; // { return NO_HASH; } + virtual uint cmp( const Node &n ) const; + public: + ProfileBooleanNode(Node *n, uint false_cnt, uint true_cnt) : Node(0, n), + _false_cnt(false_cnt), _true_cnt(true_cnt), _delay_removal(true), _consumed(false) {} + + uint false_count() const { return _false_cnt; } + uint true_count() const { return _true_cnt; } + + void consume() { _consumed = true; } + + virtual int Opcode() const; + virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual Node *Identity(PhaseTransform *phase); + virtual const Type *bottom_type() const { return TypeInt::BOOL; } +}; //----------------------PartialSubtypeCheckNode-------------------------------- // The 2nd slow-half of a subtype check. Scan the subklass's 2ndary superklass diff --git a/src/share/vm/opto/graphKit.hpp b/src/share/vm/opto/graphKit.hpp index 2fcd0d2f4e1913921d896cfda11f342cbb251081..7214167ba907cc991631256ba1153a4001219b05 100644 --- a/src/share/vm/opto/graphKit.hpp +++ b/src/share/vm/opto/graphKit.hpp @@ -708,6 +708,15 @@ class GraphKit : public Phase { klass, reason_string, must_throw, keep_exact_action); } + // Bail out to the interpreter and keep exact action (avoid switching to Action_none). + void uncommon_trap_exact(Deoptimization::DeoptReason reason, + Deoptimization::DeoptAction action, + ciKlass* klass = NULL, const char* reason_string = NULL, + bool must_throw = false) { + uncommon_trap(Deoptimization::make_trap_request(reason, action), + klass, reason_string, must_throw, /*keep_exact_action=*/true); + } + // SP when bytecode needs to be reexecuted. virtual int reexecute_sp() { return sp(); } diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp index 9278eb7eedbf4af3d6caf96c6934ec5b74ddaddc..d9cbb347000432890405038ef151eebab8446a5b 100644 --- a/src/share/vm/opto/library_call.cpp +++ b/src/share/vm/opto/library_call.cpp @@ -31,6 +31,7 @@ #include "opto/addnode.hpp" #include "opto/callGenerator.hpp" #include "opto/cfgnode.hpp" +#include "opto/connode.hpp" #include "opto/idealKit.hpp" #include "opto/mathexactnode.hpp" #include "opto/mulnode.hpp" @@ -323,6 +324,8 @@ class LibraryCallKit : public GraphKit { bool inline_updateBytesCRC32(); bool inline_updateByteBufferCRC32(); bool inline_multiplyToLen(); + + bool inline_profileBoolean(); }; @@ -934,6 +937,9 @@ bool LibraryCallKit::try_to_inline(int predicate) { case vmIntrinsics::_updateByteBufferCRC32: return inline_updateByteBufferCRC32(); + case vmIntrinsics::_profileBoolean: + return inline_profileBoolean(); + default: // If you get here, it may be that someone has added a new intrinsic // to the list in vmSymbols.hpp without implementing it here. @@ -6544,3 +6550,47 @@ Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) return instof_false; // even if it is NULL } + +bool LibraryCallKit::inline_profileBoolean() { + Node* counts = argument(1); + const TypeAryPtr* ary = NULL; + ciArray* aobj = NULL; + if (counts->is_Con() + && (ary = counts->bottom_type()->isa_aryptr()) != NULL + && (aobj = ary->const_oop()->as_array()) != NULL + && (aobj->length() == 2)) { + // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively. + jint false_cnt = aobj->element_value(0).as_int(); + jint true_cnt = aobj->element_value(1).as_int(); + + method()->set_injected_profile(true); + + if (C->log() != NULL) { + C->log()->elem("observe source='profileBoolean' false='%d' true='%d'", + false_cnt, true_cnt); + } + + if (false_cnt + true_cnt == 0) { + // According to profile, never executed. + uncommon_trap_exact(Deoptimization::Reason_intrinsic, + Deoptimization::Action_reinterpret); + return true; + } + // Stop profiling. + // MethodHandleImpl::profileBoolean() has profiling logic in it's bytecode. + // By replacing method's body with profile data (represented as ProfileBooleanNode + // on IR level) we effectively disable profiling. + // It enables full speed execution once optimized code is generated. + Node* profile = _gvn.transform(new (C) ProfileBooleanNode(argument(0), false_cnt, true_cnt)); + C->record_for_igvn(profile); + set_result(profile); + return true; + } else { + // Continue profiling. + // Profile data isn't available at the moment. So, execute method's bytecode version. + // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod + // is compiled and counters aren't available since corresponding MethodHandle + // isn't a compile-time constant. + return false; + } +} diff --git a/src/share/vm/opto/parse.hpp b/src/share/vm/opto/parse.hpp index 75cc76229f0d434025b60205c9f8792d4a785985..da870749016fac867032fde10c022d1d29183dda 100644 --- a/src/share/vm/opto/parse.hpp +++ b/src/share/vm/opto/parse.hpp @@ -549,8 +549,8 @@ class Parse : public GraphKit { void do_jsr(); void do_ret(); - float dynamic_branch_prediction(float &cnt); - float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci); + float dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test); + float branch_prediction(float &cnt, BoolTest::mask btest, int target_bci, Node* test); bool seems_never_taken(float prob) const; bool path_is_suitable_for_uncommon_trap(float prob) const; bool seems_stable_comparison() const; diff --git a/src/share/vm/opto/parse2.cpp b/src/share/vm/opto/parse2.cpp index 453ee34d29a3f31b2a2ce655bffb9e054145d4e8..a45e491fbc5a2039ef4eb688dc68a96a4c51996e 100644 --- a/src/share/vm/opto/parse2.cpp +++ b/src/share/vm/opto/parse2.cpp @@ -761,35 +761,64 @@ void Parse::do_ret() { merge_common(target, pnum); } +static bool has_injected_profile(BoolTest::mask btest, Node* test, int& taken, int& not_taken) { + if (btest != BoolTest::eq && btest != BoolTest::ne) { + // Only ::eq and ::ne are supported for profile injection. + return false; + } + if (test->is_Cmp() && + test->in(1)->Opcode() == Op_ProfileBoolean) { + ProfileBooleanNode* profile = (ProfileBooleanNode*)test->in(1); + int false_cnt = profile->false_count(); + int true_cnt = profile->true_count(); + + // Counts matching depends on the actual test operation (::eq or ::ne). + // No need to scale the counts because profile injection was designed + // to feed exact counts into VM. + taken = (btest == BoolTest::eq) ? false_cnt : true_cnt; + not_taken = (btest == BoolTest::eq) ? true_cnt : false_cnt; + + profile->consume(); + return true; + } + return false; +} //--------------------------dynamic_branch_prediction-------------------------- // Try to gather dynamic branch prediction behavior. Return a probability // of the branch being taken and set the "cnt" field. Returns a -1.0 // if we need to use static prediction for some reason. -float Parse::dynamic_branch_prediction(float &cnt) { +float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test) { ResourceMark rm; cnt = COUNT_UNKNOWN; - // Use MethodData information if it is available - // FIXME: free the ProfileData structure - ciMethodData* methodData = method()->method_data(); - if (!methodData->is_mature()) return PROB_UNKNOWN; - ciProfileData* data = methodData->bci_to_data(bci()); - if (!data->is_JumpData()) return PROB_UNKNOWN; - - // get taken and not taken values - int taken = data->as_JumpData()->taken(); + int taken = 0; int not_taken = 0; - if (data->is_BranchData()) { - not_taken = data->as_BranchData()->not_taken(); - } - // scale the counts to be commensurate with invocation counts: - taken = method()->scale_count(taken); - not_taken = method()->scale_count(not_taken); + bool use_mdo = !has_injected_profile(btest, test, taken, not_taken); + + if (use_mdo) { + // Use MethodData information if it is available + // FIXME: free the ProfileData structure + ciMethodData* methodData = method()->method_data(); + if (!methodData->is_mature()) return PROB_UNKNOWN; + ciProfileData* data = methodData->bci_to_data(bci()); + if (!data->is_JumpData()) return PROB_UNKNOWN; + + // get taken and not taken values + taken = data->as_JumpData()->taken(); + not_taken = 0; + if (data->is_BranchData()) { + not_taken = data->as_BranchData()->not_taken(); + } + + // scale the counts to be commensurate with invocation counts: + taken = method()->scale_count(taken); + not_taken = method()->scale_count(not_taken); + } // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful. - // We also check that individual counters are positive first, overwise the sum can become positive. + // We also check that individual counters are positive first, otherwise the sum can become positive. if (taken < 0 || not_taken < 0 || taken + not_taken < 40) { if (C->log() != NULL) { C->log()->elem("branch target_bci='%d' taken='%d' not_taken='%d'", iter().get_dest(), taken, not_taken); @@ -839,8 +868,9 @@ float Parse::dynamic_branch_prediction(float &cnt) { //-----------------------------branch_prediction------------------------------- float Parse::branch_prediction(float& cnt, BoolTest::mask btest, - int target_bci) { - float prob = dynamic_branch_prediction(cnt); + int target_bci, + Node* test) { + float prob = dynamic_branch_prediction(cnt, btest, test); // If prob is unknown, switch to static prediction if (prob != PROB_UNKNOWN) return prob; @@ -930,7 +960,7 @@ void Parse::do_ifnull(BoolTest::mask btest, Node *c) { Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); if (prob == PROB_UNKNOWN) { // (An earlier version of do_ifnull omitted this trap for OSR methods.) #ifndef PRODUCT @@ -1011,7 +1041,7 @@ void Parse::do_if(BoolTest::mask btest, Node* c) { Block* next_block = successor_for_bci(iter().next_bci()); float cnt; - float prob = branch_prediction(cnt, btest, target_bci); + float prob = branch_prediction(cnt, btest, target_bci, c); float untaken_prob = 1.0 - prob; if (prob == PROB_UNKNOWN) {