From 867012429e051836782c4ee6ed31dc8899638ce2 Mon Sep 17 00:00:00 2001 From: kvn Date: Mon, 26 Sep 2011 10:24:05 -0700 Subject: [PATCH] 7081933: Use zeroing elimination optimization for large array Summary: Don't zero new typeArray during runtime call if the allocation is followed by arraycopy into it. Reviewed-by: twisti --- src/cpu/sparc/vm/stubGenerator_sparc.cpp | 4 +-- src/share/vm/gc_interface/collectedHeap.hpp | 1 + .../vm/gc_interface/collectedHeap.inline.hpp | 17 ++++++++++ src/share/vm/memory/oopFactory.cpp | 9 +++++- src/share/vm/memory/oopFactory.hpp | 1 + src/share/vm/oops/typeArrayKlass.cpp | 8 +++-- src/share/vm/oops/typeArrayKlass.hpp | 3 +- src/share/vm/opto/library_call.cpp | 1 + src/share/vm/opto/macro.cpp | 14 +++++++- src/share/vm/opto/memnode.cpp | 4 +-- src/share/vm/opto/memnode.hpp | 11 +++++-- src/share/vm/opto/runtime.cpp | 32 +++++++++++++++++++ src/share/vm/opto/runtime.hpp | 3 ++ 13 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/cpu/sparc/vm/stubGenerator_sparc.cpp b/src/cpu/sparc/vm/stubGenerator_sparc.cpp index cf403040d..9415c7b6c 100644 --- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp +++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp @@ -2359,10 +2359,10 @@ class StubGenerator: public StubCodeGenerator { for (int off = 0; off < 64; off += 16) { if (use_prefetch && (off & 31) == 0) { if (ArraycopySrcPrefetchDistance > 0) { - __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads); + __ prefetch(from, ArraycopySrcPrefetchDistance+off, Assembler::severalReads); } if (ArraycopyDstPrefetchDistance > 0) { - __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads); + __ prefetch(to, ArraycopyDstPrefetchDistance+off, Assembler::severalWritesAndPossiblyReads); } } __ ldx(from, off+0, O4); diff --git a/src/share/vm/gc_interface/collectedHeap.hpp b/src/share/vm/gc_interface/collectedHeap.hpp index 36fcb40a8..5761ea554 100644 --- a/src/share/vm/gc_interface/collectedHeap.hpp +++ b/src/share/vm/gc_interface/collectedHeap.hpp @@ -322,6 +322,7 @@ class CollectedHeap : public CHeapObj { // General obj/array allocation facilities. inline static oop obj_allocate(KlassHandle klass, int size, TRAPS); inline static oop array_allocate(KlassHandle klass, int size, int length, TRAPS); + inline static oop array_allocate_nozero(KlassHandle klass, int size, int length, TRAPS); // Special obj/array allocation facilities. // Some heaps may want to manage "permanent" data uniquely. These default diff --git a/src/share/vm/gc_interface/collectedHeap.inline.hpp b/src/share/vm/gc_interface/collectedHeap.inline.hpp index fabf702ee..adced061d 100644 --- a/src/share/vm/gc_interface/collectedHeap.inline.hpp +++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp @@ -274,6 +274,23 @@ oop CollectedHeap::array_allocate(KlassHandle klass, return (oop)obj; } +oop CollectedHeap::array_allocate_nozero(KlassHandle klass, + int size, + int length, + TRAPS) { + debug_only(check_for_valid_allocation_state()); + assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); + assert(size >= 0, "int won't convert to size_t"); + HeapWord* obj = common_mem_allocate_noinit(size, CHECK_NULL); + ((oop)obj)->set_klass_gap(0); + post_allocation_setup_array(klass, obj, size, length); +#ifndef PRODUCT + const size_t hs = oopDesc::header_size()+1; + Universe::heap()->check_for_non_bad_heap_word_value(obj+hs, size-hs); +#endif + return (oop)obj; +} + oop CollectedHeap::permanent_obj_allocate(KlassHandle klass, int size, TRAPS) { oop obj = permanent_obj_allocate_no_klass_install(klass, size, CHECK_NULL); post_allocation_install_obj_klass(klass, obj, size); diff --git a/src/share/vm/memory/oopFactory.cpp b/src/share/vm/memory/oopFactory.cpp index 08f9825b3..c6e644c96 100644 --- a/src/share/vm/memory/oopFactory.cpp +++ b/src/share/vm/memory/oopFactory.cpp @@ -77,7 +77,14 @@ typeArrayOop oopFactory::new_permanent_intArray(int length, TRAPS) { typeArrayOop oopFactory::new_typeArray(BasicType type, int length, TRAPS) { klassOop type_asKlassOop = Universe::typeArrayKlassObj(type); typeArrayKlass* type_asArrayKlass = typeArrayKlass::cast(type_asKlassOop); - typeArrayOop result = type_asArrayKlass->allocate(length, THREAD); + typeArrayOop result = type_asArrayKlass->allocate_common(length, true, THREAD); + return result; +} + +typeArrayOop oopFactory::new_typeArray_nozero(BasicType type, int length, TRAPS) { + klassOop type_asKlassOop = Universe::typeArrayKlassObj(type); + typeArrayKlass* type_asArrayKlass = typeArrayKlass::cast(type_asKlassOop); + typeArrayOop result = type_asArrayKlass->allocate_common(length, false, THREAD); return result; } diff --git a/src/share/vm/memory/oopFactory.hpp b/src/share/vm/memory/oopFactory.hpp index 32e50fb40..dbb42f053 100644 --- a/src/share/vm/memory/oopFactory.hpp +++ b/src/share/vm/memory/oopFactory.hpp @@ -63,6 +63,7 @@ class oopFactory: AllStatic { static typeArrayOop new_permanent_intArray (int length, TRAPS); // used for class file structures static typeArrayOop new_typeArray(BasicType type, int length, TRAPS); + static typeArrayOop new_typeArray_nozero(BasicType type, int length, TRAPS); // Constant pools static constantPoolOop new_constantPool (int length, diff --git a/src/share/vm/oops/typeArrayKlass.cpp b/src/share/vm/oops/typeArrayKlass.cpp index 633841711..a3c58cd0e 100644 --- a/src/share/vm/oops/typeArrayKlass.cpp +++ b/src/share/vm/oops/typeArrayKlass.cpp @@ -76,7 +76,7 @@ klassOop typeArrayKlass::create_klass(BasicType type, int scale, return k(); } -typeArrayOop typeArrayKlass::allocate(int length, TRAPS) { +typeArrayOop typeArrayKlass::allocate_common(int length, bool do_zero, TRAPS) { assert(log2_element_size() >= 0, "bad scale"); if (length >= 0) { if (length <= max_length()) { @@ -84,7 +84,11 @@ typeArrayOop typeArrayKlass::allocate(int length, TRAPS) { KlassHandle h_k(THREAD, as_klassOop()); typeArrayOop t; CollectedHeap* ch = Universe::heap(); - t = (typeArrayOop)CollectedHeap::array_allocate(h_k, (int)size, length, CHECK_NULL); + if (do_zero) { + t = (typeArrayOop)CollectedHeap::array_allocate(h_k, (int)size, length, CHECK_NULL); + } else { + t = (typeArrayOop)CollectedHeap::array_allocate_nozero(h_k, (int)size, length, CHECK_NULL); + } assert(t->is_parsable(), "Don't publish unless parsable"); return t; } else { diff --git a/src/share/vm/oops/typeArrayKlass.hpp b/src/share/vm/oops/typeArrayKlass.hpp index 78a26be09..0f530668c 100644 --- a/src/share/vm/oops/typeArrayKlass.hpp +++ b/src/share/vm/oops/typeArrayKlass.hpp @@ -56,7 +56,8 @@ class typeArrayKlass : public arrayKlass { bool compute_is_subtype_of(klassOop k); // Allocation - typeArrayOop allocate(int length, TRAPS); + typeArrayOop allocate_common(int length, bool do_zero, TRAPS); + typeArrayOop allocate(int length, TRAPS) { return allocate_common(length, true, THREAD); } typeArrayOop allocate_permanent(int length, TRAPS); // used for class file structures oop multi_allocate(int rank, jint* sizes, TRAPS); diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp index 8179f379b..d5ad00e16 100644 --- a/src/share/vm/opto/library_call.cpp +++ b/src/share/vm/opto/library_call.cpp @@ -4658,6 +4658,7 @@ LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); + init->set_complete_with_arraycopy(); assert(dest->is_CheckCastPP(), "sanity"); assert(dest->in(0)->in(0) == init, "dest pinned"); adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory diff --git a/src/share/vm/opto/macro.cpp b/src/share/vm/opto/macro.cpp index 1e12e3660..b5f645359 100644 --- a/src/share/vm/opto/macro.cpp +++ b/src/share/vm/opto/macro.cpp @@ -1685,9 +1685,21 @@ void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { Node* length = alloc->in(AllocateNode::ALength); + InitializeNode* init = alloc->initialization(); + Node* klass_node = alloc->in(AllocateNode::KlassNode); + ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); + address slow_call_address; // Address of slow call + if (init != NULL && init->is_complete_with_arraycopy() && + k->is_type_array_klass()) { + // Don't zero type array during slow allocation in VM since + // it will be initialized later by arraycopy in compiled code. + slow_call_address = OptoRuntime::new_array_nozero_Java(); + } else { + slow_call_address = OptoRuntime::new_array_Java(); + } expand_allocate_common(alloc, length, OptoRuntime::new_array_Type(), - OptoRuntime::new_array_Java()); + slow_call_address); } //-----------------------mark_eliminated_locking_nodes----------------------- diff --git a/src/share/vm/opto/memnode.cpp b/src/share/vm/opto/memnode.cpp index ba0960116..4f6dc0851 100644 --- a/src/share/vm/opto/memnode.cpp +++ b/src/share/vm/opto/memnode.cpp @@ -2847,7 +2847,7 @@ Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) { //---------------------------InitializeNode------------------------------------ InitializeNode::InitializeNode(Compile* C, int adr_type, Node* rawoop) - : _is_complete(false), + : _is_complete(Incomplete), MemBarNode(C, adr_type, rawoop) { init_class_id(Class_Initialize); @@ -2885,7 +2885,7 @@ bool InitializeNode::is_non_zero() { void InitializeNode::set_complete(PhaseGVN* phase) { assert(!is_complete(), "caller responsibility"); - _is_complete = true; + _is_complete = Complete; // After this node is complete, it contains a bunch of // raw-memory initializations. There is no need for diff --git a/src/share/vm/opto/memnode.hpp b/src/share/vm/opto/memnode.hpp index a9eacdd8c..d757b13f3 100644 --- a/src/share/vm/opto/memnode.hpp +++ b/src/share/vm/opto/memnode.hpp @@ -942,7 +942,12 @@ public: class InitializeNode: public MemBarNode { friend class AllocateNode; - bool _is_complete; + enum { + Incomplete = 0, + Complete = 1, + WithArraycopy = 2 + }; + int _is_complete; public: enum { @@ -976,10 +981,12 @@ public: // An InitializeNode must completed before macro expansion is done. // Completion requires that the AllocateNode must be followed by // initialization of the new memory to zero, then to any initializers. - bool is_complete() { return _is_complete; } + bool is_complete() { return _is_complete != Incomplete; } + bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; } // Mark complete. (Must not yet be complete.) void set_complete(PhaseGVN* phase); + void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; } #ifdef ASSERT // ensure all non-degenerate stores are ordered and non-overlapping diff --git a/src/share/vm/opto/runtime.cpp b/src/share/vm/opto/runtime.cpp index 49c55212f..a4495ea74 100644 --- a/src/share/vm/opto/runtime.cpp +++ b/src/share/vm/opto/runtime.cpp @@ -102,6 +102,7 @@ // Compiled code entry points address OptoRuntime::_new_instance_Java = NULL; address OptoRuntime::_new_array_Java = NULL; +address OptoRuntime::_new_array_nozero_Java = NULL; address OptoRuntime::_multianewarray2_Java = NULL; address OptoRuntime::_multianewarray3_Java = NULL; address OptoRuntime::_multianewarray4_Java = NULL; @@ -151,6 +152,7 @@ void OptoRuntime::generate(ciEnv* env) { // ------------------------------------------------------------------------------------------------------------------------------- gen(env, _new_instance_Java , new_instance_Type , new_instance_C , 0 , true , false, false); gen(env, _new_array_Java , new_array_Type , new_array_C , 0 , true , false, false); + gen(env, _new_array_nozero_Java , new_array_Type , new_array_nozero_C , 0 , true , false, false); gen(env, _multianewarray2_Java , multianewarray2_Type , multianewarray2_C , 0 , true , false, false); gen(env, _multianewarray3_Java , multianewarray3_Type , multianewarray3_C , 0 , true , false, false); gen(env, _multianewarray4_Java , multianewarray4_Type , multianewarray4_C , 0 , true , false, false); @@ -308,6 +310,36 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(klassOopDesc* array_type, int len } JRT_END +// array allocation without zeroing +JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(klassOopDesc* array_type, int len, JavaThread *thread)) + JRT_BLOCK; +#ifndef PRODUCT + SharedRuntime::_new_array_ctr++; // new array requires GC +#endif + assert(check_compiled_frame(thread), "incorrect caller"); + + // Scavenge and allocate an instance. + oop result; + + assert(Klass::cast(array_type)->oop_is_typeArray(), "should be called only for type array"); + // The oopFactory likes to work with the element type. + BasicType elem_type = typeArrayKlass::cast(array_type)->element_type(); + result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD); + + // Pass oops back through thread local storage. Our apparent type to Java + // is that we return an oop, but we can block on exit from this routine and + // a GC can trash the oop in C's return register. The generated stub will + // fetch the oop from TLS after any possible GC. + deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION); + thread->set_vm_result(result); + JRT_BLOCK_END; + + if (GraphKit::use_ReduceInitialCardMarks()) { + // inform GC that we won't do card marks for initializing writes. + new_store_pre_barrier(thread); + } +JRT_END + // Note: multianewarray for one dimension is handled inline by GraphKit::new_array. // multianewarray for 2 dimensions diff --git a/src/share/vm/opto/runtime.hpp b/src/share/vm/opto/runtime.hpp index a691da307..f2c538b00 100644 --- a/src/share/vm/opto/runtime.hpp +++ b/src/share/vm/opto/runtime.hpp @@ -114,6 +114,7 @@ class OptoRuntime : public AllStatic { // References to generated stubs static address _new_instance_Java; static address _new_array_Java; + static address _new_array_nozero_Java; static address _multianewarray2_Java; static address _multianewarray3_Java; static address _multianewarray4_Java; @@ -143,6 +144,7 @@ class OptoRuntime : public AllStatic { // Allocate storage for a objArray or typeArray static void new_array_C(klassOopDesc* array_klass, int len, JavaThread *thread); + static void new_array_nozero_C(klassOopDesc* array_klass, int len, JavaThread *thread); // Post-slow-path-allocation, pre-initializing-stores step for // implementing ReduceInitialCardMarks @@ -208,6 +210,7 @@ private: // access to runtime stubs entry points for java code static address new_instance_Java() { return _new_instance_Java; } static address new_array_Java() { return _new_array_Java; } + static address new_array_nozero_Java() { return _new_array_nozero_Java; } static address multianewarray2_Java() { return _multianewarray2_Java; } static address multianewarray3_Java() { return _multianewarray3_Java; } static address multianewarray4_Java() { return _multianewarray4_Java; } -- GitLab