diff --git a/src/cpu/sparc/vm/stubGenerator_sparc.cpp b/src/cpu/sparc/vm/stubGenerator_sparc.cpp index cf403040dfabf6d960f63e92217074ba5e5e0db3..9415c7b6c4ceb7f74723fecf8b4fe6c049cca5cc 100644 --- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp +++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp @@ -2359,10 +2359,10 @@ class StubGenerator: public StubCodeGenerator { for (int off = 0; off < 64; off += 16) { if (use_prefetch && (off & 31) == 0) { if (ArraycopySrcPrefetchDistance > 0) { - __ prefetch(from, ArraycopySrcPrefetchDistance, Assembler::severalReads); + __ prefetch(from, ArraycopySrcPrefetchDistance+off, Assembler::severalReads); } if (ArraycopyDstPrefetchDistance > 0) { - __ prefetch(to, ArraycopyDstPrefetchDistance, Assembler::severalWritesAndPossiblyReads); + __ prefetch(to, ArraycopyDstPrefetchDistance+off, Assembler::severalWritesAndPossiblyReads); } } __ ldx(from, off+0, O4); diff --git a/src/share/vm/gc_interface/collectedHeap.hpp b/src/share/vm/gc_interface/collectedHeap.hpp index 36fcb40a80d325af89f214335015dc0e582dc341..5761ea554acc7ad71f2f6fff5c0c1e0ba5ffb93d 100644 --- a/src/share/vm/gc_interface/collectedHeap.hpp +++ b/src/share/vm/gc_interface/collectedHeap.hpp @@ -322,6 +322,7 @@ class CollectedHeap : public CHeapObj { // General obj/array allocation facilities. inline static oop obj_allocate(KlassHandle klass, int size, TRAPS); inline static oop array_allocate(KlassHandle klass, int size, int length, TRAPS); + inline static oop array_allocate_nozero(KlassHandle klass, int size, int length, TRAPS); // Special obj/array allocation facilities. // Some heaps may want to manage "permanent" data uniquely. These default diff --git a/src/share/vm/gc_interface/collectedHeap.inline.hpp b/src/share/vm/gc_interface/collectedHeap.inline.hpp index fabf702eec7bef0c97c42dc3f05b5b272cac6ab2..adced061d8ac2abdb5efab72319fe33fd9838cb2 100644 --- a/src/share/vm/gc_interface/collectedHeap.inline.hpp +++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp @@ -274,6 +274,23 @@ oop CollectedHeap::array_allocate(KlassHandle klass, return (oop)obj; } +oop CollectedHeap::array_allocate_nozero(KlassHandle klass, + int size, + int length, + TRAPS) { + debug_only(check_for_valid_allocation_state()); + assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); + assert(size >= 0, "int won't convert to size_t"); + HeapWord* obj = common_mem_allocate_noinit(size, CHECK_NULL); + ((oop)obj)->set_klass_gap(0); + post_allocation_setup_array(klass, obj, size, length); +#ifndef PRODUCT + const size_t hs = oopDesc::header_size()+1; + Universe::heap()->check_for_non_bad_heap_word_value(obj+hs, size-hs); +#endif + return (oop)obj; +} + oop CollectedHeap::permanent_obj_allocate(KlassHandle klass, int size, TRAPS) { oop obj = permanent_obj_allocate_no_klass_install(klass, size, CHECK_NULL); post_allocation_install_obj_klass(klass, obj, size); diff --git a/src/share/vm/memory/oopFactory.cpp b/src/share/vm/memory/oopFactory.cpp index 08f9825b34f1086bdc2f48d64bf7993a606c619b..c6e644c96d41e1642a028c92dc6c568a4e616de3 100644 --- a/src/share/vm/memory/oopFactory.cpp +++ b/src/share/vm/memory/oopFactory.cpp @@ -77,7 +77,14 @@ typeArrayOop oopFactory::new_permanent_intArray(int length, TRAPS) { typeArrayOop oopFactory::new_typeArray(BasicType type, int length, TRAPS) { klassOop type_asKlassOop = Universe::typeArrayKlassObj(type); typeArrayKlass* type_asArrayKlass = typeArrayKlass::cast(type_asKlassOop); - typeArrayOop result = type_asArrayKlass->allocate(length, THREAD); + typeArrayOop result = type_asArrayKlass->allocate_common(length, true, THREAD); + return result; +} + +typeArrayOop oopFactory::new_typeArray_nozero(BasicType type, int length, TRAPS) { + klassOop type_asKlassOop = Universe::typeArrayKlassObj(type); + typeArrayKlass* type_asArrayKlass = typeArrayKlass::cast(type_asKlassOop); + typeArrayOop result = type_asArrayKlass->allocate_common(length, false, THREAD); return result; } diff --git a/src/share/vm/memory/oopFactory.hpp b/src/share/vm/memory/oopFactory.hpp index 32e50fb40722b6c7daf38b0c68bd958534840488..dbb42f05332831017b9a9d717eabf7df44d14727 100644 --- a/src/share/vm/memory/oopFactory.hpp +++ b/src/share/vm/memory/oopFactory.hpp @@ -63,6 +63,7 @@ class oopFactory: AllStatic { static typeArrayOop new_permanent_intArray (int length, TRAPS); // used for class file structures static typeArrayOop new_typeArray(BasicType type, int length, TRAPS); + static typeArrayOop new_typeArray_nozero(BasicType type, int length, TRAPS); // Constant pools static constantPoolOop new_constantPool (int length, diff --git a/src/share/vm/oops/typeArrayKlass.cpp b/src/share/vm/oops/typeArrayKlass.cpp index 63384171150595e46793afd9ce4f3a7d52feef1e..a3c58cd0ea3428823db091c627b0985bf4a3abd1 100644 --- a/src/share/vm/oops/typeArrayKlass.cpp +++ b/src/share/vm/oops/typeArrayKlass.cpp @@ -76,7 +76,7 @@ klassOop typeArrayKlass::create_klass(BasicType type, int scale, return k(); } -typeArrayOop typeArrayKlass::allocate(int length, TRAPS) { +typeArrayOop typeArrayKlass::allocate_common(int length, bool do_zero, TRAPS) { assert(log2_element_size() >= 0, "bad scale"); if (length >= 0) { if (length <= max_length()) { @@ -84,7 +84,11 @@ typeArrayOop typeArrayKlass::allocate(int length, TRAPS) { KlassHandle h_k(THREAD, as_klassOop()); typeArrayOop t; CollectedHeap* ch = Universe::heap(); - t = (typeArrayOop)CollectedHeap::array_allocate(h_k, (int)size, length, CHECK_NULL); + if (do_zero) { + t = (typeArrayOop)CollectedHeap::array_allocate(h_k, (int)size, length, CHECK_NULL); + } else { + t = (typeArrayOop)CollectedHeap::array_allocate_nozero(h_k, (int)size, length, CHECK_NULL); + } assert(t->is_parsable(), "Don't publish unless parsable"); return t; } else { diff --git a/src/share/vm/oops/typeArrayKlass.hpp b/src/share/vm/oops/typeArrayKlass.hpp index 78a26be09100ea653058fd6bae76336c6c01f9d0..0f530668cc68fe2b8d4844e0850a1793e7cbb2db 100644 --- a/src/share/vm/oops/typeArrayKlass.hpp +++ b/src/share/vm/oops/typeArrayKlass.hpp @@ -56,7 +56,8 @@ class typeArrayKlass : public arrayKlass { bool compute_is_subtype_of(klassOop k); // Allocation - typeArrayOop allocate(int length, TRAPS); + typeArrayOop allocate_common(int length, bool do_zero, TRAPS); + typeArrayOop allocate(int length, TRAPS) { return allocate_common(length, true, THREAD); } typeArrayOop allocate_permanent(int length, TRAPS); // used for class file structures oop multi_allocate(int rank, jint* sizes, TRAPS); diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp index 8179f379b2da4641b42c31f1bdf7f6f14e2ad6c5..d5ad00e16e0be155ba59440ec5073cfa2ee64876 100644 --- a/src/share/vm/opto/library_call.cpp +++ b/src/share/vm/opto/library_call.cpp @@ -4658,6 +4658,7 @@ LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); + init->set_complete_with_arraycopy(); assert(dest->is_CheckCastPP(), "sanity"); assert(dest->in(0)->in(0) == init, "dest pinned"); adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory diff --git a/src/share/vm/opto/macro.cpp b/src/share/vm/opto/macro.cpp index 1e12e36602fbeda8136996a07785e123fdca69dd..b5f645359c956af0cedb4e8c3a81be57dcb28e95 100644 --- a/src/share/vm/opto/macro.cpp +++ b/src/share/vm/opto/macro.cpp @@ -1685,9 +1685,21 @@ void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { Node* length = alloc->in(AllocateNode::ALength); + InitializeNode* init = alloc->initialization(); + Node* klass_node = alloc->in(AllocateNode::KlassNode); + ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); + address slow_call_address; // Address of slow call + if (init != NULL && init->is_complete_with_arraycopy() && + k->is_type_array_klass()) { + // Don't zero type array during slow allocation in VM since + // it will be initialized later by arraycopy in compiled code. + slow_call_address = OptoRuntime::new_array_nozero_Java(); + } else { + slow_call_address = OptoRuntime::new_array_Java(); + } expand_allocate_common(alloc, length, OptoRuntime::new_array_Type(), - OptoRuntime::new_array_Java()); + slow_call_address); } //-----------------------mark_eliminated_locking_nodes----------------------- diff --git a/src/share/vm/opto/memnode.cpp b/src/share/vm/opto/memnode.cpp index ba0960116fcac04a7562e47e4c1f2335a98f1caf..4f6dc0851210576a7fd4a9cace5790dc25714bf5 100644 --- a/src/share/vm/opto/memnode.cpp +++ b/src/share/vm/opto/memnode.cpp @@ -2847,7 +2847,7 @@ Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) { //---------------------------InitializeNode------------------------------------ InitializeNode::InitializeNode(Compile* C, int adr_type, Node* rawoop) - : _is_complete(false), + : _is_complete(Incomplete), MemBarNode(C, adr_type, rawoop) { init_class_id(Class_Initialize); @@ -2885,7 +2885,7 @@ bool InitializeNode::is_non_zero() { void InitializeNode::set_complete(PhaseGVN* phase) { assert(!is_complete(), "caller responsibility"); - _is_complete = true; + _is_complete = Complete; // After this node is complete, it contains a bunch of // raw-memory initializations. There is no need for diff --git a/src/share/vm/opto/memnode.hpp b/src/share/vm/opto/memnode.hpp index a9eacdd8c9bb4461cc05d58835d937e7dd5b78db..d757b13f39e5a8fac54f67458255d1bc085eb3db 100644 --- a/src/share/vm/opto/memnode.hpp +++ b/src/share/vm/opto/memnode.hpp @@ -942,7 +942,12 @@ public: class InitializeNode: public MemBarNode { friend class AllocateNode; - bool _is_complete; + enum { + Incomplete = 0, + Complete = 1, + WithArraycopy = 2 + }; + int _is_complete; public: enum { @@ -976,10 +981,12 @@ public: // An InitializeNode must completed before macro expansion is done. // Completion requires that the AllocateNode must be followed by // initialization of the new memory to zero, then to any initializers. - bool is_complete() { return _is_complete; } + bool is_complete() { return _is_complete != Incomplete; } + bool is_complete_with_arraycopy() { return (_is_complete & WithArraycopy) != 0; } // Mark complete. (Must not yet be complete.) void set_complete(PhaseGVN* phase); + void set_complete_with_arraycopy() { _is_complete = Complete | WithArraycopy; } #ifdef ASSERT // ensure all non-degenerate stores are ordered and non-overlapping diff --git a/src/share/vm/opto/runtime.cpp b/src/share/vm/opto/runtime.cpp index 49c55212fa2752283a3f7aa8a24ca065bfc76375..a4495ea7474f676a3966a963539a7f6476b4796d 100644 --- a/src/share/vm/opto/runtime.cpp +++ b/src/share/vm/opto/runtime.cpp @@ -102,6 +102,7 @@ // Compiled code entry points address OptoRuntime::_new_instance_Java = NULL; address OptoRuntime::_new_array_Java = NULL; +address OptoRuntime::_new_array_nozero_Java = NULL; address OptoRuntime::_multianewarray2_Java = NULL; address OptoRuntime::_multianewarray3_Java = NULL; address OptoRuntime::_multianewarray4_Java = NULL; @@ -151,6 +152,7 @@ void OptoRuntime::generate(ciEnv* env) { // ------------------------------------------------------------------------------------------------------------------------------- gen(env, _new_instance_Java , new_instance_Type , new_instance_C , 0 , true , false, false); gen(env, _new_array_Java , new_array_Type , new_array_C , 0 , true , false, false); + gen(env, _new_array_nozero_Java , new_array_Type , new_array_nozero_C , 0 , true , false, false); gen(env, _multianewarray2_Java , multianewarray2_Type , multianewarray2_C , 0 , true , false, false); gen(env, _multianewarray3_Java , multianewarray3_Type , multianewarray3_C , 0 , true , false, false); gen(env, _multianewarray4_Java , multianewarray4_Type , multianewarray4_C , 0 , true , false, false); @@ -308,6 +310,36 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(klassOopDesc* array_type, int len } JRT_END +// array allocation without zeroing +JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(klassOopDesc* array_type, int len, JavaThread *thread)) + JRT_BLOCK; +#ifndef PRODUCT + SharedRuntime::_new_array_ctr++; // new array requires GC +#endif + assert(check_compiled_frame(thread), "incorrect caller"); + + // Scavenge and allocate an instance. + oop result; + + assert(Klass::cast(array_type)->oop_is_typeArray(), "should be called only for type array"); + // The oopFactory likes to work with the element type. + BasicType elem_type = typeArrayKlass::cast(array_type)->element_type(); + result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD); + + // Pass oops back through thread local storage. Our apparent type to Java + // is that we return an oop, but we can block on exit from this routine and + // a GC can trash the oop in C's return register. The generated stub will + // fetch the oop from TLS after any possible GC. + deoptimize_caller_frame(thread, HAS_PENDING_EXCEPTION); + thread->set_vm_result(result); + JRT_BLOCK_END; + + if (GraphKit::use_ReduceInitialCardMarks()) { + // inform GC that we won't do card marks for initializing writes. + new_store_pre_barrier(thread); + } +JRT_END + // Note: multianewarray for one dimension is handled inline by GraphKit::new_array. // multianewarray for 2 dimensions diff --git a/src/share/vm/opto/runtime.hpp b/src/share/vm/opto/runtime.hpp index a691da30701787fac59f3a3fefd755018b3c057c..f2c538b00753567df18e365f1f8743616f21bb5a 100644 --- a/src/share/vm/opto/runtime.hpp +++ b/src/share/vm/opto/runtime.hpp @@ -114,6 +114,7 @@ class OptoRuntime : public AllStatic { // References to generated stubs static address _new_instance_Java; static address _new_array_Java; + static address _new_array_nozero_Java; static address _multianewarray2_Java; static address _multianewarray3_Java; static address _multianewarray4_Java; @@ -143,6 +144,7 @@ class OptoRuntime : public AllStatic { // Allocate storage for a objArray or typeArray static void new_array_C(klassOopDesc* array_klass, int len, JavaThread *thread); + static void new_array_nozero_C(klassOopDesc* array_klass, int len, JavaThread *thread); // Post-slow-path-allocation, pre-initializing-stores step for // implementing ReduceInitialCardMarks @@ -208,6 +210,7 @@ private: // access to runtime stubs entry points for java code static address new_instance_Java() { return _new_instance_Java; } static address new_array_Java() { return _new_array_Java; } + static address new_array_nozero_Java() { return _new_array_nozero_Java; } static address multianewarray2_Java() { return _multianewarray2_Java; } static address multianewarray3_Java() { return _multianewarray3_Java; } static address multianewarray4_Java() { return _multianewarray4_Java; }