From 44227033c9997a2eff402cb6658441a6ad90cdde Mon Sep 17 00:00:00 2001 From: iveresov Date: Tue, 22 Feb 2011 15:25:02 -0800 Subject: [PATCH] 7020521: arraycopy stubs place prebarriers incorrectly Summary: Rearranged the pre-barrier placement in arraycopy stubs so that they are properly called in case of chained calls. Also refactored the code a little bit so that it looks uniform across the platforms and is more readable. Reviewed-by: never, kvn --- src/cpu/sparc/vm/stubGenerator_sparc.cpp | 283 +++++++++++++---------- src/cpu/x86/vm/stubGenerator_x86_32.cpp | 31 +-- src/cpu/x86/vm/stubGenerator_x86_64.cpp | 242 +++++++++---------- 3 files changed, 297 insertions(+), 259 deletions(-) diff --git a/src/cpu/sparc/vm/stubGenerator_sparc.cpp b/src/cpu/sparc/vm/stubGenerator_sparc.cpp index 7bb56140d..f99613343 100644 --- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp +++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -968,19 +968,6 @@ class StubGenerator: public StubCodeGenerator { return start; } - static address disjoint_byte_copy_entry; - static address disjoint_short_copy_entry; - static address disjoint_int_copy_entry; - static address disjoint_long_copy_entry; - static address disjoint_oop_copy_entry; - - static address byte_copy_entry; - static address short_copy_entry; - static address int_copy_entry; - static address long_copy_entry; - static address oop_copy_entry; - - static address checkcast_copy_entry; // // Verify that a register contains clean 32-bits positive value @@ -1283,7 +1270,7 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_disjoint_byte_copy(bool aligned, const char * name) { + address generate_disjoint_byte_copy(bool aligned, address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1299,9 +1286,11 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) disjoint_byte_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } // for short arrays, just do single element copy __ cmp(count, 23); // 16 + 7 @@ -1391,15 +1380,13 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_conjoint_byte_copy(bool aligned, const char * name) { + address generate_conjoint_byte_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { // Do reverse copy. __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); - address nooverlap_target = aligned ? - StubRoutines::arrayof_jbyte_disjoint_arraycopy() : - disjoint_byte_copy_entry; Label L_skip_alignment, L_align, L_aligned_copy; Label L_copy_byte, L_copy_byte_loop, L_exit; @@ -1412,9 +1399,11 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) byte_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } array_overlap_test(nooverlap_target, 0); @@ -1504,7 +1493,7 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_disjoint_short_copy(bool aligned, const char * name) { + address generate_disjoint_short_copy(bool aligned, address *entry, const char * name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1520,9 +1509,11 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) disjoint_short_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } // for short arrays, just do single element copy __ cmp(count, 11); // 8 + 3 (22 bytes) @@ -1842,15 +1833,13 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_conjoint_short_copy(bool aligned, const char * name) { + address generate_conjoint_short_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { // Do reverse copy. __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); - address nooverlap_target = aligned ? - StubRoutines::arrayof_jshort_disjoint_arraycopy() : - disjoint_short_copy_entry; Label L_skip_alignment, L_skip_alignment2, L_aligned_copy; Label L_copy_2_bytes, L_copy_2_bytes_loop, L_exit; @@ -1865,9 +1854,11 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) short_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } array_overlap_test(nooverlap_target, 1); @@ -2072,7 +2063,7 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_disjoint_int_copy(bool aligned, const char * name) { + address generate_disjoint_int_copy(bool aligned, address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -2080,9 +2071,11 @@ class StubGenerator: public StubCodeGenerator { const Register count = O2; assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) disjoint_int_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } generate_disjoint_int_copy_core(aligned); @@ -2204,20 +2197,19 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_conjoint_int_copy(bool aligned, const char * name) { + address generate_conjoint_int_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); - address nooverlap_target = aligned ? - StubRoutines::arrayof_jint_disjoint_arraycopy() : - disjoint_int_copy_entry; - assert_clean_int(O2, O3); // Make sure 'count' is clean int. - if (!aligned) int_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } array_overlap_test(nooverlap_target, 2); @@ -2336,16 +2328,18 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_disjoint_long_copy(bool aligned, const char * name) { + address generate_disjoint_long_copy(bool aligned, address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); assert_clean_int(O2, O3); // Make sure 'count' is clean int. - if (!aligned) disjoint_long_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } generate_disjoint_long_copy_core(aligned); @@ -2406,19 +2400,21 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_conjoint_long_copy(bool aligned, const char * name) { + address generate_conjoint_long_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); assert(!aligned, "usage"); - address nooverlap_target = disjoint_long_copy_entry; assert_clean_int(O2, O3); // Make sure 'count' is clean int. - if (!aligned) long_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } array_overlap_test(nooverlap_target, 3); @@ -2439,7 +2435,7 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_disjoint_oop_copy(bool aligned, const char * name) { + address generate_disjoint_oop_copy(bool aligned, address *entry, const char *name) { const Register from = O0; // source array address const Register to = O1; // destination array address @@ -2451,9 +2447,11 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) disjoint_oop_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here + BLOCK_COMMENT("Entry:"); + } // save arguments for barrier generation __ mov(to, G1); @@ -2487,7 +2485,8 @@ class StubGenerator: public StubCodeGenerator { // to: O1 // count: O2 treated as signed // - address generate_conjoint_oop_copy(bool aligned, const char * name) { + address generate_conjoint_oop_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { const Register from = O0; // source array address const Register to = O1; // destination array address @@ -2499,22 +2498,19 @@ class StubGenerator: public StubCodeGenerator { assert_clean_int(count, O3); // Make sure 'count' is clean int. - if (!aligned) oop_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here - if (!aligned) BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here + BLOCK_COMMENT("Entry:"); + } + + array_overlap_test(nooverlap_target, LogBytesPerHeapOop); // save arguments for barrier generation __ mov(to, G1); __ mov(count, G5); - gen_write_ref_array_pre_barrier(G1, G5); - address nooverlap_target = aligned ? - StubRoutines::arrayof_oop_disjoint_arraycopy() : - disjoint_oop_copy_entry; - - array_overlap_test(nooverlap_target, LogBytesPerHeapOop); - #ifdef _LP64 if (UseCompressedOops) { generate_conjoint_int_copy_core(aligned); @@ -2582,7 +2578,7 @@ class StubGenerator: public StubCodeGenerator { // ckval: O4 (super_klass) // ret: O0 zero for success; (-1^K) where K is partial transfer count // - address generate_checkcast_copy(const char* name) { + address generate_checkcast_copy(const char *name, address *entry) { const Register O0_from = O0; // source array address const Register O1_to = O1; // destination array address @@ -2600,8 +2596,6 @@ class StubGenerator: public StubCodeGenerator { StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); - gen_write_ref_array_pre_barrier(O1, O2); - #ifdef ASSERT // We sometimes save a frame (see generate_type_check below). // If this will cause trouble, let's fail now instead of later. @@ -2625,9 +2619,13 @@ class StubGenerator: public StubCodeGenerator { } #endif //ASSERT - checkcast_copy_entry = __ pc(); - // caller can pass a 64-bit byte count here (from generic stub) - BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from generic stub) + BLOCK_COMMENT("Entry:"); + } + + gen_write_ref_array_pre_barrier(O1_to, O2_count); Label load_element, store_element, do_card_marks, fail, done; __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it @@ -2700,7 +2698,11 @@ class StubGenerator: public StubCodeGenerator { // Examines the alignment of the operands and dispatches // to a long, int, short, or byte copy loop. // - address generate_unsafe_copy(const char* name) { + address generate_unsafe_copy(const char* name, + address byte_copy_entry, + address short_copy_entry, + address int_copy_entry, + address long_copy_entry) { const Register O0_from = O0; // source array address const Register O1_to = O1; // destination array address @@ -2796,8 +2798,13 @@ class StubGenerator: public StubCodeGenerator { // O0 == 0 - success // O0 == -1 - need to call System.arraycopy // - address generate_generic_copy(const char *name) { - + address generate_generic_copy(const char *name, + address entry_jbyte_arraycopy, + address entry_jshort_arraycopy, + address entry_jint_arraycopy, + address entry_oop_arraycopy, + address entry_jlong_arraycopy, + address entry_checkcast_arraycopy) { Label L_failed, L_objArray; // Input registers @@ -2970,15 +2977,15 @@ class StubGenerator: public StubCodeGenerator { BLOCK_COMMENT("choose copy loop based on element size"); __ cmp(G3_elsize, 0); - __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jbyte_arraycopy); + __ br(Assembler::equal, true, Assembler::pt, entry_jbyte_arraycopy); __ delayed()->signx(length, count); // length __ cmp(G3_elsize, LogBytesPerShort); - __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jshort_arraycopy); + __ br(Assembler::equal, true, Assembler::pt, entry_jshort_arraycopy); __ delayed()->signx(length, count); // length __ cmp(G3_elsize, LogBytesPerInt); - __ br(Assembler::equal,true,Assembler::pt,StubRoutines::_jint_arraycopy); + __ br(Assembler::equal, true, Assembler::pt, entry_jint_arraycopy); __ delayed()->signx(length, count); // length #ifdef ASSERT { Label L; @@ -2989,7 +2996,7 @@ class StubGenerator: public StubCodeGenerator { __ bind(L); } #endif - __ br(Assembler::always,false,Assembler::pt,StubRoutines::_jlong_arraycopy); + __ br(Assembler::always, false, Assembler::pt, entry_jlong_arraycopy); __ delayed()->signx(length, count); // length // objArrayKlass @@ -3013,7 +3020,7 @@ class StubGenerator: public StubCodeGenerator { __ add(src, src_pos, from); // src_addr __ add(dst, dst_pos, to); // dst_addr __ BIND(L_plain_copy); - __ br(Assembler::always, false, Assembler::pt,StubRoutines::_oop_arraycopy); + __ br(Assembler::always, false, Assembler::pt, entry_oop_arraycopy); __ delayed()->signx(length, count); // length __ BIND(L_checkcast_copy); @@ -3057,7 +3064,7 @@ class StubGenerator: public StubCodeGenerator { __ ld_ptr(G4_dst_klass, ek_offset, O4); // dest elem klass // lduw(O4, sco_offset, O3); // sco of elem klass - __ br(Assembler::always, false, Assembler::pt, checkcast_copy_entry); + __ br(Assembler::always, false, Assembler::pt, entry_checkcast_arraycopy); __ delayed()->lduw(O4, sco_offset, O3); } @@ -3068,39 +3075,76 @@ class StubGenerator: public StubCodeGenerator { } void generate_arraycopy_stubs() { - - // Note: the disjoint stubs must be generated first, some of - // the conjoint stubs use them. - StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy"); - StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy"); - StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, "jint_disjoint_arraycopy"); - StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_copy(false, "jlong_disjoint_arraycopy"); - StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, "oop_disjoint_arraycopy"); - StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, "arrayof_jbyte_disjoint_arraycopy"); - StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, "arrayof_jshort_disjoint_arraycopy"); - StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, "arrayof_jint_disjoint_arraycopy"); - StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, "arrayof_jlong_disjoint_arraycopy"); - StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, "arrayof_oop_disjoint_arraycopy"); - - StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, "jbyte_arraycopy"); - StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, "jshort_arraycopy"); - StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, "jint_arraycopy"); - StubRoutines::_jlong_arraycopy = generate_conjoint_long_copy(false, "jlong_arraycopy"); - StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, "oop_arraycopy"); - StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, "arrayof_jbyte_arraycopy"); - StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, "arrayof_jshort_arraycopy"); + address entry; + address entry_jbyte_arraycopy; + address entry_jshort_arraycopy; + address entry_jint_arraycopy; + address entry_oop_arraycopy; + address entry_jlong_arraycopy; + address entry_checkcast_arraycopy; + + StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, + "jbyte_disjoint_arraycopy"); + StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy, + "jbyte_arraycopy"); + StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, + "jshort_disjoint_arraycopy"); + StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy, + "jshort_arraycopy"); + StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry, + "jint_disjoint_arraycopy"); + StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry, &entry_jint_arraycopy, + "jint_arraycopy"); + StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_copy(false, &entry, + "jlong_disjoint_arraycopy"); + StubRoutines::_jlong_arraycopy = generate_conjoint_long_copy(false, entry, &entry_jlong_arraycopy, + "jlong_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, &entry, + "oop_disjoint_arraycopy"); + StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, entry, &entry_oop_arraycopy, + "oop_arraycopy"); + + + StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry, + "arrayof_jbyte_disjoint_arraycopy"); + StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL, + "arrayof_jbyte_arraycopy"); + + StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry, + "arrayof_jshort_disjoint_arraycopy"); + StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, entry, NULL, + "arrayof_jshort_arraycopy"); + + StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry, + "arrayof_jint_disjoint_arraycopy"); #ifdef _LP64 // since sizeof(jint) < sizeof(HeapWord), there's a different flavor: - StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, "arrayof_jint_arraycopy"); + StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, NULL, "arrayof_jint_arraycopy"); #else StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; #endif + + StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, NULL, + "arrayof_jlong_disjoint_arraycopy"); + StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, NULL, + "arrayof_oop_disjoint_arraycopy"); + StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; - StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy"); - StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy"); - StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy"); + StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_jlong_arraycopy); + StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_oop_arraycopy, + entry_jlong_arraycopy, + entry_checkcast_arraycopy); StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill"); StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill"); @@ -3224,21 +3268,6 @@ class StubGenerator: public StubCodeGenerator { }; // end class declaration - -address StubGenerator::disjoint_byte_copy_entry = NULL; -address StubGenerator::disjoint_short_copy_entry = NULL; -address StubGenerator::disjoint_int_copy_entry = NULL; -address StubGenerator::disjoint_long_copy_entry = NULL; -address StubGenerator::disjoint_oop_copy_entry = NULL; - -address StubGenerator::byte_copy_entry = NULL; -address StubGenerator::short_copy_entry = NULL; -address StubGenerator::int_copy_entry = NULL; -address StubGenerator::long_copy_entry = NULL; -address StubGenerator::oop_copy_entry = NULL; - -address StubGenerator::checkcast_copy_entry = NULL; - void StubGenerator_generate(CodeBuffer* code, bool all) { StubGenerator g(code, all); } diff --git a/src/cpu/x86/vm/stubGenerator_x86_32.cpp b/src/cpu/x86/vm/stubGenerator_x86_32.cpp index 9a7371cf6..484eff7af 100644 --- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp +++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp @@ -945,6 +945,12 @@ class StubGenerator: public StubCodeGenerator { __ movptr(from , Address(rsp, 12+ 4)); __ movptr(to , Address(rsp, 12+ 8)); __ movl(count, Address(rsp, 12+ 12)); + + if (entry != NULL) { + *entry = __ pc(); // Entry point from conjoint arraycopy stub. + BLOCK_COMMENT("Entry:"); + } + if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); @@ -952,9 +958,6 @@ class StubGenerator: public StubCodeGenerator { __ mov(saved_to, to); // save 'to' } - *entry = __ pc(); // Entry point from conjoint arraycopy stub. - BLOCK_COMMENT("Entry:"); - __ subptr(to, from); // to --> to_from __ cmpl(count, 2< rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers @@ -1479,7 +1467,8 @@ class StubGenerator: public StubCodeGenerator { // dwords or qwords that span cache line boundaries will still be loaded // and stored atomically. // - address generate_conjoint_byte_copy(bool aligned, const char *name) { + address generate_conjoint_byte_copy(bool aligned, address nooverlap_target, + address* entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1494,11 +1483,13 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - byte_copy_entry = __ pc(); - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } - array_overlap_test(disjoint_byte_copy_entry, Address::times_1); + array_overlap_test(nooverlap_target, Address::times_1); setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers @@ -1574,7 +1565,7 @@ class StubGenerator: public StubCodeGenerator { // disjoint_short_copy_entry is set to the no-overlap entry point // used by generate_conjoint_short_copy(). // - address generate_disjoint_short_copy(bool aligned, const char *name) { + address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1593,9 +1584,11 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - disjoint_short_copy_entry = __ pc(); - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers @@ -1686,7 +1679,8 @@ class StubGenerator: public StubCodeGenerator { // or qwords that span cache line boundaries will still be loaded // and stored atomically. // - address generate_conjoint_short_copy(bool aligned, const char *name) { + address generate_conjoint_short_copy(bool aligned, address nooverlap_target, + address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1701,11 +1695,13 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - short_copy_entry = __ pc(); - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); + } - array_overlap_test(disjoint_short_copy_entry, Address::times_2); + array_overlap_test(nooverlap_target, Address::times_2); setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers @@ -1773,7 +1769,7 @@ class StubGenerator: public StubCodeGenerator { // disjoint_int_copy_entry is set to the no-overlap entry point // used by generate_conjoint_int_oop_copy(). // - address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) { + address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1793,21 +1789,17 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - (is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry) = __ pc(); - - if (is_oop) { - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); } - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - if (is_oop) { __ movq(saved_to, to); + gen_write_ref_array_pre_barrier(to, count); } // 'from', 'to' and 'count' are now valid @@ -1867,7 +1859,8 @@ class StubGenerator: public StubCodeGenerator { // the hardware handle it. The two dwords within qwords that span // cache line boundaries will still be loaded and stored atomicly. // - address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) { + address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target, + address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1882,20 +1875,21 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - if (is_oop) { - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); } - (is_oop ? oop_copy_entry : int_copy_entry) = __ pc(); - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - - array_overlap_test(is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry, - Address::times_4); + array_overlap_test(nooverlap_target, Address::times_4); setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers + if (is_oop) { + // no registers are destroyed by this call + gen_write_ref_array_pre_barrier(to, count); + } + assert_clean_int(count, rax); // Make sure 'count' is clean int. // 'from', 'to' and 'count' are now valid __ movptr(dword_count, count); @@ -1959,7 +1953,7 @@ class StubGenerator: public StubCodeGenerator { // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the // no-overlap entry point used by generate_conjoint_long_oop_copy(). // - address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) { + address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1978,20 +1972,19 @@ class StubGenerator: public StubCodeGenerator { // Save no-overlap entry point for generate_conjoint_long_oop_copy() assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - if (is_oop) { - disjoint_oop_copy_entry = __ pc(); - // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2); - } else { - disjoint_long_copy_entry = __ pc(); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); } - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - // 'from', 'to' and 'qword_count' are now valid + if (is_oop) { + // no registers are destroyed by this call + gen_write_ref_array_pre_barrier(to, qword_count); + } // Copy from low to high addresses. Use 'to' as scratch. __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); @@ -2045,7 +2038,8 @@ class StubGenerator: public StubCodeGenerator { // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // - address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) { + address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, address nooverlap_target, + address *entry, const char *name) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -2059,26 +2053,16 @@ class StubGenerator: public StubCodeGenerator { __ enter(); // required for proper stackwalking of RuntimeStub frame assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int. - address disjoint_copy_entry = NULL; - if (is_oop) { - assert(!UseCompressedOops, "shouldn't be called for compressed oops"); - disjoint_copy_entry = disjoint_oop_copy_entry; - oop_copy_entry = __ pc(); - array_overlap_test(disjoint_oop_copy_entry, Address::times_8); - } else { - disjoint_copy_entry = disjoint_long_copy_entry; - long_copy_entry = __ pc(); - array_overlap_test(disjoint_long_copy_entry, Address::times_8); + if (entry != NULL) { + *entry = __ pc(); + // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) + BLOCK_COMMENT("Entry:"); } - BLOCK_COMMENT("Entry:"); - // caller can pass a 64-bit byte count here (from Unsafe.copyMemory) - array_overlap_test(disjoint_copy_entry, Address::times_8); + array_overlap_test(nooverlap_target, Address::times_8); setup_arg_regs(); // from => rdi, to => rsi, count => rdx // r9 and r10 may be used to save non-volatile registers - // 'from', 'to' and 'qword_count' are now valid - if (is_oop) { // Save to and count for store barrier __ movptr(saved_count, qword_count); @@ -2162,7 +2146,7 @@ class StubGenerator: public StubCodeGenerator { // rax == 0 - success // rax == -1^K - failure, where K is partial transfer count // - address generate_checkcast_copy(const char *name) { + address generate_checkcast_copy(const char *name, address *entry) { Label L_load_element, L_store_element, L_do_card_marks, L_done; @@ -2216,8 +2200,10 @@ class StubGenerator: public StubCodeGenerator { #endif // Caller of this entry point must set up the argument registers. - checkcast_copy_entry = __ pc(); - BLOCK_COMMENT("Entry:"); + if (entry != NULL) { + *entry = __ pc(); + BLOCK_COMMENT("Entry:"); + } // allocate spill slots for r13, r14 enum { @@ -2334,7 +2320,9 @@ class StubGenerator: public StubCodeGenerator { // Examines the alignment of the operands and dispatches // to a long, int, short, or byte copy loop. // - address generate_unsafe_copy(const char *name) { + address generate_unsafe_copy(const char *name, + address byte_copy_entry, address short_copy_entry, + address int_copy_entry, address long_copy_entry) { Label L_long_aligned, L_int_aligned, L_short_aligned; @@ -2432,7 +2420,10 @@ class StubGenerator: public StubCodeGenerator { // rax == 0 - success // rax == -1^K - failure, where K is partial transfer count // - address generate_generic_copy(const char *name) { + address generate_generic_copy(const char *name, + address byte_copy_entry, address short_copy_entry, + address int_copy_entry, address long_copy_entry, + address oop_copy_entry, address checkcast_copy_entry) { Label L_failed, L_failed_0, L_objArray; Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs; @@ -2725,33 +2716,60 @@ class StubGenerator: public StubCodeGenerator { } void generate_arraycopy_stubs() { - // Call the conjoint generation methods immediately after - // the disjoint ones so that short branches from the former - // to the latter can be generated. - StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy"); - StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, "jbyte_arraycopy"); - - StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy"); - StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, "jshort_arraycopy"); - - StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, "jint_disjoint_arraycopy"); - StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, "jint_arraycopy"); - - StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, "jlong_disjoint_arraycopy"); - StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, "jlong_arraycopy"); + address entry; + address entry_jbyte_arraycopy; + address entry_jshort_arraycopy; + address entry_jint_arraycopy; + address entry_oop_arraycopy; + address entry_jlong_arraycopy; + address entry_checkcast_arraycopy; + + StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, + "jbyte_disjoint_arraycopy"); + StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy, + "jbyte_arraycopy"); + + StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, + "jshort_disjoint_arraycopy"); + StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy, + "jshort_arraycopy"); + + StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, &entry, + "jint_disjoint_arraycopy"); + StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, entry, + &entry_jint_arraycopy, "jint_arraycopy"); + + StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, &entry, + "jlong_disjoint_arraycopy"); + StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, entry, + &entry_jlong_arraycopy, "jlong_arraycopy"); if (UseCompressedOops) { - StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, "oop_disjoint_arraycopy"); - StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, "oop_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, &entry, + "oop_disjoint_arraycopy"); + StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, entry, + &entry_oop_arraycopy, "oop_arraycopy"); } else { - StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, "oop_disjoint_arraycopy"); - StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, "oop_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, &entry, + "oop_disjoint_arraycopy"); + StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, entry, + &entry_oop_arraycopy, "oop_arraycopy"); } - StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy"); - StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy"); - StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy"); + StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_jlong_arraycopy); + StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy", + entry_jbyte_arraycopy, + entry_jshort_arraycopy, + entry_jint_arraycopy, + entry_oop_arraycopy, + entry_jlong_arraycopy, + entry_checkcast_arraycopy); StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill"); StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill"); @@ -3069,20 +3087,6 @@ class StubGenerator: public StubCodeGenerator { } }; // end class declaration -address StubGenerator::disjoint_byte_copy_entry = NULL; -address StubGenerator::disjoint_short_copy_entry = NULL; -address StubGenerator::disjoint_int_copy_entry = NULL; -address StubGenerator::disjoint_long_copy_entry = NULL; -address StubGenerator::disjoint_oop_copy_entry = NULL; - -address StubGenerator::byte_copy_entry = NULL; -address StubGenerator::short_copy_entry = NULL; -address StubGenerator::int_copy_entry = NULL; -address StubGenerator::long_copy_entry = NULL; -address StubGenerator::oop_copy_entry = NULL; - -address StubGenerator::checkcast_copy_entry = NULL; - void StubGenerator_generate(CodeBuffer* code, bool all) { StubGenerator g(code, all); } -- GitLab