From 676405f8aff21a65ca4b24bf6e66033d4abcafe8 Mon Sep 17 00:00:00 2001 From: jmasa Date: Mon, 4 Aug 2014 10:48:10 -0700 Subject: [PATCH] 8031323: Optionally align objects copied to survivor spaces Reviewed-by: brutisso, tschatzl --- .../g1/g1ParScanThreadState.cpp | 7 ++- .../parNew/parNewGeneration.cpp | 6 +-- .../parNew/parNewGeneration.hpp | 2 +- .../parallelScavenge/psPromotionLAB.hpp | 19 ++----- .../psPromotionLAB.inline.hpp | 52 +++++++++++++++++++ .../psPromotionManager.inline.hpp | 1 + .../shared/parGCAllocBuffer.hpp | 5 +- .../shared/parGCAllocBuffer.inline.hpp | 44 ++++++++++++++++ src/share/vm/gc_interface/collectedHeap.hpp | 6 +++ .../vm/gc_interface/collectedHeap.inline.hpp | 38 ++++++++++++++ src/share/vm/memory/defNewGeneration.cpp | 2 +- src/share/vm/memory/space.cpp | 22 ++++++++ src/share/vm/memory/space.hpp | 1 + src/share/vm/oops/oop.pcgc.inline.hpp | 2 - src/share/vm/runtime/arguments.cpp | 16 ++++++ src/share/vm/runtime/globals.hpp | 3 ++ 16 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp create mode 100644 src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp diff --git a/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp b/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp index 89e3e813b..bf5824f1e 100644 --- a/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp +++ b/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp @@ -288,7 +288,12 @@ void G1ParScanThreadState::undo_allocation(GCAllocPurpose purpose, HeapWord* obj } HeapWord* G1ParScanThreadState::allocate(GCAllocPurpose purpose, size_t word_sz) { - HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz); + HeapWord* obj = NULL; + if (purpose == GCAllocForSurvived) { + obj = alloc_buffer(GCAllocForSurvived)->allocate_aligned(word_sz, SurvivorAlignmentInBytes); + } else { + obj = alloc_buffer(GCAllocForTenured)->allocate(word_sz); + } if (obj != NULL) { return obj; } diff --git a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp index b30b23b97..4d407dbfa 100644 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp @@ -28,12 +28,12 @@ #include "gc_implementation/parNew/parOopClosures.inline.hpp" #include "gc_implementation/shared/adaptiveSizePolicy.hpp" #include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/parGCAllocBuffer.hpp" +#include "gc_implementation/shared/copyFailedInfo.hpp" #include "gc_implementation/shared/gcHeapSummary.hpp" #include "gc_implementation/shared/gcTimer.hpp" #include "gc_implementation/shared/gcTrace.hpp" #include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" +#include "gc_implementation/shared/parGCAllocBuffer.inline.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" #include "memory/defNewGeneration.inline.hpp" #include "memory/genCollectedHeap.hpp" @@ -251,7 +251,7 @@ HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz) { plab->set_word_size(buf_size); plab->set_buf(buf_space); record_survivor_plab(buf_space, buf_size); - obj = plab->allocate(word_sz); + obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes); // Note that we cannot compare buf_size < word_sz below // because of AlignmentReserve (see ParGCAllocBuffer::allocate()). assert(obj != NULL || plab->words_remaining() < word_sz, diff --git a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp index bde15f7b8..7685353ed 100644 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp @@ -168,7 +168,7 @@ class ParScanThreadState { HeapWord* alloc_in_to_space_slow(size_t word_sz); HeapWord* alloc_in_to_space(size_t word_sz) { - HeapWord* obj = to_space_alloc_buffer()->allocate(word_sz); + HeapWord* obj = to_space_alloc_buffer()->allocate_aligned(word_sz, SurvivorAlignmentInBytes); if (obj != NULL) return obj; else return alloc_in_to_space_slow(word_sz); } diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp index e0427ef18..3ca7ca37b 100644 --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp @@ -26,6 +26,7 @@ #define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP #include "gc_implementation/parallelScavenge/objectStartArray.hpp" +#include "gc_interface/collectedHeap.inline.hpp" #include "memory/allocation.hpp" // @@ -94,23 +95,9 @@ class PSYoungPromotionLAB : public PSPromotionLAB { PSYoungPromotionLAB() { } // Not MT safe - HeapWord* allocate(size_t size) { - // Can't assert this, when young fills, we keep the LAB around, but flushed. - // assert(_state != flushed, "Sanity"); - HeapWord* obj = top(); - HeapWord* new_top = obj + size; - // The 'new_top>obj' check is needed to detect overflow of obj+size. - if (new_top > obj && new_top <= end()) { - set_top(new_top); - assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - return obj; - } + inline HeapWord* allocate(size_t size); - return NULL; - } - - debug_only(virtual bool lab_is_valid(MemRegion lab)); + debug_only(virtual bool lab_is_valid(MemRegion lab);) }; class PSOldPromotionLAB : public PSPromotionLAB { diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp new file mode 100644 index 000000000..0e5d7e7f7 --- /dev/null +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP + +#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp" +#include "gc_interface/collectedHeap.inline.hpp" + +HeapWord* PSYoungPromotionLAB::allocate(size_t size) { + // Can't assert this, when young fills, we keep the LAB around, but flushed. + // assert(_state != flushed, "Sanity"); + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end(), SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + HeapWord* new_top = obj + size; + // The 'new_top>obj' check is needed to detect overflow of obj+size. + if (new_top > obj && new_top <= end()) { + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + +#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp index 356c25851..b2de74d41 100644 --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp @@ -27,6 +27,7 @@ #include "gc_implementation/parallelScavenge/psOldGen.hpp" #include "gc_implementation/parallelScavenge/psPromotionManager.hpp" +#include "gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp" #include "gc_implementation/parallelScavenge/psScavenge.hpp" #include "oops/oop.psgc.inline.hpp" diff --git a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp index 63f3213ce..9bce2a318 100644 --- a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp @@ -24,7 +24,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP #define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP - +#include "gc_interface/collectedHeap.hpp" #include "memory/allocation.hpp" #include "memory/blockOffsetTable.hpp" #include "memory/threadLocalAllocBuffer.hpp" @@ -84,6 +84,9 @@ public: } } + // Allocate the object aligned to "alignment_in_bytes". + HeapWord* allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes); + // Undo the last allocation in the buffer, which is required to be of the // "obj" of the given "word_sz". void undo_allocation(HeapWord* obj, size_t word_sz) { diff --git a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp new file mode 100644 index 000000000..352ce05a3 --- /dev/null +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP + +#include "gc_implementation/shared/parGCAllocBuffer.hpp" +#include "gc_interface/collectedHeap.inline.hpp" + +HeapWord* ParGCAllocBuffer::allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes) { + + HeapWord* res = CollectedHeap::align_allocation_or_fail(_top, _end, alignment_in_bytes); + if (res == NULL) { + return NULL; + } + + // Set _top so that allocate(), which expects _top to be correctly set, + // can be used below. + _top = res; + return allocate(word_sz); +} + +#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP diff --git a/src/share/vm/gc_interface/collectedHeap.hpp b/src/share/vm/gc_interface/collectedHeap.hpp index 4b56f00e7..0c4dd65a0 100644 --- a/src/share/vm/gc_interface/collectedHeap.hpp +++ b/src/share/vm/gc_interface/collectedHeap.hpp @@ -351,6 +351,12 @@ class CollectedHeap : public CHeapObj { fill_with_object(start, pointer_delta(end, start), zap); } + // Return the address "addr" aligned by "alignment_in_bytes" if such + // an address is below "end". Return NULL otherwise. + inline static HeapWord* align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes); + // Some heaps may offer a contiguous region for shared non-blocking // allocation, via inlined code (by exporting the address of the top and // end fields defining the extent of the contiguous allocation region.) diff --git a/src/share/vm/gc_interface/collectedHeap.inline.hpp b/src/share/vm/gc_interface/collectedHeap.inline.hpp index 89315a942..302d0c7cb 100644 --- a/src/share/vm/gc_interface/collectedHeap.inline.hpp +++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp @@ -241,6 +241,44 @@ inline void CollectedHeap::oop_iterate_no_header(OopClosure* cl) { oop_iterate(&no_header_cl); } + +inline HeapWord* CollectedHeap::align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes) { + if (alignment_in_bytes <= ObjectAlignmentInBytes) { + return addr; + } + + assert(is_ptr_aligned(addr, HeapWordSize), + err_msg("Address " PTR_FORMAT " is not properly aligned.", p2i(addr))); + assert(is_size_aligned(alignment_in_bytes, HeapWordSize), + err_msg("Alignment size %u is incorrect.", alignment_in_bytes)); + + HeapWord* new_addr = (HeapWord*) align_pointer_up(addr, alignment_in_bytes); + size_t padding = pointer_delta(new_addr, addr); + + if (padding == 0) { + return addr; + } + + if (padding < CollectedHeap::min_fill_size()) { + padding += alignment_in_bytes / HeapWordSize; + assert(padding >= CollectedHeap::min_fill_size(), + err_msg("alignment_in_bytes %u is expect to be larger " + "than the minimum object size", alignment_in_bytes)); + new_addr = addr + padding; + } + + assert(new_addr > addr, err_msg("Unexpected arithmetic overflow " + PTR_FORMAT " not greater than " PTR_FORMAT, p2i(new_addr), p2i(addr))); + if(new_addr < end) { + CollectedHeap::fill_with_object(addr, padding); + return new_addr; + } else { + return NULL; + } +} + #ifndef PRODUCT inline bool diff --git a/src/share/vm/memory/defNewGeneration.cpp b/src/share/vm/memory/defNewGeneration.cpp index 148ed3882..ac41f33b4 100644 --- a/src/share/vm/memory/defNewGeneration.cpp +++ b/src/share/vm/memory/defNewGeneration.cpp @@ -789,7 +789,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) { // Try allocating obj in to-space (unless too old) if (old->age() < tenuring_threshold()) { - obj = (oop) to()->allocate(s); + obj = (oop) to()->allocate_aligned(s); } // Otherwise try allocating obj tenured diff --git a/src/share/vm/memory/space.cpp b/src/share/vm/memory/space.cpp index c9c601e5a..317384f23 100644 --- a/src/share/vm/memory/space.cpp +++ b/src/share/vm/memory/space.cpp @@ -28,6 +28,7 @@ #include "gc_implementation/shared/liveRange.hpp" #include "gc_implementation/shared/markSweep.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" +#include "gc_interface/collectedHeap.inline.hpp" #include "memory/blockOffsetTable.inline.hpp" #include "memory/defNewGeneration.hpp" #include "memory/genCollectedHeap.hpp" @@ -719,6 +720,27 @@ inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size, } while (true); } +HeapWord* ContiguousSpace::allocate_aligned(size_t size) { + assert(Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), "not locked"); + HeapWord* end_value = end(); + + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end_value, SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + if (pointer_delta(end_value, obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_aligned(new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + // Requires locking. HeapWord* ContiguousSpace::allocate(size_t size) { return allocate_impl(size, end()); diff --git a/src/share/vm/memory/space.hpp b/src/share/vm/memory/space.hpp index f3723a8fb..7ee65417f 100644 --- a/src/share/vm/memory/space.hpp +++ b/src/share/vm/memory/space.hpp @@ -526,6 +526,7 @@ class ContiguousSpace: public CompactibleSpace { // Allocation (return NULL if full) virtual HeapWord* allocate(size_t word_size); virtual HeapWord* par_allocate(size_t word_size); + HeapWord* allocate_aligned(size_t word_size); // Iteration void oop_iterate(ExtendedOopClosure* cl); diff --git a/src/share/vm/oops/oop.pcgc.inline.hpp b/src/share/vm/oops/oop.pcgc.inline.hpp index 596bbc6d1..8a4603944 100644 --- a/src/share/vm/oops/oop.pcgc.inline.hpp +++ b/src/share/vm/oops/oop.pcgc.inline.hpp @@ -54,8 +54,6 @@ inline void oopDesc::follow_contents(ParCompactionManager* cm) { klass()->oop_follow_contents(cm, this); } -// Used by parallel old GC. - inline oop oopDesc::forward_to_atomic(oop p) { assert(ParNewGeneration::is_legal_forward_ptr(p), "illegal forwarding pointer value."); diff --git a/src/share/vm/runtime/arguments.cpp b/src/share/vm/runtime/arguments.cpp index 5d2c3883b..9dc68c8bd 100644 --- a/src/share/vm/runtime/arguments.cpp +++ b/src/share/vm/runtime/arguments.cpp @@ -1398,6 +1398,22 @@ bool verify_object_alignment() { (int)ObjectAlignmentInBytes, os::vm_page_size()); return false; } + if(SurvivorAlignmentInBytes == 0) { + SurvivorAlignmentInBytes = ObjectAlignmentInBytes; + } else { + if (!is_power_of_2(SurvivorAlignmentInBytes)) { + jio_fprintf(defaultStream::error_stream(), + "error: SurvivorAlignmentInBytes=%d must be power of 2\n", + (int)SurvivorAlignmentInBytes); + return false; + } + if (SurvivorAlignmentInBytes < ObjectAlignmentInBytes) { + jio_fprintf(defaultStream::error_stream(), + "error: SurvivorAlignmentInBytes=%d must be greater than ObjectAlignmentInBytes=%d \n", + (int)SurvivorAlignmentInBytes, (int)ObjectAlignmentInBytes); + return false; + } + } return true; } diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp index f0c8080b8..82f61f0ff 100644 --- a/src/share/vm/runtime/globals.hpp +++ b/src/share/vm/runtime/globals.hpp @@ -3885,6 +3885,9 @@ class CommandLineFlags { product(bool, PrintGCCause, true, \ "Include GC cause in GC logging") \ \ + experimental(intx, SurvivorAlignmentInBytes, 0, \ + "Default survivor space alignment in bytes") \ + \ product(bool , AllowNonVirtualCalls, false, \ "Obey the ACC_SUPER flag and allow invokenonvirtual calls") \ \ -- GitLab