diff --git a/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp b/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp index 89e3e813b9859509e0b34472f54e0de675dfda2b..bf5824f1ed28d5281cb05eec02845df5aca4cf4c 100644 --- a/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp +++ b/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp @@ -288,7 +288,12 @@ void G1ParScanThreadState::undo_allocation(GCAllocPurpose purpose, HeapWord* obj } HeapWord* G1ParScanThreadState::allocate(GCAllocPurpose purpose, size_t word_sz) { - HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz); + HeapWord* obj = NULL; + if (purpose == GCAllocForSurvived) { + obj = alloc_buffer(GCAllocForSurvived)->allocate_aligned(word_sz, SurvivorAlignmentInBytes); + } else { + obj = alloc_buffer(GCAllocForTenured)->allocate(word_sz); + } if (obj != NULL) { return obj; } diff --git a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp index b30b23b97923f781e7129d3a162c7dafc22d359e..4d407dbfa92ef4e98b2d1c6df28f7be035d832e5 100644 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp @@ -28,12 +28,12 @@ #include "gc_implementation/parNew/parOopClosures.inline.hpp" #include "gc_implementation/shared/adaptiveSizePolicy.hpp" #include "gc_implementation/shared/ageTable.hpp" -#include "gc_implementation/shared/parGCAllocBuffer.hpp" +#include "gc_implementation/shared/copyFailedInfo.hpp" #include "gc_implementation/shared/gcHeapSummary.hpp" #include "gc_implementation/shared/gcTimer.hpp" #include "gc_implementation/shared/gcTrace.hpp" #include "gc_implementation/shared/gcTraceTime.hpp" -#include "gc_implementation/shared/copyFailedInfo.hpp" +#include "gc_implementation/shared/parGCAllocBuffer.inline.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" #include "memory/defNewGeneration.inline.hpp" #include "memory/genCollectedHeap.hpp" @@ -251,7 +251,7 @@ HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz) { plab->set_word_size(buf_size); plab->set_buf(buf_space); record_survivor_plab(buf_space, buf_size); - obj = plab->allocate(word_sz); + obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes); // Note that we cannot compare buf_size < word_sz below // because of AlignmentReserve (see ParGCAllocBuffer::allocate()). assert(obj != NULL || plab->words_remaining() < word_sz, diff --git a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp index bde15f7b8f89ba4df9eebd099958a0319cce7033..7685353ed1e0d023b2ea4625433846805d4fb3b0 100644 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp @@ -168,7 +168,7 @@ class ParScanThreadState { HeapWord* alloc_in_to_space_slow(size_t word_sz); HeapWord* alloc_in_to_space(size_t word_sz) { - HeapWord* obj = to_space_alloc_buffer()->allocate(word_sz); + HeapWord* obj = to_space_alloc_buffer()->allocate_aligned(word_sz, SurvivorAlignmentInBytes); if (obj != NULL) return obj; else return alloc_in_to_space_slow(word_sz); } diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp index e0427ef18e9d3d7b2fe6c26fbb50ed99964bd31b..3ca7ca37bff05c9a38351a514837b0ee76701cf2 100644 --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp @@ -26,6 +26,7 @@ #define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP #include "gc_implementation/parallelScavenge/objectStartArray.hpp" +#include "gc_interface/collectedHeap.inline.hpp" #include "memory/allocation.hpp" // @@ -94,23 +95,9 @@ class PSYoungPromotionLAB : public PSPromotionLAB { PSYoungPromotionLAB() { } // Not MT safe - HeapWord* allocate(size_t size) { - // Can't assert this, when young fills, we keep the LAB around, but flushed. - // assert(_state != flushed, "Sanity"); - HeapWord* obj = top(); - HeapWord* new_top = obj + size; - // The 'new_top>obj' check is needed to detect overflow of obj+size. - if (new_top > obj && new_top <= end()) { - set_top(new_top); - assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top), - "checking alignment"); - return obj; - } + inline HeapWord* allocate(size_t size); - return NULL; - } - - debug_only(virtual bool lab_is_valid(MemRegion lab)); + debug_only(virtual bool lab_is_valid(MemRegion lab);) }; class PSOldPromotionLAB : public PSPromotionLAB { diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp new file mode 100644 index 0000000000000000000000000000000000000000..0e5d7e7f7f2eb7986a7d75065ca1f7f4012325d9 --- /dev/null +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP + +#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp" +#include "gc_interface/collectedHeap.inline.hpp" + +HeapWord* PSYoungPromotionLAB::allocate(size_t size) { + // Can't assert this, when young fills, we keep the LAB around, but flushed. + // assert(_state != flushed, "Sanity"); + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end(), SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + HeapWord* new_top = obj + size; + // The 'new_top>obj' check is needed to detect overflow of obj+size. + if (new_top > obj && new_top <= end()) { + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_object_aligned((intptr_t)new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + +#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP diff --git a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp index 356c2585168a4665de58f68f7d4e33aef33dcacd..b2de74d41754c7653dcc07acfd97331d37ca00b1 100644 --- a/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp +++ b/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp @@ -27,6 +27,7 @@ #include "gc_implementation/parallelScavenge/psOldGen.hpp" #include "gc_implementation/parallelScavenge/psPromotionManager.hpp" +#include "gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp" #include "gc_implementation/parallelScavenge/psScavenge.hpp" #include "oops/oop.psgc.inline.hpp" diff --git a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp index 63f3213ce2d4bff2e5d4415140323c82d73dbe79..9bce2a318cff785c79bc9c8f10b76740deccfefb 100644 --- a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp @@ -24,7 +24,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP #define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP - +#include "gc_interface/collectedHeap.hpp" #include "memory/allocation.hpp" #include "memory/blockOffsetTable.hpp" #include "memory/threadLocalAllocBuffer.hpp" @@ -84,6 +84,9 @@ public: } } + // Allocate the object aligned to "alignment_in_bytes". + HeapWord* allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes); + // Undo the last allocation in the buffer, which is required to be of the // "obj" of the given "word_sz". void undo_allocation(HeapWord* obj, size_t word_sz) { diff --git a/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp new file mode 100644 index 0000000000000000000000000000000000000000..352ce05a3e56c710932a04237eb971fb15012752 --- /dev/null +++ b/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP + +#include "gc_implementation/shared/parGCAllocBuffer.hpp" +#include "gc_interface/collectedHeap.inline.hpp" + +HeapWord* ParGCAllocBuffer::allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes) { + + HeapWord* res = CollectedHeap::align_allocation_or_fail(_top, _end, alignment_in_bytes); + if (res == NULL) { + return NULL; + } + + // Set _top so that allocate(), which expects _top to be correctly set, + // can be used below. + _top = res; + return allocate(word_sz); +} + +#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP diff --git a/src/share/vm/gc_interface/collectedHeap.hpp b/src/share/vm/gc_interface/collectedHeap.hpp index 4b56f00e746ed89349d711569c4d199270b965f0..0c4dd65a0069a47f7b444ea70b14c0b1eb08d97f 100644 --- a/src/share/vm/gc_interface/collectedHeap.hpp +++ b/src/share/vm/gc_interface/collectedHeap.hpp @@ -351,6 +351,12 @@ class CollectedHeap : public CHeapObj { fill_with_object(start, pointer_delta(end, start), zap); } + // Return the address "addr" aligned by "alignment_in_bytes" if such + // an address is below "end". Return NULL otherwise. + inline static HeapWord* align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes); + // Some heaps may offer a contiguous region for shared non-blocking // allocation, via inlined code (by exporting the address of the top and // end fields defining the extent of the contiguous allocation region.) diff --git a/src/share/vm/gc_interface/collectedHeap.inline.hpp b/src/share/vm/gc_interface/collectedHeap.inline.hpp index 89315a9424fe1765550ca20ed65204a34e2d0f1b..302d0c7cb3add3283725ea1d799bf4fff76ada4b 100644 --- a/src/share/vm/gc_interface/collectedHeap.inline.hpp +++ b/src/share/vm/gc_interface/collectedHeap.inline.hpp @@ -241,6 +241,44 @@ inline void CollectedHeap::oop_iterate_no_header(OopClosure* cl) { oop_iterate(&no_header_cl); } + +inline HeapWord* CollectedHeap::align_allocation_or_fail(HeapWord* addr, + HeapWord* end, + unsigned short alignment_in_bytes) { + if (alignment_in_bytes <= ObjectAlignmentInBytes) { + return addr; + } + + assert(is_ptr_aligned(addr, HeapWordSize), + err_msg("Address " PTR_FORMAT " is not properly aligned.", p2i(addr))); + assert(is_size_aligned(alignment_in_bytes, HeapWordSize), + err_msg("Alignment size %u is incorrect.", alignment_in_bytes)); + + HeapWord* new_addr = (HeapWord*) align_pointer_up(addr, alignment_in_bytes); + size_t padding = pointer_delta(new_addr, addr); + + if (padding == 0) { + return addr; + } + + if (padding < CollectedHeap::min_fill_size()) { + padding += alignment_in_bytes / HeapWordSize; + assert(padding >= CollectedHeap::min_fill_size(), + err_msg("alignment_in_bytes %u is expect to be larger " + "than the minimum object size", alignment_in_bytes)); + new_addr = addr + padding; + } + + assert(new_addr > addr, err_msg("Unexpected arithmetic overflow " + PTR_FORMAT " not greater than " PTR_FORMAT, p2i(new_addr), p2i(addr))); + if(new_addr < end) { + CollectedHeap::fill_with_object(addr, padding); + return new_addr; + } else { + return NULL; + } +} + #ifndef PRODUCT inline bool diff --git a/src/share/vm/memory/defNewGeneration.cpp b/src/share/vm/memory/defNewGeneration.cpp index 148ed38826ae3ab955ba93d390f5a7d83732ca96..ac41f33b459465668de873e9d39dd0459d100cfe 100644 --- a/src/share/vm/memory/defNewGeneration.cpp +++ b/src/share/vm/memory/defNewGeneration.cpp @@ -789,7 +789,7 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) { // Try allocating obj in to-space (unless too old) if (old->age() < tenuring_threshold()) { - obj = (oop) to()->allocate(s); + obj = (oop) to()->allocate_aligned(s); } // Otherwise try allocating obj tenured diff --git a/src/share/vm/memory/space.cpp b/src/share/vm/memory/space.cpp index c9c601e5a42258979fda02991af51d2a945dd10a..317384f23f75a1b9adbee98235e76df9d6298426 100644 --- a/src/share/vm/memory/space.cpp +++ b/src/share/vm/memory/space.cpp @@ -28,6 +28,7 @@ #include "gc_implementation/shared/liveRange.hpp" #include "gc_implementation/shared/markSweep.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" +#include "gc_interface/collectedHeap.inline.hpp" #include "memory/blockOffsetTable.inline.hpp" #include "memory/defNewGeneration.hpp" #include "memory/genCollectedHeap.hpp" @@ -719,6 +720,27 @@ inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size, } while (true); } +HeapWord* ContiguousSpace::allocate_aligned(size_t size) { + assert(Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), "not locked"); + HeapWord* end_value = end(); + + HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end_value, SurvivorAlignmentInBytes); + if (obj == NULL) { + return NULL; + } + + if (pointer_delta(end_value, obj) >= size) { + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_aligned(new_top), + "checking alignment"); + return obj; + } else { + set_top(obj); + return NULL; + } +} + // Requires locking. HeapWord* ContiguousSpace::allocate(size_t size) { return allocate_impl(size, end()); diff --git a/src/share/vm/memory/space.hpp b/src/share/vm/memory/space.hpp index f3723a8fb251f6861c63bea571f28d50898d25a6..7ee65417f2cd0c4f72c5a727307d931f8a51f05e 100644 --- a/src/share/vm/memory/space.hpp +++ b/src/share/vm/memory/space.hpp @@ -526,6 +526,7 @@ class ContiguousSpace: public CompactibleSpace { // Allocation (return NULL if full) virtual HeapWord* allocate(size_t word_size); virtual HeapWord* par_allocate(size_t word_size); + HeapWord* allocate_aligned(size_t word_size); // Iteration void oop_iterate(ExtendedOopClosure* cl); diff --git a/src/share/vm/oops/oop.pcgc.inline.hpp b/src/share/vm/oops/oop.pcgc.inline.hpp index 596bbc6d1233020d58e75869225b0d8213b0a590..8a4603944ea6f0308d6698187306c5f5bd634eba 100644 --- a/src/share/vm/oops/oop.pcgc.inline.hpp +++ b/src/share/vm/oops/oop.pcgc.inline.hpp @@ -54,8 +54,6 @@ inline void oopDesc::follow_contents(ParCompactionManager* cm) { klass()->oop_follow_contents(cm, this); } -// Used by parallel old GC. - inline oop oopDesc::forward_to_atomic(oop p) { assert(ParNewGeneration::is_legal_forward_ptr(p), "illegal forwarding pointer value."); diff --git a/src/share/vm/runtime/arguments.cpp b/src/share/vm/runtime/arguments.cpp index 5d2c3883b65ef8924c19265483b0c6721e9aee10..9dc68c8bd43c4e35c544594a81690ea0221c66ae 100644 --- a/src/share/vm/runtime/arguments.cpp +++ b/src/share/vm/runtime/arguments.cpp @@ -1398,6 +1398,22 @@ bool verify_object_alignment() { (int)ObjectAlignmentInBytes, os::vm_page_size()); return false; } + if(SurvivorAlignmentInBytes == 0) { + SurvivorAlignmentInBytes = ObjectAlignmentInBytes; + } else { + if (!is_power_of_2(SurvivorAlignmentInBytes)) { + jio_fprintf(defaultStream::error_stream(), + "error: SurvivorAlignmentInBytes=%d must be power of 2\n", + (int)SurvivorAlignmentInBytes); + return false; + } + if (SurvivorAlignmentInBytes < ObjectAlignmentInBytes) { + jio_fprintf(defaultStream::error_stream(), + "error: SurvivorAlignmentInBytes=%d must be greater than ObjectAlignmentInBytes=%d \n", + (int)SurvivorAlignmentInBytes, (int)ObjectAlignmentInBytes); + return false; + } + } return true; } diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp index f0c8080b8f3b1ded50f7f589d08fbdce8244f29d..82f61f0ff8c1816cfcf278a738f1dc735a5a50e4 100644 --- a/src/share/vm/runtime/globals.hpp +++ b/src/share/vm/runtime/globals.hpp @@ -3885,6 +3885,9 @@ class CommandLineFlags { product(bool, PrintGCCause, true, \ "Include GC cause in GC logging") \ \ + experimental(intx, SurvivorAlignmentInBytes, 0, \ + "Default survivor space alignment in bytes") \ + \ product(bool , AllowNonVirtualCalls, false, \ "Obey the ACC_SUPER flag and allow invokenonvirtual calls") \ \