diff --git a/src/os/linux/vm/os_linux.cpp b/src/os/linux/vm/os_linux.cpp index c59b702c8f3b4586d261506b1cad7fe884594845..49b8dc1fcd71f7e31b95c96accc4be368df1e225 100644 --- a/src/os/linux/vm/os_linux.cpp +++ b/src/os/linux/vm/os_linux.cpp @@ -3433,22 +3433,66 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_only(size_t bytes, char* req_ return addr; } +// Helper for os::Linux::reserve_memory_special_huge_tlbfs_mixed(). +// Allocate (using mmap, NO_RESERVE, with small pages) at either a given request address +// (req_addr != NULL) or with a given alignment. +// - bytes shall be a multiple of alignment. +// - req_addr can be NULL. If not NULL, it must be a multiple of alignment. +// - alignment sets the alignment at which memory shall be allocated. +// It must be a multiple of allocation granularity. +// Returns address of memory or NULL. If req_addr was not NULL, will only return +// req_addr or NULL. +static char* anon_mmap_aligned(size_t bytes, size_t alignment, char* req_addr) { + + size_t extra_size = bytes; + if (req_addr == NULL && alignment > 0) { + extra_size += alignment; + } + + char* start = (char*) ::mmap(req_addr, extra_size, PROT_NONE, + MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, + -1, 0); + if (start == MAP_FAILED) { + start = NULL; + } else { + if (req_addr != NULL) { + if (start != req_addr) { + ::munmap(start, extra_size); + start = NULL; + } + } else { + char* const start_aligned = (char*) align_ptr_up(start, alignment); + char* const end_aligned = start_aligned + bytes; + char* const end = start + extra_size; + if (start_aligned > start) { + ::munmap(start, start_aligned - start); + } + if (end_aligned < end) { + ::munmap(end_aligned, end - end_aligned); + } + start = start_aligned; + } + } + return start; + +} + +// Reserve memory using mmap(MAP_HUGETLB). +// - bytes shall be a multiple of alignment. +// - req_addr can be NULL. If not NULL, it must be a multiple of alignment. +// - alignment sets the alignment at which memory shall be allocated. +// It must be a multiple of allocation granularity. +// Returns address of memory or NULL. If req_addr was not NULL, will only return +// req_addr or NULL. char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t alignment, char* req_addr, bool exec) { size_t large_page_size = os::large_page_size(); - assert(bytes >= large_page_size, "Shouldn't allocate large pages for small sizes"); - // Allocate small pages. + assert(is_ptr_aligned(req_addr, alignment), "Must be"); + assert(is_size_aligned(bytes, alignment), "Must be"); - char* start; - if (req_addr != NULL) { - assert(is_ptr_aligned(req_addr, alignment), "Must be"); - assert(is_size_aligned(bytes, alignment), "Must be"); - start = os::reserve_memory(bytes, req_addr); - assert(start == NULL || start == req_addr, "Must be"); - } else { - start = os::reserve_memory_aligned(bytes, alignment); - } + // First reserve - but not commit - the address range in small pages. + char* const start = anon_mmap_aligned(bytes, alignment, req_addr); if (start == NULL) { return NULL; @@ -3456,13 +3500,6 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t al assert(is_ptr_aligned(start, alignment), "Must be"); - if (MemTracker::tracking_level() > NMT_minimal) { - // os::reserve_memory_special will record this memory area. - // Need to release it here to prevent overlapping reservations. - Tracker tkr = MemTracker::get_virtual_memory_release_tracker(); - tkr.record((address)start, bytes); - } - char* end = start + bytes; // Find the regions of the allocated chunk that can be promoted to large pages. @@ -3482,9 +3519,9 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t al int prot = exec ? PROT_READ|PROT_WRITE|PROT_EXEC : PROT_READ|PROT_WRITE; - void* result; + // Commit small-paged leading area. if (start != lp_start) { result = ::mmap(start, lp_start - start, prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, @@ -3495,11 +3532,12 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t al } } + // Commit large-paged area. result = ::mmap(lp_start, lp_bytes, prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED|MAP_HUGETLB, -1, 0); if (result == MAP_FAILED) { - warn_on_large_pages_failure(req_addr, bytes, errno); + warn_on_large_pages_failure(lp_start, lp_bytes, errno); // If the mmap above fails, the large pages region will be unmapped and we // have regions before and after with small pages. Release these regions. // @@ -3512,6 +3550,7 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t al return NULL; } + // Commit small-paged trailing area. if (lp_end != end) { result = ::mmap(lp_end, end - lp_end, prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, @@ -3528,7 +3567,7 @@ char* os::Linux::reserve_memory_special_huge_tlbfs_mixed(size_t bytes, size_t al char* os::Linux::reserve_memory_special_huge_tlbfs(size_t bytes, size_t alignment, char* req_addr, bool exec) { assert(UseLargePages && UseHugeTLBFS, "only for Huge TLBFS large pages"); assert(is_ptr_aligned(req_addr, alignment), "Must be"); - assert(is_power_of_2(alignment), "Must be"); + assert(is_size_aligned(alignment, os::vm_allocation_granularity()), "Must be"); assert(is_power_of_2(os::large_page_size()), "Must be"); assert(bytes >= os::large_page_size(), "Shouldn't allocate large pages for small sizes"); @@ -6102,47 +6141,100 @@ class TestReserveMemorySpecial : AllStatic { } } - static void test_reserve_memory_special_huge_tlbfs_mixed(size_t size, size_t alignment) { - if (!UseHugeTLBFS) { - return; - } - - test_log("test_reserve_memory_special_huge_tlbfs_mixed(" SIZE_FORMAT ", " SIZE_FORMAT ")", - size, alignment); - - assert(size >= os::large_page_size(), "Incorrect input to test"); + static void test_reserve_memory_special_huge_tlbfs_mixed() { + size_t lp = os::large_page_size(); + size_t ag = os::vm_allocation_granularity(); - char* addr = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, NULL, false); + // sizes to test + const size_t sizes[] = { + lp, lp + ag, lp + lp / 2, lp * 2, + lp * 2 + ag, lp * 2 - ag, lp * 2 + lp / 2, + lp * 10, lp * 10 + lp / 2 + }; + const int num_sizes = sizeof(sizes) / sizeof(size_t); + + // For each size/alignment combination, we test three scenarios: + // 1) with req_addr == NULL + // 2) with a non-null req_addr at which we expect to successfully allocate + // 3) with a non-null req_addr which contains a pre-existing mapping, at which we + // expect the allocation to either fail or to ignore req_addr + + // Pre-allocate two areas; they shall be as large as the largest allocation + // and aligned to the largest alignment we will be testing. + const size_t mapping_size = sizes[num_sizes - 1] * 2; + char* const mapping1 = (char*) ::mmap(NULL, mapping_size, + PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, + -1, 0); + assert(mapping1 != MAP_FAILED, "should work"); + + char* const mapping2 = (char*) ::mmap(NULL, mapping_size, + PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, + -1, 0); + assert(mapping2 != MAP_FAILED, "should work"); + + // Unmap the first mapping, but leave the second mapping intact: the first + // mapping will serve as a value for a "good" req_addr (case 2). The second + // mapping, still intact, as "bad" req_addr (case 3). + ::munmap(mapping1, mapping_size); + + // Case 1 + test_log("%s, req_addr NULL:", __FUNCTION__); + test_log("size align result"); + + for (int i = 0; i < num_sizes; i++) { + const size_t size = sizes[i]; + for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { + char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, NULL, false); + test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " -> " PTR_FORMAT " %s", + size, alignment, p, (p != NULL ? "" : "(failed)")); + if (p != NULL) { + assert(is_ptr_aligned(p, alignment), "must be"); + small_page_write(p, size); + os::Linux::release_memory_special_huge_tlbfs(p, size); + } + } + } - if (addr != NULL) { - small_page_write(addr, size); + // Case 2 + test_log("%s, req_addr non-NULL:", __FUNCTION__); + test_log("size align req_addr result"); - os::Linux::release_memory_special_huge_tlbfs(addr, size); + for (int i = 0; i < num_sizes; i++) { + const size_t size = sizes[i]; + for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { + char* const req_addr = (char*) align_ptr_up(mapping1, alignment); + char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false); + test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " -> " PTR_FORMAT " %s", + size, alignment, req_addr, p, + ((p != NULL ? (p == req_addr ? "(exact match)" : "") : "(failed)"))); + if (p != NULL) { + assert(p == req_addr, "must be"); + small_page_write(p, size); + os::Linux::release_memory_special_huge_tlbfs(p, size); + } + } } - } - static void test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(size_t size) { - size_t lp = os::large_page_size(); - size_t ag = os::vm_allocation_granularity(); + // Case 3 + test_log("%s, req_addr non-NULL with preexisting mapping:", __FUNCTION__); + test_log("size align req_addr result"); - for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { - test_reserve_memory_special_huge_tlbfs_mixed(size, alignment); + for (int i = 0; i < num_sizes; i++) { + const size_t size = sizes[i]; + for (size_t alignment = ag; is_size_aligned(size, alignment); alignment *= 2) { + char* const req_addr = (char*) align_ptr_up(mapping2, alignment); + char* p = os::Linux::reserve_memory_special_huge_tlbfs_mixed(size, alignment, req_addr, false); + test_log(SIZE_FORMAT_HEX " " SIZE_FORMAT_HEX " " PTR_FORMAT " -> " PTR_FORMAT " %s", + size, alignment, req_addr, p, + ((p != NULL ? "" : "(failed)"))); + // as the area around req_addr contains already existing mappings, the API should always + // return NULL (as per contract, it cannot return another address) + assert(p == NULL, "must be"); + } } - } - static void test_reserve_memory_special_huge_tlbfs_mixed() { - size_t lp = os::large_page_size(); - size_t ag = os::vm_allocation_granularity(); + ::munmap(mapping2, mapping_size); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp + ag); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp + lp / 2); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 + ag); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 - ag); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 2 + lp / 2); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 10); - test_reserve_memory_special_huge_tlbfs_mixed_all_alignments(lp * 10 + lp / 2); } static void test_reserve_memory_special_huge_tlbfs() {