8066875: VirtualSpace does not use large pages

Reviewed-by: stefank, tschatzl, anoll, thartmann

src/share/vm/gc_implementation/parallelScavenge/generationSizer.cpp

@@ -66,9 +66,9 @@ void GenerationSizer::initialize_flags() {
 
 void GenerationSizer::initialize_size_info() {
   trace_gen_sizes("ps heap raw");
-  const size_t max_page_sz = os::page_size_for_region(_max_heap_byte_size, 8);
+  const size_t max_page_sz = os::page_size_for_region_aligned(_max_heap_byte_size, 8);
   const size_t min_pages = 4; // 1 for eden + 1 for each survivor + 1 for old
-  const size_t min_page_sz = os::page_size_for_region(_min_heap_byte_size, min_pages);
+  const size_t min_page_sz = os::page_size_for_region_aligned(_min_heap_byte_size, min_pages);
   const size_t page_sz = MIN2(max_page_sz, min_page_sz);
 
   // Can a page size be something else than a power of two?

src/share/vm/gc_implementation/parallelScavenge/parMarkBitMap.cpp

@@ -55,7 +55,7 @@ ParMarkBitMap::initialize(MemRegion covered_region)
 
   const size_t words = bits / BitsPerWord;
   const size_t raw_bytes = words * sizeof(idx_t);
-  const size_t page_sz = os::page_size_for_region(raw_bytes, 10);
+  const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10);
   const size_t granularity = os::vm_allocation_granularity();
   _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity));
 

src/share/vm/gc_implementation/parallelScavenge/psParallelCompact.cpp

@@ -401,7 +401,7 @@ PSVirtualSpace*
 ParallelCompactData::create_vspace(size_t count, size_t element_size)
 {
   const size_t raw_bytes = count * element_size;
-  const size_t page_sz = os::page_size_for_region(raw_bytes, 10);
+  const size_t page_sz = os::page_size_for_region_aligned(raw_bytes, 10);
   const size_t granularity = os::vm_allocation_granularity();
   _reserved_byte_size = align_size_up(raw_bytes, MAX2(page_sz, granularity));
 

src/share/vm/memory/heap.cpp

@@ -100,8 +100,8 @@ bool CodeHeap::reserve(size_t reserved_size, size_t committed_size,
   size_t page_size = os::vm_page_size();
   if (os::can_execute_large_page_memory()) {
     const size_t min_pages = 8;
-    page_size = MIN2(os::page_size_for_region(committed_size, min_pages),
-                     os::page_size_for_region(reserved_size, min_pages));
+    page_size = MIN2(os::page_size_for_region_aligned(committed_size, min_pages),
+                     os::page_size_for_region_aligned(reserved_size, min_pages));
   }
 
   const size_t granularity = os::vm_allocation_granularity();

src/share/vm/runtime/os.cpp

@@ -1315,22 +1315,32 @@ bool os::stack_shadow_pages_available(Thread *thread, methodHandle method) {
   return (sp > (stack_limit + reserved_area));
 }
 
-size_t os::page_size_for_region(size_t region_size, size_t min_pages) {
+size_t os::page_size_for_region(size_t region_size, size_t min_pages, bool must_be_aligned) {
   assert(min_pages > 0, "sanity");
   if (UseLargePages) {
     const size_t max_page_size = region_size / min_pages;
 
     for (size_t i = 0; _page_sizes[i] != 0; ++i) {
       const size_t page_size = _page_sizes[i];
-      if (page_size <= max_page_size && is_size_aligned(region_size, page_size)) {
+      if (page_size <= max_page_size) {
+        if (!must_be_aligned || is_size_aligned(region_size, page_size)) {
           return page_size;
         }
       }
     }
+  }
 
   return vm_page_size();
 }
 
+size_t os::page_size_for_region_aligned(size_t region_size, size_t min_pages) {
+  return page_size_for_region(region_size, min_pages, true);
+}
+
+size_t os::page_size_for_region_unaligned(size_t region_size, size_t min_pages) {
+  return page_size_for_region(region_size, min_pages, false);
+}
+
 #ifndef PRODUCT
 void os::trace_page_sizes(const char* str, const size_t* page_sizes, int count)
 {
@@ -1579,17 +1589,17 @@ class TestOS : AllStatic {
 
   static size_t large_page_size() {
     const size_t large_page_size_example = 4 * M;
-    return os::page_size_for_region(large_page_size_example, 1);
+    return os::page_size_for_region_aligned(large_page_size_example, 1);
   }
 
-  static void test_page_size_for_region() {
+  static void test_page_size_for_region_aligned() {
     if (UseLargePages) {
       const size_t small_page = small_page_size();
       const size_t large_page = large_page_size();
 
       if (large_page > small_page) {
         size_t num_small_pages_in_large = large_page / small_page;
-        size_t page = os::page_size_for_region(large_page, num_small_pages_in_large);
+        size_t page = os::page_size_for_region_aligned(large_page, num_small_pages_in_large);
 
         assert_eq(page, small_page);
       }
@@ -1602,21 +1612,53 @@ class TestOS : AllStatic {
       const size_t large_page = large_page_size();
       if (large_page > small_page) {
         const size_t unaligned_region = large_page + 17;
-        size_t page = os::page_size_for_region(unaligned_region, 1);
+        size_t page = os::page_size_for_region_aligned(unaligned_region, 1);
         assert_eq(page, small_page);
 
         const size_t num_pages = 5;
         const size_t aligned_region = large_page * num_pages;
-        page = os::page_size_for_region(aligned_region, num_pages);
+        page = os::page_size_for_region_aligned(aligned_region, num_pages);
         assert_eq(page, large_page);
       }
     }
   }
 
+  static void test_page_size_for_region_unaligned() {
+    if (UseLargePages) {
+      // Given exact page size, should return that page size.
+      for (size_t i = 0; os::_page_sizes[i] != 0; i++) {
+        size_t expected = os::_page_sizes[i];
+        size_t actual = os::page_size_for_region_unaligned(expected, 1);
+        assert_eq(expected, actual);
+      }
+
+      // Given slightly larger size than a page size, return the page size.
+      for (size_t i = 0; os::_page_sizes[i] != 0; i++) {
+        size_t expected = os::_page_sizes[i];
+        size_t actual = os::page_size_for_region_unaligned(expected + 17, 1);
+        assert_eq(expected, actual);
+      }
+
+      // Given a slightly smaller size than a page size,
+      // return the next smaller page size.
+      if (os::_page_sizes[1] > os::_page_sizes[0]) {
+        size_t expected = os::_page_sizes[0];
+        size_t actual = os::page_size_for_region_unaligned(os::_page_sizes[1] - 17, 1);
+        assert_eq(actual, expected);
+      }
+
+      // Return small page size for values less than a small page.
+      size_t small_page = small_page_size();
+      size_t actual = os::page_size_for_region_unaligned(small_page - 17, 1);
+      assert_eq(small_page, actual);
+    }
+  }
+
  public:
   static void run_tests() {
-    test_page_size_for_region();
+    test_page_size_for_region_aligned();
     test_page_size_for_region_alignment();
+    test_page_size_for_region_unaligned();
   }
 };
 

src/share/vm/runtime/os.hpp

@@ -149,6 +149,7 @@ class os: AllStatic {
   static void   pd_free_memory(char *addr, size_t bytes, size_t alignment_hint);
   static void   pd_realign_memory(char *addr, size_t bytes, size_t alignment_hint);
 
+  static size_t page_size_for_region(size_t region_size, size_t min_pages, bool must_be_aligned);
 
  public:
   static void init(void);                      // Called before command line parsing
@@ -267,8 +268,13 @@ class os: AllStatic {
 
   // Returns the page size to use for a region of memory.
   // region_size / min_pages will always be greater than or equal to the
-  // returned value.
-  static size_t page_size_for_region(size_t region_size, size_t min_pages);
+  // returned value. The returned value will divide region_size.
+  static size_t page_size_for_region_aligned(size_t region_size, size_t min_pages);
+
+  // Returns the page size to use for a region of memory.
+  // region_size / min_pages will always be greater than or equal to the
+  // returned value. The returned value might not divide region_size.
+  static size_t page_size_for_region_unaligned(size_t region_size, size_t min_pages);
 
   // Return the largest page size that can be used
   static size_t max_page_size() {

src/share/vm/runtime/virtualspace.cpp

@@ -53,7 +53,8 @@ ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),
 }
 
 ReservedSpace::ReservedSpace(size_t size) {
-  size_t page_size = os::page_size_for_region(size, 1);
+  // Want to use large pages where possible and pad with small pages.
+  size_t page_size = os::page_size_for_region_unaligned(size, 1);
   bool large_pages = page_size != (size_t)os::vm_page_size();
   // Don't force the alignment to be large page aligned,
   // since that will waste memory.
@@ -372,7 +373,7 @@ VirtualSpace::VirtualSpace() {
 
 
 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {
-  const size_t max_commit_granularity = os::page_size_for_region(rs.size(), 1);
+  const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);
   return initialize_with_granularity(rs, committed_size, max_commit_granularity);
 }
 
@@ -995,7 +996,7 @@ class TestVirtualSpace : AllStatic {
     case Disable:
       return vs.initialize_with_granularity(rs, 0, os::vm_page_size());
     case Commit:
-      return vs.initialize_with_granularity(rs, 0, os::page_size_for_region(rs.size(), 1));
+      return vs.initialize_with_granularity(rs, 0, os::page_size_for_region_unaligned(rs.size(), 1));
     }
   }