7103665: HeapWord*ParallelScavengeHeap::failed_mem_allocate(unsigned long,bool)+0x97

Summary: Make sure that MutableNUMASpace::ensure_parsability() only calls CollectedHeap::fill_with_object() with valid sizes and make sure CollectedHeap::filler_array_max_size() returns a value that can be converted to an int without overflow
Reviewed-by: azeemj, jmasa, iveresov

src/share/vm/gc_implementation/shared/mutableNUMASpace.cpp

 
 /*
- * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,29 +91,37 @@ void MutableNUMASpace::ensure_parsability() {
     MutableSpace *s = ls->space();
     if (s->top() < top()) { // For all spaces preceding the one containing top()
       if (s->free_in_words() > 0) {
-        size_t area_touched_words = pointer_delta(s->end(), s->top());
-        CollectedHeap::fill_with_object(s->top(), area_touched_words);
+        intptr_t cur_top = (intptr_t)s->top();
+        size_t words_left_to_fill = pointer_delta(s->end(), s->top());;
+        while (words_left_to_fill > 0) {
+          size_t words_to_fill = MIN2(words_left_to_fill, CollectedHeap::filler_array_max_size());
+          assert(words_to_fill >= CollectedHeap::min_fill_size(),
+            err_msg("Remaining size ("SIZE_FORMAT ") is too small to fill (based on " SIZE_FORMAT " and " SIZE_FORMAT ")",
+            words_to_fill, words_left_to_fill, CollectedHeap::filler_array_max_size()));
+          CollectedHeap::fill_with_object((HeapWord*)cur_top, words_to_fill);
+          if (!os::numa_has_static_binding()) {
+            size_t touched_words = words_to_fill;
 #ifndef ASSERT
-        if (!ZapUnusedHeapArea) {
-          area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
-                                    area_touched_words);
-        }
+            if (!ZapUnusedHeapArea) {
+              touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)),
+                touched_words);
+            }
 #endif
-        if (!os::numa_has_static_binding()) {
-          MemRegion invalid;
-          HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size());
-          HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words),
-                                                       os::vm_page_size());
-          if (crossing_start != crossing_end) {
-            // If object header crossed a small page boundary we mark the area
-            // as invalid rounding it to a page_size().
-            HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom());
-            HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()),
-                                 s->end());
-            invalid = MemRegion(start, end);
-          }
+            MemRegion invalid;
+            HeapWord *crossing_start = (HeapWord*)round_to(cur_top, os::vm_page_size());
+            HeapWord *crossing_end = (HeapWord*)round_to(cur_top + touched_words, os::vm_page_size());
+            if (crossing_start != crossing_end) {
+              // If object header crossed a small page boundary we mark the area
+              // as invalid rounding it to a page_size().
+              HeapWord *start = MAX2((HeapWord*)round_down(cur_top, page_size()), s->bottom());
+              HeapWord *end = MIN2((HeapWord*)round_to(cur_top + touched_words, page_size()), s->end());
+              invalid = MemRegion(start, end);
+            }
 
-          ls->add_invalid_region(invalid);
+            ls->add_invalid_region(invalid);
+          }
+          cur_top = cur_top + (words_to_fill * HeapWordSize);
+          words_left_to_fill -= words_to_fill;
         }
       }
     } else {

src/share/vm/gc_interface/collectedHeap.cpp

@@ -85,7 +85,7 @@ CollectedHeap::CollectedHeap() : _n_par_threads(0)
   const size_t max_len = size_t(arrayOopDesc::max_array_length(T_INT));
   const size_t elements_per_word = HeapWordSize / sizeof(jint);
   _filler_array_max_size = align_object_size(filler_array_hdr_size() +
-                                             max_len * elements_per_word);
+                                             max_len / elements_per_word);
 
   _barrier_set = NULL;
   _is_gc_active = false;
@@ -303,10 +303,6 @@ size_t CollectedHeap::filler_array_min_size() {
   return align_object_size(filler_array_hdr_size()); // align to MinObjAlignment
 }
 
-size_t CollectedHeap::filler_array_max_size() {
-  return _filler_array_max_size;
-}
-
 #ifdef ASSERT
 void CollectedHeap::fill_args_check(HeapWord* start, size_t words)
 {
@@ -333,6 +329,7 @@ CollectedHeap::fill_with_array(HeapWord* start, size_t words, bool zap)
 
   const size_t payload_size = words - filler_array_hdr_size();
   const size_t len = payload_size * HeapWordSize / sizeof(jint);
+  assert((int)len >= 0, err_msg("size too large " SIZE_FORMAT " becomes %d", words, (int)len));
 
   // Set the length first for concurrent GC.
   ((arrayOop)start)->set_length((int)len);

src/share/vm/gc_interface/collectedHeap.hpp

@@ -128,7 +128,6 @@ class CollectedHeap : public CHeapObj {
   // Reinitialize tlabs before resuming mutators.
   virtual void resize_all_tlabs();
 
- protected:
   // Allocate from the current thread's TLAB, with broken-out slow path.
   inline static HeapWord* allocate_from_tlab(Thread* thread, size_t size);
   static HeapWord* allocate_from_tlab_slow(Thread* thread, size_t size);
@@ -169,7 +168,6 @@ class CollectedHeap : public CHeapObj {
   // Filler object utilities.
   static inline size_t filler_array_hdr_size();
   static inline size_t filler_array_min_size();
-  static inline size_t filler_array_max_size();
 
   DEBUG_ONLY(static void fill_args_check(HeapWord* start, size_t words);)
   DEBUG_ONLY(static void zap_filler_array(HeapWord* start, size_t words, bool zap = true);)
@@ -197,6 +195,10 @@ class CollectedHeap : public CHeapObj {
     G1CollectedHeap
   };
 
+  static inline size_t filler_array_max_size() {
+    return _filler_array_max_size;
+  }
+
   virtual CollectedHeap::Name kind() const { return CollectedHeap::Abstract; }
 
   /**