7114678: G1: various small fixes, code cleanup, and refactoring

Summary: Various cleanups as a prelude to introducing iterators for HeapRegions.
Reviewed-by: johnc, brutisso

src/share/vm/gc_implementation/g1/collectionSetChooser.hpp

@@ -153,4 +153,47 @@ public:
   void verify() PRODUCT_RETURN;
 };
 
+class CSetChooserParUpdater : public StackObj {
+private:
+  CollectionSetChooser* _chooser;
+  bool _parallel;
+  uint _chunk_size;
+  uint _cur_chunk_idx;
+  uint _cur_chunk_end;
+  uint _regions_added;
+  size_t _reclaimable_bytes_added;
+
+public:
+  CSetChooserParUpdater(CollectionSetChooser* chooser,
+                        bool parallel, uint chunk_size) :
+    _chooser(chooser), _parallel(parallel), _chunk_size(chunk_size),
+    _cur_chunk_idx(0), _cur_chunk_end(0),
+    _regions_added(0), _reclaimable_bytes_added(0) { }
+
+  ~CSetChooserParUpdater() {
+    if (_parallel && _regions_added > 0) {
+      _chooser->update_totals(_regions_added, _reclaimable_bytes_added);
+    }
+  }
+
+  void add_region(HeapRegion* hr) {
+    if (_parallel) {
+      if (_cur_chunk_idx == _cur_chunk_end) {
+        _cur_chunk_idx = _chooser->claim_array_chunk(_chunk_size);
+        _cur_chunk_end = _cur_chunk_idx + _chunk_size;
+      }
+      assert(_cur_chunk_idx < _cur_chunk_end, "invariant");
+      _chooser->set_region(_cur_chunk_idx, hr);
+      _cur_chunk_idx += 1;
+    } else {
+      _chooser->add_region(hr);
+    }
+    _regions_added += 1;
+    _reclaimable_bytes_added += hr->reclaimable_bytes();
+  }
+
+  bool should_add(HeapRegion* hr) { return _chooser->should_add(hr); }
+};
+
 #endif // SHARE_VM_GC_IMPLEMENTATION_G1_COLLECTIONSETCHOOSER_HPP
+

src/share/vm/gc_implementation/g1/concurrentMark.cpp

@@ -1226,9 +1226,7 @@ protected:
     } else {
       // Starts humongous case: calculate how many regions are part of
       // this humongous region and then set the bit range.
-      G1CollectedHeap* g1h = G1CollectedHeap::heap();
-      HeapRegion *last_hr = g1h->heap_region_containing_raw(hr->end() - 1);
-      BitMap::idx_t end_index = (BitMap::idx_t) last_hr->hrs_index() + 1;
+      BitMap::idx_t end_index = (BitMap::idx_t) hr->last_hc_index();
       _region_bm->par_at_put_range(index, end_index, true);
     }
   }
@@ -1645,10 +1643,12 @@ public:
   size_t freed_bytes() { return _freed_bytes; }
 
   bool doHeapRegion(HeapRegion *hr) {
+    if (hr->continuesHumongous()) {
+      return false;
+    }
     // We use a claim value of zero here because all regions
     // were claimed with value 1 in the FinalCount task.
-    hr->reset_gc_time_stamp();
-    if (!hr->continuesHumongous()) {
+    _g1->reset_gc_time_stamps(hr);
     double start = os::elapsedTime();
     _regions_claimed++;
     hr->note_end_of_marking();
@@ -1665,7 +1665,6 @@ public:
     if (region_time > _max_region_time) {
       _max_region_time = region_time;
     }
-    }
     return false;
   }
 
@@ -1881,6 +1880,7 @@ void ConcurrentMark::cleanup() {
   } else {
     g1_par_note_end_task.work(0);
   }
+  g1h->check_gc_time_stamps();
 
   if (!cleanup_list_is_empty()) {
     // The cleanup list is not empty, so we'll have to process it
@@ -2449,24 +2449,8 @@ public:
     } else {
       HeapRegion* hr  = _g1h->heap_region_containing(obj);
       guarantee(hr != NULL, "invariant");
-      bool over_tams = false;
-      bool marked = false;
-
-      switch (_vo) {
-        case VerifyOption_G1UsePrevMarking:
-          over_tams = hr->obj_allocated_since_prev_marking(obj);
-          marked = _g1h->isMarkedPrev(obj);
-          break;
-        case VerifyOption_G1UseNextMarking:
-          over_tams = hr->obj_allocated_since_next_marking(obj);
-          marked = _g1h->isMarkedNext(obj);
-          break;
-        case VerifyOption_G1UseMarkWord:
-          marked = obj->is_gc_marked();
-          break;
-        default:
-          ShouldNotReachHere();
-      }
+      bool over_tams = _g1h->allocated_since_marking(obj, hr, _vo);
+      bool marked = _g1h->is_marked(obj, _vo);
 
       if (over_tams) {
         str = " >";
@@ -2502,24 +2486,8 @@ public:
     _out(out), _vo(vo), _all(all), _hr(hr) { }
 
   void do_object(oop o) {
-    bool over_tams = false;
-    bool marked = false;
-
-    switch (_vo) {
-      case VerifyOption_G1UsePrevMarking:
-        over_tams = _hr->obj_allocated_since_prev_marking(o);
-        marked = _g1h->isMarkedPrev(o);
-        break;
-      case VerifyOption_G1UseNextMarking:
-        over_tams = _hr->obj_allocated_since_next_marking(o);
-        marked = _g1h->isMarkedNext(o);
-        break;
-      case VerifyOption_G1UseMarkWord:
-        marked = o->is_gc_marked();
-        break;
-      default:
-        ShouldNotReachHere();
-    }
+    bool over_tams = _g1h->allocated_since_marking(o, _hr, _vo);
+    bool marked = _g1h->is_marked(o, _vo);
     bool print_it = _all || over_tams || marked;
 
     if (print_it) {
@@ -2533,6 +2501,7 @@ public:
 
 class PrintReachableRegionClosure : public HeapRegionClosure {
 private:
+  G1CollectedHeap* _g1h;
   outputStream*    _out;
   VerifyOption     _vo;
   bool             _all;
@@ -2542,23 +2511,7 @@ public:
     HeapWord* b = hr->bottom();
     HeapWord* e = hr->end();
     HeapWord* t = hr->top();
-    HeapWord* p = NULL;
-
-    switch (_vo) {
-      case VerifyOption_G1UsePrevMarking:
-        p = hr->prev_top_at_mark_start();
-        break;
-      case VerifyOption_G1UseNextMarking:
-        p = hr->next_top_at_mark_start();
-        break;
-      case VerifyOption_G1UseMarkWord:
-        // When we are verifying marking using the mark word
-        // TAMS has no relevance.
-        assert(p == NULL, "post-condition");
-        break;
-      default:
-        ShouldNotReachHere();
-    }
+    HeapWord* p = _g1h->top_at_mark_start(hr, _vo);
     _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" "
                    "TAMS: "PTR_FORMAT, b, e, t, p);
     _out->cr();
@@ -2580,20 +2533,9 @@ public:
   PrintReachableRegionClosure(outputStream* out,
                               VerifyOption  vo,
                               bool          all) :
-    _out(out), _vo(vo), _all(all) { }
+    _g1h(G1CollectedHeap::heap()), _out(out), _vo(vo), _all(all) { }
 };
 
-static const char* verify_option_to_tams(VerifyOption vo) {
-  switch (vo) {
-    case VerifyOption_G1UsePrevMarking:
-      return "PTAMS";
-    case VerifyOption_G1UseNextMarking:
-      return "NTAMS";
-    default:
-      return "NONE";
-  }
-}
-
 void ConcurrentMark::print_reachable(const char* str,
                                      VerifyOption vo,
                                      bool all) {
@@ -2622,7 +2564,7 @@ void ConcurrentMark::print_reachable(const char* str,
   }
 
   outputStream* out = &fout;
-  out->print_cr("-- USING %s", verify_option_to_tams(vo));
+  out->print_cr("-- USING %s", _g1h->top_at_mark_start_str(vo));
   out->cr();
 
   out->print_cr("--- ITERATING OVER REGIONS");

src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

@@ -1149,13 +1149,16 @@ HeapWord* G1CollectedHeap::attempt_allocation_at_safepoint(size_t word_size,
 }
 
 class PostMCRemSetClearClosure: public HeapRegionClosure {
+  G1CollectedHeap* _g1h;
   ModRefBarrierSet* _mr_bs;
 public:
-  PostMCRemSetClearClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}
+  PostMCRemSetClearClosure(G1CollectedHeap* g1h, ModRefBarrierSet* mr_bs) :
+    _g1h(g1h), _mr_bs(mr_bs) { }
   bool doHeapRegion(HeapRegion* r) {
-    r->reset_gc_time_stamp();
-    if (r->continuesHumongous())
+    if (r->continuesHumongous()) {
       return false;
+    }
+    _g1h->reset_gc_time_stamps(r);
     HeapRegionRemSet* hrrs = r->rem_set();
     if (hrrs != NULL) hrrs->clear();
     // You might think here that we could clear just the cards
@@ -1168,19 +1171,10 @@ public:
   }
 };
 
-
-class PostMCRemSetInvalidateClosure: public HeapRegionClosure {
-  ModRefBarrierSet* _mr_bs;
-public:
-  PostMCRemSetInvalidateClosure(ModRefBarrierSet* mr_bs) : _mr_bs(mr_bs) {}
-  bool doHeapRegion(HeapRegion* r) {
-    if (r->continuesHumongous()) return false;
-    if (r->used_region().word_size() != 0) {
-      _mr_bs->invalidate(r->used_region(), true /*whole heap*/);
-    }
-    return false;
-  }
-};
+void G1CollectedHeap::clear_rsets_post_compaction() {
+  PostMCRemSetClearClosure rs_clear(this, mr_bs());
+  heap_region_iterate(&rs_clear);
+}
 
 class RebuildRSOutOfRegionClosure: public HeapRegionClosure {
   G1CollectedHeap*   _g1h;
@@ -1229,7 +1223,7 @@ public:
       if (!hr->isHumongous()) {
         _hr_printer->post_compaction(hr, G1HRPrinter::Old);
       } else if (hr->startsHumongous()) {
-        if (hr->capacity() == HeapRegion::GrainBytes) {
+        if (hr->region_num() == 1) {
           // single humongous region
           _hr_printer->post_compaction(hr, G1HRPrinter::SingleHumongous);
         } else {
@@ -1247,6 +1241,11 @@ public:
     : _hr_printer(hr_printer) { }
 };
 
+void G1CollectedHeap::print_hrs_post_compaction() {
+  PostCompactionPrinterClosure cl(hr_printer());
+  heap_region_iterate(&cl);
+}
+
 bool G1CollectedHeap::do_collection(bool explicit_gc,
                                     bool clear_all_soft_refs,
                                     size_t word_size) {
@@ -1402,8 +1401,8 @@ bool G1CollectedHeap::do_collection(bool explicit_gc,
     // Since everything potentially moved, we will clear all remembered
     // sets, and clear all cards.  Later we will rebuild remebered
     // sets. We will also reset the GC time stamps of the regions.
-    PostMCRemSetClearClosure rs_clear(mr_bs());
-    heap_region_iterate(&rs_clear);
+    clear_rsets_post_compaction();
+    check_gc_time_stamps();
 
     // Resize the heap if necessary.
     resize_if_necessary_after_full_collection(explicit_gc ? 0 : word_size);
@@ -1413,9 +1412,7 @@ bool G1CollectedHeap::do_collection(bool explicit_gc,
       // that all the COMMIT / UNCOMMIT events are generated before
       // the end GC event.
 
-      PostCompactionPrinterClosure cl(hr_printer());
-      heap_region_iterate(&cl);
-
+      print_hrs_post_compaction();
       _hr_printer.end_gc(true /* full */, (size_t) total_collections());
     }
 
@@ -2263,6 +2260,51 @@ size_t G1CollectedHeap::capacity() const {
   return _g1_committed.byte_size();
 }
 
+void G1CollectedHeap::reset_gc_time_stamps(HeapRegion* hr) {
+  assert(!hr->continuesHumongous(), "pre-condition");
+  hr->reset_gc_time_stamp();
+  if (hr->startsHumongous()) {
+    uint first_index = hr->hrs_index() + 1;
+    uint last_index = hr->last_hc_index();
+    for (uint i = first_index; i < last_index; i += 1) {
+      HeapRegion* chr = region_at(i);
+      assert(chr->continuesHumongous(), "sanity");
+      chr->reset_gc_time_stamp();
+    }
+  }
+}
+
+#ifndef PRODUCT
+class CheckGCTimeStampsHRClosure : public HeapRegionClosure {
+private:
+  unsigned _gc_time_stamp;
+  bool _failures;
+
+public:
+  CheckGCTimeStampsHRClosure(unsigned gc_time_stamp) :
+    _gc_time_stamp(gc_time_stamp), _failures(false) { }
+
+  virtual bool doHeapRegion(HeapRegion* hr) {
+    unsigned region_gc_time_stamp = hr->get_gc_time_stamp();
+    if (_gc_time_stamp != region_gc_time_stamp) {
+      gclog_or_tty->print_cr("Region "HR_FORMAT" has GC time stamp = %d, "
+                             "expected %d", HR_FORMAT_PARAMS(hr),
+                             region_gc_time_stamp, _gc_time_stamp);
+      _failures = true;
+    }
+    return false;
+  }
+
+  bool failures() { return _failures; }
+};
+
+void G1CollectedHeap::check_gc_time_stamps() {
+  CheckGCTimeStampsHRClosure cl(_gc_time_stamp);
+  heap_region_iterate(&cl);
+  guarantee(!cl.failures(), "all GC time stamps should have been reset");
+}
+#endif // PRODUCT
+
 void G1CollectedHeap::iterate_dirty_card_closure(CardTableEntryClosure* cl,
                                                  DirtyCardQueue* into_cset_dcq,
                                                  bool concurrent,
@@ -2530,7 +2572,7 @@ public:
   IterateOopClosureRegionClosure(MemRegion mr, OopClosure* cl)
     : _mr(mr), _cl(cl) {}
   bool doHeapRegion(HeapRegion* r) {
-    if (! r->continuesHumongous()) {
+    if (!r->continuesHumongous()) {
       r->oop_iterate(_cl);
     }
     return false;
@@ -2601,14 +2643,9 @@ void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {
   _hrs.iterate(cl);
 }
 
-void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,
-                                               HeapRegionClosure* cl) const {
-  _hrs.iterate_from(r, cl);
-}
-
 void
 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,
-                                                 uint worker,
+                                                 uint worker_id,
                                                  uint no_of_par_workers,
                                                  jint claim_value) {
   const uint regions = n_regions();
@@ -2619,7 +2656,9 @@ G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,
          no_of_par_workers == workers()->total_workers(),
          "Non dynamic should use fixed number of workers");
   // try to spread out the starting points of the workers
-  const uint start_index = regions / max_workers * worker;
+  const HeapRegion* start_hr =
+                        start_region_for_worker(worker_id, no_of_par_workers);
+  const uint start_index = start_hr->hrs_index();
 
   // each worker will actually look at all regions
   for (uint count = 0; count < regions; ++count) {
@@ -2861,6 +2900,17 @@ HeapRegion* G1CollectedHeap::start_cset_region_for_worker(int worker_i) {
   return result;
 }
 
+HeapRegion* G1CollectedHeap::start_region_for_worker(uint worker_i,
+                                                     uint no_of_par_workers) {
+  uint worker_num =
+           G1CollectedHeap::use_parallel_gc_threads() ? no_of_par_workers : 1U;
+  assert(UseDynamicNumberOfGCThreads ||
+         no_of_par_workers == workers()->total_workers(),
+         "Non dynamic should use fixed number of workers");
+  const uint start_index = n_regions() * worker_i / worker_num;
+  return region_at(start_index);
+}
+
 void G1CollectedHeap::collection_set_iterate(HeapRegionClosure* cl) {
   HeapRegion* r = g1_policy()->collection_set();
   while (r != NULL) {
@@ -2974,6 +3024,51 @@ void G1CollectedHeap::prepare_for_verify() {
   g1_rem_set()->prepare_for_verify();
 }
 
+bool G1CollectedHeap::allocated_since_marking(oop obj, HeapRegion* hr,
+                                              VerifyOption vo) {
+  switch (vo) {
+  case VerifyOption_G1UsePrevMarking:
+    return hr->obj_allocated_since_prev_marking(obj);
+  case VerifyOption_G1UseNextMarking:
+    return hr->obj_allocated_since_next_marking(obj);
+  case VerifyOption_G1UseMarkWord:
+    return false;
+  default:
+    ShouldNotReachHere();
+  }
+  return false; // keep some compilers happy
+}
+
+HeapWord* G1CollectedHeap::top_at_mark_start(HeapRegion* hr, VerifyOption vo) {
+  switch (vo) {
+  case VerifyOption_G1UsePrevMarking: return hr->prev_top_at_mark_start();
+  case VerifyOption_G1UseNextMarking: return hr->next_top_at_mark_start();
+  case VerifyOption_G1UseMarkWord:    return NULL;
+  default:                            ShouldNotReachHere();
+  }
+  return NULL; // keep some compilers happy
+}
+
+bool G1CollectedHeap::is_marked(oop obj, VerifyOption vo) {
+  switch (vo) {
+  case VerifyOption_G1UsePrevMarking: return isMarkedPrev(obj);
+  case VerifyOption_G1UseNextMarking: return isMarkedNext(obj);
+  case VerifyOption_G1UseMarkWord:    return obj->is_gc_marked();
+  default:                            ShouldNotReachHere();
+  }
+  return false; // keep some compilers happy
+}
+
+const char* G1CollectedHeap::top_at_mark_start_str(VerifyOption vo) {
+  switch (vo) {
+  case VerifyOption_G1UsePrevMarking: return "PTAMS";
+  case VerifyOption_G1UseNextMarking: return "NTAMS";
+  case VerifyOption_G1UseMarkWord:    return "NONE";
+  default:                            ShouldNotReachHere();
+  }
+  return NULL; // keep some compilers happy
+}
+
 class VerifyLivenessOopClosure: public OopClosure {
   G1CollectedHeap* _g1h;
   VerifyOption _vo;
@@ -3078,8 +3173,6 @@ public:
   }
 
   bool doHeapRegion(HeapRegion* r) {
-    guarantee(_par || r->claim_value() == HeapRegion::InitialClaimValue,
-              "Should be unclaimed at verify points.");
     if (!r->continuesHumongous()) {
       bool failures = false;
       r->verify(_vo, &failures);
@@ -5612,19 +5705,18 @@ void G1CollectedHeap::free_humongous_region(HeapRegion* hr,
   size_t hr_capacity = hr->capacity();
   size_t hr_pre_used = 0;
   _humongous_set.remove_with_proxy(hr, humongous_proxy_set);
+  // We need to read this before we make the region non-humongous,
+  // otherwise the information will be gone.
+  uint last_index = hr->last_hc_index();
   hr->set_notHumongous();
   free_region(hr, &hr_pre_used, free_list, par);
 
   uint i = hr->hrs_index() + 1;
-  uint num = 1;
-  while (i < n_regions()) {
+  while (i < last_index) {
     HeapRegion* curr_hr = region_at(i);
-    if (!curr_hr->continuesHumongous()) {
-      break;
-    }
+    assert(curr_hr->continuesHumongous(), "invariant");
     curr_hr->set_notHumongous();
     free_region(curr_hr, &hr_pre_used, free_list, par);
-    num += 1;
     i += 1;
   }
   assert(hr_pre_used == hr_used,
@@ -5732,7 +5824,6 @@ void G1CollectedHeap::verify_dirty_young_list(HeapRegion* head) {
 
 void G1CollectedHeap::verify_dirty_young_regions() {
   verify_dirty_young_list(_young_list->first_region());
-  verify_dirty_young_list(_young_list->first_survivor_region());
 }
 #endif
 

src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp

@@ -375,6 +375,13 @@ private:
   // this method will be found dead by the marking cycle).
   void allocate_dummy_regions() PRODUCT_RETURN;
 
+  // Clear RSets after a compaction. It also resets the GC time stamps.
+  void clear_rsets_post_compaction();
+
+  // If the HR printer is active, dump the state of the regions in the
+  // heap after a compaction.
+  void print_hrs_post_compaction();
+
   // These are macros so that, if the assert fires, we get the correct
   // line number, file, etc.
 
@@ -1061,11 +1068,18 @@ public:
     clear_cset_start_regions();
   }
 
+  void check_gc_time_stamps() PRODUCT_RETURN;
+
   void increment_gc_time_stamp() {
     ++_gc_time_stamp;
     OrderAccess::fence();
   }
 
+  // Reset the given region's GC timestamp. If it's starts humongous,
+  // also reset the GC timestamp of its corresponding
+  // continues humongous regions too.
+  void reset_gc_time_stamps(HeapRegion* hr);
+
   void iterate_dirty_card_closure(CardTableEntryClosure* cl,
                                   DirtyCardQueue* into_cset_dcq,
                                   bool concurrent, int worker_i);
@@ -1302,11 +1316,6 @@ public:
   // iteration early if the "doHeapRegion" method returns "true".
   void heap_region_iterate(HeapRegionClosure* blk) const;
 
-  // Iterate over heap regions starting with r (or the first region if "r"
-  // is NULL), in address order, terminating early if the "doHeapRegion"
-  // method returns "true".
-  void heap_region_iterate_from(HeapRegion* r, HeapRegionClosure* blk) const;
-
   // Return the region with the given index. It assumes the index is valid.
   HeapRegion* region_at(uint index) const { return _hrs.at(index); }
 
@@ -1351,6 +1360,11 @@ public:
   // starting region for iterating over the current collection set.
   HeapRegion* start_cset_region_for_worker(int worker_i);
 
+  // This is a convenience method that is used by the
+  // HeapRegionIterator classes to calculate the starting region for
+  // each worker so that they do not all start from the same region.
+  HeapRegion* start_region_for_worker(uint worker_i, uint no_of_par_workers);
+
   // Iterate over the regions (if any) in the current collection set.
   void collection_set_iterate(HeapRegionClosure* blk);
 
@@ -1558,24 +1572,6 @@ public:
   bool isMarkedPrev(oop obj) const;
   bool isMarkedNext(oop obj) const;
 
-  // vo == UsePrevMarking -> use "prev" marking information,
-  // vo == UseNextMarking -> use "next" marking information,
-  // vo == UseMarkWord    -> use mark word from object header
-  bool is_obj_dead_cond(const oop obj,
-                        const HeapRegion* hr,
-                        const VerifyOption vo) const {
-
-    switch (vo) {
-      case VerifyOption_G1UsePrevMarking:
-        return is_obj_dead(obj, hr);
-      case VerifyOption_G1UseNextMarking:
-        return is_obj_ill(obj, hr);
-      default:
-        assert(vo == VerifyOption_G1UseMarkWord, "must be");
-        return !obj->is_gc_marked();
-    }
-  }
-
   // Determine if an object is dead, given the object and also
   // the region to which the object belongs. An object is dead
   // iff a) it was not allocated since the last mark and b) it
@@ -1587,15 +1583,6 @@ public:
       !isMarkedPrev(obj);
   }
 
-  // This is used when copying an object to survivor space.
-  // If the object is marked live, then we mark the copy live.
-  // If the object is allocated since the start of this mark
-  // cycle, then we mark the copy live.
-  // If the object has been around since the previous mark
-  // phase, and hasn't been marked yet during this phase,
-  // then we don't mark it, we just wait for the
-  // current marking cycle to get to it.
-
   // This function returns true when an object has been
   // around since the previous marking and hasn't yet
   // been marked during this marking.
@@ -1613,23 +1600,6 @@ public:
   // Added if it is in permanent gen it isn't dead.
   // Added if it is NULL it isn't dead.
 
-  // vo == UsePrevMarking -> use "prev" marking information,
-  // vo == UseNextMarking -> use "next" marking information,
-  // vo == UseMarkWord    -> use mark word from object header
-  bool is_obj_dead_cond(const oop obj,
-                        const VerifyOption vo) const {
-
-    switch (vo) {
-      case VerifyOption_G1UsePrevMarking:
-        return is_obj_dead(obj);
-      case VerifyOption_G1UseNextMarking:
-        return is_obj_ill(obj);
-      default:
-        assert(vo == VerifyOption_G1UseMarkWord, "must be");
-        return !obj->is_gc_marked();
-    }
-  }
-
   bool is_obj_dead(const oop obj) const {
     const HeapRegion* hr = heap_region_containing(obj);
     if (hr == NULL) {
@@ -1652,6 +1622,42 @@ public:
     else return is_obj_ill(obj, hr);
   }
 
+  // The methods below are here for convenience and dispatch the
+  // appropriate method depending on value of the given VerifyOption
+  // parameter. The options for that parameter are:
+  //
+  // vo == UsePrevMarking -> use "prev" marking information,
+  // vo == UseNextMarking -> use "next" marking information,
+  // vo == UseMarkWord    -> use mark word from object header
+
+  bool is_obj_dead_cond(const oop obj,
+                        const HeapRegion* hr,
+                        const VerifyOption vo) const {
+    switch (vo) {
+    case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj, hr);
+    case VerifyOption_G1UseNextMarking: return is_obj_ill(obj, hr);
+    case VerifyOption_G1UseMarkWord:    return !obj->is_gc_marked();
+    default:                            ShouldNotReachHere();
+    }
+    return false; // keep some compilers happy
+  }
+
+  bool is_obj_dead_cond(const oop obj,
+                        const VerifyOption vo) const {
+    switch (vo) {
+    case VerifyOption_G1UsePrevMarking: return is_obj_dead(obj);
+    case VerifyOption_G1UseNextMarking: return is_obj_ill(obj);
+    case VerifyOption_G1UseMarkWord:    return !obj->is_gc_marked();
+    default:                            ShouldNotReachHere();
+    }
+    return false; // keep some compilers happy
+  }
+
+  bool allocated_since_marking(oop obj, HeapRegion* hr, VerifyOption vo);
+  HeapWord* top_at_mark_start(HeapRegion* hr, VerifyOption vo);
+  bool is_marked(oop obj, VerifyOption vo);
+  const char* top_at_mark_start_str(VerifyOption vo);
+
   // The following is just to alert the verification code
   // that a full collection has occurred and that the
   // remembered sets are no longer up to date.

src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp

@@ -1528,35 +1528,13 @@ public:
 
 class ParKnownGarbageHRClosure: public HeapRegionClosure {
   G1CollectedHeap* _g1h;
-  CollectionSetChooser* _hrSorted;
-  uint _marked_regions_added;
-  size_t _reclaimable_bytes_added;
-  uint _chunk_size;
-  uint _cur_chunk_idx;
-  uint _cur_chunk_end; // Cur chunk [_cur_chunk_idx, _cur_chunk_end)
-
-  void get_new_chunk() {
-    _cur_chunk_idx = _hrSorted->claim_array_chunk(_chunk_size);
-    _cur_chunk_end = _cur_chunk_idx + _chunk_size;
-  }
-  void add_region(HeapRegion* r) {
-    if (_cur_chunk_idx == _cur_chunk_end) {
-      get_new_chunk();
-    }
-    assert(_cur_chunk_idx < _cur_chunk_end, "postcondition");
-    _hrSorted->set_region(_cur_chunk_idx, r);
-    _marked_regions_added++;
-    _reclaimable_bytes_added += r->reclaimable_bytes();
-    _cur_chunk_idx++;
-  }
+  CSetChooserParUpdater _cset_updater;
 
 public:
   ParKnownGarbageHRClosure(CollectionSetChooser* hrSorted,
                            uint chunk_size) :
     _g1h(G1CollectedHeap::heap()),
-      _hrSorted(hrSorted), _chunk_size(chunk_size),
-      _marked_regions_added(0), _reclaimable_bytes_added(0),
-      _cur_chunk_idx(0), _cur_chunk_end(0) { }
+    _cset_updater(hrSorted, true /* parallel */, chunk_size) { }
 
   bool doHeapRegion(HeapRegion* r) {
     // Do we have any marking information for this region?
@@ -1564,14 +1542,12 @@ public:
       // We will skip any region that's currently used as an old GC
       // alloc region (we should not consider those for collection
       // before we fill them up).
-      if (_hrSorted->should_add(r) && !_g1h->is_old_gc_alloc_region(r)) {
-        add_region(r);
+      if (_cset_updater.should_add(r) && !_g1h->is_old_gc_alloc_region(r)) {
+        _cset_updater.add_region(r);
       }
     }
     return false;
   }
-  uint marked_regions_added() { return _marked_regions_added; }
-  size_t reclaimable_bytes_added() { return _reclaimable_bytes_added; }
 };
 
 class ParKnownGarbageTask: public AbstractGangTask {
@@ -1591,10 +1567,6 @@ public:
     _g1->heap_region_par_iterate_chunked(&parKnownGarbageCl, worker_id,
                                          _g1->workers()->active_workers(),
                                          HeapRegion::InitialClaimValue);
-    uint regions_added = parKnownGarbageCl.marked_regions_added();
-    size_t reclaimable_bytes_added =
-                                   parKnownGarbageCl.reclaimable_bytes_added();
-    _hrSorted->update_totals(regions_added, reclaimable_bytes_added);
   }
 };
 

src/share/vm/gc_implementation/g1/g1MarkSweep.cpp

@@ -262,18 +262,6 @@ public:
   }
 };
 
-// Finds the first HeapRegion.
-class FindFirstRegionClosure: public HeapRegionClosure {
-  HeapRegion* _a_region;
-public:
-  FindFirstRegionClosure() : _a_region(NULL) {}
-  bool doHeapRegion(HeapRegion* r) {
-    _a_region = r;
-    return true;
-  }
-  HeapRegion* result() { return _a_region; }
-};
-
 void G1MarkSweep::mark_sweep_phase2() {
   // Now all live objects are marked, compute the new object addresses.
 
@@ -294,9 +282,8 @@ void G1MarkSweep::mark_sweep_phase2() {
   TraceTime tm("phase 2", G1Log::fine() && Verbose, true, gclog_or_tty);
   GenMarkSweep::trace("2");
 
-  FindFirstRegionClosure cl;
-  g1h->heap_region_iterate(&cl);
-  HeapRegion *r = cl.result();
+  // find the first region
+  HeapRegion* r = g1h->region_at(0);
   CompactibleSpace* sp = r;
   if (r->isHumongous() && oop(r->bottom())->is_gc_marked()) {
     sp = r->next_compaction_space();
@@ -408,7 +395,3 @@ void G1MarkSweep::mark_sweep_phase4() {
   g1h->heap_region_iterate(&blk);
 
 }
-
-// Local Variables: ***
-// c-indentation-style: gnu ***
-// End: ***

src/share/vm/gc_implementation/g1/g1OopClosures.hpp

@@ -197,7 +197,6 @@ class FilterOutOfRegionClosure: public OopClosure {
   HeapWord* _r_bottom;
   HeapWord* _r_end;
   OopClosure* _oc;
-  int _out_of_region;
 public:
   FilterOutOfRegionClosure(HeapRegion* r, OopClosure* oc);
   template <class T> void do_oop_nv(T* p);
@@ -205,7 +204,6 @@ public:
   virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   bool apply_to_weak_ref_discovered_field() { return true; }
   bool do_header() { return false; }
-  int out_of_region() { return _out_of_region; }
 };
 
 // Closure for iterating over object fields during concurrent marking

src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp

@@ -36,25 +36,15 @@
  * compiler sometimes sees fit to ignore inline declarations.  Sigh.
  */
 
-// This must a ifdef'ed because the counting it controls is in a
-// perf-critical inner loop.
-#define FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT 0
-
 template <class T>
 inline void FilterIntoCSClosure::do_oop_nv(T* p) {
   T heap_oop = oopDesc::load_heap_oop(p);
   if (!oopDesc::is_null(heap_oop) &&
       _g1->obj_in_cs(oopDesc::decode_heap_oop_not_null(heap_oop))) {
     _oc->do_oop(p);
-#if FILTERINTOCSCLOSURE_DOHISTOGRAMCOUNT
-    if (_dcto_cl != NULL)
-      _dcto_cl->incr_count();
-#endif
   }
 }
 
-#define FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT 0
-
 template <class T>
 inline void FilterOutOfRegionClosure::do_oop_nv(T* p) {
   T heap_oop = oopDesc::load_heap_oop(p);
@@ -62,9 +52,6 @@ inline void FilterOutOfRegionClosure::do_oop_nv(T* p) {
     HeapWord* obj_hw = (HeapWord*)oopDesc::decode_heap_oop_not_null(heap_oop);
     if (obj_hw < _r_bottom || obj_hw >= _r_end) {
       _oc->do_oop(p);
-#if FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT
-      _out_of_region++;
-#endif
     }
   }
 }

src/share/vm/gc_implementation/g1/g1RemSet.cpp

@@ -280,62 +280,6 @@ void G1RemSet::updateRS(DirtyCardQueue* into_cset_dcq, int worker_i) {
   _g1p->phase_times()->record_update_rs_time(worker_i, (os::elapsedTime() - start) * 1000.0);
 }
 
-class CountRSSizeClosure: public HeapRegionClosure {
-  size_t _n;
-  size_t _tot;
-  size_t _max;
-  HeapRegion* _max_r;
-  enum {
-    N = 20,
-    MIN = 6
-  };
-  int _histo[N];
-public:
-  CountRSSizeClosure() : _n(0), _tot(0), _max(0), _max_r(NULL) {
-    for (int i = 0; i < N; i++) _histo[i] = 0;
-  }
-  bool doHeapRegion(HeapRegion* r) {
-    if (!r->continuesHumongous()) {
-      size_t occ = r->rem_set()->occupied();
-      _n++;
-      _tot += occ;
-      if (occ > _max) {
-        _max = occ;
-        _max_r = r;
-      }
-      // Fit it into a histo bin.
-      int s = 1 << MIN;
-      int i = 0;
-      while (occ > (size_t) s && i < (N-1)) {
-        s = s << 1;
-        i++;
-      }
-      _histo[i]++;
-    }
-    return false;
-  }
-  size_t n() { return _n; }
-  size_t tot() { return _tot; }
-  size_t mx() { return _max; }
-  HeapRegion* mxr() { return _max_r; }
-  void print_histo() {
-    int mx = N;
-    while (mx >= 0) {
-      if (_histo[mx-1] > 0) break;
-      mx--;
-    }
-    gclog_or_tty->print_cr("Number of regions with given RS sizes:");
-    gclog_or_tty->print_cr("           <= %8d   %8d", 1 << MIN, _histo[0]);
-    for (int i = 1; i < mx-1; i++) {
-      gclog_or_tty->print_cr("  %8d  - %8d   %8d",
-                    (1 << (MIN + i - 1)) + 1,
-                    1 << (MIN + i),
-                    _histo[i]);
-    }
-    gclog_or_tty->print_cr("            > %8d   %8d", (1 << (MIN+mx-2))+1, _histo[mx-1]);
-  }
-};
-
 void G1RemSet::cleanupHRRS() {
   HeapRegionRemSet::cleanup();
 }
@@ -349,17 +293,6 @@ void G1RemSet::oops_into_collection_set_do(OopsInHeapRegionClosure* oc,
     _cg1r->clear_and_record_card_counts();
   }
 
-  // Make this into a command-line flag...
-  if (G1RSCountHisto && (ParallelGCThreads == 0 || worker_i == 0)) {
-    CountRSSizeClosure count_cl;
-    _g1->heap_region_iterate(&count_cl);
-    gclog_or_tty->print_cr("Avg of %d RS counts is %f, max is %d, "
-                  "max region is " PTR_FORMAT,
-                  count_cl.n(), (float)count_cl.tot()/(float)count_cl.n(),
-                  count_cl.mx(), count_cl.mxr());
-    count_cl.print_histo();
-  }
-
   // We cache the value of 'oc' closure into the appropriate slot in the
   // _cset_rs_update_cl for this worker
   assert(worker_i < (int)n_workers(), "sanity");
@@ -568,8 +501,6 @@ void G1RemSet::scrub_par(BitMap* region_bm, BitMap* card_bm,
 }
 
 
-static IntHistogram out_of_histo(50, 50);
-
 
 G1TriggerClosure::G1TriggerClosure() :
   _triggered(false) { }
@@ -671,7 +602,6 @@ bool G1RemSet::concurrentRefineOneCard_impl(jbyte* card_ptr, int worker_i,
       sdcq->enqueue(card_ptr);
     }
   } else {
-    out_of_histo.add_entry(filter_then_update_rs_oop_cl.out_of_region());
     _conc_refine_cards++;
   }
 
@@ -862,11 +792,6 @@ void G1RemSet::print_summary_info() {
   card_repeat_count.print_on(gclog_or_tty);
 #endif
 
-  if (FILTEROUTOFREGIONCLOSURE_DOHISTOGRAMCOUNT) {
-    gclog_or_tty->print_cr("\nG1 rem-set out-of-region histogram: ");
-    gclog_or_tty->print_cr("  # of CS ptrs --> # of cards with that number.");
-    out_of_histo.print_on(gclog_or_tty);
-  }
   gclog_or_tty->print_cr("\n Concurrent RS processed %d cards",
                          _conc_refine_cards);
   DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set();
@@ -889,21 +814,24 @@ void G1RemSet::print_summary_info() {
 
   HRRSStatsIter blk;
   g1->heap_region_iterate(&blk);
-  gclog_or_tty->print_cr("  Total heap region rem set sizes = " SIZE_FORMAT "K."
-                         "  Max = " SIZE_FORMAT "K.",
+  gclog_or_tty->print_cr("  Total heap region rem set sizes = "SIZE_FORMAT"K."
+                         "  Max = "SIZE_FORMAT"K.",
                          blk.total_mem_sz()/K, blk.max_mem_sz()/K);
-  gclog_or_tty->print_cr("  Static structures = " SIZE_FORMAT "K,"
-                         " free_lists = " SIZE_FORMAT "K.",
-                         HeapRegionRemSet::static_mem_size()/K,
-                         HeapRegionRemSet::fl_mem_size()/K);
-  gclog_or_tty->print_cr("    %d occupied cards represented.",
+  gclog_or_tty->print_cr("  Static structures = "SIZE_FORMAT"K,"
+                         " free_lists = "SIZE_FORMAT"K.",
+                         HeapRegionRemSet::static_mem_size() / K,
+                         HeapRegionRemSet::fl_mem_size() / K);
+  gclog_or_tty->print_cr("    "SIZE_FORMAT" occupied cards represented.",
                          blk.occupied());
-  gclog_or_tty->print_cr("    Max sz region = [" PTR_FORMAT ", " PTR_FORMAT " )"
-                         ", cap = " SIZE_FORMAT "K, occ = " SIZE_FORMAT "K.",
-                         blk.max_mem_sz_region()->bottom(), blk.max_mem_sz_region()->end(),
-                         (blk.max_mem_sz_region()->rem_set()->mem_size() + K - 1)/K,
-                         (blk.max_mem_sz_region()->rem_set()->occupied() + K - 1)/K);
-  gclog_or_tty->print_cr("    Did %d coarsenings.", HeapRegionRemSet::n_coarsenings());
+  HeapRegion* max_mem_sz_region = blk.max_mem_sz_region();
+  HeapRegionRemSet* rem_set = max_mem_sz_region->rem_set();
+  gclog_or_tty->print_cr("    Max size region = "HR_FORMAT", "
+                         "size = "SIZE_FORMAT "K, occupied = "SIZE_FORMAT"K.",
+                         HR_FORMAT_PARAMS(max_mem_sz_region),
+                         (rem_set->mem_size() + K - 1)/K,
+                         (rem_set->occupied() + K - 1)/K);
+  gclog_or_tty->print_cr("    Did %d coarsenings.",
+                         HeapRegionRemSet::n_coarsenings());
 }
 
 void G1RemSet::prepare_for_verify() {

src/share/vm/gc_implementation/g1/heapRegion.cpp

@@ -44,14 +44,11 @@ HeapRegionDCTOC::HeapRegionDCTOC(G1CollectedHeap* g1,
                                  CardTableModRefBS::PrecisionStyle precision,
                                  FilterKind fk) :
   ContiguousSpaceDCTOC(hr, cl, precision, NULL),
-  _hr(hr), _fk(fk), _g1(g1)
-{ }
+  _hr(hr), _fk(fk), _g1(g1) { }
 
 FilterOutOfRegionClosure::FilterOutOfRegionClosure(HeapRegion* r,
                                                    OopClosure* oc) :
-  _r_bottom(r->bottom()), _r_end(r->end()),
-  _oc(oc), _out_of_region(0)
-{}
+  _r_bottom(r->bottom()), _r_end(r->end()), _oc(oc) { }
 
 class VerifyLiveClosure: public OopClosure {
 private:
@@ -512,35 +509,19 @@ HeapRegion::HeapRegion(uint hrs_index,
   assert(HeapRegionRemSet::num_par_rem_sets() > 0, "Invariant.");
 }
 
-class NextCompactionHeapRegionClosure: public HeapRegionClosure {
-  const HeapRegion* _target;
-  bool _target_seen;
-  HeapRegion* _last;
-  CompactibleSpace* _res;
-public:
-  NextCompactionHeapRegionClosure(const HeapRegion* target) :
-    _target(target), _target_seen(false), _res(NULL) {}
-  bool doHeapRegion(HeapRegion* cur) {
-    if (_target_seen) {
-      if (!cur->isHumongous()) {
-        _res = cur;
-        return true;
-      }
-    } else if (cur == _target) {
-      _target_seen = true;
-    }
-    return false;
-  }
-  CompactibleSpace* result() { return _res; }
-};
-
 CompactibleSpace* HeapRegion::next_compaction_space() const {
+  // We're not using an iterator given that it will wrap around when
+  // it reaches the last region and this is not what we want here.
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
-  // cast away const-ness
-  HeapRegion* r = (HeapRegion*) this;
-  NextCompactionHeapRegionClosure blk(r);
-  g1h->heap_region_iterate_from(r, &blk);
-  return blk.result();
+  uint index = hrs_index() + 1;
+  while (index < g1h->n_regions()) {
+    HeapRegion* hr = g1h->region_at(index);
+    if (!hr->isHumongous()) {
+      return hr;
+    }
+    index += 1;
+  }
+  return NULL;
 }
 
 void HeapRegion::save_marks() {

src/share/vm/gc_implementation/g1/heapRegion.hpp

@@ -55,7 +55,10 @@ class HeapRegionSetBase;
 #define HR_FORMAT "%u:(%s)["PTR_FORMAT","PTR_FORMAT","PTR_FORMAT"]"
 #define HR_FORMAT_PARAMS(_hr_) \
                 (_hr_)->hrs_index(), \
-                (_hr_)->is_survivor() ? "S" : (_hr_)->is_young() ? "E" : "-", \
+                (_hr_)->is_survivor() ? "S" : (_hr_)->is_young() ? "E" : \
+                (_hr_)->startsHumongous() ? "HS" : \
+                (_hr_)->continuesHumongous() ? "HC" : \
+                !(_hr_)->is_empty() ? "O" : "F", \
                 (_hr_)->bottom(), (_hr_)->top(), (_hr_)->end()
 
 // sentinel value for hrs_index
@@ -173,6 +176,7 @@ class G1OffsetTableContigSpace: public ContiguousSpace {
   virtual HeapWord* saved_mark_word() const;
   virtual void set_saved_mark();
   void reset_gc_time_stamp() { _gc_time_stamp = 0; }
+  unsigned get_gc_time_stamp() { return _gc_time_stamp; }
 
   // See the comment above in the declaration of _pre_dummy_top for an
   // explanation of what it is.
@@ -439,6 +443,25 @@ class HeapRegion: public G1OffsetTableContigSpace {
     return _humongous_start_region;
   }
 
+  // Return the number of distinct regions that are covered by this region:
+  // 1 if the region is not humongous, >= 1 if the region is humongous.
+  uint region_num() const {
+    if (!isHumongous()) {
+      return 1U;
+    } else {
+      assert(startsHumongous(), "doesn't make sense on HC regions");
+      assert(capacity() % HeapRegion::GrainBytes == 0, "sanity");
+      return (uint) (capacity() >> HeapRegion::LogOfHRGrainBytes);
+    }
+  }
+
+  // Return the index + 1 of the last HC regions that's associated
+  // with this HS region.
+  uint last_hc_index() const {
+    assert(startsHumongous(), "don't call this otherwise");
+    return hrs_index() + region_num();
+  }
+
   // Same as Space::is_in_reserved, but will use the original size of the region.
   // The original size is different only for start humongous regions. They get
   // their _end set up to be the end of the last continues region of the
@@ -622,8 +645,8 @@ class HeapRegion: public G1OffsetTableContigSpace {
   bool is_marked() { return _prev_top_at_mark_start != bottom(); }
 
   void reset_during_compaction() {
-    guarantee( isHumongous() && startsHumongous(),
-               "should only be called for humongous regions");
+    assert(isHumongous() && startsHumongous(),
+           "should only be called for starts humongous regions");
 
     zero_marked_bytes();
     init_top_at_mark_start();
@@ -774,7 +797,7 @@ class HeapRegion: public G1OffsetTableContigSpace {
   virtual void oop_since_save_marks_iterate##nv_suffix(OopClosureType* cl);
   SPECIALIZED_SINCE_SAVE_MARKS_CLOSURES(HeapRegion_OOP_SINCE_SAVE_MARKS_DECL)
 
-  CompactibleSpace* next_compaction_space() const;
+  virtual CompactibleSpace* next_compaction_space() const;
 
   virtual void reset_after_compaction();
 

src/share/vm/gc_implementation/g1/heapRegionSet.cpp

@@ -35,14 +35,6 @@ void HeapRegionSetBase::set_unrealistically_long_length(uint len) {
   _unrealistically_long_length = len;
 }
 
-uint HeapRegionSetBase::calculate_region_num(HeapRegion* hr) {
-  assert(hr->startsHumongous(), "pre-condition");
-  assert(hr->capacity() % HeapRegion::GrainBytes == 0, "invariant");
-  uint region_num = (uint) (hr->capacity() >> HeapRegion::LogOfHRGrainBytes);
-  assert(region_num > 0, "sanity");
-  return region_num;
-}
-
 void HeapRegionSetBase::fill_in_ext_msg(hrs_ext_msg* msg, const char* message) {
   msg->append("[%s] %s ln: %u rn: %u cy: "SIZE_FORMAT" ud: "SIZE_FORMAT,
               name(), message, length(), region_num(),
@@ -152,11 +144,7 @@ void HeapRegionSetBase::verify_next_region(HeapRegion* hr) {
   guarantee(verify_region(hr, this), hrs_ext_msg(this, "region verification"));
 
   _calc_length               += 1;
-  if (!hr->isHumongous()) {
-    _calc_region_num         += 1;
-  } else {
-    _calc_region_num         += calculate_region_num(hr);
-  }
+  _calc_region_num           += hr->region_num();
   _calc_total_capacity_bytes += hr->capacity();
   _calc_total_used_bytes     += hr->used();
 }

src/share/vm/gc_implementation/g1/heapRegionSet.hpp

@@ -62,8 +62,6 @@ class HeapRegionSetBase VALUE_OBJ_CLASS_SPEC {
   friend class VMStructs;
 
 protected:
-  static uint calculate_region_num(HeapRegion* hr);
-
   static uint _unrealistically_long_length;
 
   // The number of regions added to the set. If the set contains

src/share/vm/gc_implementation/g1/heapRegionSet.inline.hpp

@@ -33,11 +33,7 @@ inline void HeapRegionSetBase::update_for_addition(HeapRegion* hr) {
   // Assumes the caller has already verified the region.
 
   _length           += 1;
-  if (!hr->isHumongous()) {
-    _region_num     += 1;
-  } else {
-    _region_num     += calculate_region_num(hr);
-  }
+  _region_num       += hr->region_num();
   _total_used_bytes += hr->used();
 }
 
@@ -54,12 +50,7 @@ inline void HeapRegionSetBase::update_for_removal(HeapRegion* hr) {
   assert(_length > 0, hrs_ext_msg(this, "pre-condition"));
   _length -= 1;
 
-  uint region_num_diff;
-  if (!hr->isHumongous()) {
-    region_num_diff = 1;
-  } else {
-    region_num_diff = calculate_region_num(hr);
-  }
+  uint region_num_diff = hr->region_num();
   assert(region_num_diff <= _region_num,
          hrs_err_msg("[%s] region's region num: %u "
                      "should be <= region num: %u",