8147087: Race when reusing PerRegionTable bitmaps may result in dropped remembered set entries

Summary: Do not make reused PRTs available to other threads before the bitmap of the PRT has been cleared.
Reviewed-by: tbenson, mgerdin, brutisso
Contributed-by: NPoonam Bajaj &lt;poonam.bajaj@oracle.com&gt;, Thomas Schatzl <thomas.schatzl@oracle.com>

src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp

 /*
- * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -119,7 +119,9 @@ protected:
 
 public:
 
-  HeapRegion* hr() const { return _hr; }
+  HeapRegion* hr() const {
+    return (HeapRegion*) OrderAccess::load_ptr_acquire(&_hr);
+  }
 
   jint occupied() const {
     // Overkill, but if we ever need it...
@@ -132,10 +134,12 @@ public:
       set_next(NULL);
       set_prev(NULL);
     }
-    _hr = hr;
     _collision_list_next = NULL;
     _occupied = 0;
     _bm.clear();
+    // Make sure that the bitmap clearing above has been finished before publishing
+    // this PRT to concurrent threads.
+    OrderAccess::release_store_ptr(&_hr, hr);
   }
 
   void add_reference(OopOrNarrowOopStar from) {
@@ -441,7 +445,7 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
     if (G1TraceHeapRegionRememberedSet) {
       gclog_or_tty->print_cr("  from-card cache hit.");
     }
-    assert(contains_reference(from), "We just added it!");
+    assert(contains_reference(from), err_msg("We just found " PTR_FORMAT " in the FromCardCache", from));
     return;
   }
 
@@ -454,7 +458,7 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
     if (G1TraceHeapRegionRememberedSet) {
       gclog_or_tty->print_cr("  coarse map hit.");
     }
-    assert(contains_reference(from), "We just added it!");
+    assert(contains_reference(from), err_msg("We just found " PTR_FORMAT " in the Coarse table", from));
     return;
   }
 
@@ -488,7 +492,7 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
         if (G1TraceHeapRegionRememberedSet) {
           gclog_or_tty->print_cr("   added card to sparse table.");
         }
-        assert(contains_reference_locked(from), "We just added it!");
+        assert(contains_reference_locked(from), err_msg("We just added " PTR_FORMAT " to the Sparse table", from));
         return;
       } else {
         if (G1TraceHeapRegionRememberedSet) {
@@ -547,7 +551,7 @@ void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) {
                           hr()->bottom(), from);
     }
   }
-  assert(contains_reference(from), "We just added it!");
+  assert(contains_reference(from), err_msg("We just added " PTR_FORMAT " to the PRT", from));
 }
 
 PerRegionTable*