7068625: Testing 8 bytes of card table entries at a time speeds up card-scanning

Summary: Check clean words instead of clean bytes
Reviewed-by: jcoomes, jmasa, jwilhelm, ysr
Contributed-by: alexey.ragozin@gmail.com

src/share/vm/memory/cardTableModRefBS.hpp

 /*
- * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -72,6 +72,9 @@ class CardTableModRefBS: public ModRefBarrierSet {
     CT_MR_BS_last_reserved      = 16
   };
 
+  // a word's worth (row) of clean card values
+  static const intptr_t clean_card_row = (intptr_t)(-1);
+
   // dirty and precleaned are equivalent wrt younger_refs_iter.
   static bool card_is_dirty_wrt_gen_iter(jbyte cv) {
     return cv == dirty_card || cv == precleaned_card;

src/share/vm/memory/cardTableRS.cpp

 /*
- * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -173,6 +173,10 @@ ClearNoncleanCardWrapper::ClearNoncleanCardWrapper(
             SharedHeap::heap()->workers()->active_workers()), "Mismatch");
 }
 
+bool ClearNoncleanCardWrapper::is_word_aligned(jbyte* entry) {
+  return (((intptr_t)entry) & (BytesPerWord-1)) == 0;
+}
+
 void ClearNoncleanCardWrapper::do_MemRegion(MemRegion mr) {
   assert(mr.word_size() > 0, "Error");
   assert(_ct->is_aligned(mr.start()), "mr.start() should be card aligned");
@@ -194,6 +198,17 @@ void ClearNoncleanCardWrapper::do_MemRegion(MemRegion mr) {
         const MemRegion mrd(start_of_non_clean, end_of_non_clean);
         _dirty_card_closure->do_MemRegion(mrd);
       }
+
+      // fast forward through potential continuous whole-word range of clean cards beginning at a word-boundary
+      if (is_word_aligned(cur_entry)) {
+        jbyte* cur_row = cur_entry - BytesPerWord;
+        while (cur_row >= limit && *((intptr_t*)cur_row) ==  CardTableRS::clean_card_row()) {
+          cur_row -= BytesPerWord;
+        }
+        cur_entry = cur_row + BytesPerWord;
+        cur_hw = _ct->addr_for(cur_entry);
+      }
+
       // Reset the dirty window, while continuing to look
       // for the next dirty card that will start a
       // new dirty window.

src/share/vm/memory/cardTableRS.hpp

 /*
- * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,10 @@ class CardTableRS: public GenRemSet {
     return CardTableModRefBS::clean_card;
   }
 
+  static intptr_t clean_card_row() {
+    return CardTableModRefBS::clean_card_row;
+  }
+
   static bool
   card_is_dirty_wrt_gen_iter(jbyte cv) {
     return CardTableModRefBS::card_is_dirty_wrt_gen_iter(cv);
@@ -176,6 +180,8 @@ private:
   // Work methods called by the clear_card()
   inline bool clear_card_serial(jbyte* entry);
   inline bool clear_card_parallel(jbyte* entry);
+  // check alignment of pointer
+  bool is_word_aligned(jbyte* entry);
 
 public:
   ClearNoncleanCardWrapper(DirtyCardToOopClosure* dirty_card_closure, CardTableRS* ct);