6948539: CMS+UseCompressedOops: placement of cms_free bit interferes with promoted object link

Summary: When using compressed oops, use compressed promoted pointers in b63:b31 of the mark word, so as not to interfere with the CMS "freeness bit" at b7. Updated mark-word layout documentation.
Reviewed-by: minqi, poonam, jmasa, coleenp

src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.cpp

 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it

src/share/vm/gc_implementation/concurrentMarkSweep/promotionInfo.hpp

 /*
- * Copyright 2001-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,30 +32,75 @@ class PromotedObject VALUE_OBJ_CLASS_SPEC {
     displaced_mark = nth_bit(2),        // i.e. 0x4
     next_mask      = ~(right_n_bits(3)) // i.e. ~(0x7)
   };
-  intptr_t _next;
+
+  // Below, we want _narrow_next in the "higher" 32 bit slot,
+  // whose position will depend on endian-ness of the platform.
+  // This is so that there is no interference with the
+  // cms_free_bit occupying bit position 7 (lsb == 0)
+  // when we are using compressed oops; see FreeChunk::isFree().
+  // We cannot move the cms_free_bit down because currently
+  // biased locking code assumes that age bits are contiguous
+  // with the lock bits. Even if that assumption were relaxed,
+  // the least position we could move this bit to would be
+  // to bit position 3, which would require 16 byte alignment.
+  typedef struct {
+#ifdef VM_LITTLE_ENDIAN
+    LP64_ONLY(narrowOop _pad;)
+              narrowOop _narrow_next;
+#else
+              narrowOop _narrow_next;
+    LP64_ONLY(narrowOop _pad;)
+#endif
+  } Data;
+
+  union {
+    intptr_t _next;
+    Data     _data;
+  };
  public:
   inline PromotedObject* next() const {
-    return (PromotedObject*)(_next & next_mask);
+    assert(!((FreeChunk*)this)->isFree(), "Error");
+    PromotedObject* res;
+    if (UseCompressedOops) {
+      // The next pointer is a compressed oop stored in the top 32 bits
+      res = (PromotedObject*)oopDesc::decode_heap_oop(_data._narrow_next);
+    } else {
+      res = (PromotedObject*)(_next & next_mask);
+    }
+    assert(oop(res)->is_oop_or_null(true /* ignore mark word */), "Not an oop?");
+    return res;
   }
   inline void setNext(PromotedObject* x) {
-    assert(((intptr_t)x & ~next_mask) == 0,
-           "Conflict in bit usage, "
-           " or insufficient alignment of objects");
-    _next |= (intptr_t)x;
+    assert(((intptr_t)x & ~next_mask) == 0, "Conflict in bit usage, "
+           "or insufficient alignment of objects");
+    if (UseCompressedOops) {
+      assert(_data._narrow_next == 0, "Overwrite?");
+      _data._narrow_next = oopDesc::encode_heap_oop(oop(x));
+    } else {
+      _next |= (intptr_t)x;
+    }
+    assert(!((FreeChunk*)this)->isFree(), "Error");
   }
   inline void setPromotedMark() {
     _next |= promoted_mask;
+    assert(!((FreeChunk*)this)->isFree(), "Error");
   }
   inline bool hasPromotedMark() const {
+    assert(!((FreeChunk*)this)->isFree(), "Error");
     return (_next & promoted_mask) == promoted_mask;
   }
   inline void setDisplacedMark() {
     _next |= displaced_mark;
+    assert(!((FreeChunk*)this)->isFree(), "Error");
   }
   inline bool hasDisplacedMark() const {
+    assert(!((FreeChunk*)this)->isFree(), "Error");
     return (_next & displaced_mark) != 0;
   }
-  inline void clearNext()        { _next = 0; }
+  inline void clearNext()        {
+    _next = 0;
+    assert(!((FreeChunk*)this)->isFree(), "Error");
+  }
   debug_only(void *next_addr() { return (void *) &_next; })
 };
 

src/share/vm/oops/markOop.hpp

@@ -27,12 +27,26 @@
 // Note that the mark is not a real oop but just a word.
 // It is placed in the oop hierarchy for historical reasons.
 //
-// Bit-format of an object header (most significant first):
+// Bit-format of an object header (most significant first, big endian layout below):
 //
-//  32 bits: unused:0  hash:25 age:4 biased_lock:1 lock:2
-//  64 bits: unused:24 hash:31 cms:2 age:4 biased_lock:1 lock:2
-//           unused:20 size:35 cms:2 age:4 biased_lock:1 lock:2 (if cms
-//                                                               free chunk)
+//  32 bits:
+//  --------
+//             hash:25 ------------>| age:4    biased_lock:1 lock:2 (normal object)
+//             JavaThread*:23 epoch:2 age:4    biased_lock:1 lock:2 (biased object)
+//             size:32 ------------------------------------------>| (CMS free block)
+//             PromotedObject*:29 ---------->| promo_bits:3 ----->| (CMS promoted object)
+//
+//  64 bits:
+//  --------
+//  unused:25 hash:31 -->| unused:1   age:4    biased_lock:1 lock:2 (normal object)
+//  JavaThread*:54 epoch:2 unused:1   age:4    biased_lock:1 lock:2 (biased object)
+//  PromotedObject*:61 --------------------->| promo_bits:3 ----->| (CMS promoted object)
+//  size:64 ----------------------------------------------------->| (CMS free block)
+//
+//  unused:25 hash:31 -->| cms_free:1 age:4    biased_lock:1 lock:2 (COOPs && normal object)
+//  JavaThread*:54 epoch:2 cms_free:1 age:4    biased_lock:1 lock:2 (COOPs && biased object)
+//  narrowOop:32 unused:24 cms_free:1 unused:4 promo_bits:3 ----->| (COOPs && CMS promoted object)
+//  unused:21 size:35 -->| cms_free:1 unused:7 ------------------>| (COOPs && CMS free block)
 //
 //  - hash contains the identity hash value: largest value is
 //    31 bits, see os::random().  Also, 64-bit vm's require
@@ -61,8 +75,9 @@
 //    significant fraction of the eden semispaces and were not
 //    promoted promptly, causing an increase in the amount of copying
 //    performed. The runtime system aligns all JavaThread* pointers to
-//    a very large value (currently 128 bytes) to make room for the
-//    age bits when biased locking is enabled.
+//    a very large value (currently 128 bytes (32bVM) or 256 bytes (64bVM))
+//    to make room for the age bits & the epoch bits (used in support of
+//    biased locking), and for the CMS "freeness" bit in the 64bVM (+COOPs).
 //
 //    [JavaThread* | epoch | age | 1 | 01]       lock is biased toward given thread
 //    [0           | epoch | age | 1 | 01]       lock is anonymously biased