6695819: verify_oopx rax: broken oop in decode_heap_oop

Summary: Code in gen_subtype_check was encoding rax as an oop on a path where rax was not an oop.
Reviewed-by: never, kvn

src/cpu/sparc/vm/assembler_sparc.cpp

@@ -3622,6 +3622,7 @@ void MacroAssembler::store_heap_oop(Register d, const Address& a, int offset) {
 
 void MacroAssembler::encode_heap_oop(Register src, Register dst) {
   assert (UseCompressedOops, "must be compressed");
+  verify_oop(src);
   Label done;
   if (src == dst) {
     // optimize for frequent case src == dst
@@ -3643,12 +3644,14 @@ void MacroAssembler::encode_heap_oop(Register src, Register dst) {
 
 void MacroAssembler::encode_heap_oop_not_null(Register r) {
   assert (UseCompressedOops, "must be compressed");
+  verify_oop(r);
   sub(r, G6_heapbase, r);
   srlx(r, LogMinObjAlignmentInBytes, r);
 }
 
 void MacroAssembler::encode_heap_oop_not_null(Register src, Register dst) {
   assert (UseCompressedOops, "must be compressed");
+  verify_oop(src);
   sub(src, G6_heapbase, dst);
   srlx(dst, LogMinObjAlignmentInBytes, dst);
 }
@@ -3661,11 +3664,13 @@ void  MacroAssembler::decode_heap_oop(Register src, Register dst) {
   bpr(rc_nz, true, Assembler::pt, dst, done);
   delayed() -> add(dst, G6_heapbase, dst); // annuled if not taken
   bind(done);
+  verify_oop(dst);
 }
 
 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
   // Do not add assert code to this unless you change vtableStubs_sparc.cpp
   // pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
   assert (UseCompressedOops, "must be compressed");
   sllx(r, LogMinObjAlignmentInBytes, r);
   add(r, G6_heapbase, r);
@@ -3674,6 +3679,7 @@ void  MacroAssembler::decode_heap_oop_not_null(Register r) {
 void  MacroAssembler::decode_heap_oop_not_null(Register src, Register dst) {
   // Do not add assert code to this unless you change vtableStubs_sparc.cpp
   // pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
   assert (UseCompressedOops, "must be compressed");
   sllx(src, LogMinObjAlignmentInBytes, dst);
   add(dst, G6_heapbase, dst);

src/cpu/x86/vm/assembler_x86_64.cpp

@@ -5265,6 +5265,7 @@ void  MacroAssembler::decode_heap_oop_not_null(Register r) {
   assert (UseCompressedOops, "should only be used for compressed headers");
   // Cannot assert, unverified entry point counts instructions (see .ad file)
   // vtableStubs also counts instructions in pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
   assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
   leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
 }
@@ -5273,6 +5274,7 @@ void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
   assert (UseCompressedOops, "should only be used for compressed headers");
   // Cannot assert, unverified entry point counts instructions (see .ad file)
   // vtableStubs also counts instructions in pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
   assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
   leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
 }

src/cpu/x86/vm/interp_masm_x86_64.cpp

@@ -233,7 +233,7 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
   assert(Rsub_klass != rcx, "rcx holds 2ndary super array length");
   assert(Rsub_klass != rdi, "rdi holds 2ndary super array scan ptr");
 
-  Label not_subtype, loop;
+  Label not_subtype, not_subtype_pop, loop;
 
   // Profile the not-null value's klass.
   profile_typecheck(rcx, Rsub_klass, rdi); // blows rcx, rdi
@@ -272,12 +272,13 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
   // and we store values in objArrays always encoded, thus we need to encode value
   // before repne
   if (UseCompressedOops) {
+    pushq(rax);
     encode_heap_oop(rax);
     repne_scanl();
     // Not equal?
-    jcc(Assembler::notEqual, not_subtype);
-    // decode heap oop here for movq
-    decode_heap_oop(rax);
+    jcc(Assembler::notEqual, not_subtype_pop);
+    // restore heap oop here for movq
+    popq(rax);
   } else {
     repne_scanq();
     jcc(Assembler::notEqual, not_subtype);
@@ -287,9 +288,10 @@ void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
                Klass::secondary_super_cache_offset_in_bytes()), rax);
   jmp(ok_is_subtype);
 
+  bind(not_subtype_pop);
+  // restore heap oop here for miss
+  if (UseCompressedOops) popq(rax);
   bind(not_subtype);
-  // decode heap oop here for miss
-  if (UseCompressedOops) decode_heap_oop(rax);
   profile_typecheck_failed(rcx); // blows rcx
 }