Merge

src/cpu/sparc/vm/assembler_sparc.cpp

@@ -3647,6 +3647,12 @@ void MacroAssembler::encode_heap_oop_not_null(Register r) {
   srlx(r, LogMinObjAlignmentInBytes, r);
 }
 
+void MacroAssembler::encode_heap_oop_not_null(Register src, Register dst) {
+  assert (UseCompressedOops, "must be compressed");
+  sub(src, G6_heapbase, dst);
+  srlx(dst, LogMinObjAlignmentInBytes, dst);
+}
+
 // Same algorithm as oops.inline.hpp decode_heap_oop.
 void  MacroAssembler::decode_heap_oop(Register src, Register dst) {
   assert (UseCompressedOops, "must be compressed");
@@ -3665,6 +3671,14 @@ void  MacroAssembler::decode_heap_oop_not_null(Register r) {
   add(r, G6_heapbase, r);
 }
 
+void  MacroAssembler::decode_heap_oop_not_null(Register src, Register dst) {
+  // Do not add assert code to this unless you change vtableStubs_sparc.cpp
+  // pd_code_size_limit.
+  assert (UseCompressedOops, "must be compressed");
+  sllx(src, LogMinObjAlignmentInBytes, dst);
+  add(dst, G6_heapbase, dst);
+}
+
 void MacroAssembler::reinit_heapbase() {
   if (UseCompressedOops) {
     // call indirectly to solve generation ordering problem

src/cpu/sparc/vm/assembler_sparc.hpp

@@ -1998,6 +1998,8 @@ class MacroAssembler: public Assembler {
   }
   void encode_heap_oop_not_null(Register r);
   void decode_heap_oop_not_null(Register r);
+  void encode_heap_oop_not_null(Register src, Register dst);
+  void decode_heap_oop_not_null(Register src, Register dst);
 
   // Support for managing the JavaThread pointer (i.e.; the reference to
   // thread-local information).

src/cpu/sparc/vm/bytecodeInterpreter_sparc.hpp

@@ -78,7 +78,7 @@ public:
 
 #define LOCALS_SLOT(offset)    ((intptr_t*)&locals[-(offset)])
 #define LOCALS_ADDR(offset)    ((address)locals[-(offset)])
-#define LOCALS_INT(offset)     ((jint)(locals[-(offset)]))
+#define LOCALS_INT(offset)     (*((jint*)&locals[-(offset)]))
 #define LOCALS_FLOAT(offset)   (*((jfloat*)&locals[-(offset)]))
 #define LOCALS_OBJECT(offset)  ((oop)locals[-(offset)])
 #define LOCALS_DOUBLE(offset)  (((VMJavaVal64*)&locals[-((offset) + 1)])->d)

src/cpu/sparc/vm/cppInterpreter_sparc.cpp

@@ -159,7 +159,7 @@ address CppInterpreterGenerator::generate_tosca_to_stack_converter(BasicType typ
       break;
     case T_LONG   :
 #ifndef _LP64
-#if !defined(_LP64) && defined(COMPILER2)
+#if defined(COMPILER2)
   // All return values are where we want them, except for Longs.  C2 returns
   // longs in G1 in the 32-bit build whereas the interpreter wants them in O0/O1.
   // Since the interpreter will return longs in G1 and O0/O1 in the 32bit
@@ -173,10 +173,9 @@ address CppInterpreterGenerator::generate_tosca_to_stack_converter(BasicType typ
       // native result is in O0, O1
       __ st(O1, L1_scratch, 0);                      // Low order
       __ st(O0, L1_scratch, -wordSize);              // High order
-#endif /* !_LP64 && COMPILER2 */
+#endif /* COMPILER2 */
 #else
-      __ stx(O0, L1_scratch, 0);
-__ breakpoint_trap();
+      __ stx(O0, L1_scratch, -wordSize);
 #endif
       __ sub(L1_scratch, 2*wordSize, L1_scratch);
       break;
@@ -237,7 +236,6 @@ address CppInterpreterGenerator::generate_stack_to_stack_converter(BasicType typ
     case T_VOID:  break;
       break;
     case T_FLOAT  :
-      __ breakpoint_trap(Assembler::zero);
     case T_BOOLEAN:
     case T_CHAR   :
     case T_BYTE   :
@@ -255,11 +253,7 @@ address CppInterpreterGenerator::generate_stack_to_stack_converter(BasicType typ
       // except we allocated one extra word for this intepretState so we won't overwrite it
       // when we return a two word result.
 #ifdef _LP64
-__ breakpoint_trap();
-      // Hmm now that longs are in one entry should "_ptr" really be "x"?
       __ ld_ptr(O0, 0, O2);
-      __ ld_ptr(O0, wordSize, O3);
-      __ st_ptr(O3, O1, 0);
       __ st_ptr(O2, O1, -wordSize);
 #else
       __ ld(O0, 0, O2);
@@ -319,10 +313,7 @@ address CppInterpreterGenerator::generate_stack_to_native_abi_converter(BasicTyp
       // except we allocated one extra word for this intepretState so we won't overwrite it
       // when we return a two word result.
 #ifdef _LP64
-__ breakpoint_trap();
-      // Hmm now that longs are in one entry should "_ptr" really be "x"?
       __ ld_ptr(O0, 0, O0->after_save());
-      __ ld_ptr(O0, wordSize, O1->after_save());
 #else
       __ ld(O0, wordSize, O1->after_save());
       __ ld(O0, 0, O0->after_save());
@@ -1373,7 +1364,7 @@ void CppInterpreterGenerator::generate_more_monitors() {
   __ delayed()->ld_ptr(L1_scratch, entry_size, L3_scratch);
 
   // now zero the slot so we can find it.
-  __ st(G0, L4_scratch, BasicObjectLock::obj_offset_in_bytes());
+  __ st_ptr(G0, L4_scratch, BasicObjectLock::obj_offset_in_bytes());
 
 }
 
@@ -1713,7 +1704,7 @@ address InterpreterGenerator::generate_normal_entry(bool synchronized) {
   __ lduh(L4_scratch, in_bytes(methodOopDesc::size_of_parameters_offset()), L2_scratch); // get parameter size
   __ sll(L2_scratch, LogBytesPerWord, L2_scratch     );                           // parameter size in bytes
   __ add(L1_scratch, L2_scratch, L1_scratch);                                      // stack destination for result
-  __ ld_ptr(L4_scratch, in_bytes(methodOopDesc::result_index_offset()), L3_scratch); // called method result type index
+  __ ld(L4_scratch, in_bytes(methodOopDesc::result_index_offset()), L3_scratch); // called method result type index
 
   // tosca is really just native abi
   __ set((intptr_t)CppInterpreter::_tosca_to_stack, L4_scratch);
@@ -1757,7 +1748,7 @@ address InterpreterGenerator::generate_normal_entry(bool synchronized) {
 
   __ ld_ptr(STATE(_prev_link), L1_scratch);
   __ ld_ptr(STATE(_method), L2_scratch);                               // get method just executed
-  __ ld_ptr(L2_scratch, in_bytes(methodOopDesc::result_index_offset()), L2_scratch);
+  __ ld(L2_scratch, in_bytes(methodOopDesc::result_index_offset()), L2_scratch);
   __ tst(L1_scratch);
   __ brx(Assembler::zero, false, Assembler::pt, return_to_initial_caller);
   __ delayed()->sll(L2_scratch, LogBytesPerWord, L2_scratch);
@@ -1923,10 +1914,10 @@ address InterpreterGenerator::generate_normal_entry(bool synchronized) {
   // compute the unused java stack size
   __ sub(Gargs, L1_scratch, L2_scratch);                       // compute unused space
 
-  // Round down the unused space to that stack is always aligned
-  // by making the unused space a multiple of the size of a long.
+  // Round down the unused space to that stack is always 16-byte aligned
+  // by making the unused space a multiple of the size of two longs.
 
-  __ and3(L2_scratch, -BytesPerLong, L2_scratch);
+  __ and3(L2_scratch, -2*BytesPerLong, L2_scratch);
 
   // Now trim the stack
   __ add(SP, L2_scratch, SP);
@@ -2176,6 +2167,9 @@ int AbstractInterpreter::layout_activation(methodOop method,
     // MUCHO HACK
 
     intptr_t* frame_bottom = interpreter_frame->sp() - (full_frame_words - frame_words);
+    // 'interpreter_frame->sp()' is unbiased while 'frame_bottom' must be a biased value in 64bit mode.
+    assert(((intptr_t)frame_bottom & 0xf) == 0, "SP biased in layout_activation");
+    frame_bottom = (intptr_t*)((intptr_t)frame_bottom - STACK_BIAS);
 
     /* Now fillin the interpreterState object */
 

src/cpu/sparc/vm/sparc.ad

@@ -5957,15 +5957,27 @@ instruct storeA8B(memory mem, regD src) %{
 
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(iRegN dst, iRegP src) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (EncodeP src));
-  format %{ "SRL    $src,3,$dst\t encodeHeapOop" %}
+  format %{ "encode_heap_oop $src, $dst" %}
   ins_encode %{
     __ encode_heap_oop($src$$Register, $dst$$Register);
   %}
   ins_pipe(ialu_reg);
 %}
 
+instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (EncodeP src));
+  format %{ "encode_heap_oop_not_null $src, $dst" %}
+  ins_encode %{
+    __ encode_heap_oop_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
+
 instruct decodeHeapOop(iRegP dst, iRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (DecodeN src));
   format %{ "decode_heap_oop $src, $dst" %}
   ins_encode %{
@@ -5974,6 +5986,16 @@ instruct decodeHeapOop(iRegP dst, iRegN src) %{
   ins_pipe(ialu_reg);
 %}
 
+instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (DecodeN src));
+  format %{ "decode_heap_oop_not_null $src, $dst" %}
+  ins_encode %{
+    __ decode_heap_oop_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
+
 
 // Store Zero into Aligned Packed Bytes
 instruct storeA8B0(memory mem, immI0 zero) %{

src/cpu/x86/vm/assembler_x86_64.cpp

@@ -4150,7 +4150,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
   if (oop_result->is_valid()) {
     movq(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
     movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
-    verify_oop(oop_result);
+    verify_oop(oop_result, "broken oop in call_VM_base");
   }
 }
 
@@ -4689,6 +4689,10 @@ void MacroAssembler::warn(const char* msg) {
   popq(r12);
 }
 
+#ifndef PRODUCT
+extern "C" void findpc(intptr_t x);
+#endif
+
 void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
   // In order to get locks to work, we need to fake a in_VM state
   if (ShowMessageBoxOnError ) {
@@ -4707,6 +4711,11 @@ void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
     if (os::message_box(msg, "Execution stopped, print registers?")) {
       ttyLocker ttyl;
       tty->print_cr("rip = 0x%016lx", pc);
+#ifndef PRODUCT
+      tty->cr();
+      findpc(pc);
+      tty->cr();
+#endif
       tty->print_cr("rax = 0x%016lx", regs[15]);
       tty->print_cr("rbx = 0x%016lx", regs[12]);
       tty->print_cr("rcx = 0x%016lx", regs[14]);
@@ -5187,7 +5196,7 @@ void MacroAssembler::encode_heap_oop(Register r) {
   bind(ok);
   popq(rscratch1);
 #endif
-  verify_oop(r);
+  verify_oop(r, "broken oop in encode_heap_oop");
   testq(r, r);
   cmovq(Assembler::equal, r, r12_heapbase);
   subq(r, r12_heapbase);
@@ -5203,11 +5212,28 @@ void MacroAssembler::encode_heap_oop_not_null(Register r) {
   stop("null oop passed to encode_heap_oop_not_null");
   bind(ok);
 #endif
-  verify_oop(r);
+  verify_oop(r, "broken oop in encode_heap_oop_not_null");
   subq(r, r12_heapbase);
   shrq(r, LogMinObjAlignmentInBytes);
 }
 
+void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
+  assert (UseCompressedOops, "should be compressed");
+#ifdef ASSERT
+  Label ok;
+  testq(src, src);
+  jcc(Assembler::notEqual, ok);
+  stop("null oop passed to encode_heap_oop_not_null2");
+  bind(ok);
+#endif
+  verify_oop(src, "broken oop in encode_heap_oop_not_null2");
+  if (dst != src) {
+    movq(dst, src);
+  }
+  subq(dst, r12_heapbase);
+  shrq(dst, LogMinObjAlignmentInBytes);
+}
+
 void  MacroAssembler::decode_heap_oop(Register r) {
   assert (UseCompressedOops, "should be compressed");
 #ifdef ASSERT
@@ -5232,7 +5258,7 @@ void  MacroAssembler::decode_heap_oop(Register r) {
    leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
 #endif
   bind(done);
-  verify_oop(r);
+  verify_oop(r, "broken oop in decode_heap_oop");
 }
 
 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
@@ -5243,6 +5269,14 @@ void  MacroAssembler::decode_heap_oop_not_null(Register r) {
   leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
 }
 
+void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
+  assert (UseCompressedOops, "should only be used for compressed headers");
+  // Cannot assert, unverified entry point counts instructions (see .ad file)
+  // vtableStubs also counts instructions in pd_code_size_limit.
+  assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
+  leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
+}
+
 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
   switch (cond) {
     // Note some conditions are synonyms for others

src/cpu/x86/vm/assembler_x86_64.hpp

@@ -1111,6 +1111,8 @@ class MacroAssembler : public Assembler {
   void decode_heap_oop(Register r);
   void encode_heap_oop_not_null(Register r);
   void decode_heap_oop_not_null(Register r);
+  void encode_heap_oop_not_null(Register dst, Register src);
+  void decode_heap_oop_not_null(Register dst, Register src);
 
   // Stack frame creation/removal
   void enter();

src/cpu/x86/vm/stubGenerator_x86_64.cpp

@@ -913,11 +913,12 @@ class StubGenerator: public StubCodeGenerator {
   // Stack after saving c_rarg3:
   //    [tos + 0]: saved c_rarg3
   //    [tos + 1]: saved c_rarg2
-  //    [tos + 2]: saved flags
-  //    [tos + 3]: return address
-  //  * [tos + 4]: error message (char*)
-  //  * [tos + 5]: object to verify (oop)
-  //  * [tos + 6]: saved rax - saved by caller and bashed
+  //    [tos + 2]: saved r12 (several TemplateTable methods use it)
+  //    [tos + 3]: saved flags
+  //    [tos + 4]: return address
+  //  * [tos + 5]: error message (char*)
+  //  * [tos + 6]: object to verify (oop)
+  //  * [tos + 7]: saved rax - saved by caller and bashed
   //  * = popped on exit
   address generate_verify_oop() {
     StubCodeMark mark(this, "StubRoutines", "verify_oop");
@@ -928,12 +929,24 @@ class StubGenerator: public StubCodeGenerator {
     __ pushfq();
     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 
+    __ pushq(r12);
+
     // save c_rarg2 and c_rarg3
     __ pushq(c_rarg2);
     __ pushq(c_rarg3);
 
+    enum {
+           // After previous pushes.
+           oop_to_verify = 6 * wordSize,
+           saved_rax     = 7 * wordSize,
+
+           // Before the call to MacroAssembler::debug(), see below.
+           return_addr   = 16 * wordSize,
+           error_msg     = 17 * wordSize
+    };
+
     // get object
-    __ movq(rax, Address(rsp, 5 * wordSize));
+    __ movq(rax, Address(rsp, oop_to_verify));
 
     // make sure object is 'reasonable'
     __ testq(rax, rax);
@@ -946,6 +959,9 @@ class StubGenerator: public StubCodeGenerator {
     __ cmpq(c_rarg2, c_rarg3);
     __ jcc(Assembler::notZero, error);
 
+    // set r12 to heapbase for load_klass()
+    __ reinit_heapbase();
+
     // make sure klass is 'reasonable'
     __ load_klass(rax, rax);  // get klass
     __ testq(rax, rax);
@@ -971,40 +987,45 @@ class StubGenerator: public StubCodeGenerator {
 
     // return if everything seems ok
     __ bind(exit);
-    __ movq(rax, Address(rsp, 6 * wordSize));    // get saved rax back
-    __ popq(c_rarg3);                              // restore c_rarg3
-    __ popq(c_rarg2);                              // restore c_rarg2
+    __ movq(rax, Address(rsp, saved_rax));       // get saved rax back
+    __ popq(c_rarg3);                            // restore c_rarg3
+    __ popq(c_rarg2);                            // restore c_rarg2
+    __ popq(r12);                                // restore r12
     __ popfq();                                  // restore flags
     __ ret(3 * wordSize);                        // pop caller saved stuff
 
     // handle errors
     __ bind(error);
-    __ movq(rax, Address(rsp, 6 * wordSize));    // get saved rax back
-    __ popq(c_rarg3);                              // get saved c_rarg3 back
-    __ popq(c_rarg2);                              // get saved c_rarg2 back
+    __ movq(rax, Address(rsp, saved_rax));       // get saved rax back
+    __ popq(c_rarg3);                            // get saved c_rarg3 back
+    __ popq(c_rarg2);                            // get saved c_rarg2 back
+    __ popq(r12);                                // get saved r12 back
     __ popfq();                                  // get saved flags off stack --
                                                  // will be ignored
 
     __ pushaq();                                 // push registers
                                                  // (rip is already
                                                  // already pushed)
-    // debug(char* msg, int64_t regs[])
+    // debug(char* msg, int64_t pc, int64_t regs[])
     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
     // pushed all the registers, so now the stack looks like:
     //     [tos +  0] 16 saved registers
     //     [tos + 16] return address
-    //     [tos + 17] error message (char*)
-
-    __ movq(c_rarg0, Address(rsp, 17 * wordSize)); // pass address of error message
-    __ movq(c_rarg1, rsp);                         // pass address of regs on stack
+    //   * [tos + 17] error message (char*)
+    //   * [tos + 18] object to verify (oop)
+    //   * [tos + 19] saved rax - saved by caller and bashed
+    //   * = popped on exit
+
+    __ movq(c_rarg0, Address(rsp, error_msg));   // pass address of error message
+    __ movq(c_rarg1, Address(rsp, return_addr)); // pass return address
+    __ movq(c_rarg2, rsp);                       // pass address of regs on stack
     __ movq(r12, rsp);                           // remember rsp
     __ subq(rsp, frame::arg_reg_save_area_bytes);// windows
     __ andq(rsp, -16);                           // align stack as required by ABI
     BLOCK_COMMENT("call MacroAssembler::debug");
     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
     __ movq(rsp, r12);                           // restore rsp
-    __ reinit_heapbase();                        // r12 is heapbase
-    __ popaq();                                  // pop registers
+    __ popaq();                                  // pop registers (includes r12)
     __ ret(3 * wordSize);                        // pop caller saved stuff
 
     return start;
@@ -1038,7 +1059,7 @@ class StubGenerator: public StubCodeGenerator {
     assert_different_registers(Rtmp, Rint);
     __ movslq(Rtmp, Rint);
     __ cmpq(Rtmp, Rint);
-    __ jccb(Assembler::equal, L);
+    __ jcc(Assembler::equal, L);
     __ stop("high 32-bits of int value are not 0");
     __ bind(L);
 #endif

src/cpu/x86/vm/x86_64.ad

@@ -6080,7 +6080,8 @@ instruct loadKlassComp(rRegP dst, memory mem)
   predicate(n->in(MemNode::Address)->bottom_type()->is_narrow());
 
   ins_cost(125); // XXX
-  format %{ "movl    $dst, $mem\t# compressed class" %}
+  format %{ "movl    $dst, $mem\t# compressed class\n\t"
+            "decode_heap_oop $dst,$dst" %}
   ins_encode %{
     Address addr = build_address($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
     Register dst = as_Register($dst$$reg);
@@ -6349,7 +6350,7 @@ instruct loadConF(regF dst, immF src)
 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
   match(Set dst src);
   effect(KILL cr);
-  format %{ "xorq    $dst, $src\t# compressed ptr" %}
+  format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
   ins_encode %{
     Register dst = $dst$$Register;
     __ xorq(dst, dst);
@@ -6361,7 +6362,8 @@ instruct loadConN(rRegN dst, immN src) %{
   match(Set dst src);
 
   ins_cost(125);
-  format %{ "movl    $dst, $src\t# compressed ptr" %}
+  format %{ "movq    $dst, $src\t# compressed ptr\n\t"
+            "encode_heap_oop_not_null $dst,$dst" %}
   ins_encode %{
     address con = (address)$src$$constant;
     Register dst = $dst$$Register;
@@ -6996,6 +6998,7 @@ instruct castP2X(rRegL dst, rRegP src)
 
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (EncodeP src));
   effect(KILL cr);
   format %{ "encode_heap_oop $dst,$src" %}
@@ -7010,7 +7013,21 @@ instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
   ins_pipe(ialu_reg_long);
 %}
 
+instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (EncodeP src));
+  effect(KILL cr);
+  format %{ "encode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ encode_heap_oop_not_null(d, s);
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (DecodeN src));
   effect(KILL cr);
   format %{ "decode_heap_oop $dst,$src" %}
@@ -7025,6 +7042,18 @@ instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
   ins_pipe(ialu_reg_long);
 %}
 
+instruct decodeHeapOop_not_null(rRegP dst, rRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (DecodeN src));
+  format %{ "decode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ decode_heap_oop_not_null(d, s);
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
 
 //----------Conditional Move---------------------------------------------------
 // Jump

src/share/vm/interpreter/bytecodeInterpreter.cpp

@@ -518,16 +518,16 @@ BytecodeInterpreter::run(interpreterState istate) {
 
 /* 0xC0 */ &&opc_checkcast,   &&opc_instanceof,     &&opc_monitorenter, &&opc_monitorexit,
 /* 0xC4 */ &&opc_wide,        &&opc_multianewarray, &&opc_ifnull,       &&opc_ifnonnull,
-/* 0xC8 */ &&opc_goto_w,      &&opc_jsr_w,          &&opc_breakpoint,   &&opc_fast_igetfield,
-/* 0xCC */ &&opc_fastagetfield,&&opc_fast_aload_0,  &&opc_fast_iaccess_0, &&opc__fast_aaccess_0,
+/* 0xC8 */ &&opc_goto_w,      &&opc_jsr_w,          &&opc_breakpoint,   &&opc_default,
+/* 0xCC */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 
-/* 0xD0 */ &&opc_fast_linearswitch, &&opc_fast_binaryswitch, &&opc_return_register_finalizer,      &&opc_default,
+/* 0xD0 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 /* 0xD4 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 /* 0xD8 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 /* 0xDC */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 
 /* 0xE0 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
-/* 0xE4 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
+/* 0xE4 */ &&opc_default,     &&opc_return_register_finalizer,        &&opc_default,      &&opc_default,
 /* 0xE8 */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 /* 0xEC */ &&opc_default,     &&opc_default,        &&opc_default,      &&opc_default,
 

src/share/vm/opto/cfgnode.cpp

@@ -1419,7 +1419,8 @@ PhiNode::LoopSafety PhiNode::simple_data_loop_check(Node *in) const {
     // Check inputs of phi's inputs also.
     // It is much less expensive then full graph walk.
     uint cnt = in->req();
-    for (uint i = 1; i < cnt; ++i) {
+    uint i = (in->is_Proj() && !in->is_CFG())  ? 0 : 1;
+    for (; i < cnt; ++i) {
       Node* m = in->in(i);
       if (m == (Node*)this)
         return UnsafeLoop; // Unsafe loop
@@ -1467,7 +1468,8 @@ bool PhiNode::is_unsafe_data_reference(Node *in) const {
   while (nstack.size() != 0) {
     Node* n = nstack.pop();
     uint cnt = n->req();
-    for (uint i = 1; i < cnt; i++) { // Only data paths
+    uint i = (n->is_Proj() && !n->is_CFG()) ? 0 : 1;
+    for (; i < cnt; i++) {
       Node* m = n->in(i);
       if (m == (Node*)this) {
         return true;    // Data loop
@@ -2017,6 +2019,28 @@ Node *CreateExNode::Identity( PhaseTransform *phase ) {
 }
 
 //=============================================================================
+//------------------------------Value------------------------------------------
+// Check for being unreachable.
+const Type *NeverBranchNode::Value( PhaseTransform *phase ) const {
+  if (!in(0) || in(0)->is_top()) return Type::TOP;
+  return bottom_type();
+}
+
+//------------------------------Ideal------------------------------------------
+// Check for no longer being part of a loop
+Node *NeverBranchNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  if (can_reshape && !in(0)->is_Loop()) {
+    // Dead code elimination can sometimes delete this projection so
+    // if it's not there, there's nothing to do.
+    Node* fallthru = proj_out(0);
+    if (fallthru != NULL) {
+      phase->is_IterGVN()->subsume_node(fallthru, in(0));
+    }
+    return phase->C->top();
+  }
+  return NULL;
+}
+
 #ifndef PRODUCT
 void NeverBranchNode::format( PhaseRegAlloc *ra_, outputStream *st) const {
   st->print("%s", Name());

src/share/vm/opto/cfgnode.hpp

@@ -244,6 +244,8 @@ public:
   MultiBranchNode( uint required ) : MultiNode(required) {
     init_class_id(Class_MultiBranch);
   }
+  // returns required number of users to be well formed.
+  virtual int required_outcnt() const = 0;
 };
 
 //------------------------------IfNode-----------------------------------------
@@ -333,6 +335,7 @@ public:
   virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; }
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
   virtual const Type *Value( PhaseTransform *phase ) const;
+  virtual int required_outcnt() const { return 2; }
   virtual const RegMask &out_RegMask() const;
   void dominated_by(Node* prev_dom, PhaseIterGVN* igvn);
   int is_range_check(Node* &range, Node* &index, jint &offset);
@@ -391,6 +394,7 @@ public:
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
   virtual const Type *bottom_type() const;
   virtual bool pinned() const { return true; }
+  virtual int required_outcnt() const { return _size; }
 };
 
 //------------------------------JumpNode---------------------------------------
@@ -504,7 +508,9 @@ public:
   virtual int Opcode() const;
   virtual bool pinned() const { return true; };
   virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; }
-
+  virtual const Type *Value( PhaseTransform *phase ) const;
+  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
+  virtual int required_outcnt() const { return 2; }
   virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { }
   virtual uint size(PhaseRegAlloc *ra_) const { return 0; }
 #ifndef PRODUCT

src/share/vm/opto/compile.cpp

@@ -1981,10 +1981,6 @@ static void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &fpu ) {
 #endif
     break;
   }
-  case Op_If:
-  case Op_CountedLoopEnd:
-    fpu._tests.push(n);         // Collect CFG split points
-    break;
 
   case Op_AddP: {               // Assert sane base pointers
     const Node *addp = n->in(AddPNode::Address);
@@ -2083,10 +2079,12 @@ static void final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &fpu ) {
   default:
     assert( !n->is_Call(), "" );
     assert( !n->is_Mem(), "" );
-    if( n->is_If() || n->is_PCTable() )
-      fpu._tests.push(n);       // Collect CFG split points
     break;
   }
+
+  // Collect CFG split points
+  if (n->is_MultiBranch())
+    fpu._tests.push(n);
 }
 
 //------------------------------final_graph_reshaping_walk---------------------
@@ -2165,19 +2163,18 @@ bool Compile::final_graph_reshaping() {
 
   // Check for unreachable (from below) code (i.e., infinite loops).
   for( uint i = 0; i < fpu._tests.size(); i++ ) {
-    Node *n = fpu._tests[i];
-    assert( n->is_PCTable() || n->is_If(), "either PCTables or IfNodes" );
-    // Get number of CFG targets; 2 for IfNodes or _size for PCTables.
+    MultiBranchNode *n = fpu._tests[i]->as_MultiBranch();
+    // Get number of CFG targets.
     // Note that PCTables include exception targets after calls.
-    uint expected_kids = n->is_PCTable() ? n->as_PCTable()->_size : 2;
-    if (n->outcnt() != expected_kids) {
+    uint required_outcnt = n->required_outcnt();
+    if (n->outcnt() != required_outcnt) {
       // Check for a few special cases.  Rethrow Nodes never take the
       // 'fall-thru' path, so expected kids is 1 less.
       if (n->is_PCTable() && n->in(0) && n->in(0)->in(0)) {
         if (n->in(0)->in(0)->is_Call()) {
           CallNode *call = n->in(0)->in(0)->as_Call();
           if (call->entry_point() == OptoRuntime::rethrow_stub()) {
-            expected_kids--;      // Rethrow always has 1 less kid
+            required_outcnt--;      // Rethrow always has 1 less kid
           } else if (call->req() > TypeFunc::Parms &&
                      call->is_CallDynamicJava()) {
             // Check for null receiver. In such case, the optimizer has
@@ -2187,7 +2184,7 @@ bool Compile::final_graph_reshaping() {
             Node *arg0 = call->in(TypeFunc::Parms);
             if (arg0->is_Type() &&
                 arg0->as_Type()->type()->higher_equal(TypePtr::NULL_PTR)) {
-              expected_kids--;
+              required_outcnt--;
             }
           } else if (call->entry_point() == OptoRuntime::new_array_Java() &&
                      call->req() > TypeFunc::Parms+1 &&
@@ -2198,13 +2195,13 @@ bool Compile::final_graph_reshaping() {
             Node *arg1 = call->in(TypeFunc::Parms+1);
             if (arg1->is_Type() &&
                 arg1->as_Type()->type()->join(TypeInt::POS)->empty()) {
-              expected_kids--;
+              required_outcnt--;
             }
           }
         }
       }
-      // Recheck with a better notion of 'expected_kids'
-      if (n->outcnt() != expected_kids) {
+      // Recheck with a better notion of 'required_outcnt'
+      if (n->outcnt() != required_outcnt) {
         record_method_not_compilable("malformed control flow");
         return true;            // Not all targets reachable!
       }

src/share/vm/opto/connode.cpp

@@ -563,6 +563,26 @@ Node* DecodeNNode::Identity(PhaseTransform* phase) {
   return this;
 }
 
+const Type *DecodeNNode::Value( PhaseTransform *phase ) const {
+  if (phase->type( in(1) ) == TypeNarrowOop::NULL_PTR) {
+    return TypePtr::NULL_PTR;
+  }
+  return bottom_type();
+}
+
+Node* DecodeNNode::decode(PhaseGVN* phase, Node* value) {
+  if (value->Opcode() == Op_EncodeP) {
+    // (DecodeN (EncodeP p)) -> p
+    return value->in(1);
+  }
+  const Type* newtype = value->bottom_type();
+  if (newtype == TypeNarrowOop::NULL_PTR) {
+    return phase->transform(new (phase->C, 1) ConPNode(TypePtr::NULL_PTR));
+  } else {
+    return phase->transform(new (phase->C, 2) DecodeNNode(value, newtype->is_narrowoop()->make_oopptr()));
+  }
+}
+
 Node* EncodePNode::Identity(PhaseTransform* phase) {
   const Type *t = phase->type( in(1) );
   if( t == Type::TOP ) return in(1);
@@ -574,14 +594,26 @@ Node* EncodePNode::Identity(PhaseTransform* phase) {
   return this;
 }
 
+const Type *EncodePNode::Value( PhaseTransform *phase ) const {
+  if (phase->type( in(1) ) == TypePtr::NULL_PTR) {
+    return TypeNarrowOop::NULL_PTR;
+  }
+  return bottom_type();
+}
 
 Node* EncodePNode::encode(PhaseGVN* phase, Node* value) {
+  if (value->Opcode() == Op_DecodeN) {
+    // (EncodeP (DecodeN p)) -> p
+    return value->in(1);
+  }
   const Type* newtype = value->bottom_type();
   if (newtype == TypePtr::NULL_PTR) {
     return phase->transform(new (phase->C, 1) ConNNode(TypeNarrowOop::NULL_PTR));
+  } else if (newtype->isa_oopptr()) {
+    return phase->transform(new (phase->C, 2) EncodePNode(value, newtype->is_oopptr()->make_narrowoop()));
   } else {
-    return phase->transform(new (phase->C, 2) EncodePNode(value,
-                                                          newtype->is_oopptr()->make_narrowoop()));
+    ShouldNotReachHere();
+    return NULL; // to make C++ compiler happy.
   }
 }
 

src/share/vm/opto/connode.hpp

@@ -239,10 +239,7 @@ public:
 // cast pointer to pointer (different type)
 class CastPPNode: public ConstraintCastNode {
 public:
-  CastPPNode (Node *n, const Type *t ): ConstraintCastNode(n, t) {
-    // Only CastPP is safe.  CastII can cause optimizer loops.
-    init_flags(Flag_is_dead_loop_safe);
-  }
+  CastPPNode (Node *n, const Type *t ): ConstraintCastNode(n, t) {}
   virtual int Opcode() const;
   virtual uint ideal_reg() const { return Op_RegP; }
   virtual Node *Ideal_DU_postCCP( PhaseCCP * );
@@ -254,10 +251,10 @@ class CheckCastPPNode: public TypeNode {
 public:
   CheckCastPPNode( Node *c, Node *n, const Type *t ) : TypeNode(t,2) {
     init_class_id(Class_CheckCastPP);
-    init_flags(Flag_is_dead_loop_safe);
     init_req(0, c);
     init_req(1, n);
   }
+
   virtual Node *Identity( PhaseTransform *phase );
   virtual const Type *Value( PhaseTransform *phase ) const;
   virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
@@ -282,6 +279,7 @@ class EncodePNode : public TypeNode {
   }
   virtual int Opcode() const;
   virtual Node *Identity( PhaseTransform *phase );
+  virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegN; }
 
   static Node* encode(PhaseGVN* phase, Node* value);
@@ -300,7 +298,10 @@ class DecodeNNode : public TypeNode {
   }
   virtual int Opcode() const;
   virtual Node *Identity( PhaseTransform *phase );
+  virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegP; }
+
+  static Node* decode(PhaseGVN* phase, Node* value);
 };
 
 //------------------------------Conv2BNode-------------------------------------

src/share/vm/opto/escape.cpp

@@ -215,6 +215,10 @@ void ConnectionGraph::PointsTo(VectorSet &ptset, Node * n, PhaseTransform *phase
   VectorSet visited(Thread::current()->resource_area());
   GrowableArray<uint>  worklist;
 
+#ifdef ASSERT
+  Node *orig_n = n;
+#endif
+
   n = n->uncast();
   PointsToNode  npt = _nodes->at_grow(n->_idx);
 
@@ -223,8 +227,14 @@ void ConnectionGraph::PointsTo(VectorSet &ptset, Node * n, PhaseTransform *phase
     ptset.set(n->_idx);
     return;
   }
-  assert(npt._node != NULL, "unregistered node");
-
+#ifdef ASSERT
+  if (npt._node == NULL) {
+    if (orig_n != n)
+      orig_n->dump();
+    n->dump();
+    assert(npt._node != NULL, "unregistered node");
+  }
+#endif
   worklist.push(n->_idx);
   while(worklist.length() > 0) {
     int ni = worklist.pop();
@@ -266,7 +276,7 @@ void ConnectionGraph::remove_deferred(uint ni, GrowableArray<uint>* deferred_edg
   PointsToNode *ptn = ptnode_adr(ni);
 
   // Mark current edges as visited and move deferred edges to separate array.
-  for (; i < ptn->edge_count(); i++) {
+  while (i < ptn->edge_count()) {
     uint t = ptn->edge_target(i);
 #ifdef ASSERT
     assert(!visited->test_set(t), "expecting no duplications");
@@ -276,6 +286,8 @@ void ConnectionGraph::remove_deferred(uint ni, GrowableArray<uint>* deferred_edg
     if (ptn->edge_type(i) == PointsToNode::DeferredEdge) {
       ptn->remove_edge(t, PointsToNode::DeferredEdge);
       deferred_edges->append(t);
+    } else {
+      i++;
     }
   }
   for (int next = 0; next < deferred_edges->length(); ++next) {
@@ -1716,6 +1728,8 @@ void ConnectionGraph::record_for_escape_analysis(Node *n, PhaseTransform *phase)
     }
     case Op_CastPP:
     case Op_CheckCastPP:
+    case Op_EncodeP:
+    case Op_DecodeN:
     {
       add_node(n, PointsToNode::LocalVar, PointsToNode::UnknownEscape, false);
       int ti = n->in(1)->_idx;
@@ -1743,12 +1757,6 @@ void ConnectionGraph::record_for_escape_analysis(Node *n, PhaseTransform *phase)
       add_node(n, PointsToNode::JavaObject, es, true);
       break;
     }
-    case Op_CreateEx:
-    {
-      // assume that all exception objects globally escape
-      add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
-      break;
-    }
     case Op_ConN:
     {
       // assume all narrow oop constants globally escape except for null
@@ -1761,6 +1769,12 @@ void ConnectionGraph::record_for_escape_analysis(Node *n, PhaseTransform *phase)
       add_node(n, PointsToNode::JavaObject, es, true);
       break;
     }
+    case Op_CreateEx:
+    {
+      // assume that all exception objects globally escape
+      add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
+      break;
+    }
     case Op_LoadKlass:
     {
       add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
@@ -1976,10 +1990,11 @@ void ConnectionGraph::build_connection_graph(Node *n, PhaseTransform *phase) {
       break;
     }
     case Op_LoadP:
+    case Op_LoadN:
     {
       const Type *t = phase->type(n);
 #ifdef ASSERT
-      if (t->isa_ptr() == NULL)
+      if (!t->isa_narrowoop() && t->isa_ptr() == NULL)
         assert(false, "Op_LoadP");
 #endif
 
@@ -2060,11 +2075,16 @@ void ConnectionGraph::build_connection_graph(Node *n, PhaseTransform *phase) {
       break;
     }
     case Op_StoreP:
+    case Op_StoreN:
     case Op_StorePConditional:
     case Op_CompareAndSwapP:
+    case Op_CompareAndSwapN:
     {
       Node *adr = n->in(MemNode::Address);
       const Type *adr_type = phase->type(adr);
+      if (adr_type->isa_narrowoop()) {
+        adr_type = adr_type->is_narrowoop()->make_oopptr();
+      }
 #ifdef ASSERT
       if (!adr_type->isa_oopptr())
         assert(phase->type(adr) == TypeRawPtr::NOTNULL, "Op_StoreP");

src/share/vm/opto/lcm.cpp

@@ -629,6 +629,10 @@ bool Block::schedule_local(PhaseCFG *cfg, Matcher &matcher, int *ready_cnt, Vect
         // of the phi to be scheduled first. The select() method breaks
         // ties in scheduling by worklist order.
         delay.push(m);
+      } else if (m->is_Mach() && m->as_Mach()->ideal_Opcode() == Op_CreateEx) {
+        // Force the CreateEx to the top of the list so it's processed
+        // first and ends up at the start of the block.
+        worklist.insert(0, m);
       } else {
         worklist.push(m);         // Then on to worklist!
       }

src/share/vm/opto/macro.cpp

@@ -584,7 +584,7 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
 
       const Type *field_type;
       // The next code is taken from Parse::do_get_xxx().
-      if (basic_elem_type == T_OBJECT) {
+      if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
         if (!elem_type->is_loaded()) {
           field_type = TypeInstPtr::BOTTOM;
         } else if (field != NULL && field->is_constant()) {
@@ -597,6 +597,10 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
         } else {
           field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
         }
+        if (UseCompressedOops) {
+          field_type = field_type->is_oopptr()->make_narrowoop();
+          basic_elem_type = T_NARROWOOP;
+        }
       } else {
         field_type = Type::get_const_basic_type(basic_elem_type);
       }
@@ -659,6 +663,13 @@ bool PhaseMacroExpand::scalar_replacement(AllocateNode *alloc, GrowableArray <Sa
 #endif
         return false;
       }
+      if (UseCompressedOops && field_type->isa_narrowoop()) {
+        // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
+        // to be able scalar replace the allocation.
+        _igvn.set_delay_transform(false);
+        field_val = DecodeNNode::decode(&_igvn, field_val);
+        _igvn.set_delay_transform(true);
+      }
       sfpt->add_req(field_val);
     }
     JVMState *jvms = sfpt->jvms();

src/share/vm/opto/memnode.cpp

@@ -754,13 +754,12 @@ Node *LoadNode::make( PhaseGVN& gvn, Node *ctl, Node *mem, Node *adr, const Type
       const TypeNarrowOop* narrowtype;
       if (rt->isa_narrowoop()) {
         narrowtype = rt->is_narrowoop();
-        rt = narrowtype->make_oopptr();
       } else {
         narrowtype = rt->is_oopptr()->make_narrowoop();
       }
       Node* load  = gvn.transform(new (C, 3) LoadNNode(ctl, mem, adr, adr_type, narrowtype));
 
-      return new (C, 2) DecodeNNode(load, rt);
+      return DecodeNNode::decode(&gvn, load);
     } else
 #endif
       {
@@ -1841,15 +1840,7 @@ StoreNode* StoreNode::make( PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, cons
         (UseCompressedOops && val->bottom_type()->isa_klassptr() &&
          adr->bottom_type()->isa_rawptr())) {
       const TypePtr* type = val->bottom_type()->is_ptr();
-      Node* cp;
-      if (type->isa_oopptr()) {
-        const TypeNarrowOop* etype = type->is_oopptr()->make_narrowoop();
-        cp = gvn.transform(new (C, 2) EncodePNode(val, etype));
-      } else if (type == TypePtr::NULL_PTR) {
-        cp = gvn.transform(new (C, 1) ConNNode(TypeNarrowOop::NULL_PTR));
-      } else {
-        ShouldNotReachHere();
-      }
+      Node* cp = EncodePNode::encode(&gvn, val);
       return new (C, 4) StoreNNode(ctl, mem, adr, adr_type, cp);
     } else
 #endif

src/share/vm/opto/multnode.hpp

@@ -61,6 +61,9 @@ public:
     : Node( src ), _con(con), _is_io_use(io_use)
   {
     init_class_id(Class_Proj);
+    // Optimistic setting. Need additional checks in Node::is_dead_loop_safe().
+    if (con != TypeFunc::Memory || src->is_Start())
+      init_flags(Flag_is_dead_loop_safe);
     debug_only(check_con());
   }
   const uint _con;              // The field in the tuple we are projecting

src/share/vm/opto/node.hpp

@@ -741,8 +741,9 @@ public:
   bool is_Goto() const { return (_flags & Flag_is_Goto) != 0; }
   // The data node which is safe to leave in dead loop during IGVN optimization.
   bool is_dead_loop_safe() const {
-    return is_Phi() || is_Proj() ||
-           (_flags & (Flag_is_dead_loop_safe | Flag_is_Con)) != 0;
+    return is_Phi() || (is_Proj() && in(0) == NULL) ||
+           ((_flags & (Flag_is_dead_loop_safe | Flag_is_Con)) != 0 &&
+            (!is_Proj() || !in(0)->is_Allocate()));
   }
 
   // is_Copy() returns copied edge index (0 or 1)

src/share/vm/runtime/sharedRuntime.cpp

@@ -2229,6 +2229,8 @@ JRT_END
 #ifndef PRODUCT
 bool AdapterHandlerLibrary::contains(CodeBlob* b) {
 
+  if (_handlers == NULL) return false;
+
   for (int i = 0 ; i < _handlers->length() ; i++) {
     AdapterHandlerEntry* a = get_entry(i);
     if ( a != NULL && b == CodeCache::find_blob(a->get_i2c_entry()) ) return true;