6741940: Nonvolatile XMM registers not preserved across JNI calls

Summary: Save xmm6-xmm15 in call stub on win64
Reviewed-by: kvn, never

src/cpu/x86/vm/frame_x86.hpp

 /*
- * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -125,7 +125,7 @@
     // Entry frames
 #ifdef AMD64
 #ifdef _WIN64
-    entry_frame_after_call_words                     =  8,
+    entry_frame_after_call_words                     =  28,
     entry_frame_call_wrapper_offset                  =  2,
 
     arg_reg_save_area_bytes                          = 32, // Register argument save area

src/cpu/x86/vm/stubGenerator_x86_64.cpp

@@ -144,8 +144,11 @@ class StubGenerator: public StubCodeGenerator {
   //     [ return_from_Java     ] <--- rsp
   //     [ argument word n      ]
   //      ...
-  //  -8 [ argument word 1      ]
-  //  -7 [ saved r15            ] <--- rsp_after_call
+  // -28 [ argument word 1      ]
+  // -27 [ saved xmm15          ] <--- rsp_after_call
+  //     [ saved xmm7-xmm14     ]
+  //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
+  //  -7 [ saved r15            ]
   //  -6 [ saved r14            ]
   //  -5 [ saved r13            ]
   //  -4 [ saved r12            ]
@@ -169,8 +172,11 @@ class StubGenerator: public StubCodeGenerator {
   // Call stub stack layout word offsets from rbp
   enum call_stub_layout {
 #ifdef _WIN64
-    rsp_after_call_off = -7,
-    r15_off            = rsp_after_call_off,
+    xmm_save_first     = 6,  // save from xmm6
+    xmm_save_last      = 15, // to xmm15
+    xmm_save_base      = -9,
+    rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
+    r15_off            = -7,
     r14_off            = -6,
     r13_off            = -5,
     r12_off            = -4,
@@ -208,6 +214,13 @@ class StubGenerator: public StubCodeGenerator {
 #endif
   };
 
+#ifdef _WIN64
+  Address xmm_save(int reg) {
+    assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
+    return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
+  }
+#endif
+
   address generate_call_stub(address& return_address) {
     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
@@ -256,8 +269,11 @@ class StubGenerator: public StubCodeGenerator {
     __ movptr(r13_save, r13);
     __ movptr(r14_save, r14);
     __ movptr(r15_save, r15);
-
 #ifdef _WIN64
+    for (int i = 6; i <= 15; i++) {
+      __ movdqu(xmm_save(i), as_XMMRegister(i));
+    }
+
     const Address rdi_save(rbp, rdi_off * wordSize);
     const Address rsi_save(rbp, rsi_off * wordSize);
 
@@ -360,6 +376,11 @@ class StubGenerator: public StubCodeGenerator {
 #endif
 
     // restore regs belonging to calling function
+#ifdef _WIN64
+    for (int i = 15; i >= 6; i--) {
+      __ movdqu(as_XMMRegister(i), xmm_save(i));
+    }
+#endif
     __ movptr(r15, r15_save);
     __ movptr(r14, r14_save);
     __ movptr(r13, r13_save);