From 6a8288133ce510dd8a7c9230f2c6bc400d6e9b4f Mon Sep 17 00:00:00 2001 From: ogatak Date: Tue, 18 Jun 2019 09:33:34 -0400 Subject: [PATCH] 8185979: PPC64: Implement SHA2 intrinsic Reviewed-by: mdoerr, goetz Contributed-by: Bruno Rosa , Gustavo Serra Scalet , Igor Nunes , Martin Doerr --- src/cpu/ppc/vm/assembler_ppc.hpp | 7 +- src/cpu/ppc/vm/assembler_ppc.inline.hpp | 19 +- src/cpu/ppc/vm/macroAssembler_ppc.hpp | 34 + src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp | 1136 +++++++++++++++++ src/cpu/ppc/vm/stubGenerator_ppc.cpp | 31 + src/cpu/ppc/vm/stubRoutines_ppc_64.hpp | 2 +- src/cpu/ppc/vm/vm_version_ppc.cpp | 41 +- src/cpu/ppc/vm/vm_version_ppc.hpp | 3 + src/share/vm/opto/library_call.cpp | 18 +- src/share/vm/opto/runtime.cpp | 22 +- .../testcases/GenericTestCaseForOtherCPU.java | 3 +- .../sha/predicate/IntrinsicPredicates.java | 16 +- 12 files changed, 1308 insertions(+), 24 deletions(-) create mode 100644 src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp diff --git a/src/cpu/ppc/vm/assembler_ppc.hpp b/src/cpu/ppc/vm/assembler_ppc.hpp index b14f2b6d0..16884ca51 100644 --- a/src/cpu/ppc/vm/assembler_ppc.hpp +++ b/src/cpu/ppc/vm/assembler_ppc.hpp @@ -2000,7 +2000,8 @@ class Assembler : public AbstractAssembler { inline void vsbox( VectorRegister d, VectorRegister a); // SHA (introduced with Power 8) - // Not yet implemented. + inline void vshasigmad(VectorRegister d, VectorRegister a, bool st, int six); + inline void vshasigmaw(VectorRegister d, VectorRegister a, bool st, int six); // Vector Binary Polynomial Multiplication (introduced with Power 8) inline void vpmsumb( VectorRegister d, VectorRegister a, VectorRegister b); @@ -2096,6 +2097,10 @@ class Assembler : public AbstractAssembler { inline void lvsl( VectorRegister d, Register s2); inline void lvsr( VectorRegister d, Register s2); + // Endianess specific concatenation of 2 loaded vectors. + inline void load_perm(VectorRegister perm, Register addr); + inline void vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm); + // RegisterOrConstant versions. // These emitters choose between the versions using two registers and // those with register and immediate, depending on the content of roc. diff --git a/src/cpu/ppc/vm/assembler_ppc.inline.hpp b/src/cpu/ppc/vm/assembler_ppc.inline.hpp index 1d2c05e2a..badeecf77 100644 --- a/src/cpu/ppc/vm/assembler_ppc.inline.hpp +++ b/src/cpu/ppc/vm/assembler_ppc.inline.hpp @@ -789,7 +789,8 @@ inline void Assembler::vncipherlast(VectorRegister d, VectorRegister a, VectorRe inline void Assembler::vsbox( VectorRegister d, VectorRegister a) { emit_int32( VSBOX_OPCODE | vrt(d) | vra(a) ); } // SHA (introduced with Power 8) -// Not yet implemented. +inline void Assembler::vshasigmad(VectorRegister d, VectorRegister a, bool st, int six) { emit_int32( VSHASIGMAD_OPCODE | vrt(d) | vra(a) | vst(st) | vsix(six)); } +inline void Assembler::vshasigmaw(VectorRegister d, VectorRegister a, bool st, int six) { emit_int32( VSHASIGMAW_OPCODE | vrt(d) | vra(a) | vst(st) | vsix(six)); } // Vector Binary Polynomial Multiplication (introduced with Power 8) inline void Assembler::vpmsumb( VectorRegister d, VectorRegister a, VectorRegister b) { emit_int32( VPMSUMB_OPCODE | vrt(d) | vra(a) | vrb(b)); } @@ -887,6 +888,22 @@ inline void Assembler::stvxl( VectorRegister d, Register s2) { emit_int32( STVXL inline void Assembler::lvsl( VectorRegister d, Register s2) { emit_int32( LVSL_OPCODE | vrt(d) | rb(s2)); } inline void Assembler::lvsr( VectorRegister d, Register s2) { emit_int32( LVSR_OPCODE | vrt(d) | rb(s2)); } +inline void Assembler::load_perm(VectorRegister perm, Register addr) { +#if defined(VM_LITTLE_ENDIAN) + lvsr(perm, addr); +#else + lvsl(perm, addr); +#endif +} + +inline void Assembler::vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm) { +#if defined(VM_LITTLE_ENDIAN) + vperm(first_dest, second, first_dest, perm); +#else + vperm(first_dest, first_dest, second, perm); +#endif +} + inline void Assembler::load_const(Register d, void* x, Register tmp) { load_const(d, (long)x, tmp); } diff --git a/src/cpu/ppc/vm/macroAssembler_ppc.hpp b/src/cpu/ppc/vm/macroAssembler_ppc.hpp index 3c6cea51b..afcec9a47 100644 --- a/src/cpu/ppc/vm/macroAssembler_ppc.hpp +++ b/src/cpu/ppc/vm/macroAssembler_ppc.hpp @@ -667,6 +667,40 @@ class MacroAssembler: public Assembler { void kernel_crc32_singleByte(Register crc, Register buf, Register len, Register table, Register tmp); + // SHA-2 auxiliary functions and public interfaces + private: + void sha256_deque(const VectorRegister src, + const VectorRegister dst1, const VectorRegister dst2, const VectorRegister dst3); + void sha256_load_h_vec(const VectorRegister a, const VectorRegister e, const Register hptr); + void sha256_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw); + void sha256_load_w_plus_k_vec(const Register buf_in, const VectorRegister* ws, + const int total_ws, const Register k, const VectorRegister* kpws, + const int total_kpws); + void sha256_calc_4w(const VectorRegister w0, const VectorRegister w1, + const VectorRegister w2, const VectorRegister w3, const VectorRegister kpw0, + const VectorRegister kpw1, const VectorRegister kpw2, const VectorRegister kpw3, + const Register j, const Register k); + void sha256_update_sha_state(const VectorRegister a, const VectorRegister b, + const VectorRegister c, const VectorRegister d, const VectorRegister e, + const VectorRegister f, const VectorRegister g, const VectorRegister h, + const Register hptr); + + void sha512_load_w_vec(const Register buf_in, const VectorRegister* ws, const int total_ws); + void sha512_update_sha_state(const Register state, const VectorRegister* hs, const int total_hs); + void sha512_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw); + void sha512_load_h_vec(const Register state, const VectorRegister* hs, const int total_hs); + void sha512_calc_2w(const VectorRegister w0, const VectorRegister w1, + const VectorRegister w2, const VectorRegister w3, + const VectorRegister w4, const VectorRegister w5, + const VectorRegister w6, const VectorRegister w7, + const VectorRegister kpw0, const VectorRegister kpw1, const Register j, + const VectorRegister vRb, const Register k); + + public: + void sha256(bool multi_block); + void sha512(bool multi_block); + + // // Debugging // diff --git a/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp b/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp new file mode 100644 index 000000000..7a82ed3f9 --- /dev/null +++ b/src/cpu/ppc/vm/macroAssembler_ppc_sha.cpp @@ -0,0 +1,1136 @@ +// Copyright (c) 2017 Instituto de Pesquisas Eldorado. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. + +// Implemented according to "Descriptions of SHA-256, SHA-384, and SHA-512" +// (http://www.iwar.org.uk/comsec/resources/cipher/sha256-384-512.pdf). + +#include "asm/macroAssembler.inline.hpp" +#include "runtime/stubRoutines.hpp" + +/********************************************************************** + * SHA 256 + *********************************************************************/ + +void MacroAssembler::sha256_deque(const VectorRegister src, + const VectorRegister dst1, + const VectorRegister dst2, + const VectorRegister dst3) { + vsldoi (dst1, src, src, 12); + vsldoi (dst2, src, src, 8); + vsldoi (dst3, src, src, 4); +} + +void MacroAssembler::sha256_round(const VectorRegister* hs, + const int total_hs, + int& h_cnt, + const VectorRegister kpw) { + // convenience registers: cycle from 0-7 downwards + const VectorRegister a = hs[(total_hs + 0 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister b = hs[(total_hs + 1 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister c = hs[(total_hs + 2 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister d = hs[(total_hs + 3 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister e = hs[(total_hs + 4 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister f = hs[(total_hs + 5 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister g = hs[(total_hs + 6 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister h = hs[(total_hs + 7 - (h_cnt % total_hs)) % total_hs]; + // temporaries + VectorRegister ch = VR0; + VectorRegister maj = VR1; + VectorRegister bsa = VR2; + VectorRegister bse = VR3; + VectorRegister vt0 = VR4; + VectorRegister vt1 = VR5; + VectorRegister vt2 = VR6; + VectorRegister vt3 = VR7; + + vsel (ch, g, f, e); + vxor (maj, a, b); + vshasigmaw (bse, e, 1, 0xf); + vadduwm (vt2, ch, kpw); + vadduwm (vt1, h, bse); + vsel (maj, b, c, maj); + vadduwm (vt3, vt1, vt2); + vshasigmaw (bsa, a, 1, 0); + vadduwm (vt0, bsa, maj); + + vadduwm (d, d, vt3); + vadduwm (h, vt3, vt0); + + // advance vector pointer to the next iteration + h_cnt++; +} + +void MacroAssembler::sha256_load_h_vec(const VectorRegister a, + const VectorRegister e, + const Register hptr) { + // temporaries + Register tmp = R8; + VectorRegister vt0 = VR0; + VectorRegister vRb = VR6; + // labels + Label sha256_aligned; + + andi_ (tmp, hptr, 0xf); + lvx (a, hptr); + addi (tmp, hptr, 16); + lvx (e, tmp); + beq (CCR0, sha256_aligned); + + // handle unaligned accesses + load_perm(vRb, hptr); + addi (tmp, hptr, 32); + vec_perm(a, e, vRb); + + lvx (vt0, tmp); + vec_perm(e, vt0, vRb); + + // aligned accesses + bind(sha256_aligned); +} + +void MacroAssembler::sha256_load_w_plus_k_vec(const Register buf_in, + const VectorRegister* ws, + const int total_ws, + const Register k, + const VectorRegister* kpws, + const int total_kpws) { + Label w_aligned, after_w_load; + + Register tmp = R8; + VectorRegister vt0 = VR0; + VectorRegister vt1 = VR1; + VectorRegister vRb = VR6; + + andi_ (tmp, buf_in, 0xF); + beq (CCR0, w_aligned); // address ends with 0x0, not 0x8 + + // deal with unaligned addresses + lvx (ws[0], buf_in); + load_perm(vRb, buf_in); + + for (int n = 1; n < total_ws; n++) { + VectorRegister w_cur = ws[n]; + VectorRegister w_prev = ws[n-1]; + + addi (tmp, buf_in, n * 16); + lvx (w_cur, tmp); + vec_perm(w_prev, w_cur, vRb); + } + addi (tmp, buf_in, total_ws * 16); + lvx (vt0, tmp); + vec_perm(ws[total_ws-1], vt0, vRb); + b (after_w_load); + + bind(w_aligned); + + // deal with aligned addresses + lvx(ws[0], buf_in); + for (int n = 1; n < total_ws; n++) { + VectorRegister w = ws[n]; + addi (tmp, buf_in, n * 16); + lvx (w, tmp); + } + + bind(after_w_load); + +#if defined(VM_LITTLE_ENDIAN) + // Byte swapping within int values + li (tmp, 8); + lvsl (vt0, tmp); + vspltisb (vt1, 0xb); + vxor (vt1, vt0, vt1); + for (int n = 0; n < total_ws; n++) { + VectorRegister w = ws[n]; + vec_perm(w, w, vt1); + } +#endif + + // Loading k, which is always aligned to 16-bytes + lvx (kpws[0], k); + for (int n = 1; n < total_kpws; n++) { + VectorRegister kpw = kpws[n]; + addi (tmp, k, 16 * n); + lvx (kpw, tmp); + } + + // Add w to K + assert(total_ws == total_kpws, "Redesign the loop below"); + for (int n = 0; n < total_kpws; n++) { + VectorRegister kpw = kpws[n]; + VectorRegister w = ws[n]; + + vadduwm (kpw, kpw, w); + } +} + +void MacroAssembler::sha256_calc_4w(const VectorRegister w0, + const VectorRegister w1, + const VectorRegister w2, + const VectorRegister w3, + const VectorRegister kpw0, + const VectorRegister kpw1, + const VectorRegister kpw2, + const VectorRegister kpw3, + const Register j, + const Register k) { + // Temporaries + const VectorRegister vt0 = VR0; + const VectorRegister vt1 = VR1; + const VectorSRegister vsrt1 = vt1->to_vsr(); + const VectorRegister vt2 = VR2; + const VectorRegister vt3 = VR3; + const VectorSRegister vst3 = vt3->to_vsr(); + const VectorRegister vt4 = VR4; + + // load to k[j] + lvx (vt0, j, k); + + // advance j + addi (j, j, 16); // 16 bytes were read + +#if defined(VM_LITTLE_ENDIAN) + // b = w[j-15], w[j-14], w[j-13], w[j-12] + vsldoi (vt1, w1, w0, 12); + + // c = w[j-7], w[j-6], w[j-5], w[j-4] + vsldoi (vt2, w3, w2, 12); + +#else + // b = w[j-15], w[j-14], w[j-13], w[j-12] + vsldoi (vt1, w0, w1, 4); + + // c = w[j-7], w[j-6], w[j-5], w[j-4] + vsldoi (vt2, w2, w3, 4); +#endif + + // d = w[j-2], w[j-1], w[j-4], w[j-3] + vsldoi (vt3, w3, w3, 8); + + // b = s0(w[j-15]) , s0(w[j-14]) , s0(w[j-13]) , s0(w[j-12]) + vshasigmaw (vt1, vt1, 0, 0); + + // d = s1(w[j-2]) , s1(w[j-1]) , s1(w[j-4]) , s1(w[j-3]) + vshasigmaw (vt3, vt3, 0, 0xf); + + // c = s0(w[j-15]) + w[j-7], + // s0(w[j-14]) + w[j-6], + // s0(w[j-13]) + w[j-5], + // s0(w[j-12]) + w[j-4] + vadduwm (vt2, vt1, vt2); + + // c = s0(w[j-15]) + w[j-7] + w[j-16], + // s0(w[j-14]) + w[j-6] + w[j-15], + // s0(w[j-13]) + w[j-5] + w[j-14], + // s0(w[j-12]) + w[j-4] + w[j-13] + vadduwm (vt2, vt2, w0); + + // e = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j] + // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1] + // s0(w[j-13]) + w[j-5] + w[j-14] + s1(w[j-4]), // UNDEFINED + // s0(w[j-12]) + w[j-4] + w[j-13] + s1(w[j-3]) // UNDEFINED + vadduwm (vt4, vt2, vt3); + + // At this point, e[0] and e[1] are the correct values to be stored at w[j] + // and w[j+1]. + // e[2] and e[3] are not considered. + // b = s1(w[j]) , s1(s(w[j+1]) , UNDEFINED , UNDEFINED + vshasigmaw (vt1, vt4, 0, 0xf); + + // v5 = s1(w[j-2]) , s1(w[j-1]) , s1(w[j]) , s1(w[j+1]) +#if defined(VM_LITTLE_ENDIAN) + xxmrgld (vst3, vsrt1, vst3); +#else + xxmrghd (vst3, vst3, vsrt1); +#endif + + // c = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j] + // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1] + // s0(w[j-13]) + w[j-5] + w[j-14] + s1(w[j]), // w[j+2] + // s0(w[j-12]) + w[j-4] + w[j-13] + s1(w[j+1]) // w[j+4] + vadduwm (vt2, vt2, vt3); + + // Updating w0 to w3 to hold the new previous 16 values from w. + vmr (w0, w1); + vmr (w1, w2); + vmr (w2, w3); + vmr (w3, vt2); + + // store k + w to v9 (4 values at once) +#if defined(VM_LITTLE_ENDIAN) + vadduwm (kpw0, vt2, vt0); + + vsldoi (kpw1, kpw0, kpw0, 12); + vsldoi (kpw2, kpw0, kpw0, 8); + vsldoi (kpw3, kpw0, kpw0, 4); +#else + vadduwm (kpw3, vt2, vt0); + + vsldoi (kpw2, kpw3, kpw3, 12); + vsldoi (kpw1, kpw3, kpw3, 8); + vsldoi (kpw0, kpw3, kpw3, 4); +#endif +} + +void MacroAssembler::sha256_update_sha_state(const VectorRegister a, + const VectorRegister b_, + const VectorRegister c, + const VectorRegister d, + const VectorRegister e, + const VectorRegister f, + const VectorRegister g, + const VectorRegister h, + const Register hptr) { + // temporaries + VectorRegister vt0 = VR0; + VectorRegister vt1 = VR1; + VectorRegister vt2 = VR2; + VectorRegister vt3 = VR3; + VectorRegister vt4 = VR4; + VectorRegister vt5 = VR5; + VectorRegister vaux = VR6; + VectorRegister vRb = VR6; + Register tmp = R8; + Register of16 = R8; + Register of32 = R9; + Label state_load_aligned; + + // Load hptr + andi_ (tmp, hptr, 0xf); + li (of16, 16); + lvx (vt0, hptr); + lvx (vt5, of16, hptr); + beq (CCR0, state_load_aligned); + + // handle unaligned accesses + li (of32, 32); + load_perm(vRb, hptr); + + vec_perm(vt0, vt5, vRb); // vt0 = hptr[0]..hptr[3] + + lvx (vt1, hptr, of32); + vec_perm(vt5, vt1, vRb); // vt5 = hptr[4]..hptr[7] + + // aligned accesses + bind(state_load_aligned); + +#if defined(VM_LITTLE_ENDIAN) + vmrglw (vt1, b_, a); // vt1 = {a, b, ?, ?} + vmrglw (vt2, d, c); // vt2 = {c, d, ?, ?} + vmrglw (vt3, f, e); // vt3 = {e, f, ?, ?} + vmrglw (vt4, h, g); // vt4 = {g, h, ?, ?} + xxmrgld (vt1->to_vsr(), vt2->to_vsr(), vt1->to_vsr()); // vt1 = {a, b, c, d} + xxmrgld (vt3->to_vsr(), vt4->to_vsr(), vt3->to_vsr()); // vt3 = {e, f, g, h} + vadduwm (a, vt0, vt1); // a = {a+hptr[0], b+hptr[1], c+hptr[2], d+hptr[3]} + vadduwm (e, vt5, vt3); // e = {e+hptr[4], f+hptr[5], g+hptr[6], h+hptr[7]} + + // Save hptr back, works for any alignment + xxswapd (vt0->to_vsr(), a->to_vsr()); + stxvd2x (vt0->to_vsr(), hptr); + xxswapd (vt5->to_vsr(), e->to_vsr()); + stxvd2x (vt5->to_vsr(), of16, hptr); +#else + vmrglw (vt1, a, b_); // vt1 = {a, b, ?, ?} + vmrglw (vt2, c, d); // vt2 = {c, d, ?, ?} + vmrglw (vt3, e, f); // vt3 = {e, f, ?, ?} + vmrglw (vt4, g, h); // vt4 = {g, h, ?, ?} + xxmrgld (vt1->to_vsr(), vt1->to_vsr(), vt2->to_vsr()); // vt1 = {a, b, c, d} + xxmrgld (vt3->to_vsr(), vt3->to_vsr(), vt4->to_vsr()); // vt3 = {e, f, g, h} + vadduwm (d, vt0, vt1); // d = {a+hptr[0], b+hptr[1], c+hptr[2], d+hptr[3]} + vadduwm (h, vt5, vt3); // h = {e+hptr[4], f+hptr[5], g+hptr[6], h+hptr[7]} + + // Save hptr back, works for any alignment + stxvd2x (d->to_vsr(), hptr); + stxvd2x (h->to_vsr(), of16, hptr); +#endif +} + +static const uint32_t sha256_round_table[64] __attribute((aligned(16))) = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; +static const uint32_t *sha256_round_consts = sha256_round_table; + +// R3_ARG1 - byte[] Input string with padding but in Big Endian +// R4_ARG2 - int[] SHA.state (at first, the root of primes) +// R5_ARG3 - int offset +// R6_ARG4 - int limit +// +// Internal Register usage: +// R7 - k +// R8 - tmp | j | of16 +// R9 - of32 +// VR0-VR8 - ch, maj, bsa, bse, vt0-vt3 | vt0-vt5, vaux/vRb +// VR9-VR16 - a-h +// VR17-VR20 - w0-w3 +// VR21-VR23 - vRb | vaux0-vaux2 +// VR24-VR27 - kpw0-kpw3 +void MacroAssembler::sha256(bool multi_block) { + static const ssize_t buf_size = 64; + static const uint8_t w_size = sizeof(sha256_round_table)/sizeof(uint32_t); +#ifdef AIX + // malloc provides 16 byte alignment + if (((uintptr_t)sha256_round_consts & 0xF) != 0) { + uint32_t *new_round_consts = (uint32_t*)malloc(sizeof(sha256_round_table)); + guarantee(new_round_consts, "oom"); + memcpy(new_round_consts, sha256_round_consts, sizeof(sha256_round_table)); + sha256_round_consts = (const uint32_t*)new_round_consts; + } +#endif + + Register buf_in = R3_ARG1; + Register state = R4_ARG2; + Register ofs = R5_ARG3; + Register limit = R6_ARG4; + + Label sha_loop, core_loop; + + // Save non-volatile vector registers in the red zone + static const VectorRegister nv[] = { + VR20, VR21, VR22, VR23, VR24, VR25, VR26, VR27/*, VR28, VR29, VR30, VR31*/ + }; + static const uint8_t nv_size = sizeof(nv) / sizeof (VectorRegister); + + for (int c = 0; c < nv_size; c++) { + Register tmp = R8; + li (tmp, (c - (nv_size)) * 16); + stvx(nv[c], tmp, R1); + } + + // Load hash state to registers + VectorRegister a = VR9; + VectorRegister b = VR10; + VectorRegister c = VR11; + VectorRegister d = VR12; + VectorRegister e = VR13; + VectorRegister f = VR14; + VectorRegister g = VR15; + VectorRegister h = VR16; + static const VectorRegister hs[] = {a, b, c, d, e, f, g, h}; + static const int total_hs = sizeof(hs)/sizeof(VectorRegister); + // counter for cycling through hs vector to avoid register moves between iterations + int h_cnt = 0; + + // Load a-h registers from the memory pointed by state +#if defined(VM_LITTLE_ENDIAN) + sha256_load_h_vec(a, e, state); +#else + sha256_load_h_vec(d, h, state); +#endif + + // keep k loaded also during MultiBlock loops + Register k = R7; + assert(((uintptr_t)sha256_round_consts & 0xF) == 0, "k alignment"); + load_const_optimized(k, (address)sha256_round_consts, R0); + + // Avoiding redundant loads + if (multi_block) { + align(OptoLoopAlignment); + } + bind(sha_loop); +#if defined(VM_LITTLE_ENDIAN) + sha256_deque(a, b, c, d); + sha256_deque(e, f, g, h); +#else + sha256_deque(d, c, b, a); + sha256_deque(h, g, f, e); +#endif + + // Load 16 elements from w out of the loop. + // Order of the int values is Endianess specific. + VectorRegister w0 = VR17; + VectorRegister w1 = VR18; + VectorRegister w2 = VR19; + VectorRegister w3 = VR20; + static const VectorRegister ws[] = {w0, w1, w2, w3}; + static const int total_ws = sizeof(ws)/sizeof(VectorRegister); + + VectorRegister kpw0 = VR24; + VectorRegister kpw1 = VR25; + VectorRegister kpw2 = VR26; + VectorRegister kpw3 = VR27; + static const VectorRegister kpws[] = {kpw0, kpw1, kpw2, kpw3}; + static const int total_kpws = sizeof(kpws)/sizeof(VectorRegister); + + sha256_load_w_plus_k_vec(buf_in, ws, total_ws, k, kpws, total_kpws); + + // Cycle through the first 16 elements + assert(total_ws == total_kpws, "Redesign the loop below"); + for (int n = 0; n < total_ws; n++) { + VectorRegister vaux0 = VR21; + VectorRegister vaux1 = VR22; + VectorRegister vaux2 = VR23; + + sha256_deque(kpws[n], vaux0, vaux1, vaux2); + +#if defined(VM_LITTLE_ENDIAN) + sha256_round(hs, total_hs, h_cnt, kpws[n]); + sha256_round(hs, total_hs, h_cnt, vaux0); + sha256_round(hs, total_hs, h_cnt, vaux1); + sha256_round(hs, total_hs, h_cnt, vaux2); +#else + sha256_round(hs, total_hs, h_cnt, vaux2); + sha256_round(hs, total_hs, h_cnt, vaux1); + sha256_round(hs, total_hs, h_cnt, vaux0); + sha256_round(hs, total_hs, h_cnt, kpws[n]); +#endif + } + + Register tmp = R8; + // loop the 16th to the 64th iteration by 8 steps + li (tmp, (w_size - 16) / total_hs); + mtctr(tmp); + + // j will be aligned to 4 for loading words. + // Whenever read, advance the pointer (e.g: when j is used in a function) + Register j = R8; + li (j, 16*4); + + align(OptoLoopAlignment); + bind(core_loop); + + // due to VectorRegister rotate, always iterate in multiples of total_hs + for (int n = 0; n < total_hs/4; n++) { + sha256_calc_4w(w0, w1, w2, w3, kpw0, kpw1, kpw2, kpw3, j, k); + sha256_round(hs, total_hs, h_cnt, kpw0); + sha256_round(hs, total_hs, h_cnt, kpw1); + sha256_round(hs, total_hs, h_cnt, kpw2); + sha256_round(hs, total_hs, h_cnt, kpw3); + } + + bdnz (core_loop); + + // Update hash state + sha256_update_sha_state(a, b, c, d, e, f, g, h, state); + + if (multi_block) { + addi(buf_in, buf_in, buf_size); + addi(ofs, ofs, buf_size); + cmplw(CCR0, ofs, limit); + ble(CCR0, sha_loop); + + // return ofs + mr(R3_RET, ofs); + } + + // Restore non-volatile registers + for (int c = 0; c < nv_size; c++) { + Register tmp = R8; + li (tmp, (c - (nv_size)) * 16); + lvx(nv[c], tmp, R1); + } +} + + +/********************************************************************** + * SHA 512 + *********************************************************************/ + +void MacroAssembler::sha512_load_w_vec(const Register buf_in, + const VectorRegister* ws, + const int total_ws) { + Register tmp = R8; + VectorRegister vRb = VR8; + VectorRegister aux = VR9; + Label is_aligned, after_alignment; + + andi_ (tmp, buf_in, 0xF); + beq (CCR0, is_aligned); // address ends with 0x0, not 0x8 + + // deal with unaligned addresses + lvx (ws[0], buf_in); + load_perm(vRb, buf_in); + + for (int n = 1; n < total_ws; n++) { + VectorRegister w_cur = ws[n]; + VectorRegister w_prev = ws[n-1]; + addi (tmp, buf_in, n * 16); + lvx (w_cur, tmp); + vec_perm(w_prev, w_cur, vRb); + } + addi (tmp, buf_in, total_ws * 16); + lvx (aux, tmp); + vec_perm(ws[total_ws-1], aux, vRb); + b (after_alignment); + + bind(is_aligned); + lvx (ws[0], buf_in); + for (int n = 1; n < total_ws; n++) { + VectorRegister w = ws[n]; + addi (tmp, buf_in, n * 16); + lvx (w, tmp); + } + + bind(after_alignment); +} + +// Update hash state +void MacroAssembler::sha512_update_sha_state(const Register state, + const VectorRegister* hs, + const int total_hs) { + +#if defined(VM_LITTLE_ENDIAN) + int start_idx = 0; +#else + int start_idx = 1; +#endif + + // load initial hash from the memory pointed by state + VectorRegister ini_a = VR10; + VectorRegister ini_c = VR12; + VectorRegister ini_e = VR14; + VectorRegister ini_g = VR16; + static const VectorRegister inis[] = {ini_a, ini_c, ini_e, ini_g}; + static const int total_inis = sizeof(inis)/sizeof(VectorRegister); + + Label state_save_aligned, after_state_save_aligned; + + Register addr = R7; + Register tmp = R8; + VectorRegister vRb = VR8; + VectorRegister aux = VR9; + + andi_(tmp, state, 0xf); + beq(CCR0, state_save_aligned); + // deal with unaligned addresses + + { + VectorRegister a = hs[0]; + VectorRegister b_ = hs[1]; + VectorRegister c = hs[2]; + VectorRegister d = hs[3]; + VectorRegister e = hs[4]; + VectorRegister f = hs[5]; + VectorRegister g = hs[6]; + VectorRegister h = hs[7]; + load_perm(vRb, state); + lvx (ini_a, state); + addi (addr, state, 16); + + lvx (ini_c, addr); + addi (addr, state, 32); + vec_perm(ini_a, ini_c, vRb); + + lvx (ini_e, addr); + addi (addr, state, 48); + vec_perm(ini_c, ini_e, vRb); + + lvx (ini_g, addr); + addi (addr, state, 64); + vec_perm(ini_e, ini_g, vRb); + + lvx (aux, addr); + vec_perm(ini_g, aux, vRb); + +#if defined(VM_LITTLE_ENDIAN) + xxmrgld(a->to_vsr(), b_->to_vsr(), a->to_vsr()); + xxmrgld(c->to_vsr(), d->to_vsr(), c->to_vsr()); + xxmrgld(e->to_vsr(), f->to_vsr(), e->to_vsr()); + xxmrgld(g->to_vsr(), h->to_vsr(), g->to_vsr()); +#else + xxmrgld(b_->to_vsr(), a->to_vsr(), b_->to_vsr()); + xxmrgld(d->to_vsr(), c->to_vsr(), d->to_vsr()); + xxmrgld(f->to_vsr(), e->to_vsr(), f->to_vsr()); + xxmrgld(h->to_vsr(), g->to_vsr(), h->to_vsr()); +#endif + + for (int n = start_idx; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + VectorRegister ini_cur = inis[n/2]; + + vaddudm(h_cur, ini_cur, h_cur); + } + + for (int n = start_idx; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + + mfvrd (tmp, h_cur); +#if defined(VM_LITTLE_ENDIAN) + std (tmp, 8*n + 8, state); +#else + std (tmp, 8*n - 8, state); +#endif + vsldoi (aux, h_cur, h_cur, 8); + mfvrd (tmp, aux); + std (tmp, 8*n + 0, state); + } + + b (after_state_save_aligned); + } + + bind(state_save_aligned); + { + for (int n = 0; n < total_hs; n += 2) { +#if defined(VM_LITTLE_ENDIAN) + VectorRegister h_cur = hs[n]; + VectorRegister h_next = hs[n+1]; +#else + VectorRegister h_cur = hs[n+1]; + VectorRegister h_next = hs[n]; +#endif + VectorRegister ini_cur = inis[n/2]; + + if (n/2 == 0) { + lvx(ini_cur, state); + } else { + addi(addr, state, (n/2) * 16); + lvx(ini_cur, addr); + } + xxmrgld(h_cur->to_vsr(), h_next->to_vsr(), h_cur->to_vsr()); + } + + for (int n = start_idx; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + VectorRegister ini_cur = inis[n/2]; + + vaddudm(h_cur, ini_cur, h_cur); + } + + for (int n = start_idx; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + + if (n/2 == 0) { + stvx(h_cur, state); + } else { + addi(addr, state, (n/2) * 16); + stvx(h_cur, addr); + } + } + } + + bind(after_state_save_aligned); +} + +// Use h_cnt to cycle through hs elements but also increment it at the end +void MacroAssembler::sha512_round(const VectorRegister* hs, + const int total_hs, int& h_cnt, + const VectorRegister kpw) { + + // convenience registers: cycle from 0-7 downwards + const VectorRegister a = hs[(total_hs + 0 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister b = hs[(total_hs + 1 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister c = hs[(total_hs + 2 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister d = hs[(total_hs + 3 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister e = hs[(total_hs + 4 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister f = hs[(total_hs + 5 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister g = hs[(total_hs + 6 - (h_cnt % total_hs)) % total_hs]; + const VectorRegister h = hs[(total_hs + 7 - (h_cnt % total_hs)) % total_hs]; + // temporaries + const VectorRegister Ch = VR20; + const VectorRegister Maj = VR21; + const VectorRegister bsa = VR22; + const VectorRegister bse = VR23; + const VectorRegister tmp1 = VR24; + const VectorRegister tmp2 = VR25; + + vsel (Ch, g, f, e); + vxor (Maj, a, b); + vshasigmad(bse, e, 1, 0xf); + vaddudm (tmp2, Ch, kpw); + vaddudm (tmp1, h, bse); + vsel (Maj, b, c, Maj); + vaddudm (tmp1, tmp1, tmp2); + vshasigmad(bsa, a, 1, 0); + vaddudm (tmp2, bsa, Maj); + vaddudm (d, d, tmp1); + vaddudm (h, tmp1, tmp2); + + // advance vector pointer to the next iteration + h_cnt++; +} + +void MacroAssembler::sha512_calc_2w(const VectorRegister w0, + const VectorRegister w1, + const VectorRegister w2, + const VectorRegister w3, + const VectorRegister w4, + const VectorRegister w5, + const VectorRegister w6, + const VectorRegister w7, + const VectorRegister kpw0, + const VectorRegister kpw1, + const Register j, + const VectorRegister vRb, + const Register k) { + // Temporaries + const VectorRegister VR_a = VR20; + const VectorRegister VR_b = VR21; + const VectorRegister VR_c = VR22; + const VectorRegister VR_d = VR23; + + // load to k[j] + lvx (VR_a, j, k); + // advance j + addi (j, j, 16); // 16 bytes were read + +#if defined(VM_LITTLE_ENDIAN) + // v6 = w[j-15], w[j-14] + vperm (VR_b, w1, w0, vRb); + // v12 = w[j-7], w[j-6] + vperm (VR_c, w5, w4, vRb); +#else + // v6 = w[j-15], w[j-14] + vperm (VR_b, w0, w1, vRb); + // v12 = w[j-7], w[j-6] + vperm (VR_c, w4, w5, vRb); +#endif + + // v6 = s0(w[j-15]) , s0(w[j-14]) + vshasigmad (VR_b, VR_b, 0, 0); + // v5 = s1(w[j-2]) , s1(w[j-1]) + vshasigmad (VR_d, w7, 0, 0xf); + // v6 = s0(w[j-15]) + w[j-7] , s0(w[j-14]) + w[j-6] + vaddudm (VR_b, VR_b, VR_c); + // v8 = s1(w[j-2]) + w[j-16] , s1(w[j-1]) + w[j-15] + vaddudm (VR_d, VR_d, w0); + // v9 = s0(w[j-15]) + w[j-7] + w[j-16] + s1(w[j-2]), // w[j] + // s0(w[j-14]) + w[j-6] + w[j-15] + s1(w[j-1]), // w[j+1] + vaddudm (VR_c, VR_d, VR_b); + // Updating w0 to w7 to hold the new previous 16 values from w. + vmr (w0, w1); + vmr (w1, w2); + vmr (w2, w3); + vmr (w3, w4); + vmr (w4, w5); + vmr (w5, w6); + vmr (w6, w7); + vmr (w7, VR_c); + +#if defined(VM_LITTLE_ENDIAN) + // store k + w to kpw0 (2 values at once) + vaddudm (kpw0, VR_c, VR_a); + // kpw1 holds (k + w)[1] + vsldoi (kpw1, kpw0, kpw0, 8); +#else + // store k + w to kpw0 (2 values at once) + vaddudm (kpw1, VR_c, VR_a); + // kpw1 holds (k + w)[1] + vsldoi (kpw0, kpw1, kpw1, 8); +#endif +} + +void MacroAssembler::sha512_load_h_vec(const Register state, + const VectorRegister* hs, + const int total_hs) { +#if defined(VM_LITTLE_ENDIAN) + VectorRegister a = hs[0]; + VectorRegister g = hs[6]; + int start_idx = 0; +#else + VectorRegister a = hs[1]; + VectorRegister g = hs[7]; + int start_idx = 1; +#endif + + Register addr = R7; + VectorRegister vRb = VR8; + Register tmp = R8; + Label state_aligned, after_state_aligned; + + andi_(tmp, state, 0xf); + beq(CCR0, state_aligned); + + // deal with unaligned addresses + VectorRegister aux = VR9; + + lvx(hs[start_idx], state); + load_perm(vRb, state); + + for (int n = start_idx + 2; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + VectorRegister h_prev2 = hs[n - 2]; + addi(addr, state, (n/2) * 16); + lvx(h_cur, addr); + vec_perm(h_prev2, h_cur, vRb); + } + addi(addr, state, (total_hs/2) * 16); + lvx (aux, addr); + vec_perm(hs[total_hs - 2 + start_idx], aux, vRb); + b (after_state_aligned); + + bind(state_aligned); + + // deal with aligned addresses + lvx(hs[start_idx], state); + + for (int n = start_idx + 2; n < total_hs; n += 2) { + VectorRegister h_cur = hs[n]; + addi(addr, state, (n/2) * 16); + lvx(h_cur, addr); + } + + bind(after_state_aligned); +} + +static const uint64_t sha512_round_table[80] __attribute((aligned(16))) = { + 0x428a2f98d728ae22, 0x7137449123ef65cd, + 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc, + 0x3956c25bf348b538, 0x59f111f1b605d019, + 0x923f82a4af194f9b, 0xab1c5ed5da6d8118, + 0xd807aa98a3030242, 0x12835b0145706fbe, + 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2, + 0x72be5d74f27b896f, 0x80deb1fe3b1696b1, + 0x9bdc06a725c71235, 0xc19bf174cf692694, + 0xe49b69c19ef14ad2, 0xefbe4786384f25e3, + 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, + 0x2de92c6f592b0275, 0x4a7484aa6ea6e483, + 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5, + 0x983e5152ee66dfab, 0xa831c66d2db43210, + 0xb00327c898fb213f, 0xbf597fc7beef0ee4, + 0xc6e00bf33da88fc2, 0xd5a79147930aa725, + 0x06ca6351e003826f, 0x142929670a0e6e70, + 0x27b70a8546d22ffc, 0x2e1b21385c26c926, + 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df, + 0x650a73548baf63de, 0x766a0abb3c77b2a8, + 0x81c2c92e47edaee6, 0x92722c851482353b, + 0xa2bfe8a14cf10364, 0xa81a664bbc423001, + 0xc24b8b70d0f89791, 0xc76c51a30654be30, + 0xd192e819d6ef5218, 0xd69906245565a910, + 0xf40e35855771202a, 0x106aa07032bbd1b8, + 0x19a4c116b8d2d0c8, 0x1e376c085141ab53, + 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8, + 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, + 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, + 0x748f82ee5defb2fc, 0x78a5636f43172f60, + 0x84c87814a1f0ab72, 0x8cc702081a6439ec, + 0x90befffa23631e28, 0xa4506cebde82bde9, + 0xbef9a3f7b2c67915, 0xc67178f2e372532b, + 0xca273eceea26619c, 0xd186b8c721c0c207, + 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, + 0x06f067aa72176fba, 0x0a637dc5a2c898a6, + 0x113f9804bef90dae, 0x1b710b35131c471b, + 0x28db77f523047d84, 0x32caab7b40c72493, + 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c, + 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, + 0x5fcb6fab3ad6faec, 0x6c44198c4a475817, +}; +static const uint64_t *sha512_round_consts = sha512_round_table; + +// R3_ARG1 - byte[] Input string with padding but in Big Endian +// R4_ARG2 - int[] SHA.state (at first, the root of primes) +// R5_ARG3 - int offset +// R6_ARG4 - int limit +// +// Internal Register usage: +// R7 R8 R9 - volatile temporaries +// VR0-VR7 - a-h +// VR8 - vRb +// VR9 - aux (highly volatile, use with care) +// VR10-VR17 - w0-w7 | ini_a-ini_h +// VR18 - vsp16 | kplusw0 +// VR19 - vsp32 | kplusw1 +// VR20-VR25 - sha512_calc_2w and sha512_round temporaries +void MacroAssembler::sha512(bool multi_block) { + static const ssize_t buf_size = 128; + static const uint8_t w_size = sizeof(sha512_round_table)/sizeof(uint64_t); +#ifdef AIX + // malloc provides 16 byte alignment + if (((uintptr_t)sha512_round_consts & 0xF) != 0) { + uint64_t *new_round_consts = (uint64_t*)malloc(sizeof(sha512_round_table)); + guarantee(new_round_consts, "oom"); + memcpy(new_round_consts, sha512_round_consts, sizeof(sha512_round_table)); + sha512_round_consts = (const uint64_t*)new_round_consts; + } +#endif + + Register buf_in = R3_ARG1; + Register state = R4_ARG2; + Register ofs = R5_ARG3; + Register limit = R6_ARG4; + + Label sha_loop, core_loop; + + // Save non-volatile vector registers in the red zone + static const VectorRegister nv[] = { + VR20, VR21, VR22, VR23, VR24, VR25/*, VR26, VR27, VR28, VR29, VR30, VR31*/ + }; + static const uint8_t nv_size = sizeof(nv) / sizeof (VectorRegister); + + for (int c = 0; c < nv_size; c++) { + Register idx = R7; + li (idx, (c - (nv_size)) * 16); + stvx(nv[c], idx, R1); + } + + // Load hash state to registers + VectorRegister a = VR0; + VectorRegister b = VR1; + VectorRegister c = VR2; + VectorRegister d = VR3; + VectorRegister e = VR4; + VectorRegister f = VR5; + VectorRegister g = VR6; + VectorRegister h = VR7; + static const VectorRegister hs[] = {a, b, c, d, e, f, g, h}; + static const int total_hs = sizeof(hs)/sizeof(VectorRegister); + // counter for cycling through hs vector to avoid register moves between iterations + int h_cnt = 0; + + // Load a-h registers from the memory pointed by state + sha512_load_h_vec(state, hs, total_hs); + + Register k = R9; + assert(((uintptr_t)sha512_round_consts & 0xF) == 0, "k alignment"); + load_const_optimized(k, (address)sha512_round_consts, R0); + + if (multi_block) { + align(OptoLoopAlignment); + } + bind(sha_loop); + + for (int n = 0; n < total_hs; n += 2) { +#if defined(VM_LITTLE_ENDIAN) + VectorRegister h_cur = hs[n]; + VectorRegister h_next = hs[n + 1]; +#else + VectorRegister h_cur = hs[n + 1]; + VectorRegister h_next = hs[n]; +#endif + vsldoi (h_next, h_cur, h_cur, 8); + } + + // Load 16 elements from w out of the loop. + // Order of the long values is Endianess specific. + VectorRegister w0 = VR10; + VectorRegister w1 = VR11; + VectorRegister w2 = VR12; + VectorRegister w3 = VR13; + VectorRegister w4 = VR14; + VectorRegister w5 = VR15; + VectorRegister w6 = VR16; + VectorRegister w7 = VR17; + static const VectorRegister ws[] = {w0, w1, w2, w3, w4, w5, w6, w7}; + static const int total_ws = sizeof(ws)/sizeof(VectorRegister); + + // Load 16 w into vectors and setup vsl for vperm + sha512_load_w_vec(buf_in, ws, total_ws); + +#if defined(VM_LITTLE_ENDIAN) + VectorRegister vsp16 = VR18; + VectorRegister vsp32 = VR19; + VectorRegister shiftarg = VR9; + + vspltisw(vsp16, 8); + vspltisw(shiftarg, 1); + vsl (vsp16, vsp16, shiftarg); + vsl (vsp32, vsp16, shiftarg); + + VectorRegister vsp8 = VR9; + vspltish(vsp8, 8); + + // Convert input from Big Endian to Little Endian + for (int c = 0; c < total_ws; c++) { + VectorRegister w = ws[c]; + vrlh (w, w, vsp8); + } + for (int c = 0; c < total_ws; c++) { + VectorRegister w = ws[c]; + vrlw (w, w, vsp16); + } + for (int c = 0; c < total_ws; c++) { + VectorRegister w = ws[c]; + vrld (w, w, vsp32); + } +#endif + + Register Rb = R10; + VectorRegister vRb = VR8; + li (Rb, 8); + load_perm(vRb, Rb); + + VectorRegister kplusw0 = VR18; + VectorRegister kplusw1 = VR19; + + Register addr = R7; + + for (int n = 0; n < total_ws; n++) { + VectorRegister w = ws[n]; + + if (n == 0) { + lvx (kplusw0, k); + } else { + addi (addr, k, n * 16); + lvx (kplusw0, addr); + } +#if defined(VM_LITTLE_ENDIAN) + vaddudm(kplusw0, kplusw0, w); + vsldoi (kplusw1, kplusw0, kplusw0, 8); +#else + vaddudm(kplusw1, kplusw0, w); + vsldoi (kplusw0, kplusw1, kplusw1, 8); +#endif + + sha512_round(hs, total_hs, h_cnt, kplusw0); + sha512_round(hs, total_hs, h_cnt, kplusw1); + } + + Register tmp = R8; + li (tmp, (w_size-16)/total_hs); + mtctr (tmp); + // j will be aligned to 4 for loading words. + // Whenever read, advance the pointer (e.g: when j is used in a function) + Register j = tmp; + li (j, 8*16); + + align(OptoLoopAlignment); + bind(core_loop); + + // due to VectorRegister rotate, always iterate in multiples of total_hs + for (int n = 0; n < total_hs/2; n++) { + sha512_calc_2w(w0, w1, w2, w3, w4, w5, w6, w7, kplusw0, kplusw1, j, vRb, k); + sha512_round(hs, total_hs, h_cnt, kplusw0); + sha512_round(hs, total_hs, h_cnt, kplusw1); + } + + bdnz (core_loop); + + sha512_update_sha_state(state, hs, total_hs); + + if (multi_block) { + addi(buf_in, buf_in, buf_size); + addi(ofs, ofs, buf_size); + cmplw(CCR0, ofs, limit); + ble(CCR0, sha_loop); + + // return ofs + mr(R3_RET, ofs); + } + + // Restore non-volatile registers + for (int c = 0; c < nv_size; c++) { + Register idx = R7; + li (idx, (c - (nv_size)) * 16); + lvx(nv[c], idx, R1); + } +} diff --git a/src/cpu/ppc/vm/stubGenerator_ppc.cpp b/src/cpu/ppc/vm/stubGenerator_ppc.cpp index 12dd846ce..12cff00c8 100644 --- a/src/cpu/ppc/vm/stubGenerator_ppc.cpp +++ b/src/cpu/ppc/vm/stubGenerator_ppc.cpp @@ -2652,6 +2652,28 @@ class StubGenerator: public StubCodeGenerator { return start; } + address generate_sha256_implCompress(bool multi_block, const char *name) { + assert(UseSHA, "need SHA instructions"); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ function_entry(); + + __ sha256 (multi_block); + + __ blr(); + return start; + } + + address generate_sha512_implCompress(bool multi_block, const char *name) { + assert(UseSHA, "need SHA instructions"); + StubCodeMark mark(this, "StubRoutines", name); + address start = __ function_entry(); + + __ sha512 (multi_block); + + __ blr(); + return start; + } + void generate_arraycopy_stubs() { // Note: the disjoint stubs must be generated first, some of // the conjoint stubs use them. @@ -2881,6 +2903,15 @@ class StubGenerator: public StubCodeGenerator { StubRoutines::_montgomerySquare = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square); } + + if (UseSHA256Intrinsics) { + StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress"); + StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB"); + } + if (UseSHA512Intrinsics) { + StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress"); + StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB"); + } } public: diff --git a/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp b/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp index 3655c1f78..cf9ebc4e6 100644 --- a/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp +++ b/src/cpu/ppc/vm/stubRoutines_ppc_64.hpp @@ -34,7 +34,7 @@ static bool returns_to_call_stub(address return_pc) { return return_pc == _call_ enum platform_dependent_constants { code_size1 = 20000, // simply increase if too small (assembler will crash if too small) - code_size2 = 20000 // simply increase if too small (assembler will crash if too small) + code_size2 = 22000 // simply increase if too small (assembler will crash if too small) }; // CRC32 Intrinsics. diff --git a/src/cpu/ppc/vm/vm_version_ppc.cpp b/src/cpu/ppc/vm/vm_version_ppc.cpp index 6761c02a2..e7fad4843 100644 --- a/src/cpu/ppc/vm/vm_version_ppc.cpp +++ b/src/cpu/ppc/vm/vm_version_ppc.cpp @@ -110,7 +110,7 @@ void VM_Version::initialize() { // Create and print feature-string. char buf[(num_features+1) * 16]; // Max 16 chars per feature. jio_snprintf(buf, sizeof(buf), - "ppc64%s%s%s%s%s%s%s%s%s%s%s%s%s", + "ppc64%s%s%s%s%s%s%s%s%s%s%s%s%s%s", (has_fsqrt() ? " fsqrt" : ""), (has_isel() ? " isel" : ""), (has_lxarxeh() ? " lxarxeh" : ""), @@ -124,7 +124,8 @@ void VM_Version::initialize() { (has_vcipher() ? " aes" : ""), (has_vpmsumb() ? " vpmsumb" : ""), (has_mfdscr() ? " mfdscr" : ""), - (has_vsx() ? " vsx" : "") + (has_vsx() ? " vsx" : ""), + (has_vshasig() ? " sha" : "") // Make sure number of %s matches num_features! ); _features_str = strdup(buf); @@ -206,17 +207,43 @@ void VM_Version::initialize() { } #endif - if (UseSHA) { - warning("SHA instructions are not available on this CPU"); + if (has_vshasig()) { + if (FLAG_IS_DEFAULT(UseSHA)) { + UseSHA = true; + } + } else if (UseSHA) { + if (!FLAG_IS_DEFAULT(UseSHA)) + warning("SHA instructions are not available on this CPU"); FLAG_SET_DEFAULT(UseSHA, false); } - if (UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics) { - warning("SHA intrinsics are not available on this CPU"); + + if (UseSHA1Intrinsics) { + warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU."); FLAG_SET_DEFAULT(UseSHA1Intrinsics, false); + } + + if (UseSHA && has_vshasig()) { + if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) { + FLAG_SET_DEFAULT(UseSHA256Intrinsics, true); + } + } else if (UseSHA256Intrinsics) { + warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU."); FLAG_SET_DEFAULT(UseSHA256Intrinsics, false); + } + + if (UseSHA && has_vshasig()) { + if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) { + FLAG_SET_DEFAULT(UseSHA512Intrinsics, true); + } + } else if (UseSHA512Intrinsics) { + warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU."); FLAG_SET_DEFAULT(UseSHA512Intrinsics, false); } + if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics)) { + FLAG_SET_DEFAULT(UseSHA, false); + } + if (FLAG_IS_DEFAULT(UseMontgomeryMultiplyIntrinsic)) { UseMontgomeryMultiplyIntrinsic = true; } @@ -503,6 +530,7 @@ void VM_Version::determine_features() { a->vpmsumb(VR0, VR1, VR2); // code[12] -> vpmsumb a->mfdscr(R0); // code[13] -> mfdscr a->lxvd2x(VSR0, R3_ARG1); // code[14] -> vsx + a->vshasigmaw(VR0, VR1, 1, 0xF); // code[15] -> vshasig a->blr(); // Emit function to set one cache line to zero. Emit function descriptor and get pointer to it. @@ -551,6 +579,7 @@ void VM_Version::determine_features() { if (code[feature_cntr++]) features |= vpmsumb_m; if (code[feature_cntr++]) features |= mfdscr_m; if (code[feature_cntr++]) features |= vsx_m; + if (code[feature_cntr++]) features |= vshasig_m; // Print the detection code. if (PrintAssembly) { diff --git a/src/cpu/ppc/vm/vm_version_ppc.hpp b/src/cpu/ppc/vm/vm_version_ppc.hpp index 6245e1c64..86243a1a8 100644 --- a/src/cpu/ppc/vm/vm_version_ppc.hpp +++ b/src/cpu/ppc/vm/vm_version_ppc.hpp @@ -47,6 +47,7 @@ protected: vpmsumb, mfdscr, vsx, + vshasig, num_features // last entry to count features }; enum Feature_Flag_Set { @@ -63,6 +64,7 @@ protected: dcba_m = (1 << dcba ), lqarx_m = (1 << lqarx ), vcipher_m = (1 << vcipher), + vshasig_m = (1 << vshasig), vpmsumb_m = (1 << vpmsumb), mfdscr_m = (1 << mfdscr ), vsx_m = (1 << vsx ), @@ -99,6 +101,7 @@ public: static bool has_vpmsumb() { return (_features & vpmsumb_m) != 0; } static bool has_mfdscr() { return (_features & mfdscr_m) != 0; } static bool has_vsx() { return (_features & vsx_m) != 0; } + static bool has_vshasig() { return (_features & vshasig_m) != 0; } static const char* cpu_features() { return _features_str; } diff --git a/src/share/vm/opto/library_call.cpp b/src/share/vm/opto/library_call.cpp index 5f2bb3084..db36fb349 100644 --- a/src/share/vm/opto/library_call.cpp +++ b/src/share/vm/opto/library_call.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -6759,10 +6759,18 @@ bool LibraryCallKit::inline_sha_implCompressMB(Node* digestBase_obj, ciInstanceK if (state == NULL) return false; // Call the stub. - Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, - OptoRuntime::digestBase_implCompressMB_Type(), - stubAddr, stubName, TypePtr::BOTTOM, - src_start, state, ofs, limit); + Node *call; + if (CCallingConventionRequiresIntsAsLongs) { + call = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::digestBase_implCompressMB_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + src_start, state, ofs XTOP, limit XTOP); + } else { + call = make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::digestBase_implCompressMB_Type(), + stubAddr, stubName, TypePtr::BOTTOM, + src_start, state, ofs, limit); + } // return ofs (int) Node* result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms)); set_result(result); diff --git a/src/share/vm/opto/runtime.cpp b/src/share/vm/opto/runtime.cpp index 364152c56..57d2f5764 100644 --- a/src/share/vm/opto/runtime.cpp +++ b/src/share/vm/opto/runtime.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -930,12 +930,24 @@ const TypeFunc* OptoRuntime::digestBase_implCompressMB_Type() { // create input type (domain) int num_args = 4; int argcnt = num_args; + if(CCallingConventionRequiresIntsAsLongs) { + argcnt += 2; + } const Type** fields = TypeTuple::fields(argcnt); int argp = TypeFunc::Parms; - fields[argp++] = TypePtr::NOTNULL; // buf - fields[argp++] = TypePtr::NOTNULL; // state - fields[argp++] = TypeInt::INT; // ofs - fields[argp++] = TypeInt::INT; // limit + if(CCallingConventionRequiresIntsAsLongs) { + fields[argp++] = TypePtr::NOTNULL; // buf + fields[argp++] = TypePtr::NOTNULL; // state + fields[argp++] = TypeLong::LONG; // ofs + fields[argp++] = Type::HALF; + fields[argp++] = TypeLong::LONG; // limit + fields[argp++] = Type::HALF; + } else { + fields[argp++] = TypePtr::NOTNULL; // buf + fields[argp++] = TypePtr::NOTNULL; // state + fields[argp++] = TypeInt::INT; // ofs + fields[argp++] = TypeInt::INT; // limit + } assert(argp == TypeFunc::Parms+argcnt, "correct decoding"); const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields); diff --git a/test/compiler/intrinsics/sha/cli/testcases/GenericTestCaseForOtherCPU.java b/test/compiler/intrinsics/sha/cli/testcases/GenericTestCaseForOtherCPU.java index c5d21ed4b..32850711a 100644 --- a/test/compiler/intrinsics/sha/cli/testcases/GenericTestCaseForOtherCPU.java +++ b/test/compiler/intrinsics/sha/cli/testcases/GenericTestCaseForOtherCPU.java @@ -36,7 +36,8 @@ public class GenericTestCaseForOtherCPU extends public GenericTestCaseForOtherCPU(String optionName) { // Execute the test case on any CPU except SPARC and X86 super(optionName, new NotPredicate(new OrPredicate(Platform::isSparc, - new OrPredicate(Platform::isX64, Platform::isX86)))); + new OrPredicate(Platform::isPPC, + new OrPredicate(Platform::isX64, Platform::isX86))))); } @Override diff --git a/test/compiler/testlibrary/sha/predicate/IntrinsicPredicates.java b/test/compiler/testlibrary/sha/predicate/IntrinsicPredicates.java index 51ab609dc..bac466575 100644 --- a/test/compiler/testlibrary/sha/predicate/IntrinsicPredicates.java +++ b/test/compiler/testlibrary/sha/predicate/IntrinsicPredicates.java @@ -63,12 +63,20 @@ public class IntrinsicPredicates { null); public static final BooleanSupplier SHA256_INSTRUCTION_AVAILABLE - = new CPUSpecificPredicate("sparc.*", new String[] { "sha256" }, - null); + = new OrPredicate(new CPUSpecificPredicate("sparc.*", new String[] { "sha256" }, + null), + new OrPredicate(new CPUSpecificPredicate("ppc64.*", new String[] { "sha" }, + null), + new CPUSpecificPredicate("ppc64le.*", new String[] { "sha" }, + null))); public static final BooleanSupplier SHA512_INSTRUCTION_AVAILABLE - = new CPUSpecificPredicate("sparc.*", new String[] { "sha512" }, - null); + = new OrPredicate(new CPUSpecificPredicate("sparc.*", new String[] { "sha512" }, + null), + new OrPredicate(new CPUSpecificPredicate("ppc64.*", new String[] { "sha" }, + null), + new CPUSpecificPredicate("ppc64le.*", new String[] { "sha" }, + null))); public static final BooleanSupplier ANY_SHA_INSTRUCTION_AVAILABLE = new OrPredicate(IntrinsicPredicates.SHA1_INSTRUCTION_AVAILABLE, -- GitLab