From dea8068015d99786304d1e7c9dc63a1126ff8da6 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Sun, 14 Oct 2012 14:25:00 +0000 Subject: [PATCH] perlasm/sparcv9_modes.pl: "cooperative" optimizations based on suggestions from David Miller. --- crypto/perlasm/sparcv9_modes.pl | 320 ++++++++++++++++++++++++++++++-- 1 file changed, 309 insertions(+), 11 deletions(-) diff --git a/crypto/perlasm/sparcv9_modes.pl b/crypto/perlasm/sparcv9_modes.pl index e6b8a24cc2..d372586bbc 100644 --- a/crypto/perlasm/sparcv9_modes.pl +++ b/crypto/perlasm/sparcv9_modes.pl @@ -1,9 +1,28 @@ #!/usr/bin/env perl -# Specific mode implementations for SPARC T4 modules. +# Specific modes implementations for SPARC Architecture 2011. There +# is T4 dependency though, an ASI value that is not specified in the +# Architecture Manual. But as SPARC universe is rather monocultural, +# we imply that processor capable of executing crypto instructions +# can handle the ASI in question as well. This means that we ought to +# keep eyes open when new processors emerge... +# +# As for above mentioned ASI. It's so called "block initializing +# store" which cancels "read" in "read-update-write" on cache lines. +# This is "cooperative" optimization, as it reduces overall pressure +# on memory interface. Benefits can't be observed/quantified with +# usual benchmarks, on the contrary you can notice that single-thread +# performance for parallelizable modes is ~1.5% worse. Special note +# about commented 'membar' instructions, otherwise recommended by +# manual. Rationale is following. Memory view is consistent from +# viewpoint of processor executing the code even when ASI in question +# is used. If thread on another processor has to access the result, +# its availability would have to be mediated and it can be done only +# through a syncronization operation which would requre ... 'membar'. +# All this based on suggestions from David Miller. my ($inp,$out,$len,$key,$ivec,$enc)=map("%i$_",(0..5)); -my ($ileft,$iright,$ooff,$omask,$ivoff)=map("%l$_",(1..7)); +my ($ileft,$iright,$ooff,$omask,$ivoff,$blk_init)=map("%l$_",(0..7)); sub alg_cbc_encrypt_implement { my ($alg,$bits) = @_; @@ -13,6 +32,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_encrypt: save %sp, -$::frame, %sp + sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff @@ -33,8 +53,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f3 ___ $::code.=<<___; + prefetch [$inp], 20 + prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey - srlx $len, 4, $len and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft @@ -42,9 +63,16 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff - alignaddrl $out, %g0, $out + cmp $len, 127 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<128 || + brnz,pn $blk_init, .L${bits}cbc_enc_blk ! $inp==$out) srl $omask, $ooff, $omask + alignaddrl $out, %g0, $out + srlx $len, 4, $len + prefetch [$out], 22 + .L${bits}_cbc_enc_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f @@ -65,6 +93,8 @@ $::code.=<<___; fxor %f12, %f0, %f0 ! ^= ivec fxor %f14, %f2, %f2 + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 call _${alg}${bits}_encrypt_1x add $inp, 16, $inp @@ -138,6 +168,68 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f8, [$ivec + $omask]0xc0 ___ +$::code.=<<___; + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}cbc_enc_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + +.L${bits}_cbc_enc_blk_loop: + ldx [$inp + 0], %o0 + brz,pt $ileft, 5f + ldx [$inp + 8], %o1 + + ldx [$inp + 16], %o2 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + sllx %o1, $ileft, %o1 + or %g1, %o0, %o0 + srlx %o2, $iright, %o2 + or %o2, %o1, %o1 +5: + xor %g4, %o0, %o0 ! ^= rk[0] + xor %g5, %o1, %o1 + movxtod %o0, %f12 + movxtod %o1, %f14 + + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + prefetch [$inp + 16+63], 20 + call _${alg}${bits}_encrypt_1x + add $inp, 16, $inp + sub $len, 1, $len + + stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + brnz,pt $len, .L${bits}_cbc_enc_blk_loop + add $out, 8, $out + + !membar 0x0f + brnz,pt $blk_init, .L${bits}_cbc_enc_loop + mov $blk_init, $len +___ +$::code.=<<___ if ($::evp); + st %f0, [$ivec + 0] + st %f1, [$ivec + 4] + st %f2, [$ivec + 8] + st %f3, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3b + nop + + std %f0, [$ivec + 0] ! write out ivec + std %f2, [$ivec + 8] +___ $::code.=<<___; ret restore @@ -154,6 +246,7 @@ $::code.=<<___; .align 32 ${alg}${bits}_t4_cbc_decrypt: save %sp, -$::frame, %sp + sub $inp, $out, $blk_init ! $inp!=$out ___ $::code.=<<___ if (!$::evp); andcc $ivec, 7, $ivoff @@ -174,9 +267,9 @@ $::code.=<<___ if ($::evp); ld [$ivec + 12], %f15 ___ $::code.=<<___; + prefetch [$inp], 20 + prefetch [$inp + 63], 20 call _${alg}${bits}_load_deckey - srlx $len, 4, $len - andcc $len, 1, %g0 ! is number of blocks even? and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft @@ -184,9 +277,17 @@ $::code.=<<___; mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff + cmp $len, 127 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<128 || + brnz,pn $blk_init, .L${bits}cbc_dec_blk ! $inp==$out) + srl $omask, $ooff, $omask + + andcc $len, 16, %g0 ! is number of blocks even? + srlx $len, 4, $len alignaddrl $out, %g0, $out bz %icc, .L${bits}_cbc_dec_loop2x - srl $omask, $ooff, $omask + prefetch [$out], 22 .L${bits}_cbc_dec_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f @@ -205,6 +306,8 @@ $::code.=<<___; movxtod %o2, %f0 movxtod %o3, %f2 + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 call _${alg}${bits}_decrypt_1x add $inp, 16, $inp @@ -304,6 +407,8 @@ $::code.=<<___; movxtod %o4, %f4 movxtod %o5, %f6 + prefetch [$out + 63], 22 + prefetch [$inp + 32+63], 20 call _${alg}${bits}_decrypt_2x add $inp, 32, $inp @@ -392,6 +497,97 @@ $::code.=<<___ if (!$::evp); orn %g0, $omask, $omask stda %f4, [$ivec + $omask]0xc0 ___ +$::code.=<<___; + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}cbc_dec_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + sub $len, 1, $len + add $blk_init, 1, $blk_init + +.L${bits}_cbc_dec_blk_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 5f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +5: + xor %g4, %o0, %o4 ! ^= rk[0] + xor %g5, %o1, %o5 + movxtod %o4, %f0 + movxtod %o5, %f2 + xor %g4, %o2, %o4 + xor %g5, %o3, %o5 + movxtod %o4, %f4 + movxtod %o5, %f6 + + prefetch [$inp + 32+63], 20 + call _${alg}${bits}_decrypt_2x + add $inp, 32, $inp + subcc $len, 2, $len + + movxtod %o0, %f8 + movxtod %o1, %f10 + fxor %f12, %f0, %f0 ! ^= ivec + fxor %f14, %f2, %f2 + movxtod %o2, %f12 + movxtod %o3, %f14 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + bgu,pt $::size_t_cc, .L${bits}_cbc_dec_blk_loop2x + add $out, 8, $out + + add $blk_init, $len, $len + andcc $len, 1, %g0 ! is number of blocks even? + !membar 0x0f + bnz,pt %icc, .L${bits}_cbc_dec_loop + srl $len, 0, $len + brnz,pn $len, .L${bits}_cbc_dec_loop2x + nop +___ +$::code.=<<___ if ($::evp); + st %f0, [$ivec + 0] + st %f1, [$ivec + 4] + st %f2, [$ivec + 8] + st %f3, [$ivec + 12] +___ +$::code.=<<___ if (!$::evp); + brnz,pn $ivoff, 3b + nop + + std %f0, [$ivec + 0] ! write out ivec + std %f2, [$ivec + 8] +___ $::code.=<<___; ret restore @@ -409,8 +605,10 @@ $::code.=<<___; ${alg}${bits}_t4_ctr32_encrypt: save %sp, -$::frame, %sp + prefetch [$inp], 20 + prefetch [$inp + 63], 20 call _${alg}${bits}_load_enckey - nop + sllx $len, 4, $len ld [$ivec + 0], %l4 ! counter ld [$ivec + 4], %l5 @@ -424,7 +622,7 @@ ${alg}${bits}_t4_ctr32_encrypt: xor %g1, %g5, %g5 movxtod %g4, %f14 ! most significant 64 bits - andcc $len, 1, %g0 ! is number of blocks even? + sub $inp, $out, $blk_init ! $inp!=$out and $inp, 7, $ileft andn $inp, 7, $inp sll $ileft, 3, $ileft @@ -432,9 +630,16 @@ ${alg}${bits}_t4_ctr32_encrypt: mov 0xff, $omask sub $iright, $ileft, $iright and $out, 7, $ooff + cmp $len, 127 + movrnz $ooff, 0, $blk_init ! if ( $out&7 || + movleu $::size_t_cc, 0, $blk_init ! $len<128 || + brnz,pn $blk_init, .L${bits}_ctr32_blk ! $inp==$out) + srl $omask, $ooff, $omask + + andcc $len, 16, %g0 ! is number of blocks even? alignaddrl $out, %g0, $out bz %icc, .L${bits}_ctr32_loop2x - srl $omask, $ooff, $omask + srlx $len, 4, $len .L${bits}_ctr32_loop: ldx [$inp + 0], %o0 brz,pt $ileft, 4f @@ -452,6 +657,8 @@ ${alg}${bits}_t4_ctr32_encrypt: add %l7, 1, %l7 movxtod %g1, %f2 srl %l7, 0, %l7 ! clruw + prefetch [$out + 63], 22 + prefetch [$inp + 16+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f4 @@ -531,7 +738,8 @@ $::code.=<<___; add %l7, 1, %l7 movxtod %g1, %f6 srl %l7, 0, %l7 ! clruw - + prefetch [$out + 63], 22 + prefetch [$inp + 32+63], 20 ___ $::code.=<<___ if ($alg eq "aes"); aes_eround01 %f16, %f14, %f2, %f8 @@ -592,6 +800,96 @@ $::code.=<<___; brnz,pt $len, .L${bits}_ctr32_loop2x+4 orn %g0, $omask, $omask + ret + restore + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +.align 32 +.L${bits}_ctr32_blk: + add $out, $len, $blk_init + and $blk_init, 63, $blk_init ! tail + sub $len, $blk_init, $len + add $blk_init, 15, $blk_init ! round up to 16n + srlx $len, 4, $len + srl $blk_init, 4, $blk_init + sub $len, 1, $len + add $blk_init, 1, $blk_init + +.L${bits}_ctr32_blk_loop2x: + ldx [$inp + 0], %o0 + ldx [$inp + 8], %o1 + ldx [$inp + 16], %o2 + brz,pt $ileft, 5f + ldx [$inp + 24], %o3 + + ldx [$inp + 32], %o4 + sllx %o0, $ileft, %o0 + srlx %o1, $iright, %g1 + or %g1, %o0, %o0 + sllx %o1, $ileft, %o1 + srlx %o2, $iright, %g1 + or %g1, %o1, %o1 + sllx %o2, $ileft, %o2 + srlx %o3, $iright, %g1 + or %g1, %o2, %o2 + sllx %o3, $ileft, %o3 + srlx %o4, $iright, %o4 + or %o4, %o3, %o3 +5: + xor %g5, %l7, %g1 ! ^= rk[0] + add %l7, 1, %l7 + movxtod %g1, %f2 + srl %l7, 0, %l7 ! clruw + xor %g5, %l7, %g1 + add %l7, 1, %l7 + movxtod %g1, %f6 + srl %l7, 0, %l7 ! clruw + prefetch [$inp + 32+63], 20 +___ +$::code.=<<___ if ($alg eq "aes"); + aes_eround01 %f16, %f14, %f2, %f8 + aes_eround23 %f18, %f14, %f2, %f2 + aes_eround01 %f16, %f14, %f6, %f10 + aes_eround23 %f18, %f14, %f6, %f6 +___ +$::code.=<<___ if ($alg eq "cmll"); + camellia_f %f16, %f2, %f14, %f2 + camellia_f %f16, %f6, %f14, %f6 + camellia_f %f18, %f14, %f2, %f0 + camellia_f %f18, %f14, %f6, %f4 +___ +$::code.=<<___; + call _${alg}${bits}_encrypt_2x+16 + add $inp, 32, $inp + subcc $len, 2, $len + + movxtod %o0, %f8 + movxtod %o1, %f10 + movxtod %o2, %f12 + fxor %f8, %f0, %f0 ! ^= inp + movxtod %o3, %f8 + fxor %f10, %f2, %f2 + fxor %f12, %f4, %f4 + fxor %f8, %f6, %f6 + + stda %f0, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f2, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f4, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + add $out, 8, $out + stda %f6, [$out]0xf2 ! ASI_BLK_INIT, T4-specific + bgu,pt $::size_t_cc, .L${bits}_ctr32_blk_loop2x + add $out, 8, $out + + add $blk_init, $len, $len + andcc $len, 1, %g0 ! is number of blocks even? + !membar 0x0f + bnz,pt %icc, .L${bits}_ctr32_loop + srl $len, 0, $len + brnz,pn $len, .L${bits}_ctr32_loop2x + nop + ret restore .type ${alg}${bits}_t4_ctr32_encrypt,#function -- GitLab