Add ARMv8 Montgomery multiplication module.

Reviewed-by: NRich Salz <rsalz@openssl.org>

crypto/bn/asm/armv8-mont.pl

+#!/usr/bin/env perl
+
+# ====================================================================
+# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
+# project. The module is, however, dual licensed under OpenSSL and
+# CRYPTOGAMS licenses depending on where you obtain it. For further
+# details see http://www.openssl.org/~appro/cryptogams/.
+# ====================================================================
+
+# March 2015
+#
+# "Teaser" Montgomery multiplication module for ARMv8. Needs more
+# work. While it does improve RSA sign performance by 20-30% (less for
+# longer keys) on most processors, for some reason RSA2048 is not
+# faster and RSA4096 goes 15-20% slower on Cortex-A57. Multiplication
+# instruction issue rate is limited on processor in question, meaning
+# that dedicated squaring procedure is a must. Well, actually all
+# contemporary AArch64 processors seem to have limited multiplication
+# issue rate, i.e. they can't issue multiplication every cycle, which
+# explains moderate improvement coefficients in comparison to
+# compiler-generated code. Recall that compiler is instructed to use
+# umulh and therefore uses same amount of multiplication instructions
+# to do the job. Assembly's edge is to minimize number of "collateral"
+# instructions and of course instruction scheduling.
+
+$flavour = shift;
+$output  = shift;
+
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
+( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
+( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
+die "can't locate arm-xlate.pl";
+
+open OUT,"| \"$^X\" $xlate $flavour $output";
+*STDOUT=*OUT;
+
+($lo0,$hi0,$aj,$m0,$alo,$ahi,
+ $lo1,$hi1,$nj,$m1,$nlo,$nhi,
+ $ovf, $i,$j,$tp,$tj) = map("x$_",6..17,19..24);
+
+# int bn_mul_mont(
+$rp="x0";	# BN_ULONG *rp,
+$ap="x1";	# const BN_ULONG *ap,
+$bp="x2";	# const BN_ULONG *bp,
+$np="x3";	# const BN_ULONG *np,
+$n0="x4";	# const BN_ULONG *n0,
+$num="x5";	# int num);
+
+$code.=<<___;
+.text
+
+.globl	bn_mul_mont
+.type	bn_mul_mont,%function
+.align	5
+bn_mul_mont:
+	stp	x29,x30,[sp,#-64]!
+	add	x29,sp,#0
+	stp	x19,x20,[sp,#16]
+	stp	x21,x22,[sp,#32]
+	stp	x23,x24,[sp,#48]
+
+	ldr	$m0,[$bp],#8		// bp[0]
+	sub	$tp,sp,$num,lsl#3
+	ldp	$hi0,$aj,[$ap],#16	// ap[0..1]
+	lsl	$num,$num,#3
+	ldr	$n0,[$n0]		// *n0
+	and	$tp,$tp,#-16		// ABI says so
+	ldp	$hi1,$nj,[$np],#16	// np[0..1]
+
+	mul	$lo0,$hi0,$m0		// ap[0]*bp[0]
+	sub	$j,$num,#16		// j=num-2
+	umulh	$hi0,$hi0,$m0
+	mul	$alo,$aj,$m0		// ap[1]*bp[0]
+	umulh	$ahi,$aj,$m0
+
+	mul	$m1,$lo0,$n0		// "tp[0]"*n0
+	mov	sp,$tp			// alloca
+
+	mul	$lo1,$hi1,$m1		// np[0]*m1
+	umulh	$hi1,$hi1,$m1
+	mul	$nlo,$nj,$m1		// np[1]*m1
+	adds	$lo1,$lo1,$lo0		// discarded
+	umulh	$nhi,$nj,$m1
+	adc	$hi1,$hi1,xzr
+	cbz	$j,.L1st_skip
+
+.L1st:
+	ldr	$aj,[$ap],#8
+	adds	$lo0,$alo,$hi0
+	sub	$j,$j,#8		// j--
+	adc	$hi0,$ahi,xzr
+
+	ldr	$nj,[$np],#8
+	adds	$lo1,$nlo,$hi1
+	mul	$alo,$aj,$m0		// ap[j]*bp[0]
+	adc	$hi1,$nhi,xzr
+	umulh	$ahi,$aj,$m0
+
+	adds	$lo1,$lo1,$lo0
+	mul	$nlo,$nj,$m1		// np[j]*m1
+	adc	$hi1,$hi1,xzr
+	umulh	$nhi,$nj,$m1
+	str	$lo1,[$tp],#8		// tp[j-1]
+	cbnz	$j,.L1st
+
+.L1st_skip:
+	adds	$lo0,$alo,$hi0
+	sub	$ap,$ap,$num		// rewind $ap
+	adc	$hi0,$ahi,xzr
+
+	adds	$lo1,$nlo,$hi1
+	sub	$np,$np,$num		// rewind $np
+	adc	$hi1,$nhi,xzr
+
+	adds	$lo1,$lo1,$lo0
+	sub	$i,$num,#8		// i=num-1
+	adcs	$hi1,$hi1,$hi0
+
+	adc	$ovf,xzr,xzr		// upmost overflow bit
+	stp	$lo1,$hi1,[$tp]
+
+.Louter:
+	ldr	$m0,[$bp],#8		// bp[i]
+	ldp	$hi0,$aj,[$ap],#16
+	ldr	$tj,[sp]		// tp[0]
+	add	$tp,sp,#8
+
+	mul	$lo0,$hi0,$m0		// ap[0]*bp[i]
+	sub	$j,$num,#16		// j=num-2
+	umulh	$hi0,$hi0,$m0
+	ldp	$hi1,$nj,[$np],#16
+	mul	$alo,$aj,$m0		// ap[1]*bp[i]
+	adds	$lo0,$lo0,$tj
+	umulh	$ahi,$aj,$m0
+	adc	$hi0,$hi0,xzr
+
+	mul	$m1,$lo0,$n0
+	sub	$i,$i,#8		// i--
+
+	mul	$lo1,$hi1,$m1		// np[0]*m1
+	umulh	$hi1,$hi1,$m1
+	mul	$nlo,$nj,$m1		// np[1]*m1
+	adds	$lo1,$lo1,$lo0
+	umulh	$nhi,$nj,$m1
+	cbz	$j,.Linner_skip
+
+.Linner:
+	ldr	$aj,[$ap],#8
+	adc	$hi1,$hi1,xzr
+	ldr	$tj,[$tp],#8		// tp[j]
+	adds	$lo0,$alo,$hi0
+	sub	$j,$j,#8		// j--
+	adc	$hi0,$ahi,xzr
+
+	adds	$lo1,$nlo,$hi1
+	ldr	$nj,[$np],#8
+	adc	$hi1,$nhi,xzr
+
+	mul	$alo,$aj,$m0		// ap[j]*bp[i]
+	adds	$lo0,$lo0,$tj
+	umulh	$ahi,$aj,$m0
+	adc	$hi0,$hi0,xzr
+
+	mul	$nlo,$nj,$m1		// np[j]*m1
+	adds	$lo1,$lo1,$lo0
+	umulh	$nhi,$nj,$m1
+	str	$lo1,[$tp,#-16]		// tp[j-1]
+	cbnz	$j,.Linner
+
+.Linner_skip:
+	ldr	$tj,[$tp],#8		// tp[j]
+	adc	$hi1,$hi1,xzr
+	adds	$lo0,$alo,$hi0
+	sub	$ap,$ap,$num		// rewind $ap
+	adc	$hi0,$ahi,xzr
+
+	adds	$lo1,$nlo,$hi1
+	sub	$np,$np,$num		// rewind $np
+	adc	$hi1,$nhi,$ovf
+
+	adds	$lo0,$lo0,$tj
+	adc	$hi0,$hi0,xzr
+
+	adds	$lo1,$lo1,$lo0
+	adcs	$hi1,$hi1,$hi0
+	adc	$ovf,xzr,xzr		// upmost overflow bit
+	stp	$lo1,$hi1,[$tp,#-16]
+
+	cbnz	$i,.Louter
+
+	// Final step. We see if result is larger than modulus, and
+	// if it is, subtract the modulus. But comparison implies
+	// subtraction. So we subtract modulus, see if it borrowed,
+	// and conditionally copy original value. 
+	ldr	$tj,[sp]		// tp[0]
+	add	$tp,sp,#8
+	ldr	$nj,[$np],#8		// np[0]
+	subs	$j,$num,#8		// j=num-1 and clear borrow
+	mov	$ap,$rp
+.Lsub:
+	sbcs	$aj,$tj,$nj		// tp[j]-np[j]
+	ldr	$tj,[$tp],#8
+	sub	$j,$j,#8		// j--
+	ldr	$nj,[$np],#8
+	str	$aj,[$ap],#8		// rp[j]=tp[j]-np[j]
+	cbnz	$j,.Lsub
+
+	sbcs	$aj,$tj,$nj
+	sbcs	$ovf,$ovf,xzr		// did it borrow?
+	str	$aj,[$ap],#8		// rp[num-1]
+
+	ldr	$tj,[sp]		// tp[0]
+	add	$tp,sp,#8
+	ldr	$aj,[$rp],#8		// rp[0]
+	sub	$num,$num,#8		// num--
+	nop
+.Lcond_copy:
+	sub	$num,$num,#8		// num--
+	csel	$nj,$aj,$tj,cs		// did it borrow?
+	ldr	$tj,[$tp],#8
+	ldr	$aj,[$rp],#8
+	str	xzr,[$tp,#-16]		// wipe tp
+	str	$nj,[$rp,#-16]
+	cbnz	$num,.Lcond_copy
+
+	csel	$nj,$aj,$tj,cs
+	str	xzr,[$tp,#-8]		// wipe tp
+	str	$nj,[$rp,#-8]
+
+	ldp	x19,x20,[x29,#16]
+	mov	sp,x29
+	ldp	x21,x22,[x29,#32]
+	ldp	x23,x24,[x29,#48]
+	ldr	x29,[sp],#64
+	ret
+.size	bn_mul_mont,.-bn_mul_mont
+
+.asciz	"Montgomery Multiplication for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
+.align	4
+___
+
+print $code;
+
+close STDOUT;