Add AES-NI GCM stitch.

Configure

@@ -128,7 +128,7 @@ my $x86_asm="x86cpuid.o:bn-586.o co-586.o x86-mont.o x86-gf2m.o:des-586.o crypt5
 
 my $x86_elf_asm="$x86_asm:elf";
 
-my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o:e_padlock-x86_64.o";
+my $x86_64_asm="x86_64cpuid.o:x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o modexp512-x86_64.o::aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o::md5-x86_64.o:sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o::rc4-x86_64.o rc4-md5-x86_64.o:::wp-x86_64.o:cmll-x86_64.o cmll_misc.o:ghash-x86_64.o aesni-gcm-x86_64.o:e_padlock-x86_64.o";
 my $ia64_asm="ia64cpuid.o:bn-ia64.o ia64-mont.o::aes_core.o aes_cbc.o aes-ia64.o::md5-ia64.o:sha1-ia64.o sha256-ia64.o sha512-ia64.o::rc4-ia64.o rc4_skey.o:::::ghash-ia64.o::void";
 my $sparcv9_asm="sparcv9cap.o sparccpuid.o:bn-sparcv9.o sparcv9-mont.o sparcv9a-mont.o vis3-mont.o sparct4-mont.o sparcv9-gf2m.o:des_enc-sparc.o fcrypt_b.o:aes_core.o aes_cbc.o aes-sparcv9.o aest4-sparcv9.o::md5-sparcv9.o:sha1-sparcv9.o sha256-sparcv9.o sha512-sparcv9.o::::::camellia.o cmll_misc.o cmll_cbc.o cmllt4-sparcv9.o:ghash-sparcv9.o::void";
 my $sparcv8_asm=":sparcv8.o:des_enc-sparc.o fcrypt_b.o:::::::::::::void";

TABLE

@@ -318,7 +318,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -813,7 +813,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = auto
 $dso_scheme   = win32
@@ -1506,7 +1506,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = macosx
 $dso_scheme   = dlfcn
@@ -1671,7 +1671,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = auto
 $dso_scheme   = win32
@@ -1770,7 +1770,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = macosx
 $dso_scheme   = dlfcn
@@ -1836,7 +1836,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -2034,7 +2034,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -2562,7 +2562,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -2760,7 +2760,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -2826,7 +2826,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -4476,7 +4476,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -4509,7 +4509,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -4542,7 +4542,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -4740,7 +4740,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = mingw64
 $dso_scheme   = win32
@@ -5730,7 +5730,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn
@@ -5763,7 +5763,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = wp-x86_64.o
 $cmll_obj     = cmll-x86_64.o cmll_misc.o
-$modes_obj    = ghash-x86_64.o
+$modes_obj    = ghash-x86_64.o aesni-gcm-x86_64.o
 $engines_obj  = e_padlock-x86_64.o
 $perlasm_scheme = elf
 $dso_scheme   = dlfcn

crypto/evp/e_aes.c

@@ -227,6 +227,26 @@ void aesni_ccm64_decrypt_blocks (const unsigned char *in,
 			const unsigned char ivec[16],
 			unsigned char cmac[16]);
 
+#if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
+size_t aesni_gcm_encrypt(const unsigned char *in,
+			unsigned char *out,
+			size_t len,
+			const void *key,
+			unsigned char ivec[16],
+			u64 *Xi);
+#define AES_gcm_encrypt	aesni_gcm_encrypt
+size_t aesni_gcm_decrypt(const unsigned char *in,
+			unsigned char *out,
+			size_t len,
+			const void *key,
+			unsigned char ivec[16],
+			u64 *Xi);
+#define AES_gcm_decrypt	aesni_gcm_decrypt
+void gcm_ghash_avx(u64 Xi[2],const u128 Htable[16],const u8 *in,size_t len);
+#define AES_GCM_ASM(gctx)	(gctx->ctr==aesni_ctr32_encrypt_blocks && \
+				 gctx->gcm.ghash==gcm_ghash_avx)
+#endif
+
 static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 		   const unsigned char *iv, int enc)
 	{
@@ -1297,8 +1317,24 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 		/* Encrypt payload */
 		if (gctx->ctr)
 			{
+			size_t bulk=0;
+#if defined(AES_GCM_ASM)
+			if (len>=32 && AES_GCM_ASM(gctx))
+				{
+				if (CRYPTO_gcm128_encrypt(&gctx->gcm,NULL,NULL,0))
+					return -1;
+
+				bulk = AES_gcm_encrypt(in,out,len,
+							gctx->gcm.key,
+							gctx->gcm.Yi.c,
+							gctx->gcm.Xi.u);
+				gctx->gcm.len.u[1] += bulk;
+				}
+#endif
 			if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
-							in, out, len,
+							in +bulk,
+							out+bulk,
+							len-bulk,
 							gctx->ctr))
 				goto err;
 			}
@@ -1316,8 +1352,24 @@ static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 		/* Decrypt */
 		if (gctx->ctr)
 			{
+			size_t bulk=0;
+#if defined(AES_GCM_ASM)
+			if (len>=16 && AES_GCM_ASM(gctx))
+				{
+				if (CRYPTO_gcm128_decrypt(&gctx->gcm,NULL,NULL,0))
+					return -1;
+
+				bulk = AES_gcm_decrypt(in,out,len,
+							gctx->gcm.key,
+							gctx->gcm.Yi.c,
+							gctx->gcm.Xi.u);
+				gctx->gcm.len.u[1] += bulk;
+				}
+#endif
 			if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
-							in, out, len,
+							in +bulk,
+							out+bulk,
+							len-bulk,
 							gctx->ctr))
 				goto err;
 			}
@@ -1367,8 +1419,28 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			{
 			if (gctx->ctr)
 				{
+				size_t bulk=0;
+#if defined(AES_GCM_ASM)
+				if (len>=32 && AES_GCM_ASM(gctx))
+					{
+					size_t res = (16-gctx->gcm.mres)%16;
+
+					if (CRYPTO_gcm128_encrypt(&gctx->gcm,
+							in,out,res))
+						return -1;
+
+					bulk = AES_gcm_encrypt(in+res,
+							out+res,len-res,								gctx->gcm.key,
+							gctx->gcm.Yi.c,
+							gctx->gcm.Xi.u);
+					gctx->gcm.len.u[1] += bulk;
+					bulk += res;
+					}
+#endif
 				if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
-							in, out, len,
+							in +bulk,
+							out+bulk,
+							len-bulk,
 							gctx->ctr))
 					return -1;
 				}
@@ -1381,8 +1453,29 @@ static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			{
 			if (gctx->ctr)
 				{
+				size_t bulk=0;
+#if defined(AES_GCM_ASM)
+				if (len>=16 && AES_GCM_ASM(gctx))
+					{
+					size_t res = (16-gctx->gcm.mres)%16;
+
+					if (CRYPTO_gcm128_decrypt(&gctx->gcm,
+							in,out,res))
+						return -1;
+
+					bulk = AES_gcm_decrypt(in+res,
+							out+res,len-res,
+							gctx->gcm.key,
+							gctx->gcm.Yi.c,
+							gctx->gcm.Xi.u);
+					gctx->gcm.len.u[1] += bulk;
+					bulk += res;
+					}
+#endif
 				if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
-							in, out, len,
+							in +bulk,
+							out+bulk,
+							len-bulk,
 							gctx->ctr))
 					return -1;
 				}

crypto/evp/evptests.txt

@@ -367,5 +367,7 @@ aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000
 aes-128-gcm:00000000000000000000000000000000:000000000000000000000000:0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:0388dace60b6a392f328c2b971b2fe78f795aaab494b5923f7fd89ff948bc1e0200211214e7394da2089b6acd093abe0c94da219118e297d7b7ebcbcc9c388f28ade7d85a8ee35616f7124a9d527029195b84d1b96c690ff2f2de30bf2ec89e00253786e126504f0dab90c48a30321de3345e6b0461e7c9e6c6b7afedde83f40::cac45f60e31efd3b5a43b98a22ce1aa1
 # 192 bytes plaintext, iv is chosen so that initial counter LSB is 0xFF
 aes-128-gcm:00000000000000000000000000000000:ffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:56b3373ca9ef6e4a2b64fe1e9a17b61425f10d47a75a5fce13efc6bc784af24f4141bdd48cf7c770887afd573cca5418a9aeffcd7c5ceddfc6a78397b9a85b499da558257267caab2ad0b23ca476a53cb17fb41c4b8b475cb4f3f7165094c229c9e8c4dc0a2a5ff1903e501511221376a1cdb8364c5061a20cae74bc4acd76ceb0abc9fd3217ef9f8c90be402ddf6d8697f4f880dff15bfb7a6b28241ec8fe183c2d59e3f9dfff653c7126f0acb9e64211f42bae12af462b1070bef1ab5e3606::566f8ef683078bfdeeffa869d751a017
+# 288 bytes plaintext, iv is chosen so that initial counter LSB is 0xFF
+aes-128-gcm:00000000000000000000000000000000:ffffffff000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000:56b3373ca9ef6e4a2b64fe1e9a17b61425f10d47a75a5fce13efc6bc784af24f4141bdd48cf7c770887afd573cca5418a9aeffcd7c5ceddfc6a78397b9a85b499da558257267caab2ad0b23ca476a53cb17fb41c4b8b475cb4f3f7165094c229c9e8c4dc0a2a5ff1903e501511221376a1cdb8364c5061a20cae74bc4acd76ceb0abc9fd3217ef9f8c90be402ddf6d8697f4f880dff15bfb7a6b28241ec8fe183c2d59e3f9dfff653c7126f0acb9e64211f42bae12af462b1070bef1ab5e3606872ca10dee15b3249b1a1b958f23134c4bccb7d03200bce420a2f8eb66dcf3644d1423c1b5699003c13ecef4bf38a3b60eedc34033bac1902783dc6d89e2e774188a439c7ebcc0672dbda4ddcfb2794613b0be41315ef778708a70ee7d75165c::8b307f6b33286d0ab026a9ed3fe1e85f
 # 80 bytes plaintext, submitted by Intel
 aes-128-gcm:843ffcf5d2b72694d19ed01d01249412:dbcca32ebf9b804617c3aa9e:000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f:6268c6fa2a80b2d137467f092f657ac04d89be2beaa623d61b5a868c8f03ff95d3dcee23ad2f1ab3a6c80eaf4b140eb05de3457f0fbc111a6b43d0763aa422a3013cf1dc37fe417d1fbfc449b75d4cc5:00000000000000000000000000000000101112131415161718191a1b1c1d1e1f:3b629ccfbc1119b7319e1dce2cd6fd6d

crypto/modes/Makefile

@@ -50,6 +50,8 @@ ghash-x86.s:	asm/ghash-x86.pl
 	$(PERL) asm/ghash-x86.pl $(PERLASM_SCHEME) $(CFLAGS) $(PROCESSOR) > $@
 ghash-x86_64.s:	asm/ghash-x86_64.pl
 	$(PERL) asm/ghash-x86_64.pl $(PERLASM_SCHEME) > $@
+aesni-gcm-x86_64.s:	asm/aesni-gcm-x86_64.pl
+	$(PERL) asm/aesni-gcm-x86_64.pl $(PERLASM_SCHEME) > $@
 ghash-sparcv9.s:	asm/ghash-sparcv9.pl
 	$(PERL) asm/ghash-sparcv9.pl $@ $(CFLAGS)
 ghash-alpha.s:	asm/ghash-alpha.pl