aesni-sha1-x86_64.pl: add stiched decrypt procedure,

but keep it disabled, too little gain... Add some Atom-specific
optimization.

crypto/evp/e_aes_cbc_hmac_sha1.c

@@ -113,6 +113,10 @@ void aesni_cbc_sha1_enc (const void *inp, void *out, size_t blocks,
 		const AES_KEY *key, unsigned char iv[16],
 		SHA_CTX *ctx,const void *in0);
 
+void aesni256_cbc_sha1_dec (const void *inp, void *out, size_t blocks,
+		const AES_KEY *key, unsigned char iv[16],
+		SHA_CTX *ctx,const void *in0);
+
 #define data(ctx) ((EVP_AES_HMAC_SHA1 *)(ctx)->cipher_data)
 
 static int aesni_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx,
@@ -137,6 +141,7 @@ static int aesni_cbc_hmac_sha1_init_key(EVP_CIPHER_CTX *ctx,
 	}
 
 #define	STITCHED_CALL
+#undef	STITCHED_DECRYPT_CALL
 
 #if !defined(STITCHED_CALL)
 #define	aes_off 0
@@ -435,28 +440,45 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 		/* arrange cache line alignment */
 		pmac = (void *)(((size_t)mac.c+31)&((size_t)0-32));
 
-		/* decrypt HMAC|padding at once */
-		aesni_cbc_encrypt(in,out,len,
-				&key->ks,ctx->iv,0);
-
-		if (plen) {	/* "TLS" mode of operation */
+		if (plen != NO_PAYLOAD_LENGTH) {	/* "TLS" mode of operation */
 			size_t inp_len, mask, j, i;
 			unsigned int res, maxpad, pad, bitlen;
 			int ret = 1;
 			union {	unsigned int  u[SHA_LBLOCK];
 				unsigned char c[SHA_CBLOCK]; }
 				*data = (void *)key->md.data;
+#if defined(STITCHED_DECRYPT_CALL)
+			unsigned char tail_iv[AES_BLOCK_SIZE];
+			int stitch=0;
+#endif
 
 			if ((key->aux.tls_aad[plen-4]<<8|key->aux.tls_aad[plen-3])
-			    >= TLS1_1_VERSION)
-				iv = AES_BLOCK_SIZE;
-
-			if (len<(iv+SHA_DIGEST_LENGTH+1))
+			    >= TLS1_1_VERSION) {
+			    	if (len<(AES_BLOCK_SIZE+SHA_DIGEST_LENGTH+1))
+					return 0;
+
+				/* omit explicit iv */
+				memcpy(ctx->iv,in,AES_BLOCK_SIZE);
+				in  += AES_BLOCK_SIZE;
+				out += AES_BLOCK_SIZE;
+				len -= AES_BLOCK_SIZE;
+			}
+			else if (len<(SHA_DIGEST_LENGTH+1))
 				return 0;
 
-			/* omit explicit iv */
-			out += iv;
-			len -= iv;
+#if defined(STITCHED_DECRYPT_CALL)
+			if (len>=1024 && ctx->key_len==32) {
+				/* decrypt last block */
+				memcpy(tail_iv,in+len-2*AES_BLOCK_SIZE,AES_BLOCK_SIZE);
+				aesni_cbc_encrypt(in+len-AES_BLOCK_SIZE,
+						out+len-AES_BLOCK_SIZE,AES_BLOCK_SIZE,
+						&key->ks,tail_iv,0);
+				stitch=1;
+			} else
+#endif
+			/* decrypt HMAC|padding at once */
+			aesni_cbc_encrypt(in,out,len,
+					&key->ks,ctx->iv,0);
 
 			/* figure out payload length */
 			pad = out[len-1];
@@ -476,6 +498,30 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			key->md = key->head;
 			SHA1_Update(&key->md,key->aux.tls_aad,plen);
 
+#if defined(STITCHED_DECRYPT_CALL)
+			if (stitch) {
+				blocks = (len-(256+32+SHA_CBLOCK))/SHA_CBLOCK;
+				aes_off = len-AES_BLOCK_SIZE-blocks*SHA_CBLOCK;
+				sha_off = SHA_CBLOCK-plen;
+
+				aesni_cbc_encrypt(in,out,aes_off,
+					&key->ks,ctx->iv,0);
+
+				SHA1_Update(&key->md,out,sha_off);
+				aesni256_cbc_sha1_dec(in+aes_off,
+					out+aes_off,blocks,&key->ks,ctx->iv,
+					&key->md,out+sha_off);
+
+				sha_off += blocks*=SHA_CBLOCK;
+				out += sha_off;
+				len -= sha_off;
+				inp_len -= sha_off;
+
+				key->md.Nl += (blocks<<3);	/* at most 18 bits */
+				memcpy(ctx->iv,tail_iv,AES_BLOCK_SIZE);
+			}
+#endif
+
 #if 1
 			len -= SHA_DIGEST_LENGTH;		/* amend mac */
 			if (len>=(256+SHA_CBLOCK)) {
@@ -630,6 +676,34 @@ static int aesni_cbc_hmac_sha1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 #endif
 			return ret;
 		} else {
+#if defined(STITCHED_DECRYPT_CALL)
+			if (len>=1024 && ctx->key_len==32) {
+				if (sha_off%=SHA_CBLOCK)
+					blocks = (len-3*SHA_CBLOCK)/SHA_CBLOCK;
+				else
+					blocks = (len-2*SHA_CBLOCK)/SHA_CBLOCK;
+				aes_off = len-blocks*SHA_CBLOCK;
+
+				aesni_cbc_encrypt(in,out,aes_off,
+					&key->ks,ctx->iv,0);
+				SHA1_Update(&key->md,out,sha_off);
+				aesni256_cbc_sha1_dec(in+aes_off,
+					out+aes_off,blocks,&key->ks,ctx->iv,
+					&key->md,out+sha_off);
+
+				sha_off += blocks*=SHA_CBLOCK;
+				out += sha_off;
+				len -= sha_off;
+
+				key->md.Nh += blocks>>29;
+				key->md.Nl += blocks<<=3;
+				if (key->md.Nl<(unsigned int)blocks) key->md.Nh++;
+			} else
+#endif
+			/* decrypt HMAC|padding at once */
+			aesni_cbc_encrypt(in,out,len,
+					&key->ks,ctx->iv,0);
+
 			SHA1_Update(&key->md,out,len);
 		}
 	}

crypto/evp/e_aes_cbc_hmac_sha256.c

@@ -400,7 +400,7 @@ static int aesni_cbc_hmac_sha256_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 			iv = AES_BLOCK_SIZE;
 
 #if defined(STITCHED_CALL)
-		if (OPENSSL_ia32cap_P[1]&(1<<(60-32)) &&
+		if (OPENSSL_ia32cap_P[1]&(1<<(60-32)) && /* AVX? */
 		    plen>(sha_off+iv) &&
 		    (blocks=(plen-(sha_off+iv))/SHA256_CBLOCK)) {
 			SHA256_Update(&key->md,in+iv,sha_off);
@@ -451,7 +451,7 @@ static int aesni_cbc_hmac_sha256_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 		aesni_cbc_encrypt(in,out,len,
 				&key->ks,ctx->iv,0);
 
-		if (plen) {	/* "TLS" mode of operation */
+		if (plen != NO_PAYLOAD_LENGTH) {	/* "TLS" mode of operation */
 			size_t inp_len, mask, j, i;
 			unsigned int res, maxpad, pad, bitlen;
 			int ret = 1;