Use more flexible method of determining output length, by setting &outlen

value of the passed output buffer is NULL.

The old method of using EVP_PKEY_size(pkey) isn't flexible enough to cover all
cases where the output length may depend on the operation or the parameters
associated with it.

apps/pkeyutl.c

@@ -79,6 +79,10 @@ static EVP_PKEY_CTX *init_ctx(int *pkeysize,
 static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
 							const char *file);
 
+static int do_keyop(EVP_PKEY_CTX *ctx, int pkey_op,
+		unsigned char *out, int *poutlen,
+		unsigned char *in, int inlen);
+
 int MAIN(int argc, char **);
 
 int MAIN(int argc, char **argv)
@@ -299,8 +303,6 @@ int MAIN(int argc, char **argv)
 			}
 		}
 	
-	buf_out = OPENSSL_malloc(keysize);
-
 	if (in)
 		{
 		/* Read the input data */
@@ -323,29 +325,8 @@ int MAIN(int argc, char **argv)
 			}
 		}
 
-	switch(pkey_op)
+	if(pkey_op == EVP_PKEY_OP_VERIFY)
 		{
-		case EVP_PKEY_OP_VERIFYRECOVER:
-		rv  = EVP_PKEY_verify_recover(ctx, buf_out, &buf_outlen,
-							buf_in, buf_inlen);
-		break;
-
-		case EVP_PKEY_OP_SIGN:
-		rv  = EVP_PKEY_sign(ctx, buf_out, &buf_outlen,
-							buf_in, buf_inlen);
-		break;
-
-		case EVP_PKEY_OP_ENCRYPT:
-		rv  = EVP_PKEY_encrypt(ctx, buf_out, &buf_outlen,
-							buf_in, buf_inlen);
-		break;
-
-		case EVP_PKEY_OP_DECRYPT:
-		rv  = EVP_PKEY_decrypt(ctx, buf_out, &buf_outlen,
-							buf_in, buf_inlen);
-		break; 
-
-		case EVP_PKEY_OP_VERIFY:
 		rv  = EVP_PKEY_verify(ctx, sig, siglen, buf_in, buf_inlen);
 		if (rv == 0)
 			BIO_puts(out, "Signature Verification Failure\n");
@@ -353,12 +334,21 @@ int MAIN(int argc, char **argv)
 			BIO_puts(out, "Signature Verified Successfully\n");
 		if (rv >= 0)
 			goto end;
-		break; 
-
-		case EVP_PKEY_OP_DERIVE:
-		rv  = EVP_PKEY_derive(ctx, buf_out, &buf_outlen);
-		break;
-
+		}
+	else
+		{	
+		rv = do_keyop(ctx, pkey_op, NULL, &buf_outlen,
+							buf_in, buf_inlen);
+		if (rv > 0)
+			{
+			buf_out = OPENSSL_malloc(buf_outlen);
+			if (!buf_out)
+				rv = -1;
+			else
+				rv = do_keyop(ctx, pkey_op,
+						buf_out, &buf_outlen,
+						buf_in, buf_inlen);
+			}
 		}
 
 	if(rv <= 0)
@@ -541,4 +531,34 @@ static int setup_peer(BIO *err, EVP_PKEY_CTX *ctx, int peerform,
 		ERR_print_errors(err);
 	return ret;
 	}
-			
+
+static int do_keyop(EVP_PKEY_CTX *ctx, int pkey_op,
+		unsigned char *out, int *poutlen,
+		unsigned char *in, int inlen)
+	{
+	int rv;
+	switch(pkey_op)
+		{
+		case EVP_PKEY_OP_VERIFYRECOVER:
+		rv  = EVP_PKEY_verify_recover(ctx, out, poutlen, in, inlen);
+		break;
+
+		case EVP_PKEY_OP_SIGN:
+		rv  = EVP_PKEY_sign(ctx, out, poutlen, in, inlen);
+		break;
+
+		case EVP_PKEY_OP_ENCRYPT:
+		rv  = EVP_PKEY_encrypt(ctx, out, poutlen, in, inlen);
+		break;
+
+		case EVP_PKEY_OP_DECRYPT:
+		rv  = EVP_PKEY_decrypt(ctx, out, poutlen, in, inlen);
+		break; 
+
+		case EVP_PKEY_OP_DERIVE:
+		rv  = EVP_PKEY_derive(ctx, out, poutlen);
+		break;
+
+		}
+	return rv;
+	}

crypto/dh/dh_pmeth.c

@@ -206,7 +206,7 @@ static int pkey_dh_derive(EVP_PKEY_CTX *ctx, unsigned char *key, int *keylen)
 const EVP_PKEY_METHOD dh_pkey_meth = 
 	{
 	EVP_PKEY_DH,
-	0,
+	EVP_PKEY_FLAG_AUTOARGLEN,
 	pkey_dh_init,
 	pkey_dh_cleanup,
 

crypto/dsa/dsa_pmeth.c

@@ -221,7 +221,7 @@ static int pkey_dsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 const EVP_PKEY_METHOD dsa_pkey_meth = 
 	{
 	EVP_PKEY_DSA,
-	0,
+	EVP_PKEY_FLAG_AUTOARGLEN,
 	pkey_dsa_init,
 	pkey_dsa_cleanup,
 

crypto/evp/evp.h

@@ -937,6 +937,8 @@ void EVP_PKEY_asn1_set_ctrl(EVP_PKEY_ASN1_METHOD *ameth,
 
 #define EVP_PKEY_ALG_CTRL		0x1000
 
+#define EVP_PKEY_FLAG_AUTOARGLEN	2
+
 const EVP_PKEY_METHOD *EVP_PKEY_meth_find(int type, ENGINE *e);
 EVP_PKEY_CTX *EVP_PKEY_CTX_new(EVP_PKEY *pkey, ENGINE *e);
 EVP_PKEY_CTX *EVP_PKEY_CTX_new_id(int id, ENGINE *e);
@@ -1119,6 +1121,7 @@ void ERR_load_EVP_strings(void);
 #define EVP_R_BAD_KEY_LENGTH				 137
 #define EVP_R_BN_DECODE_ERROR				 112
 #define EVP_R_BN_PUBKEY_ERROR				 113
+#define EVP_R_BUFFER_TOO_SMALL				 155
 #define EVP_R_CIPHER_PARAMETER_ERROR			 122
 #define EVP_R_COMMAND_NOT_SUPPORTED			 147
 #define EVP_R_CTRL_NOT_IMPLEMENTED			 132

crypto/evp/evp_err.c

@@ -95,9 +95,9 @@ static ERR_STRING_DATA EVP_str_functs[]=
 {ERR_FUNC(EVP_F_EVP_PKEY_DECRYPT),	"EVP_PKEY_decrypt"},
 {ERR_FUNC(EVP_F_EVP_PKEY_DECRYPT_INIT),	"EVP_PKEY_decrypt_init"},
 {ERR_FUNC(EVP_F_EVP_PKEY_DECRYPT_OLD),	"EVP_PKEY_decrypt_old"},
-{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE),	"EVP_PKEY_DERIVE"},
-{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE_INIT),	"EVP_PKEY_DERIVE_INIT"},
-{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE_SET_PEER),	"EVP_PKEY_DERIVE_SET_PEER"},
+{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE),	"EVP_PKEY_derive"},
+{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE_INIT),	"EVP_PKEY_derive_init"},
+{ERR_FUNC(EVP_F_EVP_PKEY_DERIVE_SET_PEER),	"EVP_PKEY_derive_set_peer"},
 {ERR_FUNC(EVP_F_EVP_PKEY_ENCRYPT),	"EVP_PKEY_encrypt"},
 {ERR_FUNC(EVP_F_EVP_PKEY_ENCRYPT_INIT),	"EVP_PKEY_encrypt_init"},
 {ERR_FUNC(EVP_F_EVP_PKEY_ENCRYPT_OLD),	"EVP_PKEY_encrypt_old"},
@@ -137,6 +137,7 @@ static ERR_STRING_DATA EVP_str_reasons[]=
 {ERR_REASON(EVP_R_BAD_KEY_LENGTH)        ,"bad key length"},
 {ERR_REASON(EVP_R_BN_DECODE_ERROR)       ,"bn decode error"},
 {ERR_REASON(EVP_R_BN_PUBKEY_ERROR)       ,"bn pubkey error"},
+{ERR_REASON(EVP_R_BUFFER_TOO_SMALL)      ,"buffer too small"},
 {ERR_REASON(EVP_R_CIPHER_PARAMETER_ERROR),"cipher parameter error"},
 {ERR_REASON(EVP_R_COMMAND_NOT_SUPPORTED) ,"command not supported"},
 {ERR_REASON(EVP_R_CTRL_NOT_IMPLEMENTED)  ,"ctrl not implemented"},

crypto/evp/evp_locl.h

@@ -256,7 +256,7 @@ struct evp_pkey_ctx_st
 	int keygen_info_count;
 	} /* EVP_PKEY_CTX */;
 
-#define EVP_PKEY_DYNAMIC	1
+#define EVP_PKEY_FLAG_DYNAMIC	1
 
 struct evp_pkey_method_st
 	{

crypto/evp/pmeth_fn.c

@@ -63,6 +63,22 @@
 #include <openssl/evp.h>
 #include "evp_locl.h"
 
+#define M_check_autoarg(ctx, arg, arglen, err) \
+	if (ctx->pmeth->flags & EVP_PKEY_FLAG_AUTOARGLEN) \
+		{ \
+		int pksize = EVP_PKEY_size(ctx->pkey); \
+		if (!arg) \
+			{ \
+			*arglen = pksize; \
+			return 1; \
+			} \
+		else if (*arglen < pksize) \
+			{ \
+			EVPerr(err, EVP_R_BUFFER_TOO_SMALL); \
+			return 0; \
+			} \
+		}
+
 int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx)
 	{
 	int ret;
@@ -96,6 +112,7 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx,
 		EVPerr(EVP_F_EVP_PKEY_SIGN, EVP_R_OPERATON_NOT_INITIALIZED);
 		return -1;
 		}
+	M_check_autoarg(ctx, sig, siglen, EVP_F_EVP_PKEY_SIGN)
 	return ctx->pmeth->sign(ctx, sig, siglen, tbs, tbslen);
 	}
 
@@ -168,6 +185,7 @@ int EVP_PKEY_verify_recover(EVP_PKEY_CTX *ctx,
 		EVPerr(EVP_F_EVP_PKEY_VERIFY_RECOVER, EVP_R_OPERATON_NOT_INITIALIZED);
 		return -1;
 		}
+	M_check_autoarg(ctx, rout, routlen, EVP_F_EVP_PKEY_VERIFY_RECOVER)
 	return ctx->pmeth->verify_recover(ctx, rout, routlen, sig, siglen);
 	}
 
@@ -204,6 +222,7 @@ int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx,
 		EVPerr(EVP_F_EVP_PKEY_ENCRYPT, EVP_R_OPERATON_NOT_INITIALIZED);
 		return -1;
 		}
+	M_check_autoarg(ctx, out, outlen, EVP_F_EVP_PKEY_ENCRYPT)
 	return ctx->pmeth->encrypt(ctx, out, outlen, in, inlen);
 	}
 
@@ -240,6 +259,7 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx,
 		EVPerr(EVP_F_EVP_PKEY_DECRYPT, EVP_R_OPERATON_NOT_INITIALIZED);
 		return -1;
 		}
+	M_check_autoarg(ctx, out, outlen, EVP_F_EVP_PKEY_DECRYPT)
 	return ctx->pmeth->decrypt(ctx, out, outlen, in, inlen);
 	}
 
@@ -335,6 +355,7 @@ int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, unsigned char *key, int *pkeylen)
 		EVPerr(EVP_F_EVP_PKEY_DERIVE, EVP_R_OPERATON_NOT_INITIALIZED);
 		return -1;
 		}
+	M_check_autoarg(ctx, key, pkeylen, EVP_F_EVP_PKEY_DERIVE)
 	return ctx->pmeth->derive(ctx, key, pkeylen);
 	}
 

crypto/evp/pmeth_lib.c

@@ -146,7 +146,7 @@ EVP_PKEY_METHOD* EVP_PKEY_meth_new(int id, int flags)
 		return NULL;
 
 	pmeth->pkey_id = id;
-	pmeth->flags = flags | EVP_PKEY_DYNAMIC;
+	pmeth->flags = flags | EVP_PKEY_FLAG_DYNAMIC;
 
 	pmeth->init = 0;
 	pmeth->cleanup = 0;
@@ -178,7 +178,7 @@ EVP_PKEY_METHOD* EVP_PKEY_meth_new(int id, int flags)
 
 void EVP_PKEY_meth_free(EVP_PKEY_METHOD *pmeth)
 	{
-	if (pmeth && (pmeth->flags & EVP_PKEY_DYNAMIC))
+	if (pmeth && (pmeth->flags & EVP_PKEY_FLAG_DYNAMIC))
 		OPENSSL_free(pmeth);
 	}
 

crypto/rsa/rsa_pmeth.c

@@ -498,7 +498,7 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
 const EVP_PKEY_METHOD rsa_pkey_meth = 
 	{
 	EVP_PKEY_RSA,
-	0,
+	EVP_PKEY_FLAG_AUTOARGLEN,
 	pkey_rsa_init,
 	pkey_rsa_cleanup,