Replace RSA specific PKCS7_RECIP_INFO set up with an public key algorithm

ctrl.

crypto/evp/evp.h

@@ -804,6 +804,7 @@ void EVP_PBE_cleanup(void);
 #define ASN1_PKEY_SIGPARAM_NULL	0x4
 
 #define ASN1_PKEY_CTRL_PKCS7_SIGN	0x1
+#define ASN1_PKEY_CTRL_PKCS7_ENCRYPT	0x2
 
 int EVP_PKEY_asn1_get_count(void);
 const EVP_PKEY_ASN1_METHOD *EVP_PKEY_asn1_get0(int idx);

crypto/pkcs7/pk7_lib.c

@@ -456,9 +456,11 @@ PKCS7_RECIP_INFO *PKCS7_add_recipient(PKCS7 *p7, X509 *x509)
 	if ((ri=PKCS7_RECIP_INFO_new()) == NULL) goto err;
 	if (!PKCS7_RECIP_INFO_set(ri,x509)) goto err;
 	if (!PKCS7_add_recipient_info(p7,ri)) goto err;
-	return(ri);
+	return ri;
 err:
-	return(NULL);
+	if (ri)
+		PKCS7_RECIP_INFO_free(ri);
+	return NULL;
 	}
 
 int PKCS7_add_recipient_info(PKCS7 *p7, PKCS7_RECIP_INFO *ri)
@@ -486,6 +488,8 @@ int PKCS7_add_recipient_info(PKCS7 *p7, PKCS7_RECIP_INFO *ri)
 
 int PKCS7_RECIP_INFO_set(PKCS7_RECIP_INFO *p7i, X509 *x509)
 	{
+	int ret;
+	EVP_PKEY *pkey = NULL;
 	if (!ASN1_INTEGER_set(p7i->version,0))
 		return 0;
 	if (!X509_NAME_set(&p7i->issuer_and_serial->issuer,
@@ -497,14 +501,41 @@ int PKCS7_RECIP_INFO_set(PKCS7_RECIP_INFO *p7i, X509 *x509)
 		M_ASN1_INTEGER_dup(X509_get_serialNumber(x509))))
 		return 0;
 
-	X509_ALGOR_free(p7i->key_enc_algor);
-	if (!(p7i->key_enc_algor= X509_ALGOR_dup(x509->cert_info->key->algor)))
-		return 0;
+	pkey = X509_get_pubkey(x509);
+
+	if (!pkey || !pkey->ameth || !pkey->ameth->pkey_ctrl)
+		{
+		PKCS7err(PKCS7_F_PKCS7_RECIP_INFO_SET,
+			PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
+		goto err;
+		}
+
+	ret = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_PKCS7_ENCRYPT,
+						0, p7i);
+	if (ret == -2)
+		{
+		PKCS7err(PKCS7_F_PKCS7_RECIP_INFO_SET,
+			PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
+		goto err;
+		}
+	if (ret <= 0)
+		{
+		PKCS7err(PKCS7_F_PKCS7_RECIP_INFO_SET,
+				PKCS7_R_ENCRYPTION_CTRL_FAILURE);
+		goto err;
+		}
+
+	EVP_PKEY_free(pkey);
 
 	CRYPTO_add(&x509->references,1,CRYPTO_LOCK_X509);
 	p7i->cert=x509;
 
-	return(1);
+	return 1;
+
+	err:
+	if (pkey)
+		EVP_PKEY_free(pkey);
+	return 0;
 	}
 
 X509 *PKCS7_cert_from_signer_info(PKCS7 *p7, PKCS7_SIGNER_INFO *si)

crypto/pkcs7/pkcs7.h

@@ -406,6 +406,7 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_F_PKCS7_ENCRYPT				 115
 #define PKCS7_F_PKCS7_FIND_DIGEST			 127
 #define PKCS7_F_PKCS7_GET0_SIGNERS			 124
+#define PKCS7_F_PKCS7_RECIP_INFO_SET			 130
 #define PKCS7_F_PKCS7_SET_CIPHER			 108
 #define PKCS7_F_PKCS7_SET_CONTENT			 109
 #define PKCS7_F_PKCS7_SET_DIGEST			 126
@@ -427,6 +428,8 @@ void ERR_load_PKCS7_strings(void);
 #define PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH		 100
 #define PKCS7_R_DECRYPT_ERROR				 119
 #define PKCS7_R_DIGEST_FAILURE				 101
+#define PKCS7_R_ENCRYPTION_CTRL_FAILURE			 149
+#define PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE 150
 #define PKCS7_R_ERROR_ADDING_RECIPIENT			 120
 #define PKCS7_R_ERROR_SETTING_CIPHER			 121
 #define PKCS7_R_INVALID_MIME_TYPE			 131

crypto/pkcs7/pkcs7err.c

@@ -88,6 +88,7 @@ static ERR_STRING_DATA PKCS7_str_functs[]=
 {ERR_FUNC(PKCS7_F_PKCS7_ENCRYPT),	"PKCS7_encrypt"},
 {ERR_FUNC(PKCS7_F_PKCS7_FIND_DIGEST),	"PKCS7_FIND_DIGEST"},
 {ERR_FUNC(PKCS7_F_PKCS7_GET0_SIGNERS),	"PKCS7_GET0_SIGNERS"},
+{ERR_FUNC(PKCS7_F_PKCS7_RECIP_INFO_SET),	"PKCS7_RECIP_INFO_set"},
 {ERR_FUNC(PKCS7_F_PKCS7_SET_CIPHER),	"PKCS7_set_cipher"},
 {ERR_FUNC(PKCS7_F_PKCS7_SET_CONTENT),	"PKCS7_set_content"},
 {ERR_FUNC(PKCS7_F_PKCS7_SET_DIGEST),	"PKCS7_set_digest"},
@@ -112,6 +113,8 @@ static ERR_STRING_DATA PKCS7_str_reasons[]=
 {ERR_REASON(PKCS7_R_DECRYPTED_KEY_IS_WRONG_LENGTH),"decrypted key is wrong length"},
 {ERR_REASON(PKCS7_R_DECRYPT_ERROR)       ,"decrypt error"},
 {ERR_REASON(PKCS7_R_DIGEST_FAILURE)      ,"digest failure"},
+{ERR_REASON(PKCS7_R_ENCRYPTION_CTRL_FAILURE),"encryption ctrl failure"},
+{ERR_REASON(PKCS7_R_ENCRYPTION_NOT_SUPPORTED_FOR_THIS_KEY_TYPE),"encryption not supported for this key type"},
 {ERR_REASON(PKCS7_R_ERROR_ADDING_RECIPIENT),"error adding recipient"},
 {ERR_REASON(PKCS7_R_ERROR_SETTING_CIPHER),"error setting cipher"},
 {ERR_REASON(PKCS7_R_INVALID_MIME_TYPE)   ,"invalid mime type"},

crypto/rsa/rsa_ameth.c

@@ -266,6 +266,7 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 	{
 	switch (op)
 		{
+
 		case ASN1_PKEY_CTRL_PKCS7_SIGN:
 		if (arg1 == 0)
 			{
@@ -276,6 +277,16 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 			}
 		return 1;
 
+		case ASN1_PKEY_CTRL_PKCS7_ENCRYPT:
+		if (arg1 == 0)
+			{
+			X509_ALGOR *alg;
+			PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
+			X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
+							V_ASN1_NULL, 0);
+			}
+		return 1;
+
 		default:
 		return -2;