Only allow PSS padding for PSS keys.

Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

crypto/rsa/rsa_ameth.c

@@ -673,10 +673,15 @@ static int rsa_cms_verify(CMS_SignerInfo *si)
     EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
     CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
     nid = OBJ_obj2nid(alg->algorithm);
-    if (nid == NID_rsaEncryption)
-        return 1;
     if (nid == EVP_PKEY_RSA_PSS)
         return rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
+    /* Only PSS allowed for PSS keys */
+    if (pkey_ctx_is_pss(pkctx)) {
+        RSAerr(RSA_F_RSA_CMS_VERIFY, RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+        return 0;
+    }
+    if (nid == NID_rsaEncryption)
+        return 1;
     /* Workaround for some implementation that use a signature OID */
     if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
         if (nid2 == NID_rsaEncryption)

crypto/rsa/rsa_err.c

@@ -33,6 +33,7 @@ static ERR_STRING_DATA RSA_str_functs[] = {
     {ERR_FUNC(RSA_F_RSA_CHECK_KEY), "RSA_check_key"},
     {ERR_FUNC(RSA_F_RSA_CHECK_KEY_EX), "RSA_check_key_ex"},
     {ERR_FUNC(RSA_F_RSA_CMS_DECRYPT), "rsa_cms_decrypt"},
+    {ERR_FUNC(RSA_F_RSA_CMS_VERIFY), "rsa_cms_verify"},
     {ERR_FUNC(RSA_F_RSA_ITEM_VERIFY), "rsa_item_verify"},
     {ERR_FUNC(RSA_F_RSA_METH_DUP), "RSA_meth_dup"},
     {ERR_FUNC(RSA_F_RSA_METH_NEW), "RSA_meth_new"},

include/openssl/rsa.h

@@ -490,6 +490,7 @@ int ERR_load_RSA_strings(void);
 # define RSA_F_RSA_CHECK_KEY                              123
 # define RSA_F_RSA_CHECK_KEY_EX                           160
 # define RSA_F_RSA_CMS_DECRYPT                            159
+# define RSA_F_RSA_CMS_VERIFY                             158
 # define RSA_F_RSA_ITEM_VERIFY                            148
 # define RSA_F_RSA_METH_DUP                               161
 # define RSA_F_RSA_METH_NEW                               162