RSA: Be less strict on PSS parameters when exporting to provider

We have a key in test/recipes/30-test_evp_data/evppkey.txt with bad
PSS parameters (RSA-PSS-BAD), which is supposed to trigger signature
computation faults.  However, if this key needs to be exported to the
RSA provider implementation, the result would be an earlier error,
giving the computation that's supposed to be checked n chance to even
be reached.

Either way, the legacy to provider export is no place to validate the
values of the key.

We also ensure that the provider implementation can handle and detect
signed (negative) saltlen values.
Reviewed-by: NPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12583)

crypto/rsa/rsa_ameth.c

@@ -1218,10 +1218,11 @@ static int rsa_int_export_to(const EVP_PKEY *from, int rsa_type,
 
     if (rsa->pss != NULL) {
         const EVP_MD *md = NULL, *mgf1md = NULL;
-        int md_nid, mgf1md_nid, saltlen;
+        int md_nid, mgf1md_nid, saltlen, trailerfield;
         RSA_PSS_PARAMS_30 pss_params;
 
-        if (!rsa_pss_get_param(rsa->pss, &md, &mgf1md, &saltlen))
+        if (!rsa_pss_get_param_unverified(rsa->pss, &md, &mgf1md,
+                                          &saltlen, &trailerfield))
             goto err;
         md_nid = EVP_MD_type(md);
         mgf1md_nid = EVP_MD_type(mgf1md);

include/crypto/rsa.h

@@ -19,8 +19,8 @@ typedef struct rsa_pss_params_30_st {
         int algorithm_nid;       /* Currently always NID_mgf1 */
         int hash_algorithm_nid;
     } mask_gen;
-    unsigned int salt_len;
-    unsigned int trailer_field;
+    int salt_len;
+    int trailer_field;
 } RSA_PSS_PARAMS_30;
 
 RSA_PSS_PARAMS_30 *rsa_get0_pss_params_30(RSA *r);

providers/implementations/signature/rsa.c

@@ -176,16 +176,16 @@ static int rsa_check_padding(int mdnid, int padding)
     return 1;
 }
 
-static int rsa_check_parameters(EVP_MD *md, PROV_RSA_CTX *prsactx)
+static int rsa_check_parameters(PROV_RSA_CTX *prsactx)
 {
     if (prsactx->pad_mode == RSA_PKCS1_PSS_PADDING) {
         int max_saltlen;
 
         /* See if minimum salt length exceeds maximum possible */
-        max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(md);
+        max_saltlen = RSA_size(prsactx->rsa) - EVP_MD_size(prsactx->md);
         if ((RSA_bits(prsactx->rsa) & 0x7) == 1)
             max_saltlen--;
-        if (prsactx->min_saltlen > max_saltlen) {
+        if (prsactx->min_saltlen < 0 || prsactx->min_saltlen > max_saltlen) {
             ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_SALT_LENGTH);
             return 0;
         }
@@ -230,7 +230,6 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
         if (md == NULL
             || md_nid == NID_undef
             || !rsa_check_padding(md_nid, ctx->pad_mode)
-            || !rsa_check_parameters(md, ctx)
             || mdname_len >= sizeof(ctx->mdname)) {
             if (md == NULL)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
@@ -365,7 +364,8 @@ static int rsa_signature_init(void *vprsactx, void *vrsa, int operation)
                 prsactx->saltlen = min_saltlen;
 
                 return rsa_setup_md(prsactx, mdname, prsactx->propq)
-                    && rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq);
+                    && rsa_setup_mgf1_md(prsactx, mgf1mdname, prsactx->propq)
+                    && rsa_check_parameters(prsactx);
             }
         }
 
@@ -1151,7 +1151,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
         }
 
         if (rsa_pss_restricted(prsactx)) {
-            switch (prsactx->saltlen) {
+            switch (saltlen) {
             case RSA_PSS_SALTLEN_AUTO:
                 if (prsactx->operation == EVP_PKEY_OP_VERIFY) {
                     ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_PSS_SALTLEN);
@@ -1168,7 +1168,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
                                    EVP_MD_size(prsactx->md));
                     return 0;
                 }
-                /* FALLTHRU */
+                break;
             default:
                 if (saltlen >= 0 && saltlen < prsactx->min_saltlen) {
                     ERR_raise_data(ERR_LIB_PROV,