PROV & ASYM_CIPHER: Adapt the RSA asymmetric cipher code for PSS-parameters

Reviewed-by: NShane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

providers/implementations/asymciphers/rsa_enc.c

@@ -95,7 +95,16 @@ static int rsa_init(void *vprsactx, void *vrsa)
         return 0;
     RSA_free(prsactx->rsa);
     prsactx->rsa = vrsa;
-    prsactx->pad_mode = RSA_PKCS1_PADDING;
+
+    switch (RSA_test_flags(prsactx->rsa, RSA_FLAG_TYPE_MASK)) {
+    case RSA_FLAG_TYPE_RSA:
+        prsactx->pad_mode = RSA_PKCS1_PADDING;
+        break;
+    default:
+        ERR_raise(ERR_LIB_PROV, PROV_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+        return 0;
+    }
+
     return 1;
 }