Support RSA operations in PSS.

Add support for common operations in PSS by adding a new function
RSA_pkey_ctx_ctrl() which calls EVP_PKEY_CTX_ctrl if the key type
is RSA or PSS.
Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

crypto/rsa/rsa_lib.c

@@ -13,6 +13,8 @@
 #include <openssl/lhash.h>
 #include "internal/bn_int.h"
 #include <openssl/engine.h>
+#include <openssl/evp.h>
+#include "internal/evp_int.h"
 #include "rsa_locl.h"
 
 static const RSA_METHOD *default_RSA_meth = NULL;
@@ -309,3 +311,13 @@ ENGINE *RSA_get0_engine(const RSA *r)
 {
     return r->engine;
 }
+
+int RSA_pkey_ctx_ctrl(EVP_PKEY_CTX *ctx, int optype, int cmd, int p1, void *p2)
+{
+    /* If key type not RSA or RSA-PSS return error */
+    if (ctx != NULL && ctx->pmeth != NULL
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+        return -1;
+     return EVP_PKEY_CTX_ctrl(ctx, -1, optype, cmd, p1, p2);
+}

include/openssl/rsa.h

@@ -94,38 +94,32 @@ extern "C" {
                                 EVP_PKEY_CTRL_GET_RSA_PADDING, 0, ppad)
 
 # define EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, len) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \
-                                (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
-                                EVP_PKEY_CTRL_RSA_PSS_SALTLEN, \
-                                len, NULL)
+        RSA_pkey_ctx_ctrl(ctx, (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
+                          EVP_PKEY_CTRL_RSA_PSS_SALTLEN, len, NULL)
 
 # define EVP_PKEY_CTX_get_rsa_pss_saltlen(ctx, plen) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \
-                                (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
-                                EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN, \
-                                0, plen)
+        RSA_pkey_ctx_ctrl(ctx, (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
+                          EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN, 0, plen)
 
 # define EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_KEYGEN, \
-                                EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
+        RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_KEYGEN, \
+                          EVP_PKEY_CTRL_RSA_KEYGEN_BITS, bits, NULL)
 
 # define EVP_PKEY_CTX_set_rsa_keygen_pubexp(ctx, pubexp) \
-        EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_KEYGEN, \
-                                EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP, 0, pubexp)
+        RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_KEYGEN, \
+                          EVP_PKEY_CTRL_RSA_KEYGEN_PUBEXP, 0, pubexp)
 
 # define  EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md)  \
-                EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \
-                        EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \
-                                EVP_PKEY_CTRL_RSA_MGF1_MD, 0, (void *)md)
+        RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \
+                          EVP_PKEY_CTRL_RSA_MGF1_MD, 0, (void *)md)
 
 # define  EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md)  \
                 EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT,  \
                                 EVP_PKEY_CTRL_RSA_OAEP_MD, 0, (void *)md)
 
 # define  EVP_PKEY_CTX_get_rsa_mgf1_md(ctx, pmd) \
-                EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, \
-                        EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \
-                                EVP_PKEY_CTRL_GET_RSA_MGF1_MD, 0, (void *)pmd)
+        RSA_pkey_ctx_ctrl(ctx, EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT, \
+                          EVP_PKEY_CTRL_GET_RSA_MGF1_MD, 0, (void *)pmd)
 
 # define  EVP_PKEY_CTX_get_rsa_oaep_md(ctx, pmd) \
                 EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA, EVP_PKEY_OP_TYPE_CRYPT,  \
@@ -231,6 +225,8 @@ const RSA_METHOD *RSA_PKCS1_OpenSSL(void);
 
 const RSA_METHOD *RSA_null_method(void);
 
+int RSA_pkey_ctx_ctrl(EVP_PKEY_CTX *ctx, int optype, int cmd, int p1, void *p2);
+
 DECLARE_ASN1_ENCODE_FUNCTIONS_const(RSA, RSAPublicKey)
 DECLARE_ASN1_ENCODE_FUNCTIONS_const(RSA, RSAPrivateKey)