Use method key type instead of EVP_PKEY_RSA

Make RSA method more flexible by using the key type from the
method instead of hard coding EVP_PKEY_RSA: by doing this the
same code supports both RSA and RSA-PSS.
Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

crypto/rsa/rsa_ameth.c

@@ -51,7 +51,7 @@ static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey)
         RSAerr(RSA_F_RSA_PUB_DECODE, ERR_R_RSA_LIB);
         return 0;
     }
-    EVP_PKEY_assign_RSA(pkey, rsa);
+    EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
     return 1;
 }
 
@@ -72,7 +72,7 @@ static int old_rsa_priv_decode(EVP_PKEY *pkey,
         RSAerr(RSA_F_OLD_RSA_PRIV_DECODE, ERR_R_RSA_LIB);
         return 0;
     }
-    EVP_PKEY_assign_RSA(pkey, rsa);
+    EVP_PKEY_assign(pkey, pkey->ameth->pkey_id, rsa);
     return 1;
 }
 
@@ -92,7 +92,7 @@ static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
         return 0;
     }
 
-    if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(NID_rsaEncryption), 0,
+    if (!PKCS8_pkey_set0(p8, OBJ_nid2obj(pkey->ameth->pkey_id), 0,
                          V_ASN1_NULL, NULL, rk, rklen)) {
         RSAerr(RSA_F_RSA_PRIV_ENCODE, ERR_R_MALLOC_FAILURE);
         return 0;

crypto/rsa/rsa_pmeth.c

@@ -631,7 +631,7 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey)
     ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, pcb);
     BN_GENCB_free(pcb);
     if (ret > 0)
-        EVP_PKEY_assign_RSA(pkey, rsa);
+        EVP_PKEY_assign(pkey, ctx->pmeth->pkey_id, rsa);
     else
         RSA_free(rsa);
     return ret;