Return errors PKCS#7/CMS enveloped data ctrls and PSS

Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

crypto/rsa/rsa_pmeth.c

@@ -526,14 +526,18 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         return rctx->oaep_labellen;
 
     case EVP_PKEY_CTRL_DIGESTINIT:
-    case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
-    case EVP_PKEY_CTRL_PKCS7_DECRYPT:
     case EVP_PKEY_CTRL_PKCS7_SIGN:
+#ifndef OPENSSL_NO_CMS
+    case EVP_PKEY_CTRL_CMS_SIGN:
+#endif
     return 1;
+
+    case EVP_PKEY_CTRL_PKCS7_ENCRYPT:
+    case EVP_PKEY_CTRL_PKCS7_DECRYPT:
 #ifndef OPENSSL_NO_CMS
     case EVP_PKEY_CTRL_CMS_DECRYPT:
     case EVP_PKEY_CTRL_CMS_ENCRYPT:
-    case EVP_PKEY_CTRL_CMS_SIGN:
+    if (!pkey_ctx_is_pss(ctx))
         return 1;
 #endif
     case EVP_PKEY_CTRL_PEER_KEY: