Remove CMS recipient info information out of the algorithm implementations

Low level algorithm implementations have no business knowing about details
of the higher level CMS concept. This knowledge is therefore moved into the
CMS layer.
Reviewed-by: NShane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13088)

crypto/cms/cms_env.c

@@ -1303,6 +1303,20 @@ err:
  */
 int cms_pkey_get_ri_type(EVP_PKEY *pk)
 {
+    /* Check types that we know about */
+    if (EVP_PKEY_is_a(pk, "DH"))
+        return CMS_RECIPINFO_AGREE;
+    else if (EVP_PKEY_is_a(pk, "DSA"))
+        return CMS_RECIPINFO_NONE;
+    else if (EVP_PKEY_is_a(pk, "EC"))
+        return CMS_RECIPINFO_AGREE;
+    else if (EVP_PKEY_is_a(pk, "RSA"))
+        return CMS_RECIPINFO_TRANS;
+
+    /*
+     * Otherwise this might ben an engine implementation, so see if we can get
+     * the type from the ameth.
+     */
     if (pk->ameth && pk->ameth->pkey_ctrl) {
         int i, r;
         i = pk->ameth->pkey_ctrl(pk, ASN1_PKEY_CTRL_CMS_RI_TYPE, 0, &r);

crypto/dh/dh_ameth.c

@@ -19,7 +19,6 @@
 #include <openssl/bn.h>
 #include <openssl/core_names.h>
 #include <openssl/param_build.h>
-#include <openssl/cms.h>
 #include "internal/ffc.h"
 #include "internal/cryptlib.h"
 #include "crypto/asn1.h"
@@ -449,11 +448,6 @@ static int dh_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 static int dhx_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 {
     switch (op) {
-#ifndef OPENSSL_NO_CMS
-    case ASN1_PKEY_CTRL_CMS_RI_TYPE:
-        *(int *)arg2 = CMS_RECIPINFO_AGREE;
-        return 1;
-#endif
     default:
         return -2;
     }

crypto/dsa/dsa_ameth.c

@@ -17,7 +17,6 @@
 #include <openssl/x509.h>
 #include <openssl/asn1.h>
 #include <openssl/bn.h>
-#include <openssl/cms.h>
 #include <openssl/core_names.h>
 #include <openssl/param_build.h>
 #include "internal/cryptlib.h"
@@ -481,11 +480,6 @@ static int dsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
             X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0);
         }
         return 1;
-#ifndef OPENSSL_NO_CMS
-    case ASN1_PKEY_CTRL_CMS_RI_TYPE:
-        *(int *)arg2 = CMS_RECIPINFO_NONE;
-        return 1;
-#endif
 
     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
         *(int *)arg2 = NID_sha256;

crypto/ec/ec_ameth.c

@@ -18,7 +18,6 @@
 #include <openssl/x509.h>
 #include <openssl/ec.h>
 #include <openssl/bn.h>
-#include <openssl/cms.h>
 #include <openssl/asn1t.h>
 #include "crypto/asn1.h"
 #include "crypto/evp.h"
@@ -489,11 +488,6 @@ static int ec_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
             X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0);
         }
         return 1;
-#ifndef OPENSSL_NO_CMS
-    case ASN1_PKEY_CTRL_CMS_RI_TYPE:
-        *(int *)arg2 = CMS_RECIPINFO_AGREE;
-        return 1;
-#endif
 
     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
         if (EVP_PKEY_id(pkey) == EVP_PKEY_SM2) {

crypto/rsa/rsa_ameth.c

@@ -18,7 +18,6 @@
 #include <openssl/asn1t.h>
 #include <openssl/x509.h>
 #include <openssl/bn.h>
-#include <openssl/cms.h>
 #include <openssl/core_names.h>
 #include <openssl/param_build.h>
 #include "crypto/asn1.h"
@@ -505,13 +504,6 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         if (arg1 == 0)
             PKCS7_RECIP_INFO_get0_alg(arg2, &alg);
         break;
-#ifndef OPENSSL_NO_CMS
-    case ASN1_PKEY_CTRL_CMS_RI_TYPE:
-        if (pkey_is_pss(pkey))
-            return -2;
-        *(int *)arg2 = CMS_RECIPINFO_TRANS;
-        return 1;
-#endif
 
     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
         if (pkey->pkey.rsa->pss != NULL) {