enable CMS sign/verify for provider-implemented PKEYs

We need to handle signatures with and without digest algs
and we generalize the ossl_cms_ecdsa_dsa_sign() function
to other algorithms that are handled in the same way.
Reviewed-by: NMatt Caswell <matt@openssl.org>
Reviewed-by: NTomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17733)

crypto/cms/cms_ec.c

@@ -388,26 +388,3 @@ int ossl_cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt)
     ERR_raise(ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
     return 0;
 }
-
-/* ECDSA and DSA implementation is the same */
-int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify)
-{
-    assert(verify == 0 || verify == 1);
-
-    if (!verify) {
-        int snid, hnid;
-        X509_ALGOR *alg1, *alg2;
-        EVP_PKEY *pkey = si->pkey;
-
-        CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2);
-        if (alg1 == NULL || alg1->algorithm == NULL)
-            return -1;
-        hnid = OBJ_obj2nid(alg1->algorithm);
-        if (hnid == NID_undef)
-            return -1;
-        if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_get_id(pkey)))
-            return -1;
-        return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL);
-    }
-    return 1;
-}

crypto/cms/cms_local.h

@@ -479,7 +479,6 @@ int ossl_cms_check_signing_certs(const CMS_SignerInfo *si,
 int ossl_cms_dh_envelope(CMS_RecipientInfo *ri, int decrypt);
 int ossl_cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt);
 int ossl_cms_rsa_envelope(CMS_RecipientInfo *ri, int decrypt);
-int ossl_cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify);
 int ossl_cms_rsa_sign(CMS_SignerInfo *si, int verify);
 
 DECLARE_ASN1_ITEM(CMS_CertificateChoices)

crypto/cms/cms_sd.c

@@ -227,19 +227,50 @@ int ossl_cms_SignerIdentifier_cert_cmp(CMS_SignerIdentifier *sid, X509 *cert)
         return -1;
 }
 
+/* Method to map any, incl. provider-implemented PKEY types to OIDs */
+/* ECDSA and DSA and all provider-delivered signatures implementation is the same */
+static int cms_generic_sign(CMS_SignerInfo *si, int verify)
+{
+    if (!ossl_assert(verify == 0 || verify == 1))
+        return -1;
+
+    if (!verify) {
+        int snid, hnid, pknid;
+        X509_ALGOR *alg1, *alg2;
+        EVP_PKEY *pkey = si->pkey;
+        pknid = EVP_PKEY_get_id(pkey);
+
+        CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2);
+        if (alg1 == NULL || alg1->algorithm == NULL)
+            return -1;
+        hnid = OBJ_obj2nid(alg1->algorithm);
+        if (hnid == NID_undef)
+            return -1;
+        if (pknid <= 0) { /* check whether a provider registered a NID */
+            const char *typename = EVP_PKEY_get0_type_name(pkey);
+            if (typename != NULL)
+                pknid = OBJ_txt2nid(typename);
+        }
+        if (!OBJ_find_sigid_by_algs(&snid, hnid, pknid))
+            return -1;
+        return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL);
+    }
+    return 1;
+}
+
 static int cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd)
 {
     EVP_PKEY *pkey = si->pkey;
     int i;
 
     if (EVP_PKEY_is_a(pkey, "DSA") || EVP_PKEY_is_a(pkey, "EC"))
-        return ossl_cms_ecdsa_dsa_sign(si, cmd);
+        return cms_generic_sign(si, cmd);
     else if (EVP_PKEY_is_a(pkey, "RSA") || EVP_PKEY_is_a(pkey, "RSA-PSS"))
         return ossl_cms_rsa_sign(si, cmd);
 
-    /* Something else? We'll give engines etc a chance to handle this */
+    /* Now give engines, providers, etc a chance to handle this */
     if (pkey->ameth == NULL || pkey->ameth->pkey_ctrl == NULL)
-        return 1;
+        return cms_generic_sign(si, cmd);
     i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_SIGN, cmd, si);
     if (i == -2) {
         ERR_raise(ERR_LIB_CMS, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);

crypto/objects/obj_xref.c

@@ -36,7 +36,14 @@ static int sigx_cmp(const nid_triple *const *a, const nid_triple *const *b)
     int ret;
 
     ret = (*a)->hash_id - (*b)->hash_id;
-    if (ret != 0)
+    /* The "b" side of the comparison carries the algorithms already
+     * registered. A NID_undef for 'hash_id' there means that the
+     * signature algorithm doesn't need a digest to operate OK. In
+     * such case, any hash_id/digest algorithm on the test side (a),
+     * incl. NID_undef, is acceptable. signature algorithm NID
+     * (pkey_id) must match in any case.
+     */
+    if ((ret != 0) && ((*b)->hash_id != NID_undef))
         return ret;
     return (*a)->pkey_id - (*b)->pkey_id;
 }