EVP: add internal evp_keymgmt_util_get_deflt_digest_name() and use it

evp_keymgmt_util_get_deflt_digest_name() is a refactor of the provider
side key part of EVP_PKEY_get_default_digest_name(), that takes
EVP_KEYMGMT and provider keydata pointers instead of an EVP_PKEY
pointer.

We also ensure that it uses SN_undef as the default name if the
provider implementation gave us an empty string, since this is what
EVP_PKEY_get_default_digest_name() responds when getting the digest
name via a EVP_PKEY_ASN1_METHOD ctrl call that returns NID_undef.
Reviewed-by: NPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11576)

crypto/evp/keymgmt_lib.c

@@ -412,3 +412,51 @@ void *evp_keymgmt_util_gen(EVP_PKEY *target, EVP_KEYMGMT *keymgmt,
 
     return keydata;
 }
+
+/*
+ * Returns the same numbers as EVP_PKEY_get_default_digest_name()
+ * When the string from the EVP_KEYMGMT implementation is "", we use
+ * SN_undef, since that corresponds to what EVP_PKEY_get_default_nid()
+ * returns for no digest.
+ */
+int evp_keymgmt_util_get_deflt_digest_name(EVP_KEYMGMT *keymgmt,
+                                           void *keydata,
+                                           char *mdname, size_t mdname_sz)
+{
+    OSSL_PARAM params[3];
+    char mddefault[100] = "";
+    char mdmandatory[100] = "";
+    char *result = NULL;
+    int rv = -2;
+
+    params[0] =
+        OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_DEFAULT_DIGEST,
+                                         mddefault, sizeof(mddefault));
+    params[0].return_size = sizeof(mddefault) + 1;
+    params[1] =
+        OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_MANDATORY_DIGEST,
+                                         mdmandatory,
+                                         sizeof(mdmandatory));
+    params[1].return_size = sizeof(mdmandatory) + 1;
+    params[2] = OSSL_PARAM_construct_end();
+
+    if (!evp_keymgmt_get_params(keymgmt, keydata, params))
+        return 0;
+
+    if (params[1].return_size != sizeof(mdmandatory) + 1) {
+        if (params[1].return_size == 1) /* Only a NUL byte */
+            result = SN_undef;
+        else
+            result = mdmandatory;
+        rv = 2;
+    } else if (params[0].return_size != sizeof(mddefault) + 1) {
+        if (params[0].return_size == 1) /* Only a NUL byte */
+            result = SN_undef;
+        else
+            result = mddefault;
+        rv = 1;
+    }
+    if (rv > 0)
+        OPENSSL_strlcpy(mdname, result, mdname_sz);
+    return rv;
+}

crypto/evp/p_lib.c

@@ -1007,28 +1007,10 @@ int EVP_PKEY_get_default_digest_nid(EVP_PKEY *pkey, int *pnid)
 int EVP_PKEY_get_default_digest_name(EVP_PKEY *pkey,
                                      char *mdname, size_t mdname_sz)
 {
-    if (pkey->ameth == NULL) {
-        OSSL_PARAM params[3];
-        char mddefault[100] = "";
-        char mdmandatory[100] = "";
-
-        params[0] =
-            OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_DEFAULT_DIGEST,
-                                             mddefault, sizeof(mddefault));
-        params[1] =
-            OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_MANDATORY_DIGEST,
-                                             mdmandatory,
-                                             sizeof(mdmandatory));
-        params[2] = OSSL_PARAM_construct_end();
-        if (!evp_keymgmt_get_params(pkey->keymgmt, pkey->keydata, params))
-            return 0;
-        if (mdmandatory[0] != '\0') {
-            OPENSSL_strlcpy(mdname, mdmandatory, mdname_sz);
-            return 2;
-        }
-        OPENSSL_strlcpy(mdname, mddefault, mdname_sz);
-        return 1;
-    }
+    if (pkey->ameth == NULL)
+        return evp_keymgmt_util_get_deflt_digest_name(pkey->keymgmt,
+                                                      pkey->keydata,
+                                                      mdname, mdname_sz);
 
     {
         int nid = NID_undef;

doc/man3/EVP_PKEY_get_default_digest_nid.pod

@@ -18,7 +18,8 @@ EVP_PKEY_get_default_digest_nid, EVP_PKEY_get_default_digest_name
 EVP_PKEY_get_default_digest_name() fills in the default message digest
 name for the public key signature operations associated with key
 I<pkey> into I<mdname>, up to at most I<mdname_sz> bytes including the
-ending NUL byte.
+ending NUL byte.  The name could be C<"UNDEF">, signifying that no digest
+should be used.
 
 EVP_PKEY_get_default_digest_nid() sets I<pnid> to the default message
 digest NID for the public key signature operations associated with key

include/crypto/evp.h

@@ -655,7 +655,9 @@ int evp_keymgmt_util_match(EVP_PKEY *pk1, EVP_PKEY *pk2, int selection);
 int evp_keymgmt_util_copy(EVP_PKEY *to, EVP_PKEY *from, int selection);
 void *evp_keymgmt_util_gen(EVP_PKEY *target, EVP_KEYMGMT *keymgmt,
                            void *genctx, OSSL_CALLBACK *cb, void *cbarg);
-
+int evp_keymgmt_util_get_deflt_digest_name(EVP_KEYMGMT *keymgmt,
+                                           void *keydata,
+                                           char *mdname, size_t mdname_sz);
 
 /*
  * KEYMGMT provider interface functions