Support calling EVP_DigestUpdate instead of EVP_Digest[Sign|Verify]Update

Prior to OpenSSL 3.0 EVP_Digest[Sign|Verify|Update were just macros for
EVP_DigestUpdate. They are now separate functions. Unfortunately some
code assumes that EVP_Digest[Sign|Verify]Update is interchangeable with
EVP_DigestUpdate. For example the dgst app uses an MD bio which always
calls EVP_DigestUpdate(). However the dgst app supports signing instead
of digesting and may initialise with EVP_DigestSignInit_ex() instead of
just EVP_DigestInit().

We now detect these differences and redirect to the correct function
where appropriate.

Fixes #10114
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10116)

crypto/evp/build.info

 LIBS=../../libcrypto
 $COMMON=digest.c evp_enc.c evp_lib.c evp_fetch.c cmeth_lib.c evp_utils.c \
-        mac_lib.c mac_meth.c keymgmt_meth.c keymgmt_lib.c kdf_lib.c kdf_meth.c
+        mac_lib.c mac_meth.c keymgmt_meth.c keymgmt_lib.c kdf_lib.c kdf_meth.c \
+        m_sigver.c
 SOURCE[../../libcrypto]=$COMMON\
         encode.c evp_key.c evp_cnf.c \
         e_des.c e_bf.c e_idea.c e_des3.c e_camellia.c\
@@ -13,7 +14,7 @@ SOURCE[../../libcrypto]=$COMMON\
         c_allc.c c_alld.c bio_ok.c \
         evp_pkey.c evp_pbe.c p5_crpt.c p5_crpt2.c pbe_scrypt.c \
         pkey_kdf.c \
-        e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c m_sigver.c \
+        e_old.c pmeth_lib.c pmeth_fn.c pmeth_gn.c \
         e_aes_cbc_hmac_sha1.c e_aes_cbc_hmac_sha256.c e_rc4_hmac_md5.c \
         e_chacha20_poly1305.c \
         pkey_mac.c exchange.c \

crypto/evp/digest.c

@@ -285,6 +285,24 @@ int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data, size_t count)
     if (count == 0)
         return 1;
 
+    if (ctx->pctx != NULL
+            && EVP_PKEY_CTX_IS_SIGNATURE_OP(ctx->pctx)
+            && ctx->pctx->op.sig.sigprovctx != NULL) {
+        /*
+         * Prior to OpenSSL 3.0 EVP_DigestSignUpdate() and
+         * EVP_DigestVerifyUpdate() were just macros for EVP_DigestUpdate().
+         * Some code calls EVP_DigestUpdate() directly even when initialised
+         * with EVP_DigestSignInit_ex() or EVP_DigestVerifyInit_ex(), so we
+         * detect that and redirect to the correct EVP_Digest*Update() function
+         */
+        if (ctx->pctx->operation == EVP_PKEY_OP_SIGNCTX)
+            return EVP_DigestSignUpdate(ctx, data, count);
+        if (ctx->pctx->operation == EVP_PKEY_OP_VERIFYCTX)
+            return EVP_DigestVerifyUpdate(ctx, data, count);
+        EVPerr(EVP_F_EVP_DIGESTUPDATE, EVP_R_UPDATE_ERROR);
+        return 0;
+    }
+
     if (ctx->digest == NULL || ctx->digest->prov == NULL)
         goto legacy;
 

crypto/evp/m_sigver.c

@@ -16,6 +16,8 @@
 #include "internal/provider.h"
 #include "evp_local.h"
 
+#ifndef FIPS_MODE
+
 static int update(EVP_MD_CTX *ctx, const void *data, size_t datalen)
 {
     EVPerr(EVP_F_UPDATE, EVP_R_ONLY_ONESHOT_SUPPORTED);
@@ -220,6 +222,7 @@ int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
 {
     return do_sigver_init(ctx, pctx, type, NULL, NULL, e, pkey, NULL, 1);
 }
+#endif /* FIPS_MDOE */
 
 int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
 {
@@ -255,7 +258,7 @@ int EVP_DigestVerifyUpdate(EVP_MD_CTX *ctx, const void *data, size_t dsize)
     return EVP_DigestUpdate(ctx, data, dsize);
 }
 
-
+#ifndef FIPS_MODE
 int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
                         size_t *siglen)
 {
@@ -397,3 +400,4 @@ int EVP_DigestVerify(EVP_MD_CTX *ctx, const unsigned char *sigret,
         return -1;
     return EVP_DigestVerifyFinal(ctx, sigret, siglen);
 }
+#endif /* FIPS_MODE */