Prevent an overflow if an application supplies a buffer that is too small

If an application bug means that a buffer smaller than is necessary is
passed to various functions then OpenSSL does not spot that the buffer
is too small and fills it anyway. This PR prevents that.

Since it requires an application bug to hit this problem, no CVE is
allocated.

Thanks to David Benjamin for reporting this issue.
Reviewed-by: NTomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16789)

crypto/evp/m_sigver.c

@@ -411,14 +411,14 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, unsigned char *sigret,
     if (sigret == NULL || (ctx->flags & EVP_MD_CTX_FLAG_FINALISE) != 0)
         return pctx->op.sig.signature->digest_sign_final(pctx->op.sig.algctx,
                                                          sigret, siglen,
-                                                         SIZE_MAX);
+                                                         (sigret == NULL) ? 0 : *siglen);
     dctx = EVP_PKEY_CTX_dup(pctx);
     if (dctx == NULL)
         return 0;
 
     r = dctx->op.sig.signature->digest_sign_final(dctx->op.sig.algctx,
                                                   sigret, siglen,
-                                                  SIZE_MAX);
+                                                  (sigret == NULL) ? 0 : *siglen);
     EVP_PKEY_CTX_free(dctx);
     return r;
 
@@ -506,7 +506,8 @@ int EVP_DigestSign(EVP_MD_CTX *ctx, unsigned char *sigret, size_t *siglen,
             && pctx->op.sig.signature != NULL) {
         if (pctx->op.sig.signature->digest_sign != NULL)
             return pctx->op.sig.signature->digest_sign(pctx->op.sig.algctx,
-                                                       sigret, siglen, SIZE_MAX,
+                                                       sigret, siglen,
+                                                       sigret == NULL ? 0 : *siglen,
                                                        tbs, tbslen);
     } else {
         /* legacy */

crypto/evp/p_lib.c

@@ -529,12 +529,14 @@ static int get_raw_key_details(const OSSL_PARAM params[], void *arg)
         if ((p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_PRIV_KEY))
                 != NULL)
             return OSSL_PARAM_get_octet_string(p, (void **)raw_key->key,
-                                               SIZE_MAX, raw_key->len);
+                                               raw_key->key == NULL ? 0 : *raw_key->len,
+                                               raw_key->len);
     } else if (raw_key->selection == OSSL_KEYMGMT_SELECT_PUBLIC_KEY) {
         if ((p = OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_PUB_KEY))
                 != NULL)
             return OSSL_PARAM_get_octet_string(p, (void **)raw_key->key,
-                                               SIZE_MAX, raw_key->len);
+                                               raw_key->key == NULL ? 0 : *raw_key->len,
+                                               raw_key->len);
     }
 
     return 0;

crypto/evp/signature.c

@@ -582,7 +582,7 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx,
         goto legacy;
 
     ret = ctx->op.sig.signature->sign(ctx->op.sig.algctx, sig, siglen,
-                                      SIZE_MAX, tbs, tbslen);
+                                      (sig == NULL) ? 0 : *siglen, tbs, tbslen);
 
     return ret;
  legacy: