digests: add FIPS error state handling

Check for providering being runnable in init, final, newctx and dupctx.
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12801)

providers/implementations/digests/sha3_prov.c

@@ -47,6 +47,8 @@ static sha3_final_fn generic_sha3_final;
 
 static int keccak_init(void *vctx)
 {
+    if (!ossl_prov_is_running())
+        return 0;
     /* The newctx() handles most of the ctx fixed setup. */
     sha3_reset((KECCAK1600_CTX *)vctx);
     return 1;
@@ -95,6 +97,8 @@ static int keccak_final(void *vctx, unsigned char *out, size_t *outl,
     int ret = 1;
     KECCAK1600_CTX *ctx = vctx;
 
+    if (!ossl_prov_is_running())
+        return 0;
     if (outsz > 0)
         ret = ctx->meth.final(out, ctx);
 
@@ -145,6 +149,8 @@ static int s390x_sha3_final(unsigned char *md, void *vctx)
 {
     KECCAK1600_CTX *ctx = vctx;
 
+    if (!ossl_prov_is_running())
+        return 0;
     s390x_klmd(ctx->buf, ctx->bufsz, NULL, 0, ctx->pad, ctx->A);
     memcpy(md, ctx->A, ctx->md_size);
     return 1;
@@ -154,6 +160,8 @@ static int s390x_shake_final(unsigned char *md, void *vctx)
 {
     KECCAK1600_CTX *ctx = vctx;
 
+    if (!ossl_prov_is_running())
+        return 0;
     s390x_klmd(ctx->buf, ctx->bufsz, md, ctx->md_size, ctx->pad, ctx->A);
     return 1;
 }
@@ -185,7 +193,8 @@ static PROV_SHA3_METHOD shake_s390x_md =
 static OSSL_FUNC_digest_newctx_fn name##_newctx;                               \
 static void *name##_newctx(void *provctx)                                      \
 {                                                                              \
-    KECCAK1600_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));                        \
+    KECCAK1600_CTX *ctx = ossl_prov_is_running() ? OPENSSL_zalloc(sizeof(*ctx)) \
+                                                : NULL;                        \
                                                                                \
     if (ctx == NULL)                                                           \
         return NULL;                                                           \
@@ -198,7 +207,8 @@ static void *name##_newctx(void *provctx)                                      \
 static OSSL_FUNC_digest_newctx_fn uname##_newctx;                              \
 static void *uname##_newctx(void *provctx)                                     \
 {                                                                              \
-    KECCAK1600_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));                        \
+    KECCAK1600_CTX *ctx = ossl_prov_is_running() ? OPENSSL_zalloc(sizeof(*ctx)) \
+                                                : NULL;                        \
                                                                                \
     if (ctx == NULL)                                                           \
         return NULL;                                                           \
@@ -239,7 +249,8 @@ static void keccak_freectx(void *vctx)
 static void *keccak_dupctx(void *ctx)
 {
     KECCAK1600_CTX *in = (KECCAK1600_CTX *)ctx;
-    KECCAK1600_CTX *ret = OPENSSL_malloc(sizeof(*ret));
+    KECCAK1600_CTX *ret = ossl_prov_is_running() ? OPENSSL_malloc(sizeof(*ret))
+                                                : NULL;
 
     if (ret != NULL)
         *ret = *in;

providers/implementations/include/prov/digestcommon.h

@@ -13,6 +13,7 @@
 # include <openssl/core_dispatch.h>
 # include <openssl/core_names.h>
 # include <openssl/params.h>
+# include "prov/providercommon.h"
 
 # ifdef __cplusplus
 extern "C" {
@@ -37,7 +38,7 @@ static OSSL_FUNC_digest_freectx_fn name##_freectx;
 static OSSL_FUNC_digest_dupctx_fn name##_dupctx;                                 \
 static void *name##_newctx(void *prov_ctx)                                     \
 {                                                                              \
-    CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));                                   \
+    CTX *ctx = ossl_prov_is_running() ? OPENSSL_zalloc(sizeof(*ctx)) : NULL;    \
     return ctx;                                                                \
 }                                                                              \
 static void name##_freectx(void *vctx)                                         \
@@ -48,16 +49,21 @@ static void name##_freectx(void *vctx)                                         \
 static void *name##_dupctx(void *ctx)                                          \
 {                                                                              \
     CTX *in = (CTX *)ctx;                                                      \
-    CTX *ret = OPENSSL_malloc(sizeof(*ret));                                   \
+    CTX *ret = ossl_prov_is_running() ? OPENSSL_malloc(sizeof(*ret)) : NULL;    \
     if (ret != NULL)                                                           \
         *ret = *in;                                                            \
     return ret;                                                                \
 }                                                                              \
-static OSSL_FUNC_digest_final_fn name##_internal_final;                          \
+static OSSL_FUNC_digest_init_fn name##_internal_init;                          \
+static int name##_internal_init(void *ctx)                                     \
+{                                                                              \
+    return ossl_prov_is_running() ? init(ctx) : 0;                              \
+}                                                                              \
+static OSSL_FUNC_digest_final_fn name##_internal_final;                        \
 static int name##_internal_final(void *ctx, unsigned char *out, size_t *outl,  \
                                  size_t outsz)                                 \
 {                                                                              \
-    if (outsz >= dgstsize && fin(out, ctx)) {                                  \
+    if (ossl_prov_is_running() && outsz >= dgstsize && fin(out, ctx)) {         \
         *outl = dgstsize;                                                      \
         return 1;                                                              \
     }                                                                          \
@@ -66,7 +72,7 @@ static int name##_internal_final(void *ctx, unsigned char *out, size_t *outl,  \
 PROV_FUNC_DIGEST_GET_PARAM(name, blksize, dgstsize, flags)                     \
 const OSSL_DISPATCH name##_functions[] = {                                     \
     { OSSL_FUNC_DIGEST_NEWCTX, (void (*)(void))name##_newctx },                \
-    { OSSL_FUNC_DIGEST_INIT, (void (*)(void))init },                           \
+    { OSSL_FUNC_DIGEST_INIT, (void (*)(void))name##_internal_init },           \
     { OSSL_FUNC_DIGEST_UPDATE, (void (*)(void))upd },                          \
     { OSSL_FUNC_DIGEST_FINAL, (void (*)(void))name##_internal_final },         \
     { OSSL_FUNC_DIGEST_FREECTX, (void (*)(void))name##_freectx },              \