Add key length check to rsa_kem operation.

This uses similiar code used by other rsa related operations.
Reviewed-by: NTomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12955)

providers/implementations/kem/rsa_kem.c

@@ -25,11 +25,12 @@
 #include "prov/providercommonerr.h"
 #include "prov/provider_ctx.h"
 #include "prov/implementations.h"
+#include "prov/securitycheck.h"
 
 static OSSL_FUNC_kem_newctx_fn rsakem_newctx;
-static OSSL_FUNC_kem_encapsulate_init_fn rsakem_init;
+static OSSL_FUNC_kem_encapsulate_init_fn rsakem_encapsulate_init;
 static OSSL_FUNC_kem_encapsulate_fn rsakem_generate;
-static OSSL_FUNC_kem_decapsulate_init_fn rsakem_init;
+static OSSL_FUNC_kem_decapsulate_init_fn rsakem_decapsulate_init;
 static OSSL_FUNC_kem_decapsulate_fn rsakem_recover;
 static OSSL_FUNC_kem_freectx_fn rsakem_freectx;
 static OSSL_FUNC_kem_dupctx_fn rsakem_dupctx;
@@ -116,7 +117,7 @@ static void *rsakem_dupctx(void *vprsactx)
     return dstctx;
 }
 
-static int rsakem_init(void *vprsactx, void *vrsa)
+static int rsakem_init(void *vprsactx, void *vrsa, int operation)
 {
     PROV_RSA_CTX *prsactx = (PROV_RSA_CTX *)vprsactx;
 
@@ -124,10 +125,24 @@ static int rsakem_init(void *vprsactx, void *vrsa)
         return 0;
     RSA_free(prsactx->rsa);
     prsactx->rsa = vrsa;
-    /* TODO(3.0) Add a RSA keylength check here for fips */
+
+    if (!rsa_check_key(vrsa, operation == EVP_PKEY_OP_ENCAPSULATE)) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_INVALID_KEY_LENGTH);
+        return 0;
+    }
     return 1;
 }
 
+static int rsakem_encapsulate_init(void *vprsactx, void *vrsa)
+{
+    return rsakem_init(vprsactx, vrsa, EVP_PKEY_OP_ENCAPSULATE);
+}
+
+static int rsakem_decapsulate_init(void *vprsactx, void *vrsa)
+{
+    return rsakem_init(vprsactx, vrsa, EVP_PKEY_OP_DECAPSULATE);
+}
+
 static int rsakem_get_ctx_params(void *vprsactx, OSSL_PARAM *params)
 {
     PROV_RSA_CTX *ctx = (PROV_RSA_CTX *)vprsactx;
@@ -322,10 +337,10 @@ static int rsakem_recover(void *vprsactx, unsigned char *out, size_t *outlen,
 const OSSL_DISPATCH rsa_asym_kem_functions[] = {
     { OSSL_FUNC_KEM_NEWCTX, (void (*)(void))rsakem_newctx },
     { OSSL_FUNC_KEM_ENCAPSULATE_INIT,
-      (void (*)(void))rsakem_init },
+      (void (*)(void))rsakem_encapsulate_init },
     { OSSL_FUNC_KEM_ENCAPSULATE, (void (*)(void))rsakem_generate },
     { OSSL_FUNC_KEM_DECAPSULATE_INIT,
-      (void (*)(void))rsakem_init },
+      (void (*)(void))rsakem_decapsulate_init },
     { OSSL_FUNC_KEM_DECAPSULATE, (void (*)(void))rsakem_recover },
     { OSSL_FUNC_KEM_FREECTX, (void (*)(void))rsakem_freectx },
     { OSSL_FUNC_KEM_DUPCTX, (void (*)(void))rsakem_dupctx },