CHANGES

@@ -6,6 +6,38 @@
  For a full list of changes, see the git commit log; for example,
  https://github.com/openssl/openssl/commits/ and pick the appropriate
  release branch.
+ *) Fix excessive time spent checking DH q parameter value.
+
+    The function DH_check() performs various checks on DH parameters. After
+    fixing CVE-2023-3446 it was discovered that a large q parameter value can
+    also trigger an overly long computation during some of these checks.
+    A correct q value, if present, cannot be larger than the modulus p
+    parameter, thus it is unnecessary to perform these checks if q is larger
+    than p.
+
+    If DH_check() is called with such q parameter value,
+    DH_CHECK_INVALID_Q_VALUE return flag is set and the computationally
+    intensive checks are skipped.
+
+    (CVE-2023-3817)
+    [Tomáš Mráz]
+
+ *) Fix DH_check() excessive time with over sized modulus
+
+    The function DH_check() performs various checks on DH parameters. One of
+    those checks confirms that the modulus ("p" parameter) is not too large.
+    Trying to use a very large modulus is slow and OpenSSL will not normally use
+    a modulus which is over 10,000 bits in length.
+
+    However the DH_check() function checks numerous aspects of the key or
+    parameters that have been supplied. Some of those checks use the supplied
+    modulus value even if it has already been found to be too large.
+
+    A new limit has been added to DH_check of 32,768 bits. Supplying a
+    key/parameters with a modulus over this size will simply cause DH_check()
+    to fail.
+    (CVE-2023-3446)
+    [Matt Caswell]
 
   *) Mitigate for the time it takes for `OBJ_obj2txt` to translate gigantic
      OBJECT IDENTIFIER sub-identifiers to canonical numeric text form.

NEWS

@@ -4,6 +4,8 @@
 
   This file gives a brief overview of the major changes between each OpenSSL
   release. For more details please read the CHANGES file.
+      o Fix excessive time spent checking DH q parameter value (CVE-2023-3817)
+      o Fix DH_check() excessive time with over sized modulus (CVE-2023-3446)
       o Fixed documentation of X509_VERIFY_PARAM_add0_policy() (CVE-2023-0466)
       o Mitigate for very slow `OBJ_obj2txt()` performance with gigantic
         OBJECT IDENTIFIER sub-identities.  (CVE-2023-2650)

crypto/dh/dh_check.c

@@ -97,10 +97,16 @@ int DH_check_ex(const DH *dh)
 
 int DH_check(const DH *dh, int *ret)
 {
-    int ok = 0, r;
+    int ok = 0, r, q_good = 0;
     BN_CTX *ctx = NULL;
     BIGNUM *t1 = NULL, *t2 = NULL;
 
+    /* Don't do any checks at all with an excessively large modulus */
+    if (BN_num_bits(dh->p) > OPENSSL_DH_CHECK_MAX_MODULUS_BITS) {
+        DHerr(DH_F_DH_CHECK, DH_R_MODULUS_TOO_LARGE);
+        return 0;
+    }
+
     if (!DH_check_params(dh, ret))
         return 0;
 
@@ -113,7 +119,14 @@ int DH_check(const DH *dh, int *ret)
     if (t2 == NULL)
         goto err;
 
-    if (dh->q) {
+    if (dh->q != NULL) {
+        if (BN_ucmp(dh->p, dh->q) > 0)
+            q_good = 1;
+        else
+            *ret |= DH_CHECK_INVALID_Q_VALUE;
+    }
+
+    if (q_good) {
         if (BN_cmp(dh->g, BN_value_one()) <= 0)
             *ret |= DH_NOT_SUITABLE_GENERATOR;
         else if (BN_cmp(dh->g, dh->p) >= 0)

crypto/dh/dh_err.c

 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -18,6 +18,7 @@ static const ERR_STRING_DATA DH_str_functs[] = {
     {ERR_PACK(ERR_LIB_DH, DH_F_DHPARAMS_PRINT_FP, 0), "DHparams_print_fp"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_BUILTIN_GENPARAMS, 0),
      "dh_builtin_genparams"},
+    {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK, 0), "DH_check"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_EX, 0), "DH_check_ex"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_PARAMS_EX, 0), "DH_check_params_ex"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_PUB_KEY_EX, 0), "DH_check_pub_key_ex"},

crypto/err/openssl.txt

@@ -402,6 +402,7 @@ CT_F_SCT_SET_VERSION:104:SCT_set_version
 DH_F_COMPUTE_KEY:102:compute_key
 DH_F_DHPARAMS_PRINT_FP:101:DHparams_print_fp
 DH_F_DH_BUILTIN_GENPARAMS:106:dh_builtin_genparams
+DH_F_DH_CHECK:126:DH_check
 DH_F_DH_CHECK_EX:121:DH_check_ex
 DH_F_DH_CHECK_PARAMS_EX:122:DH_check_params_ex
 DH_F_DH_CHECK_PUB_KEY_EX:123:DH_check_pub_key_ex

include/openssl/dh.h

@@ -29,6 +29,9 @@ extern "C" {
 # ifndef OPENSSL_DH_MAX_MODULUS_BITS
 #  define OPENSSL_DH_MAX_MODULUS_BITS    10000
 # endif
+# ifndef OPENSSL_DH_CHECK_MAX_MODULUS_BITS
+#  define OPENSSL_DH_CHECK_MAX_MODULUS_BITS  32768
+# endif
 
 # define OPENSSL_DH_FIPS_MIN_MODULUS_BITS 1024
 

include/openssl/dherr.h

 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2023 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -30,6 +30,7 @@ int ERR_load_DH_strings(void);
 #  define DH_F_COMPUTE_KEY                                 102
 #  define DH_F_DHPARAMS_PRINT_FP                           101
 #  define DH_F_DH_BUILTIN_GENPARAMS                        106
+#  define DH_F_DH_CHECK                                    126
 #  define DH_F_DH_CHECK_EX                                 121
 #  define DH_F_DH_CHECK_PARAMS_EX                          122
 #  define DH_F_DH_CHECK_PUB_KEY_EX                         123

test/dhtest.c

@@ -63,7 +63,7 @@ static int dh_test(void)
         || !TEST_true(DH_set0_pqg(dh, p, q, g)))
         goto err1;
 
-    if (!DH_check(dh, &i))
+    if (!TEST_true(DH_check(dh, &i)))
         goto err2;
     if (!TEST_false(i & DH_CHECK_P_NOT_PRIME)
             || !TEST_false(i & DH_CHECK_P_NOT_SAFE_PRIME)
@@ -123,6 +123,17 @@ static int dh_test(void)
     /* check whether the public key was calculated correctly */
     TEST_uint_eq(BN_get_word(pub_key2), 3331L);
 
+    /* Modulus of size: dh check max modulus bits + 1 */
+    if (!TEST_true(BN_set_word(p, 1))
+            || !TEST_true(BN_lshift(p, p, OPENSSL_DH_CHECK_MAX_MODULUS_BITS)))
+        goto err3;
+
+    /*
+     * We expect no checks at all for an excessively large modulus
+     */
+    if (!TEST_false(DH_check(dh, &i)))
+        goto err3;
+
     /*
      * II) key generation
      */
@@ -137,7 +148,7 @@ static int dh_test(void)
         goto err3;
 
     /* ... and check whether it is valid */
-    if (!DH_check(a, &i))
+    if (!TEST_true(DH_check(a, &i)))
         goto err3;
     if (!TEST_false(i & DH_CHECK_P_NOT_PRIME)
             || !TEST_false(i & DH_CHECK_P_NOT_SAFE_PRIME)