Prevent small subgroup attacks on DH/DHE

Historically OpenSSL only ever generated DH parameters based on "safe"
primes. More recently (in version 1.0.2) support was provided for
generating X9.42 style parameter files such as those required for RFC
5114 support. The primes used in such files may not be "safe". Where an
application is using DH configured with parameters based on primes that
are not "safe" then an attacker could use this fact to find a peer's
private DH exponent. This attack requires that the attacker complete
multiple handshakes in which the peer uses the same DH exponent.

A simple mitigation is to ensure that y^q (mod p) == 1

CVE-2016-0701

Issue reported by Antonio Sanso.
Reviewed-by: NViktor Dukhovni <viktor@openssl.org>

crypto/dh/dh_check.c

@@ -142,22 +142,38 @@ int DH_check(const DH *dh, int *ret)
 int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret)
 {
     int ok = 0;
-    BIGNUM *q = NULL;
+    BIGNUM *tmp = NULL;
+    BN_CTX *ctx = NULL;
 
     *ret = 0;
-    q = BN_new();
-    if (q == NULL)
+    ctx = BN_CTX_new();
+    if (ctx == NULL)
         goto err;
-    BN_set_word(q, 1);
-    if (BN_cmp(pub_key, q) <= 0)
+    BN_CTX_start(ctx);
+    tmp = BN_CTX_get(ctx);
+    if (tmp == NULL)
+        goto err;
+    BN_set_word(tmp, 1);
+    if (BN_cmp(pub_key, tmp) <= 0)
         *ret |= DH_CHECK_PUBKEY_TOO_SMALL;
-    BN_copy(q, dh->p);
-    BN_sub_word(q, 1);
-    if (BN_cmp(pub_key, q) >= 0)
+    BN_copy(tmp, dh->p);
+    BN_sub_word(tmp, 1);
+    if (BN_cmp(pub_key, tmp) >= 0)
         *ret |= DH_CHECK_PUBKEY_TOO_LARGE;
 
+    if (dh->q != NULL) {
+        /* Check pub_key^q == 1 mod p */
+        if (!BN_mod_exp(tmp, pub_key, dh->q, dh->p, ctx))
+            goto err;
+        if (!BN_is_one(tmp))
+            *ret |= DH_CHECK_PUBKEY_INVALID;
+    }
+
     ok = 1;
  err:
-    BN_free(q);
+    if (ctx != NULL) {
+        BN_CTX_end(ctx);
+        BN_CTX_free(ctx);
+    }
     return (ok);
 }

include/openssl/dh.h

@@ -174,6 +174,7 @@ struct dh_st {
 /* DH_check_pub_key error codes */
 # define DH_CHECK_PUBKEY_TOO_SMALL       0x01
 # define DH_CHECK_PUBKEY_TOO_LARGE       0x02
+# define DH_CHECK_PUBKEY_INVALID         0x03
 
 /*
  * primes p where (p-1)/2 is prime too are called "safe"; we define this for