Update DH_check() to peform sensible checks when q parameter is present.

crypto/dh/dh.h

@@ -165,6 +165,9 @@ struct dh_st
 #define DH_CHECK_P_NOT_SAFE_PRIME	0x02
 #define DH_UNABLE_TO_CHECK_GENERATOR	0x04
 #define DH_NOT_SUITABLE_GENERATOR	0x08
+#define DH_CHECK_Q_NOT_PRIME		0x10
+#define DH_CHECK_INVALID_Q_VALUE	0x20
+#define DH_CHECK_INVALID_J_VALUE	0x40
 
 /* DH_check_pub_key error codes */
 #define DH_CHECK_PUBKEY_TOO_SMALL	0x01

crypto/dh/dh_check.c

@@ -75,15 +75,42 @@ int DH_check(const DH *dh, int *ret)
 	int ok=0;
 	BN_CTX *ctx=NULL;
 	BN_ULONG l;
-	BIGNUM *q=NULL;
+	BIGNUM *t1=NULL, *t2 = NULL;
 
 	*ret=0;
 	ctx=BN_CTX_new();
 	if (ctx == NULL) goto err;
-	q=BN_new();
-	if (q == NULL) goto err;
+	BN_CTX_start(ctx);
+	t1=BN_CTX_get(ctx);
+	t2=BN_CTX_get(ctx);
+	if (t2 == NULL) goto err;
 
-	if (BN_is_word(dh->g,DH_GENERATOR_2))
+	if (dh->q)
+		{
+		if (BN_cmp(dh->g, BN_value_one()) <= 0)
+			*ret|=DH_NOT_SUITABLE_GENERATOR;
+		else if (BN_cmp(dh->g, dh->p) >= 0)
+			*ret|=DH_NOT_SUITABLE_GENERATOR;
+		else
+			{
+			/* Check g^q == 1 mod p */
+			if (!BN_mod_exp(t1, dh->g, dh->q, dh->p, ctx))
+				goto err;
+			if (!BN_is_one(t1))
+				*ret|=DH_NOT_SUITABLE_GENERATOR;
+			}
+		if (!BN_is_prime_ex(dh->q,BN_prime_checks,ctx,NULL))
+			*ret|=DH_CHECK_Q_NOT_PRIME;
+		/* Check p == 1 mod q  i.e. q divides p - 1 */
+		if (!BN_div(t1, t2, dh->p, dh->q, ctx))
+			goto err;
+		if (!BN_is_one(t2))
+			*ret|=DH_CHECK_INVALID_Q_VALUE;
+		if (dh->j && BN_cmp(dh->j, t1))
+			*ret|=DH_CHECK_INVALID_J_VALUE;
+			
+		}
+	else if (BN_is_word(dh->g,DH_GENERATOR_2))
 		{
 		l=BN_mod_word(dh->p,24);
 		if (l != 11) *ret|=DH_NOT_SUITABLE_GENERATOR;
@@ -106,16 +133,19 @@ int DH_check(const DH *dh, int *ret)
 
 	if (!BN_is_prime_ex(dh->p,BN_prime_checks,ctx,NULL))
 		*ret|=DH_CHECK_P_NOT_PRIME;
-	else
+	else if (!dh->q)
 		{
-		if (!BN_rshift1(q,dh->p)) goto err;
-		if (!BN_is_prime_ex(q,BN_prime_checks,ctx,NULL))
+		if (!BN_rshift1(t1,dh->p)) goto err;
+		if (!BN_is_prime_ex(t1,BN_prime_checks,ctx,NULL))
 			*ret|=DH_CHECK_P_NOT_SAFE_PRIME;
 		}
 	ok=1;
 err:
-	if (ctx != NULL) BN_CTX_free(ctx);
-	if (q != NULL) BN_free(q);
+	if (ctx != NULL)
+		{
+		BN_CTX_end(ctx);
+		BN_CTX_free(ctx);
+		}
 	return(ok);
 	}