Don't allow BIGNUMs to become so large that computations with dmax

might overflow.

crypto/bn/bn.h

@@ -541,6 +541,7 @@ int BN_bntest_rand(BIGNUM *rnd, int bits, int top,int bottom);
 #define BN_R_NOT_INITIALIZED				 107
 #define BN_R_NO_INVERSE					 108
 #define BN_R_P_IS_NOT_PRIME				 112
+#define BN_R_TOO_LARGE					 114
 #define BN_R_TOO_MANY_ITERATIONS			 113
 #define BN_R_TOO_MANY_TEMPORARY_VARIABLES		 109
 

crypto/bn/bn_err.c

@@ -105,6 +105,7 @@ static ERR_STRING_DATA BN_str_reasons[]=
 {BN_R_NOT_INITIALIZED                    ,"not initialized"},
 {BN_R_NO_INVERSE                         ,"no inverse"},
 {BN_R_P_IS_NOT_PRIME                     ,"p is not prime"},
+{BN_R_TOO_LARGE                          ,"too large"},
 {BN_R_TOO_MANY_ITERATIONS                ,"too many iterations"},
 {BN_R_TOO_MANY_TEMPORARY_VARIABLES       ,"too many temporary variables"},
 {0,NULL}

crypto/bn/bn_lib.c

@@ -312,6 +312,12 @@ static BN_ULONG *bn_expand_internal(const BIGNUM *b, int words)
 	const BN_ULONG *B;
 	int i;
 
+	if (words > (INT_MAX/(4*BN_BITS2)))
+		{
+		BNerr(BN_F_BN_EXPAND_INTERNAL,BN_R_TOO_LARGE);
+		return NULL;
+		}
+
 	bn_check_top(b);	
 	if (BN_get_flags(b,BN_FLG_STATIC_DATA))
 		{