Allow RSA key-generation to specify an arbitrary public exponent. Jelte

proposed the change and submitted the patch, I jiggled it slightly and
adjusted the other parts of openssl that were affected.

PR: 867
Submitted by: Jelte Jansen
Reviewed by: Geoff Thorpe

CHANGES

@@ -4,6 +4,11 @@
 
  Changes between 0.9.7c and 0.9.8  [xx XXX xxxx]
 
+  *) The new 'RSA_generate_key_ex' function now takes a BIGNUM for the public
+     exponent rather than 'unsigned long'. There is a corresponding change to
+     the new 'rsa_keygen' element of the RSA_METHOD structure.
+     [Jelte Jansen, Geoff Thorpe]
+
   *) Reduce the chances of duplicate issuer name and serial numbers (in
      violation of RFC3280) using the OpenSSL certificate creation utilities. 
      This is done by creating a random 64 bit value for the initial serial

apps/genrsa.c

@@ -92,7 +92,6 @@ int MAIN(int argc, char **argv)
 	ENGINE *e = NULL;
 #endif
 	int ret=1;
-	RSA *rsa=NULL;
 	int i,num=DEFBITS;
 	long l;
 	const EVP_CIPHER *enc=NULL;
@@ -104,6 +103,10 @@ int MAIN(int argc, char **argv)
 #endif
 	char *inrand=NULL;
 	BIO *out=NULL;
+	BIGNUM *bn = BN_new();
+	RSA *rsa = RSA_new();
+
+	if(!bn || !rsa) goto err;
 
 	apps_startup();
 	BN_GENCB_set(&cb, genrsa_cb, bio_err);
@@ -242,13 +245,11 @@ bad:
 	BIO_printf(bio_err,"Generating RSA private key, %d bit long modulus\n",
 		num);
 
-	if(((rsa = RSA_new()) == NULL) || !RSA_generate_key_ex(rsa, num, f4, &cb))
+	if(!BN_set_word(bn, f4) || !RSA_generate_key_ex(rsa, num, bn, &cb))
 		goto err;
 		
 	app_RAND_write_file(NULL, bio_err);
 
-	if (rsa == NULL) goto err;
-	
 	/* We need to do the following for when the base number size is <
 	 * long, esp windows 3.1 :-(. */
 	l=0L;
@@ -272,8 +273,9 @@ bad:
 
 	ret=0;
 err:
-	if (rsa != NULL) RSA_free(rsa);
-	if (out != NULL) BIO_free_all(out);
+	if (bn) BN_free(bn);
+	if (rsa) RSA_free(rsa);
+	if (out) BIO_free_all(out);
 	if(passout) OPENSSL_free(passout);
 	if (ret != 0)
 		ERR_print_errors(bio_err);

apps/req.c

@@ -748,12 +748,16 @@ bad:
 		if (pkey_type == TYPE_RSA)
 			{
 			RSA *rsa = RSA_new();
-			if(!rsa || !RSA_generate_key_ex(rsa, newkey, 0x10001, &cb) ||
+			BIGNUM *bn = BN_new();
+			if(!bn || !rsa || !BN_set_word(bn, 0x10001) ||
+					!RSA_generate_key_ex(rsa, newkey, bn, &cb) ||
 					!EVP_PKEY_assign_RSA(pkey, rsa))
 				{
+				if(bn) BN_free(bn);
 				if(rsa) RSA_free(rsa);
 				goto end;
 				}
+			BN_free(bn);
 			}
 		else
 #endif

apps/s_server.c

@@ -1782,17 +1782,20 @@ err:
 #ifndef OPENSSL_NO_RSA
 static RSA MS_CALLBACK *tmp_rsa_cb(SSL *s, int is_export, int keylength)
 	{
+	BIGNUM *bn = NULL;
 	static RSA *rsa_tmp=NULL;
 
-	if (rsa_tmp == NULL)
+	if (!rsa_tmp && ((bn = BN_new()) == NULL))
+		BIO_printf(bio_err,"Allocation error in generating RSA key\n");
+	if (!rsa_tmp && bn)
 		{
 		if (!s_quiet)
 			{
 			BIO_printf(bio_err,"Generating temp (%d bit) RSA key...",keylength);
 			(void)BIO_flush(bio_err);
 			}
-		if(((rsa_tmp = RSA_new()) == NULL) || !RSA_generate_key_ex(
-					rsa_tmp, keylength,RSA_F4,NULL))
+		if(!BN_set_word(bn, RSA_F4) || ((rsa_tmp = RSA_new()) == NULL) ||
+				!RSA_generate_key_ex(rsa_tmp, keylength, bn, NULL))
 			{
 			if(rsa_tmp) RSA_free(rsa_tmp);
 			rsa_tmp = NULL;
@@ -1802,6 +1805,7 @@ static RSA MS_CALLBACK *tmp_rsa_cb(SSL *s, int is_export, int keylength)
 			BIO_printf(bio_err,"\n");
 			(void)BIO_flush(bio_err);
 			}
+		BN_free(bn);
 		}
 	return(rsa_tmp);
 	}

crypto/rsa/rsa.h

@@ -120,7 +120,7 @@ struct rsa_meth_st
  * is for behavioural compatibility whilst the code gets rewired, but one day
  * it would be nice to assume there are no such things as "builtin software"
  * implementations. */
-	int (*rsa_keygen)(RSA *rsa, int bits, unsigned long e, BN_GENCB *cb);
+	int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
 	};
 
 struct rsa_st
@@ -203,7 +203,7 @@ RSA *	RSA_generate_key(int bits, unsigned long e,void
 #endif /* !defined(OPENSSL_NO_DEPRECATED) */
 
 /* New version */
-int	RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e, BN_GENCB *cb);
+int	RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
 
 int	RSA_check_key(const RSA *);
 	/* next 4 return -1 on error */

crypto/rsa/rsa_depr.c

@@ -62,23 +62,39 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
+#ifdef OPENSSL_NO_DEPRECATED
+
 static void *dummy=&dummy;
 
-#ifndef OPENSSL_NO_DEPRECATED
+#else
+
 RSA *RSA_generate_key(int bits, unsigned long e_value,
 	     void (*callback)(int,int,void *), void *cb_arg)
 	{
 	BN_GENCB cb;
-	RSA *rsa;
+	int i;
+	RSA *rsa = RSA_new();
+	BIGNUM *e = BN_new();
 
-	if((rsa=RSA_new()) == NULL)
-		return 0;
+	if(!rsa || !e) goto err;
+
+	/* The problem is when building with 8, 16, or 32 BN_ULONG,
+	 * unsigned long can be larger */
+	for (i=0; i<sizeof(unsigned long)*8; i++)
+		{
+		if (e_value & (1UL<<i))
+			BN_set_bit(e,i);
+		}
 
 	BN_GENCB_set_old(&cb, callback, cb_arg);
 
-	if(RSA_generate_key_ex(rsa, bits, e_value, &cb))
+	if(RSA_generate_key_ex(rsa, bits, e, &cb)) {
+		BN_free(e);
 		return rsa;
-	RSA_free(rsa);
+	}
+err:
+	if(e) BN_free(e);
+	if(rsa) RSA_free(rsa);
 	return 0;
 	}
 #endif

crypto/rsa/rsa_gen.c

@@ -68,25 +68,24 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb);
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb);
 
 /* NB: this wrapper would normally be placed in rsa_lib.c and the static
  * implementation would probably be in rsa_eay.c. Nonetheless, is kept here so
  * that we don't introduce a new linker dependency. Eg. any application that
  * wasn't previously linking object code related to key-generation won't have to
  * now just because key-generation is part of RSA_METHOD. */
-int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
+int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 	{
 	if(rsa->meth->rsa_keygen)
 		return rsa->meth->rsa_keygen(rsa, bits, e_value, cb);
 	return rsa_builtin_keygen(rsa, bits, e_value, cb);
 	}
 
-static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
 	{
 	BIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;
 	int bitsp,bitsq,ok= -1,n=0;
-	unsigned int i;
 	BN_CTX *ctx=NULL,*ctx2=NULL;
 
 	ctx=BN_CTX_new();
@@ -113,17 +112,7 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, unsigned long e_value, BN_GENC
 	if(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;
 	if(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;
 
-#if 1
-	/* The problem is when building with 8, 16, or 32 BN_ULONG,
-	 * unsigned long can be larger */
-	for (i=0; i<sizeof(unsigned long)*8; i++)
-		{
-		if (e_value & (1UL<<i))
-			BN_set_bit(rsa->e,i);
-		}
-#else
-	if (!BN_set_word(rsa->e,e_value)) goto err;
-#endif
+	BN_copy(rsa->e, e_value);
 
 	/* generate p and q */
 	for (;;)

ssl/ssltest.c

@@ -1612,17 +1612,19 @@ static RSA *rsa_tmp=NULL;
 
 static RSA MS_CALLBACK *tmp_rsa_cb(SSL *s, int is_export, int keylength)
 	{
+	BIGNUM *bn = NULL;
 	if (rsa_tmp == NULL)
 		{
+		bn = BN_new();
 		rsa_tmp = RSA_new();
-		if(!rsa_tmp)
+		if(!bn || !rsa_tmp || !BN_set_word(bn, RSA_F4))
 			{
 			BIO_printf(bio_err, "Memory error...");
 			goto end;
 			}
 		BIO_printf(bio_err,"Generating temp (%d bit) RSA key...",keylength);
 		(void)BIO_flush(bio_err);
-		if(!RSA_generate_key_ex(rsa_tmp,keylength,RSA_F4,NULL))
+		if(!RSA_generate_key_ex(rsa_tmp,keylength,bn,NULL))
 			{
 			BIO_printf(bio_err, "Error generating key.");
 			RSA_free(rsa_tmp);
@@ -1632,6 +1634,7 @@ end:
 		BIO_printf(bio_err,"\n");
 		(void)BIO_flush(bio_err);
 		}
+	if(bn) BN_free(bn);
 	return(rsa_tmp);
 	}