always reject data >= n

CHANGES

@@ -12,6 +12,18 @@
          *) applies to 0.9.6a/0.9.6b and 0.9.7
          +) applies to 0.9.7 only
 
+  *) In
+
+          RSA_eay_public_encrypt
+          RSA_eay_private_decrypt
+          RSA_eay_private_encrypt (signing)
+          RSA_eay_public_decrypt (signature verification)
+
+     (default implementations for RSA_public_encrypt,
+     RSA_private_decrypt, RSA_private_encrypt, RSA_public_decrypt),
+     always reject numbers >= n.
+     [Bodo Moeller]
+
   *) In crypto/rand/md_rand.c, set 'locking_thread' to current thread's ID
      *before* setting the 'crypto_lock_rand' flag.  The previous code had
      a race condition if 0 is a valid thread ID.

crypto/rsa/rsa.h

@@ -219,8 +219,6 @@ const RSA_METHOD *RSA_PKCS1_SSLeay(void);
 
 const RSA_METHOD *RSA_null_method(void);
 
-void	ERR_load_RSA_strings(void );
-
 DECLARE_ASN1_ENCODE_FUNCTIONS_const(RSA, RSAPublicKey)
 DECLARE_ASN1_ENCODE_FUNCTIONS_const(RSA, RSAPrivateKey)
 
@@ -285,10 +283,12 @@ int RSA_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new *new_func,
 int RSA_set_ex_data(RSA *r,int idx,void *arg);
 void *RSA_get_ex_data(const RSA *r, int idx);
 
+
 /* BEGIN ERROR CODES */
 /* The following lines are auto generated by the script mkerr.pl. Any changes
  * made after this point may be overwritten when the script is next run.
  */
+void ERR_load_RSA_strings(void);
 
 /* Error codes for the RSA functions. */
 
@@ -330,6 +330,7 @@ void *RSA_get_ex_data(const RSA *r, int idx);
 #define RSA_R_DATA_GREATER_THAN_MOD_LEN			 108
 #define RSA_R_DATA_TOO_LARGE				 109
 #define RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE		 110
+#define RSA_R_DATA_TOO_LARGE_FOR_MODULUS		 132
 #define RSA_R_DATA_TOO_SMALL				 111
 #define RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE		 122
 #define RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY		 112
@@ -356,4 +357,3 @@ void *RSA_get_ex_data(const RSA *r, int idx);
 }
 #endif
 #endif
-

crypto/rsa/rsa_eay.c

@@ -79,8 +79,8 @@ static int RSA_eay_finish(RSA *rsa);
 static RSA_METHOD rsa_pkcs1_eay_meth={
 	"Eric Young's PKCS#1 RSA",
 	RSA_eay_public_encrypt,
-	RSA_eay_public_decrypt,
-	RSA_eay_private_encrypt,
+	RSA_eay_public_decrypt, /* signature verification */
+	RSA_eay_private_encrypt, /* signing */
 	RSA_eay_private_decrypt,
 	RSA_eay_mod_exp,
 	BN_mod_exp_mont, /* XXX probably we should not use Montgomery if  e == 3 */
@@ -139,6 +139,13 @@ static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 
 	if (BN_bin2bn(buf,num,&f) == NULL) goto err;
 	
+	if (BN_ucmp(&f, rsa->n) >= 0)
+		{	
+		/* usually the padding functions would catch this */
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+		goto err;
+		}
+
 	if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
 		{
 		BN_MONT_CTX* bn_mont_ctx;
@@ -186,6 +193,7 @@ err:
 	return(r);
 	}
 
+/* signing */
 static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
@@ -223,6 +231,13 @@ static int RSA_eay_private_encrypt(int flen, const unsigned char *from,
 	if (i <= 0) goto err;
 
 	if (BN_bin2bn(buf,num,&f) == NULL) goto err;
+	
+	if (BN_ucmp(&f, rsa->n) >= 0)
+		{	
+		/* usually the padding functions would catch this */
+		RSAerr(RSA_F_RSA_EAY_PRIVATE_ENCRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+		goto err;
+		}
 
 	if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
 		RSA_blinding_on(rsa,ctx);
@@ -299,6 +314,12 @@ static int RSA_eay_private_decrypt(int flen, const unsigned char *from,
 	/* make data into a big number */
 	if (BN_bin2bn(from,(int)flen,&f) == NULL) goto err;
 
+	if (BN_ucmp(&f, rsa->n) >= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PRIVATE_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+		goto err;
+		}
+
 	if ((rsa->flags & RSA_FLAG_BLINDING) && (rsa->blinding == NULL))
 		RSA_blinding_on(rsa,ctx);
 	if (rsa->flags & RSA_FLAG_BLINDING)
@@ -359,6 +380,7 @@ err:
 	return(r);
 	}
 
+/* signature verification */
 static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 	     unsigned char *to, RSA *rsa, int padding)
 	{
@@ -392,6 +414,13 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 		}
 
 	if (BN_bin2bn(from,flen,&f) == NULL) goto err;
+
+	if (BN_ucmp(&f, rsa->n) >= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT,RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+		goto err;
+		}
+
 	/* do the decrypt */
 	if ((rsa->_method_mod_n == NULL) && (rsa->flags & RSA_FLAG_CACHE_PUBLIC))
 		{

crypto/rsa/rsa_err.c

@@ -106,6 +106,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {RSA_R_DATA_GREATER_THAN_MOD_LEN         ,"data greater than mod len"},
 {RSA_R_DATA_TOO_LARGE                    ,"data too large"},
 {RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE       ,"data too large for key size"},
+{RSA_R_DATA_TOO_LARGE_FOR_MODULUS        ,"data too large for modulus"},
 {RSA_R_DATA_TOO_SMALL                    ,"data too small"},
 {RSA_R_DATA_TOO_SMALL_FOR_KEY_SIZE       ,"data too small for key size"},
 {RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY        ,"digest too big for rsa key"},