The logic in the main signing and verifying functions to check lengths was

incorrect.  Fortunately, there is a second check that's correct, when adding
the pads.
PR: 355

crypto/rsa/rsa.h

@@ -175,6 +175,8 @@ struct rsa_st
 #define RSA_NO_PADDING		3
 #define RSA_PKCS1_OAEP_PADDING	4
 
+#define RSA_PKCS1_PADDING_SIZE	11
+
 #define RSA_set_app_data(s,arg)         RSA_set_ex_data(s,0,arg)
 #define RSA_get_app_data(s)             RSA_get_ex_data(s,0)
 

crypto/rsa/rsa_pk1.c

@@ -68,7 +68,7 @@ int RSA_padding_add_PKCS1_type_1(unsigned char *to, int tlen,
 	int j;
 	unsigned char *p;
 
-	if (flen > (tlen-11))
+	if (flen > (tlen-RSA_PKCS1_PADDING_SIZE))
 		{
 		RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_TYPE_1,RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
 		return(0);

crypto/rsa/rsa_saos.c

@@ -77,7 +77,7 @@ int RSA_sign_ASN1_OCTET_STRING(int type,
 
 	i=i2d_ASN1_OCTET_STRING(&sig,NULL);
 	j=RSA_size(rsa);
-	if ((i-RSA_PKCS1_PADDING) > j)
+	if (i > (j-RSA_PKCS1_PADDING_SIZE))
 		{
 		RSAerr(RSA_F_RSA_SIGN_ASN1_OCTET_STRING,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
 		return(0);

crypto/rsa/rsa_sign.c

@@ -113,7 +113,7 @@ int RSA_sign(int type, const unsigned char *m, unsigned int m_len,
 		i=i2d_X509_SIG(&sig,NULL);
 	}
 	j=RSA_size(rsa);
-	if ((i-RSA_PKCS1_PADDING) > j)
+	if (i > (j-RSA_PKCS1_PADDING_SIZE))
 		{
 		RSAerr(RSA_F_RSA_SIGN,RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
 		return(0);