Fix buffer overflow when generating large RSA keys in FIPS mode.

A pairwise test runs only in FIPS mode.
An assumption about the size of the 'to' buffer passed to
RSA_private_decrypt() was incorrect. It needs to be up to RSA_size()
bytes long - so a fixed buffer of 256 bytes was not large enough.
An exiting malloc has increased in size to allocate buffer space for
both the encrypt and decrypt buffer.

The existing test used 2080 bits which was not quite large enough to
trigger the issue. A test using 3072 bits has been added.

Reported by Mark Powers from Acumen.
Reviewed-by: NTomas Mraz <tomas@openssl.org>
Reviewed-by: NPaul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15447)

crypto/rsa/rsa_gen.c

@@ -479,7 +479,7 @@ static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
     unsigned int ciphertxt_len;
     unsigned char *ciphertxt = NULL;
     const unsigned char plaintxt[16] = {0};
-    unsigned char decoded[256];
+    unsigned char *decoded = NULL;
     unsigned int decoded_len;
     unsigned int plaintxt_len = (unsigned int)sizeof(plaintxt_len);
     int padding = RSA_PKCS1_PADDING;
@@ -492,9 +492,14 @@ static int rsa_keygen_pairwise_test(RSA *rsa, OSSL_CALLBACK *cb, void *cbarg)
                            OSSL_SELF_TEST_DESC_PCT_RSA_PKCS1);
 
     ciphertxt_len = RSA_size(rsa);
-    ciphertxt = OPENSSL_zalloc(ciphertxt_len);
+    /*
+     * RSA_private_encrypt() and RSA_private_decrypt() requires the 'to'
+     * parameter to be a maximum of RSA_size() - allocate space for both.
+     */
+    ciphertxt = OPENSSL_zalloc(ciphertxt_len * 2);
     if (ciphertxt == NULL)
         goto err;
+    decoded = ciphertxt + ciphertxt_len;
 
     ciphertxt_len = RSA_public_encrypt(plaintxt_len, plaintxt, ciphertxt, rsa,
                                        padding);

test/recipes/15-test_genrsa.t

@@ -24,7 +24,7 @@ use lib bldtop_dir('.');
 my $no_fips = disabled('fips') || ($ENV{NO_FIPS} // 0);
 
 plan tests =>
-    ($no_fips ? 0 : 1)          # Extra FIPS related test
+    ($no_fips ? 0 : 2)          # Extra FIPS related test
     + 13;
 
 # We want to know that an absurdly small number of bits isn't support
@@ -131,4 +131,10 @@ unless ($no_fips) {
                '-pkeyopt', 'bits:2080',
                '-out', 'genrsatest2080.pem'])),
        "Generating RSA key with > 2048 bits and < 3072 bits");
+    ok(run(app(['openssl', 'genpkey',
+                @prov,
+               '-algorithm', 'RSA',
+               '-pkeyopt', 'bits:3072',
+               '-out', 'genrsatest3072.pem'])),
+       "Generating RSA key with 3072 bits");
 }