bio_enc.c: add check for BIO_new_mem_buf

Since the memory allocation may fail, the BIO_new_mem_buf() may
return NULL pointer.
Therefore, it should be better to check it and return error if fails.
Signed-off-by: NJiasheng Jiang <jiasheng@iscas.ac.cn>
Reviewed-by: NTomas Mraz <tomas@openssl.org>
Reviewed-by: NPaul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17730)

test/bio_enc_test.c

@@ -38,7 +38,7 @@ static const unsigned char IV[] = {
 static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
     const unsigned char* iv)
 {
-    BIO *b;
+    BIO *b, *mem;
     static unsigned char inp[BUF_SIZE] = { 0 };
     unsigned char out[BUF_SIZE], ref[BUF_SIZE];
     int i, lref, len;
@@ -54,8 +54,11 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
     if (!TEST_ptr(b))
         return 0;
     if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, ENCRYPT)))
-        return 0;
-    BIO_push(b, BIO_new_mem_buf(inp, DATA_SIZE));
+        goto err;
+    mem = BIO_new_mem_buf(inp, DATA_SIZE);
+    if (!TEST_ptr(mem))
+        goto err;
+    BIO_push(b, mem);
     lref = BIO_read(b, ref, sizeof(ref));
     BIO_free_all(b);
 
@@ -66,16 +69,19 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
             return 0;
         if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, ENCRYPT))) {
             TEST_info("Split encrypt failed @ operation %d", i);
-            return 0;
+            goto err;
         }
-        BIO_push(b, BIO_new_mem_buf(inp, DATA_SIZE));
+        mem = BIO_new_mem_buf(inp, DATA_SIZE);
+        if (!TEST_ptr(mem))
+            goto err;
+        BIO_push(b, mem);
         memset(out, 0, sizeof(out));
         out[i] = ~ref[i];
         len = BIO_read(b, out, i);
         /* check for overstep */
         if (!TEST_uchar_eq(out[i], (unsigned char)~ref[i])) {
             TEST_info("Encrypt overstep check failed @ operation %d", i);
-            return 0;
+            goto err;
         }
         len += BIO_read(b, out + len, sizeof(out) - len);
         BIO_free_all(b);
@@ -95,9 +101,12 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
             return 0;
         if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, ENCRYPT))) {
             TEST_info("Small chunk encrypt failed @ operation %d", i);
-            return 0;
+            goto err;
         }
-        BIO_push(b, BIO_new_mem_buf(inp, DATA_SIZE));
+        mem = BIO_new_mem_buf(inp, DATA_SIZE);
+        if (!TEST_ptr(mem))
+            goto err;
+        BIO_push(b, mem);
         memset(out, 0, sizeof(out));
         for (len = 0; (delta = BIO_read(b, out + len, i)); ) {
             len += delta;
@@ -117,9 +126,12 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
     if (!TEST_ptr(b))
         return 0;
     if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, DECRYPT)))
-        return 0;
+        goto err;
     /* Use original reference output as input */
-    BIO_push(b, BIO_new_mem_buf(ref, lref));
+    mem = BIO_new_mem_buf(ref, lref);
+    if (!TEST_ptr(mem))
+        goto err;
+    BIO_push(b, mem);
     (void)BIO_flush(b);
     memset(out, 0, sizeof(out));
     len = BIO_read(b, out, sizeof(out));
@@ -135,16 +147,19 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
             return 0;
         if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, DECRYPT))) {
             TEST_info("Split decrypt failed @ operation %d", i);
-            return 0;
+            goto err;
         }
-        BIO_push(b, BIO_new_mem_buf(ref, lref));
+        mem = BIO_new_mem_buf(ref, lref);
+        if (!TEST_ptr(mem))
+            goto err;
+        BIO_push(b, mem);
         memset(out, 0, sizeof(out));
         out[i] = ~ref[i];
         len = BIO_read(b, out, i);
         /* check for overstep */
         if (!TEST_uchar_eq(out[i], (unsigned char)~ref[i])) {
             TEST_info("Decrypt overstep check failed @ operation %d", i);
-            return 0;
+            goto err;
         }
         len += BIO_read(b, out + len, sizeof(out) - len);
         BIO_free_all(b);
@@ -164,9 +179,12 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
             return 0;
         if (!TEST_true(BIO_set_cipher(b, cipher, key, iv, DECRYPT))) {
             TEST_info("Small chunk decrypt failed @ operation %d", i);
-            return 0;
+            goto err;
         }
-        BIO_push(b, BIO_new_mem_buf(ref, lref));
+        mem = BIO_new_mem_buf(ref, lref);
+        if (!TEST_ptr(mem))
+            goto err;
+        BIO_push(b, mem);
         memset(out, 0, sizeof(out));
         for (len = 0; (delta = BIO_read(b, out + len, i)); ) {
             len += delta;
@@ -180,6 +198,10 @@ static int do_bio_cipher(const EVP_CIPHER* cipher, const unsigned char* key,
     }
 
     return 1;
+
+err:
+    BIO_free_all(b);
+    return 0;
 }
 
 static int do_test_bio_cipher(const EVP_CIPHER* cipher, int idx)