Refactor the test framework testutil

It's now built as a static library, and greatly simplified for test
programs, which no longer need to include test_main_custom.h or
test_main.h and link with the corresponding object files.
Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3243)

test/README

@@ -99,14 +99,13 @@ test):
 to modify the include paths and source files if you don't want to use the
 basic test framework:
 
-    SOURCE[{name}]={name}.c testutil.c test_main.c
+    SOURCE[{name}]={name}.c
     INCLUDE[{name}]=.. ../include
-    DEPEND[{name}]=../libcrypto
+    DEPEND[{name}]=../libcrypto libtestutil.a
 
 Generic form of C test executables
 ==================================
 
-    #include "test_main.h"
     #include "testutil.h"
 
     static int my_test(void)

test/build.info

@@ -5,11 +5,16 @@
          my ($base, $files) = @_;
          return join(" ", map { "$base/$_" } split(/\s+/, $files));
      }
-     our $apps_extra =
-         $config{target} =~ /^vms-/ ? "../apps/vms_term_sock.c" : "";
      ""
 -}
 IF[{- !$disabled{tests} -}]
+  LIBS_NO_INST=libtestutil.a
+  SOURCE[libtestutil.a]=testutil/basic_output.c testutil/driver.c \
+          testutil/tests.c testutil/test_main.c testutil/main.c \
+          {- rebase_files("../apps", $target{apps_aux_src}) -}
+  INCLUDE[libtestutil.a]=../include
+  DEPEND[libtestutil.a]=../libcrypto
+
   PROGRAMS_NO_INST=\
           aborttest test_test \
           sanitytest exdatatest bntest \

test/test_main.h

-/*
- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the OpenSSL license (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#ifndef HEADER_TEST_MAIN_H
-# define HEADER_TEST_MAIN_H
-
-/*
- * Simple unit tests should implement register_tests() and link to test_main.c.
- */
-extern void register_tests(void);
-
-#endif  /* HEADER_TEST_MAIN_H */

test/test_main_custom.h

-/*
- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
- *
- * Licensed under the OpenSSL license (the "License").  You may not use
- * this file except in compliance with the License.  You can obtain a copy
- * in the file LICENSE in the source distribution or at
- * https://www.openssl.org/source/license.html
- */
-
-#ifndef HEADER_TEST_MAIN_CUSTOM_H
-# define HEADER_TEST_MAIN_CUSTOM_H
-
-/*
- * Unit tests that need a custom main() should implement test_main and link to
- * test_main_custom.c
- * test_main() should return the result of run_tests().
- */
-extern int test_main(int argc, char *argv[]);
-
-#endif  /* HEADER_TEST_MAIN_CUSTOM_H */

test/testutil.h

@@ -16,12 +16,9 @@
 #include <openssl/e_os2.h>
 
 /*-
- * Simple unit tests should implement register_tests() from test_main.h
- * and link against test_main.c.
+ * Simple unit tests should implement register_tests().
  * To register tests, call ADD_TEST or ADD_ALL_TESTS:
  *
- * #include "test_main.h"
- *
  * void register_tests(void)
  * {
  *     ADD_TEST(test_foo);
@@ -29,8 +26,7 @@
  * }
  *
  * Tests that need to perform custom setup or read command-line arguments should
- * implement test_main() from test_main_custom.h and link against
- * test_main_custom.c:
+ * implement test_main():
  *
  * int test_main(int argc, char *argv[])
  * {
@@ -138,14 +134,27 @@ void add_test(const char *test_case_name, int (*test_fn) ());
 void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);
 __owur int run_tests(const char *test_prog_name);
 
+/*
+ * Declarations for user defined functions
+ */
+void register_tests(void);
+int test_main(int argc, char *argv[]);
+
+
 /*
  *  Test assumption verification helpers.
  */
 
-# if defined(__GNUC__)
-#define PRINTF_FORMAT(a, b)   __attribute__ ((format(printf, a, b)))
-# else
 #define PRINTF_FORMAT(a, b)
+#if defined(__GNUC__) && defined(__STDC_VERSION__)
+  /*
+   * Because we support the 'z' modifier, which made its appearance in C99,
+   * we can't use __attribute__ with pre C99 dialects.
+   */
+# if __STDC_VERSION__ >= 199901L
+#  undef PRINTF_FORMAT
+#  define PRINTF_FORMAT(a, b)   __attribute__ ((format(printf, a, b)))
+# endif
 #endif
 
 #  define DECLARE_COMPARISON(type, name, opname)                        \
@@ -335,3 +344,22 @@ void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2);
         }                                       \
     } while (0)
 #endif                          /* HEADER_TESTUTIL_H */
+
+
+/*
+ * The basic I/O functions used by the test framework.  These can be
+ * overriden when needed. Note that if one is, then all must be.
+ */
+void test_open_streams(void);
+void test_close_streams(void);
+/* The following ALL return the number of characters written */
+int test_puts_stdout(const char *str);
+int test_puts_stderr(const char *str);
+int test_vprintf_stdout(const char *fmt, va_list ap);
+int test_vprintf_stderr(const char *fmt, va_list ap);
+/* These return failure or success */
+int test_flush_stdout(void);
+int test_flush_stderr(void);
+
+extern BIO *bio_out;
+extern BIO *bio_err;

test/testutil/basic_output.c

+/*
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <openssl/crypto.h>
+#include <openssl/bio.h>
+
+BIO *bio_out = NULL;
+BIO *bio_err = NULL;
+
+#ifdef OPENSSL_USE_APPLINK
+/*
+ * Using BIO_new_fd() obligates the use of applinks on platforms where it's
+ * relevant.  Because it becomes a module of the libtestutil library and would
+ * be disregarded if not actively referred to, we have this dummy that does
+ * exactly this.  For any module that uses the rest of the routines here,
+ * OPENSSL_Applink should tag along for sure.
+ */
+void Applink_dummy(void);
+void Applink_dummy(void)
+{
+    OPENSSL_EXTERN void OPENSSL_Applink(void);
+
+    OPENSSL_Applink();
+}
+/* Generate an error for anyone who tries to actually use this dummy */
+# define Applink_dummy "DON'T USE THIS"
+#endif
+
+void test_open_streams(void)
+{
+    bio_out = BIO_new_fd(1, 0);
+    bio_err = BIO_new_fd(2, 0);
+
+    OPENSSL_assert(bio_out != NULL);
+    OPENSSL_assert(bio_err != NULL);
+}
+
+void test_close_streams(void)
+{
+    BIO_free(bio_out);
+    BIO_free(bio_err);
+}
+
+int test_puts_stdout(const char *str)
+{
+    return BIO_puts(bio_out, str);
+}
+
+int test_puts_stderr(const char *str)
+{
+    return BIO_puts(bio_err, str);
+}
+
+int test_vprintf_stdout(const char *fmt, va_list ap)
+{
+    return BIO_vprintf(bio_out, fmt, ap);
+}
+
+int test_vprintf_stderr(const char *fmt, va_list ap)
+{
+    return BIO_vprintf(bio_err, fmt, ap);
+}
+
+int test_flush_stdout(void)
+{
+    return BIO_flush(bio_out);
+}
+
+int test_flush_stderr(void)
+{
+    return BIO_flush(bio_err);
+}

test/testutil/driver.c

+/*
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include "../testutil.h"
+
+#include <string.h>
+#include <assert.h>
+
+#include "../../e_os.h"
+#include <openssl/bio.h>
+
+/*
+ * Declares the structures needed to register each test case function.
+ */
+typedef struct test_info {
+    const char *test_case_name;
+    int (*test_fn) ();
+    int (*param_test_fn)(int idx);
+    int num;
+} TEST_INFO;
+
+static TEST_INFO all_tests[1024];
+static int num_tests = 0;
+/*
+ * A parameterised tests runs a loop of test cases.
+ * |num_test_cases| counts the total number of test cases
+ * across all tests.
+ */
+static int num_test_cases = 0;
+
+void add_test(const char *test_case_name, int (*test_fn) ())
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].test_fn = test_fn;
+    all_tests[num_tests].num = -1;
+    ++num_tests;
+    ++num_test_cases;
+}
+
+void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
+                   int num)
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].param_test_fn = test_fn;
+    all_tests[num_tests].num = num;
+    ++num_tests;
+    num_test_cases += num;
+}
+
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+static int should_report_leaks()
+{
+    /*
+     * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
+     * can be used to disable leak checking at runtime.
+     * Note this only works when running the test binary manually;
+     * the test harness always enables OPENSSL_DEBUG_MEMORY.
+     */
+    char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
+
+    return mem_debug_env == NULL
+        || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
+}
+#endif
+
+
+static int err_cb(const char *str, size_t len, void *u)
+{
+    return test_puts_stderr(str);
+}
+
+void setup_test()
+{
+    test_open_streams();
+
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+    if (should_report_leaks()) {
+        CRYPTO_set_mem_debug(1);
+        CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
+    }
+#endif
+}
+
+int finish_test(int ret)
+{
+#ifndef OPENSSL_NO_CRYPTO_MDEBUG
+    if (should_report_leaks() && CRYPTO_mem_leaks_cb(err_cb, NULL) <= 0)
+        return EXIT_FAILURE;
+#endif
+
+    test_close_streams();
+
+    return ret;
+}
+
+static void finalize(int success)
+{
+    if (success)
+        ERR_clear_error();
+    else
+        ERR_print_errors_cb(err_cb, NULL);
+}
+
+static void helper_printf_stdout(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    test_vprintf_stdout(fmt, ap);
+    va_end(ap);
+}
+
+int run_tests(const char *test_prog_name)
+{
+    int num_failed = 0;
+    int i, j;
+
+    helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases,
+                         num_test_cases == 1 ? "" : "s");
+    test_flush_stdout();
+
+    for (i = 0; i != num_tests; ++i) {
+        if (all_tests[i].num == -1) {
+            int ret = all_tests[i].test_fn();
+
+            if (!ret) {
+                helper_printf_stdout("** %s failed **\n--------\n",
+                                     all_tests[i].test_case_name);
+                test_flush_stdout();
+                ++num_failed;
+            }
+            finalize(ret);
+        } else {
+            for (j = 0; j < all_tests[i].num; j++) {
+                int ret = all_tests[i].param_test_fn(j);
+
+                if (!ret) {
+                    helper_printf_stdout("** %s failed test %d\n--------\n",
+                                         all_tests[i].test_case_name, j);
+                    test_flush_stdout();
+                    ++num_failed;
+                }
+                finalize(ret);
+            }
+        }
+    }
+
+    if (num_failed != 0) {
+        helper_printf_stdout("%s: %d test%s failed (out of %d)\n",
+                             test_prog_name, num_failed,
+                             num_failed != 1 ? "s" : "", num_test_cases);
+        test_flush_stdout();
+        return EXIT_FAILURE;
+    }
+    helper_printf_stdout("  All tests passed.\n");
+    test_flush_stdout();
+    return EXIT_SUCCESS;
+}
+

test/test_main_custom.c → test/testutil/main.c

 /*
- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -7,11 +7,7 @@
  * https://www.openssl.org/source/license.html
  */
 
-#include "test_main_custom.h"
-#include "testutil.h"
-
-#include <stdio.h>
-#include <stdlib.h>
+#include "../testutil.h"
 
 int main(int argc, char *argv[])
 {

test/test_main.c → test/testutil/test_main.c

 /*
- * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -7,20 +7,15 @@
  * https://www.openssl.org/source/license.html
  */
 
-#include "test_main.h"
-#include "testutil.h"
+#include "../testutil.h"
 
 #include <stdio.h>
 
-int main(int argc, char *argv[])
+int test_main(int argc, char *argv[])
 {
-    int ret;
     if (argc > 1)
-        printf("Warning: ignoring extra command-line arguments.\n");
+        test_puts_stderr("Warning: ignoring extra command-line arguments.\n");
 
-    setup_test();
     register_tests();
-    ret = run_tests(argv[0]);
-
-    return finish_test(ret);
+    return run_tests(argv[0]);
 }

test/testutil.c → test/testutil/tests.c

 /*
- * Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -7,152 +7,14 @@
  * https://www.openssl.org/source/license.html
  */
 
-#include "testutil.h"
+#include "../testutil.h"
 
-#include <assert.h>
-#include <stdlib.h>
-#include <stdio.h>
 #include <string.h>
-#include "e_os.h"
-
-#include <openssl/opensslconf.h>
-#include <openssl/crypto.h>
-#include <openssl/err.h>
+#include "../../e_os.h"
 
 /* The size of memory buffers to display on failure */
 #define MEM_BUFFER_SIZE     (21)
 
-/*
- * Declares the structures needed to register each test case function.
- */
-typedef struct test_info {
-    const char *test_case_name;
-    int (*test_fn) ();
-    int (*param_test_fn)(int idx);
-    int num;
-} TEST_INFO;
-
-static TEST_INFO all_tests[1024];
-static int num_tests = 0;
-/*
- * A parameterised tests runs a loop of test cases.
- * |num_test_cases| counts the total number of test cases
- * across all tests.
- */
-static int num_test_cases = 0;
-
-void add_test(const char *test_case_name, int (*test_fn) ())
-{
-    assert(num_tests != OSSL_NELEM(all_tests));
-    all_tests[num_tests].test_case_name = test_case_name;
-    all_tests[num_tests].test_fn = test_fn;
-    all_tests[num_tests].num = -1;
-    ++num_test_cases;
-    ++num_tests;
-}
-
-void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
-                   int num)
-{
-    assert(num_tests != OSSL_NELEM(all_tests));
-    all_tests[num_tests].test_case_name = test_case_name;
-    all_tests[num_tests].param_test_fn = test_fn;
-    all_tests[num_tests].num = num;
-    ++num_tests;
-    num_test_cases += num;
-}
-
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-static int should_report_leaks()
-{
-    /*
-     * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0
-     * can be used to disable leak checking at runtime.
-     * Note this only works when running the test binary manually;
-     * the test harness always enables OPENSSL_DEBUG_MEMORY.
-     */
-    char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY");
-
-    return mem_debug_env == NULL
-        || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, ""));
-}
-#endif
-
-
-void setup_test()
-{
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (should_report_leaks()) {
-        CRYPTO_set_mem_debug(1);
-        CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON);
-    }
-#endif
-}
-
-int finish_test(int ret)
-{
-#ifndef OPENSSL_NO_CRYPTO_MDEBUG
-    if (should_report_leaks() && CRYPTO_mem_leaks_fp(stderr) <= 0)
-        return EXIT_FAILURE;
-#endif
-    return ret;
-}
-
-static void finalize(int success)
-{
-    if (success)
-        ERR_clear_error();
-    else
-        ERR_print_errors_fp(stderr);
-}
-
-int run_tests(const char *test_prog_name)
-{
-    int num_failed = 0;
-
-    int i, j;
-
-    printf("%s: %d test case%s\n", test_prog_name, num_test_cases,
-           num_test_cases == 1 ? "" : "s");
-    fflush(stdout);
-
-    for (i = 0; i != num_tests; ++i) {
-        if (all_tests[i].num == -1) {
-            int ret = all_tests[i].test_fn();
-
-            if (!ret) {
-                printf("** %s failed **\n--------\n",
-                       all_tests[i].test_case_name);
-                fflush(stdout);
-                ++num_failed;
-            }
-            finalize(ret);
-        } else {
-            for (j = 0; j < all_tests[i].num; j++) {
-                int ret = all_tests[i].param_test_fn(j);
-
-                if (!ret) {
-                    printf("** %s failed test %d\n--------\n",
-                           all_tests[i].test_case_name, j);
-                    fflush(stdout);
-                    ++num_failed;
-                }
-                finalize(ret);
-            }
-        }
-    }
-
-    if (num_failed != 0) {
-        printf("%s: %d test%s failed (out of %d)\n", test_prog_name,
-               num_failed, num_failed != 1 ? "s" : "", num_test_cases);
-        fflush(stdout);
-        return EXIT_FAILURE;
-    }
-    printf("  All tests passed.\n");
-    fflush(stdout);
-    return EXIT_SUCCESS;
-}
-
 /*
  * A common routine to output test failure messages.  Generally this should not
  * be called directly, rather it should be called by the following functions.
@@ -182,27 +44,38 @@ static void test_fail_message(const char *prefix, const char *file, int line,
                               const char *type, const char *fmt, ...)
             PRINTF_FORMAT(5, 6);
 
+static void helper_printf_stderr(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    test_vprintf_stderr(fmt, ap);
+    va_end(ap);
+}
+
 static void test_fail_message_va(const char *prefix, const char *file, int line,
                                  const char *type, const char *fmt, va_list ap)
 {
-    fputs(prefix != NULL ? prefix : "ERROR", stderr);
-    fputs(":", stderr);
+    test_puts_stderr(prefix != NULL ? prefix : "ERROR");
+    test_puts_stderr(":");
     if (type)
-        fprintf(stderr, " (%s)", type);
+        helper_printf_stderr(" (%s)", type);
     if (fmt != NULL) {
-        fputc(' ', stderr);
-        vfprintf(stderr, fmt, ap);
+        test_puts_stderr(" ");
+        test_vprintf_stderr(fmt, ap);
     }
     if (file != NULL) {
-        fprintf(stderr, " @ %s:%d", file, line);
+        helper_printf_stderr(" @ %s:%d", file, line);
     }
-    fputc('\n', stderr);
+    test_puts_stderr("\n");
+    test_flush_stderr();
 }
 
 static void test_fail_message(const char *prefix, const char *file, int line,
                               const char *type, const char *fmt, ...)
 {
     va_list ap;
+
     va_start(ap, fmt);
     test_fail_message_va(prefix, file, line, type, fmt, ap);
     va_end(ap);
@@ -293,7 +166,7 @@ DEFINE_COMPARISONS(char, char, "%c")
 DEFINE_COMPARISONS(unsigned char, uchar, "%u")
 DEFINE_COMPARISONS(long, long, "%ld")
 DEFINE_COMPARISONS(unsigned long, ulong, "%lu")
-DEFINE_COMPARISONS(size_t, size_t, "%" OSSLzu)
+DEFINE_COMPARISONS(size_t, size_t, "%zu")
 
 DEFINE_COMPARISON(void *, ptr, eq, ==, "%p")
 DEFINE_COMPARISON(void *, ptr, ne, !=, "%p")
@@ -434,14 +307,14 @@ int test_mem_eq(const char *file, int line, const char *st1, const char *st2,
         return 1;
     if (n1 != n2) {
         test_fail_message(NULL, file, line, "memory",
-                          "size mismatch %s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
+                          "size mismatch %s %s [%zu] != %s %s [%zu]",
                           st1, print_mem_maybe_null(s1, n1, b1), n1,
                           st2, print_mem_maybe_null(s2, n2, b2), n2);
         return 0;
     }
     if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) {
         test_fail_message(NULL, file, line, "memory",
-                          "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
+                          "%s %s [%zu] != %s %s [%zu]",
                           st1, print_mem_maybe_null(s1, n1, b1), n1,
                           st2, print_mem_maybe_null(s2, n2, b2), n2);
         return 0;
@@ -460,7 +333,7 @@ int test_mem_ne(const char *file, int line, const char *st1, const char *st2,
         return 1;
     if (s1 == NULL || memcmp(s1, s2, n1) == 0) {
         test_fail_message(NULL, file, line, "memory",
-                          "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]",
+                          "%s %s [%zu] != %s %s [%zu]",
                           st1, print_mem_maybe_null(s1, n1, b1), n1,
                           st2, print_mem_maybe_null(s2, n2, b2), n2);
         return 0;