TAP line filter BIO.

This is an implementation of a BIO filter that produce TAP compatible output
for the test framework.  The current test indentation level is honoured.

The test output functions have been modified to not attempt to indent
their output and to not include the leading '#' character.

The filter is applied to bio_err only.  bio_out is left unchanged, although
tests using bio_out have been modified to use bio_err instead.
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3732)

test/asn1_encode_test.c

@@ -717,7 +717,7 @@ static int do_print_item(const TEST_PACKAGE *package)
     OPENSSL_assert(package->encode_expectations_elem_size <= DATA_BUF_SIZE);
 
     (void)RAND_bytes(buf, (int)package->encode_expectations_elem_size);
-    ret = ASN1_item_print(bio_out, o, 0, i, NULL);
+    ret = ASN1_item_print(bio_err, o, 0, i, NULL);
 
     return ret;
 }

test/build.info

@@ -11,7 +11,7 @@ IF[{- !$disabled{tests} -}]
   LIBS_NO_INST=libtestutil.a
   SOURCE[libtestutil.a]=testutil/basic_output.c testutil/output_helpers.c \
           testutil/driver.c testutil/tests.c testutil/cb.c testutil/stanza.c \
-          testutil/format_output.c \
+          testutil/format_output.c testutil/tap_bio.c \
           {- rebase_files("../apps", $target{apps_aux_src}) -} \
           testutil/test_main.c testutil/main.c
   INCLUDE[libtestutil.a]=.. ../include

test/ecdsatest.c

@@ -14,13 +14,7 @@
 
 #include <openssl/opensslconf.h> /* To see if OPENSSL_NO_EC is defined */
 
-#ifdef OPENSSL_NO_EC
-int main(int argc, char *argv[])
-{
-    puts("Elliptic curves are disabled.");
-    return 0;
-}
-#else
+#ifndef OPENSSL_NO_EC
 
 # include <openssl/crypto.h>
 # include <openssl/bio.h>
@@ -403,9 +397,13 @@ static int test_builtin(void)
 
 void register_tests(void)
 {
+#ifdef OPENSSL_NO_EC
+    TEST_note("Elliptic curves are disabled.");
+#else
     /* initialize the prng */
     RAND_seed(rnd_seed, sizeof(rnd_seed));
     ADD_TEST(x9_62_tests);
     ADD_TEST(test_builtin);
+#endif
 }
 #endif

test/testutil/basic_output.c

@@ -9,6 +9,7 @@
 
 #include "../testutil.h"
 #include "output.h"
+#include "tu_local.h"
 
 #include <openssl/crypto.h>
 #include <openssl/bio.h>
@@ -20,6 +21,7 @@ void test_open_streams(void)
 {
     bio_out = BIO_new_fp(stdout, BIO_NOCLOSE | BIO_FP_TEXT);
     bio_err = BIO_new_fp(stderr, BIO_NOCLOSE | BIO_FP_TEXT);
+    bio_err = BIO_push(BIO_new(BIO_f_tap()), bio_err);
 
     OPENSSL_assert(bio_out != NULL);
     OPENSSL_assert(bio_err != NULL);

test/testutil/cb.c

@@ -12,5 +12,5 @@
 
 int openssl_error_cb(const char *str, size_t len, void *u)
 {
-    return test_printf_stderr("%*s# %s", subtest_level(), "", str);
+    return test_printf_stderr("%s", str);
 }

test/testutil/format_output.c

+/*
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
 #include "../testutil.h"
 #include "output.h"
 #include "tu_local.h"
@@ -14,17 +23,17 @@
 /* Output a diff header */
 static void test_diff_header(const char *left, const char *right)
 {
-    test_printf_stderr("%*s# --- %s\n", subtest_level(), "", left);
-    test_printf_stderr("%*s# +++ %s\n", subtest_level(), "", right);
+    test_printf_stderr("--- %s\n", left);
+    test_printf_stderr("+++ %s\n", right);
 }
 
 /* Formatted string output routines */
 static void test_string_null_empty(const char *m, char c)
 {
     if (m == NULL)
-        test_printf_stderr("%*s# % 4s %c NULL\n", subtest_level(), "", "", c);
+        test_printf_stderr("% 4s %c NULL\n", "", c);
     else
-        test_printf_stderr("%*s# % 4u:%c ''\n", subtest_level(), "", 0u, c);
+        test_printf_stderr("% 4u:%c ''\n", 0u, c);
 }
 
 static void test_fail_string_common(const char *prefix, const char *file,
@@ -33,8 +42,7 @@ static void test_fail_string_common(const char *prefix, const char *file,
                                     const char *op, const char *m1, size_t l1,
                                     const char *m2, size_t l2)
 {
-    const int indent = subtest_level();
-    const size_t width = (MAX_STRING_WIDTH - indent - 12) / 16 * 16;
+    const size_t width = (MAX_STRING_WIDTH - subtest_level() - 12) / 16 * 16;
     char b1[MAX_STRING_WIDTH + 1], b2[MAX_STRING_WIDTH + 1];
     char bdiff[MAX_STRING_WIDTH + 1];
     size_t n1, n2, i;
@@ -86,19 +94,18 @@ static void test_fail_string_common(const char *prefix, const char *file,
             bdiff[i] = '\0';
         }
         if (n1 == n2 && !diff) {
-            test_printf_stderr("%*s# % 4u:  '%s'\n", indent, "", cnt,
-                               n2 > n1 ? b2 : b1);
+            test_printf_stderr("% 4u:  '%s'\n", cnt, n2 > n1 ? b2 : b1);
         } else {
             if (cnt == 0 && (m1 == NULL || *m1 == '\0'))
                 test_string_null_empty(m1, '-');
             else if (n1 > 0)
-                test_printf_stderr("%*s# % 4u:- '%s'\n", indent, "", cnt, b1);
+                test_printf_stderr("% 4u:- '%s'\n", cnt, b1);
             if (cnt == 0 && (m2 == NULL || *m2 == '\0'))
                test_string_null_empty(m2, '+');
             else if (n2 > 0)
-                test_printf_stderr("%*s# % 4u:+ '%s'\n", indent, "", cnt, b2);
+                test_printf_stderr("% 4u:+ '%s'\n", cnt, b2);
             if (diff && i > 0)
-                test_printf_stderr("%*s# % 4s    %s\n", indent, "", "", bdiff);
+                test_printf_stderr("% 4s    %s\n", "", bdiff);
         }
         m1 += n1;
         m2 += n2;
@@ -168,8 +175,7 @@ static const int bn_chars = (MAX_STRING_WIDTH - 9) / (BN_OUTPUT_SIZE * 2 + 1)
  */
 static void test_bignum_header_line(void)
 {
-    test_printf_stderr("%*s#  %*s\n", subtest_level(), "", bn_chars + 6,
-                       "bit position");
+    test_printf_stderr(" %*s\n", bn_chars + 6, "bit position");
 }
 
 static const char *test_bignum_zero_null(const BIGNUM *bn)
@@ -188,8 +194,7 @@ static void test_bignum_zero_print(const BIGNUM *bn, char sep)
     const char *v = test_bignum_zero_null(bn);
     const char *suf = bn != NULL ? ":    0" : "";
 
-    test_printf_stderr("%*s# %c%*s%s\n", subtest_level(), "", sep, bn_chars,
-                       v, suf);
+    test_printf_stderr("%c%*s%s\n", sep, bn_chars, v, suf);
 }
 
 /*
@@ -260,7 +265,6 @@ static void test_fail_bignum_common(const char *prefix, const char *file,
                                     const char *op,
                                     const BIGNUM *bn1, const BIGNUM *bn2)
 {
-    const int indent = subtest_level();
     const size_t bytes = bn_bytes;
     char b1[MAX_STRING_WIDTH + 1], b2[MAX_STRING_WIDTH + 1];
     char *p, bdiff[MAX_STRING_WIDTH + 1];
@@ -295,8 +299,7 @@ static void test_fail_bignum_common(const char *prefix, const char *file,
     if (len > MEM_BUFFER_SIZE && (bufp = OPENSSL_malloc(len * 2)) == NULL) {
         bufp = buffer;
         len = MEM_BUFFER_SIZE;
-        test_printf_stderr("%*s# WARNING: these BIGNUMs have been truncated",
-                           indent, "");
+        test_printf_stderr("WARNING: these BIGNUMs have been truncated");
     }
 
     if (bn1 != NULL) {
@@ -326,20 +329,19 @@ static void test_fail_bignum_common(const char *prefix, const char *file,
             }
         *p++ = '\0';
         if (!diff) {
-            test_printf_stderr("%*s#  %s:% 5d\n", indent, "",
-                               n2 > n1 ? b2 : b1, cnt);
+            test_printf_stderr(" %s:% 5d\n", n2 > n1 ? b2 : b1, cnt);
         } else {
             if (cnt == 0 && bn1 == NULL)
-                test_printf_stderr("%*s# -%s\n", indent, "", b1);
+                test_printf_stderr("-%s\n", b1);
             else if (cnt == 0 || n1 > 0)
-                test_printf_stderr("%*s# -%s:% 5d\n", indent, "", b1, cnt);
+                test_printf_stderr("-%s:% 5d\n", b1, cnt);
             if (cnt == 0 && bn2 == NULL)
-                test_printf_stderr("%*s# +%s\n", indent, "", b2);
+                test_printf_stderr("+%s\n", b2);
             else if (cnt == 0 || n2 > 0)
-                test_printf_stderr("%*s# +%s:% 5d\n", indent, "", b2, cnt);
+                test_printf_stderr("+%s:% 5d\n", b2, cnt);
             if (real_diff && (cnt == 0 || (n1 > 0 && n2 > 0))
                     && bn1 != NULL && bn2 != NULL)
-                test_printf_stderr("%*s#  %s\n", indent, "", bdiff);
+                test_printf_stderr(" %s\n", bdiff);
         }
         if (m1 != NULL)
             m1 += bytes;
@@ -380,8 +382,8 @@ void test_fail_bignum_mono_message(const char *prefix, const char *file,
 void test_output_bignum(const char *name, const BIGNUM *bn)
 {
     if (bn == NULL || BN_is_zero(bn)) {
-        test_printf_stderr("%*s# bignum: '%s' = %s\n", subtest_level(), "",
-                           name, test_bignum_zero_null(bn));
+        test_printf_stderr("bignum: '%s' = %s", name,
+                           test_bignum_zero_null(bn));
     } else if (BN_num_bytes(bn) <= BN_OUTPUT_SIZE) {
         unsigned char buf[BN_OUTPUT_SIZE];
         char out[2 * sizeof(buf) + 1];
@@ -391,8 +393,8 @@ void test_output_bignum(const char *name, const BIGNUM *bn)
         hex_convert_memory(buf, n, p, BN_OUTPUT_SIZE);
         while (*p == '0' && *++p != '\0')
             ;
-        test_printf_stderr("%*s# bignum: '%s' = %s0x%s\n", subtest_level(), "",
-                           name, BN_is_negative(bn) ? "-" : "", p);
+        test_printf_stderr("bignum: '%s' = %s0x%s\n", name,
+                           BN_is_negative(bn) ? "-" : "", p);
     } else {
         test_fail_bignum_common("bignum", NULL, 0, NULL, NULL, NULL, name,
                                 bn, bn);
@@ -404,12 +406,12 @@ void test_output_bignum(const char *name, const BIGNUM *bn)
 /*
  * Handle zero length blocks of memory or NULL pointers to memory
  */
-static void test_memory_null_empty(const unsigned char *m, int indent, char c)
+static void test_memory_null_empty(const unsigned char *m, char c)
 {
     if (m == NULL)
-        test_printf_stderr("%*s# % 4s %c%s\n", indent, "", "", c, "NULL");
+        test_printf_stderr("% 4s %c%s\n", "", c, "NULL");
     else
-        test_printf_stderr("%*s# %04x %c%s\n", indent, "", 0u, c, "empty");
+        test_printf_stderr("%04x %c%s\n", 0u, c, "empty");
 }
 
 /*
@@ -422,7 +424,6 @@ static void test_fail_memory_common(const char *prefix, const char *file,
                                     const unsigned char *m1, size_t l1,
                                     const unsigned char *m2, size_t l2)
 {
-    const int indent = subtest_level();
     const size_t bytes = (MAX_STRING_WIDTH - 9) / 17 * 8;
     char b1[MAX_STRING_WIDTH + 1], b2[MAX_STRING_WIDTH + 1];
     char *p, bdiff[MAX_STRING_WIDTH + 1];
@@ -436,11 +437,11 @@ static void test_fail_memory_common(const char *prefix, const char *file,
         l2 = 0;
     if (l1 == 0 && l2 == 0) {
         if ((m1 == NULL) == (m2 == NULL)) {
-            test_memory_null_empty(m1, indent, ' ');
+            test_memory_null_empty(m1, ' ');
         } else {
             test_diff_header(left, right);
-            test_memory_null_empty(m1, indent, '-');
-            test_memory_null_empty(m2, indent, '+');
+            test_memory_null_empty(m1, '-');
+            test_memory_null_empty(m2, '+');
         }
         goto fin;
     }
@@ -481,18 +482,18 @@ static void test_fail_memory_common(const char *prefix, const char *file,
         }
 
         if (n1 == n2 && !diff) {
-            test_printf_stderr("%*s# %04x: %s\n", indent, "", cnt, b1);
+            test_printf_stderr("%04x: %s\n", cnt, b1);
         } else {
             if (cnt == 0 && (m1 == NULL || l1 == 0))
-                test_memory_null_empty(m1, indent, '-');
+                test_memory_null_empty(m1, '-');
             else if (n1 > 0)
-                test_printf_stderr("%*s# %04x:-%s\n", indent, "", cnt, b1);
+                test_printf_stderr("%04x:-%s\n", cnt, b1);
             if (cnt == 0 && (m2 == NULL || l2 == 0))
-                test_memory_null_empty(m2, indent, '+');
+                test_memory_null_empty(m2, '+');
             else if (n2 > 0)
-                test_printf_stderr("%*s# %04x:+%s\n", indent, "", cnt, b2);
+                test_printf_stderr("%04x:+%s\n", cnt, b2);
             if (diff && i > 0)
-                test_printf_stderr("%*s# % 4s  %s\n", indent, "", "", bdiff);
+                test_printf_stderr("% 4s  %s\n", "", bdiff);
         }
         m1 += n1;
         m2 += n2;

test/testutil/tap_bio.c

+/*
+ * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright (c) 2017, Oracle and/or its affiliates.  All rights reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <string.h>
+#include "tu_local.h"
+
+static int tap_write_ex(BIO *b, const char *buf, size_t size, size_t *in_size);
+static int tap_read_ex(BIO *b, char *buf, size_t size, size_t *out_size);
+static int tap_puts(BIO *b, const char *str);
+static int tap_gets(BIO *b, char *str, int size);
+static long tap_ctrl(BIO *b, int cmd, long arg1, void *arg2);
+static int tap_new(BIO *b);
+static int tap_free(BIO *b);
+static long tap_callback_ctrl(BIO *h, int cmd, bio_info_cb *fp);
+
+const BIO_METHOD *BIO_f_tap(void)
+{
+    static BIO_METHOD *tap = NULL;
+
+    if (tap == NULL) {
+        tap = BIO_meth_new(BIO_TYPE_START | BIO_TYPE_FILTER, "tap");
+        if (tap != NULL) {
+            BIO_meth_set_write_ex(tap, tap_write_ex);
+            BIO_meth_set_read_ex(tap, tap_read_ex);
+            BIO_meth_set_puts(tap, tap_puts);
+            BIO_meth_set_gets(tap, tap_gets);
+            BIO_meth_set_ctrl(tap, tap_ctrl);
+            BIO_meth_set_create(tap, tap_new);
+            BIO_meth_set_destroy(tap, tap_free);
+            BIO_meth_set_callback_ctrl(tap, tap_callback_ctrl);
+        }
+    }
+    return tap;
+}
+
+static int tap_new(BIO *b)
+{
+    BIO_set_data(b, NULL);
+    BIO_set_init(b, 1);
+    return 1;
+}
+
+static int tap_free(BIO *b)
+{
+    if (b == NULL)
+        return 0;
+    BIO_set_data(b, NULL);
+    BIO_set_init(b, 0);
+    return 1;
+}
+
+static int tap_read_ex(BIO *b, char *buf, size_t size, size_t *out_size)
+{
+    BIO *next = BIO_next(b);
+    int ret = 0;
+
+    ret = BIO_read_ex(next, buf, size, out_size);
+    BIO_clear_retry_flags(b);
+    BIO_copy_next_retry(b);
+    return ret;
+}
+
+/*
+ * Output a string to the specified bio and return 1 if successful.
+ */
+static int write_string(BIO *b, const char *buf, size_t n)
+{
+    size_t m;
+
+    return BIO_write_ex(b, buf, n, &m) != 0 && m == n;
+}
+
+/*
+ * Write some data.
+ *
+ * This function implements a simple state machine that detects new lines.
+ * It indents the output and prefixes it with a '#' character.
+ *
+ * It returns the number of input characters that were output in in_size.
+ * More characters than this will likely have been output however any calling
+ * code will be unable to correctly assess the actual number of characters
+ * emitted and would be prone to failure if the actual number were returned.
+ *
+ * The BIO_data field is used as our state.  If it is NULL, we've just
+ * seen a new line.  If it is not NULL, we're processing characters in a line.
+ */
+static int tap_write_ex(BIO *b, const char *buf, size_t size, size_t *in_size)
+{
+    BIO *next = BIO_next(b);
+    size_t i;
+    int j;
+
+    for (i = 0; i < size; i++) {
+        if (BIO_get_data(b) == NULL) {
+            BIO_set_data(b, "");
+            for (j = 0; j < subtest_level(); j++)
+                if (!write_string(next, " ", 1))
+                    goto err;
+            if (!write_string(next, "# ", 2))
+                goto err;
+        }
+        if (!write_string(next, buf + i, 1))
+            goto err;
+        if (buf[i] == '\n')
+            BIO_set_data(b, NULL);
+    }
+    *in_size = i;
+    return 1;
+
+err:
+    *in_size = i;
+    return 0;
+}
+
+static long tap_ctrl(BIO *b, int cmd, long num, void *ptr)
+{
+    BIO *next = BIO_next(b);
+ 
+    switch (cmd) {
+    case BIO_CTRL_RESET:
+        BIO_set_data(b, NULL);
+        break;
+
+    default:
+        break;
+    }
+    return BIO_ctrl(next, cmd, num, ptr);
+}
+
+static long tap_callback_ctrl(BIO *b, int cmd, bio_info_cb *fp)
+{
+    return BIO_callback_ctrl(BIO_next(b), cmd, fp);
+}
+
+static int tap_gets(BIO *b, char *buf, int size)
+{
+    return BIO_gets(BIO_next(b), buf, size);
+}
+
+static int tap_puts(BIO *b, const char *str)
+{
+    size_t m;
+
+    if (!tap_write_ex(b, str, strlen(str), &m))
+        return 0;
+    return m;
+}

test/testutil/tests.c

@@ -27,8 +27,7 @@ void test_fail_message_prefix(const char *prefix, const char *file,
                               const char *left, const char *right,
                               const char *op)
 {
-    test_printf_stderr("%*s# %s: ", subtest_level(), "",
-                       prefix != NULL ? prefix : "ERROR");
+    test_printf_stderr("%s: ", prefix != NULL ? prefix : "ERROR");
     if (type)
         test_printf_stderr("(%s) ", type);
     if (op != NULL) {
@@ -79,7 +78,6 @@ static void test_fail_message_va(const char *prefix, const char *file,
 {
     test_fail_message_prefix(prefix, file, line, type, left, right, op);
     if (fmt != NULL) {
-        test_printf_stderr("%*s# ", subtest_level(), "");
         test_vprintf_stderr(fmt, ap);
         test_printf_stderr("\n");
     }
@@ -150,7 +148,6 @@ void test_note(const char *fmt, ...)
     if (fmt != NULL) {
         va_list ap;
 
-        test_printf_stderr("%*s# ", subtest_level(), "");
         va_start(ap, fmt);
         test_vprintf_stderr(fmt, ap);
         va_end(ap);

test/testutil/tu_local.h

@@ -9,9 +9,11 @@
 
 #include <stdlib.h>              /* size_t */
 #include <openssl/bn.h>
+#include <openssl/bio.h>
 
 int subtest_level(void);
 int openssl_error_cb(const char *str, size_t len, void *u);
+const BIO_METHOD *BIO_f_tap(void);
 
 void test_fail_message_prefix(const char *prefix, const char *file,
                               int line, const char *type,