trace: apps/openssl: print the correct category name

Previously, if the openssl application was run with OPENSSL_TRACE=any,
all trace output would just show 'ANY' as the category name, which was
not very useful. To get the correct category name printed in the trace
output, the openssl application now registers separate channels for
each category.

The trace API is unchanged, it is still possible for an application to
register a single channel for the 'ANY' category to see all outputt,
if it does not need this level of detail.
Reviewed-by: NPaul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8552)

apps/openssl.c

@@ -185,6 +185,33 @@ static void cleanup_trace(void)
     sk_tracedata_pop_free(trace_data_stack, tracedata_free);
 }
 
+static void setup_trace_category(int category)
+{
+    BIO *channel;
+    tracedata *trace_data;
+
+    if (OSSL_trace_enabled(category))
+        return;
+
+    channel = BIO_push(BIO_new(apps_bf_prefix()),
+                       dup_bio_err(FORMAT_TEXT));
+    trace_data = OPENSSL_zalloc(sizeof(*trace_data));
+
+    if (trace_data == NULL
+        || (trace_data->bio = channel) == NULL
+        || OSSL_trace_set_callback(category, internal_trace_cb,
+                                   trace_data) == 0
+        || sk_tracedata_push(trace_data_stack, trace_data) == 0) {
+
+        fprintf(stderr,
+                "warning: unable to setup trace callback for category '%s'.\n",
+                OSSL_trace_get_category_name(category));
+
+        OSSL_trace_set_callback(category, NULL, NULL);
+        BIO_free_all(channel);
+    }
+}
+
 static void setup_trace(const char *str)
 {
     char *val;
@@ -199,26 +226,15 @@ static void setup_trace(const char *str)
         for (valp = val; (item = strtok(valp, ",")) != NULL; valp = NULL) {
             int category = OSSL_trace_get_category_num(item);
 
-            if (category >= 0) {
-                BIO *channel = BIO_push(BIO_new(apps_bf_prefix()),
-                                        dup_bio_err(FORMAT_TEXT));
-                tracedata *trace_data = OPENSSL_zalloc(sizeof(*trace_data));
-
-                if (trace_data == NULL
-                    || (trace_data->bio = channel) == NULL
-                    || OSSL_trace_set_callback(category, internal_trace_cb,
-                                               trace_data) == 0
-                    || sk_tracedata_push(trace_data_stack, trace_data) == 0) {
-                    OSSL_trace_set_callback(category, NULL, NULL);
-                    BIO_free_all(channel);
-                    fprintf(stderr,
-                            "warning: unable to setup trace callback for category '%s'.\n",
-                            item);
-                }
+            if (category == OSSL_TRACE_CATEGORY_ANY) {
+                while (++category < OSSL_TRACE_CATEGORY_NUM)
+                    setup_trace_category(category);
+                break;
+            } else if (category > 0) {
+                setup_trace_category(category);
             } else {
                 fprintf(stderr,
-                        "warning: unknown trace category: '%s'.\n",
-                        item);
+                        "warning: unknown trace category: '%s'.\n", item);
             }
         }
     }

crypto/trace.c

@@ -330,7 +330,7 @@ int OSSL_trace_set_channel(int category, BIO *channel)
 #ifndef OPENSSL_NO_TRACE
     if (category >= 0 && category < OSSL_TRACE_CATEGORY_NUM)
         return set_trace_data(category, SIMPLE_CHANNEL, &channel, NULL, NULL,
-                              trace_attach_cb, trace_detach_cb))
+                              trace_attach_cb, trace_detach_cb);
 #endif
     return 0;
 }

doc/man3/OSSL_trace_set_channel.pod

@@ -181,6 +181,12 @@ Traces BIGNUM context operations.
 There is also C<OSSL_TRACE_CATEGORY_ANY>, which works as a fallback
 and can be used to get I<all> trace output.
 
+Note, however, that in this case all trace output will effectively be
+associated with the 'ALL' category, which is undesirable if the
+application intends to include the category name in the trace output.
+In this case it is better to register separate channels for each
+trace category instead.
+
 =head1 RETURN VALUES
 
 OSSL_trace_set_channel(), OSSL_trace_set_prefix(),

include/openssl/trace.h

@@ -30,7 +30,11 @@ extern "C" {
  * BIO which sends all trace output it receives to the registered application
  * callback.
  *
- * The ANY category is used as a fallback category.
+ * The ANY category can be used as a fallback category to register a single
+ * channel which receives the output from all categories. However, if the
+ * application intends to print the trace channel name in the line prefix,
+ * it is better to register channels for all categories separately.
+ * (This is how the openssl application does it.)
  */
 # define OSSL_TRACE_CATEGORY_ANY                 0 /* The fallback */
 # define OSSL_TRACE_CATEGORY_TRACE               1