New SSL test framework

Currently, SSL tests are configured via command-line switches to
ssltest.c. This results in a lot of duplication between ssltest.c and
apps, and a complex setup. ssltest.c is also simply old and needs
maintenance.

Instead, we already have a way to configure SSL servers and clients, so
we leverage that. SSL tests can now be configured from a configuration
file. Test servers and clients are configured using the standard
ssl_conf module. Additional test settings are configured via a test
configuration.

Moreover, since the CONF language involves unnecessary boilerplate, the
test conf itself is generated from a shorter Perl syntax.

The generated testcase files are checked in to the repo to make
it easier to verify that the intended test cases are in fact run; and to
simplify debugging failures.

To demonstrate the approach, min/max protocol tests are converted to the
new format. This change also fixes MinProtocol and MaxProtocol
handling. It was previously requested that an SSL_CTX have both the
server and client flags set for these commands; this clearly can never work.

Guide to this PR:
 - test/ssl_test.c - test framework
 - test/ssl_test_ctx.* - test configuration structure
 - test/handshake_helper.* - new SSL test handshaking code
 - test/ssl-tests/ - test configurations
 - test/generate_ssl_tests.pl - script for generating CONF-style test
   configurations from perl inputs
Reviewed-by: NRichard Levitte <levitte@openssl.org>

Configurations/unix-Makefile.tmpl

@@ -711,7 +711,8 @@ dist:
 
 # Helper targets #####################################################
 
-link-utils: $(BLDDIR)/util/opensslwrap.sh $(BLDDIR)/util/shlib_wrap.sh
+link-utils: $(BLDDIR)/util/opensslwrap.sh $(BLDDIR)/util/shlib_wrap.sh \
+	    $(BLDDIR)/test/generate_ssl_tests.pl
 
 $(BLDDIR)/util/opensslwrap.sh: configdata.pm
 	@if [ "$(SRCDIR)" != "$(BLDDIR)" ]; then \
@@ -723,7 +724,11 @@ $(BLDDIR)/util/shlib_wrap.sh: configdata.pm
 	    mkdir -p "$(BLDDIR)/util"; \
 	    ln -sf "../$(SRCDIR)/util/shlib_wrap.sh" "$(BLDDIR)/util"; \
 	fi
-
+$(BLDDIR)/test/generate_ssl_tests.pl: configdata.pm
+	@if [ "$(SRCDIR)" != "$(BLDDIR)" ]; then \
+	    mkdir -p "$(BLDDIR)/test"; \
+	    ln -sf "../$(SRCDIR)/test/generate_ssl_tests.pl" "$(BLDDIR)/test"; \
+	fi
 FORCE:
 
 # Building targets ###################################################

ssl/ssl_conf.c

@@ -594,8 +594,8 @@ static const ssl_conf_cmd_tbl ssl_conf_cmds[] = {
 #endif
     SSL_CONF_CMD_STRING(CipherString, "cipher", 0),
     SSL_CONF_CMD_STRING(Protocol, NULL, 0),
-    SSL_CONF_CMD_STRING(MinProtocol, "min_protocol", SSL_CONF_FLAG_SERVER | SSL_CONF_FLAG_CLIENT),
-    SSL_CONF_CMD_STRING(MaxProtocol, "max_protocol", SSL_CONF_FLAG_SERVER | SSL_CONF_FLAG_CLIENT),
+    SSL_CONF_CMD_STRING(MinProtocol, "min_protocol", 0),
+    SSL_CONF_CMD_STRING(MaxProtocol, "max_protocol", 0),
     SSL_CONF_CMD_STRING(Options, NULL, 0),
     SSL_CONF_CMD_STRING(VerifyMode, NULL, 0),
     SSL_CONF_CMD(Certificate, "cert", SSL_CONF_FLAG_CERTIFICATE,

test/Makefile.in

@@ -85,6 +85,8 @@ CTTEST=	ct_test
 THREADSTEST=	threadstest
 AFALGTEST= afalgtest
 D2ITEST = d2i_test
+SSLTESTCTXTEST = ssl_test_ctx_test
+NEWSSLTEST = ssl_test
 
 TESTS=		alltests
 
@@ -107,7 +109,8 @@ EXE=	$(NPTEST)$(EXE_EXT) $(MEMLEAKTEST)$(EXE_EXT) \
 	$(CONSTTIMETEST)$(EXE_EXT) $(VERIFYEXTRATEST)$(EXE_EXT) \
 	$(CLIENTHELLOTEST)$(EXE_EXT) $(PACKETTEST)$(EXE_EXT) $(ASYNCTEST)$(EXE_EXT) \
 	$(DTLSV1LISTENTEST)$(EXE_EXT) $(CTTEST)$(EXE_EXT) $(THREADSTEST)$(EXE_EXT) \
-	$(AFALGTEST)$(EXE_EXT) $(D2ITEST)$(EXE_EXT)
+	$(AFALGTEST)$(EXE_EXT) $(D2ITEST)$(EXE_EXT) $(SSLTESTCTXTEST)$(EXE_EXT) \
+	$(NEWSSLTEST)$(EXE_EXT)
 
 # $(METHTEST)$(EXE_EXT)
 
@@ -125,7 +128,8 @@ OBJ=	$(NPTEST).o $(MEMLEAKTEST).o \
 	$(HEARTBEATTEST).o $(P5_CRPT2_TEST).o \
 	$(CONSTTIMETEST).o $(VERIFYEXTRATEST).o $(CLIENTHELLOTEST).o \
 	$(PACKETTEST).o $(ASYNCTEST).o $(DTLSV1LISTENTEST).o $(CTTEST).o \
-	$(THREADSTEST).o testutil.o $(AFALGTEST).o $(D2ITEST).o
+	$(THREADSTEST).o testutil.o $(AFALGTEST).o $(D2ITEST).o ssl_test_ctx.o \
+	$(SSLTESTCTXTEST).o $(NEWSSLTEST).o handshake_helper.o
 
 SRC=	$(NPTEST).c $(MEMLEAKTEST).c \
 	$(BNTEST).c $(ECTEST).c \
@@ -140,9 +144,10 @@ SRC=	$(NPTEST).c $(MEMLEAKTEST).c \
 	$(HEARTBEATTEST).c $(P5_CRPT2_TEST).c \
 	$(CONSTTIMETEST).c $(VERIFYEXTRATEST).c $(CLIENTHELLOTEST).c \
 	$(PACKETTEST).c $(ASYNCTEST).c $(DTLSV1LISTENTEST).c $(CTTEST).c \
-	$(THREADSTEST).c testutil.c $(AFALGTEST).c $(D2ITEST).c
+	$(THREADSTEST).c testutil.c $(AFALGTEST).c $(D2ITEST).c ssl_test_ctx.c \
+	$(SSLTESTCTXTEST).c $(NEWSSLTEST).c handshake_helper.c
 
-HEADER=	testutil.h
+HEADER=	testutil.h ssl_test_ctx.h handshake_helper.h
 
 ALL=	$(GENERAL) $(SRC) $(HEADER)
 
@@ -389,4 +394,17 @@ $(AFALGTEST)$(EXE_EXT): $(AFALGTEST).o $(DLIBCRYPTO)
 $(D2ITEST)$(EXE_EXT): $(D2ITEST).o $(DLIBCRYPTO) testutil.o
 	@target=$(D2ITEST) testutil=testutil.o; $(BUILD_CMD)
 
+$(SSLTESTCTXTEST)$(EXE_EXT): $(SSLTESTCTXTEST).o testutil.o $(DLIBCRYPTO)
+	@target=$(SSLTESTCTXTEST); $(BUILD_CMD)
+
+$(SSLTESTCTXTEST)$(EXE_EXT): $(SSLTESTCTXTEST).o testutil.o ssl_test_ctx.o \
+	$(DLIBCRYPTO)
+	@target=$(SSLTESTCTXTEST) testutil="testutil.o ssl_test_ctx.o"; \
+	$(BUILD_CMD)
+
+$(NEWSSLTEST)$(EXE_EXT): $(NEWSSLTEST).o testutil.o ssl_test_ctx.o \
+	handshake_helper.o $(DLIBSSL) $(DLIBCRYPTO)
+	@target=$(NEWSSLTEST) testutil="testutil.o ssl_test_ctx.o \
+	handshake_helper.o"; $(BUILD_CMD)
+
 # DO NOT DELETE THIS LINE -- make depend depends on it.

test/README.ssltest.md

+# SSL tests
+
+SSL testcases are configured in the `ssl-tests` directory.
+
+Each `ssl_*.conf.in` file contains a number of test configurations. These files
+are used to generate testcases in the OpenSSL CONF format.
+
+The precise test output can be dependent on the library configuration. The test
+harness generates the output files on the fly.
+
+However, for verification, we also include checked-in configuration outputs
+corresponding to the default configuration. These testcases live in
+`test/ssl-tests/*.conf` files. Therefore, whenever you're adding or updating a
+generated test, you should run
+
+```
+$ ./config
+$ cd test
+$ TOP=.. perl -I testlib/ generate_ssl_tests.pl ssl-tests/my.conf.in \
+  > ssl-tests/my.conf
+```
+
+where `my.conf.in` is your test input file.
+
+For example, to generate the test cases in `ssl-tests/01-simple.conf.in`, do
+
+```
+$ TOP=.. perl generate_ssl_tests.pl ssl-tests/01-simple.conf.in > ssl-tests/01-simple.conf
+```
+
+For more details, see `ssl-tests/01-simple.conf.in` for an example.
+
+## Configuring the test
+
+First, give your test a name. The names do not have to be unique.
+
+An example test input looks like this:
+
+```
+    {
+        name => "test-default",
+        server => { "CipherString" => "DEFAULT" },
+        client => { "CipherString" => "DEFAULT" },
+        test   => { "ExpectedResult" => "Success" },
+    }
+```
+
+The test section supports the following options:
+
+* ExpectedResult - expected handshake outcome. One of
+  - Success - handshake success
+  - ServerFail - serverside handshake failure
+  - ClientFail - clientside handshake failure
+  - InternalError - some other error
+
+* ClientAlert, ServerAlert - expected alert. See `ssl_test_ctx.c` for known
+  values.
+
+* Protocol - expected negotiated protocol. One of
+  SSLv3, TLSv1, TLSv1.1, TLSv1.2.
+
+## Configuring the client and server
+
+The client and server configurations can be any valid `SSL_CTX`
+configurations. For details, see the manpages for `SSL_CONF_cmd`.
+
+Give your configurations as a dictionary of CONF commands, e.g.
+
+```
+server => {
+    "CipherString" => "DEFAULT",
+    "MinProtocol" => "TLSv1",
+}
+```
+
+### Default server and client configurations
+
+The default server certificate and CA files are added to the configurations
+automatically. Server certificate verification is requested by default.
+
+You can override these options by redefining them:
+
+```
+client => {
+    "VerifyCAFile" => "/path/to/custom/file"
+}
+```
+
+or by deleting them
+
+```
+client => {
+    "VerifyCAFile" => undef
+}
+```
+
+## Adding a test to the test harness
+
+Add your configuration file to `test/recipes/80-test_ssl_new.t`.
+
+## Running the tests with the test harness
+
+```
+HARNESS_VERBOSE=yes make TESTS=test_ssl_new test
+```
+
+## Running a test manually
+
+These steps are only needed during development. End users should run `make test`
+or follow the instructions above to run the SSL test suite.
+
+To run an SSL test manually from the command line, the `TEST_CERTS_DIR`
+environment variable to point to the location of the certs. E.g., from the root
+OpenSSL directory, do
+
+```
+$ TEST_CERTS_DIR=test/certs test/ssl_test test/ssl-tests/01-simple.conf
+```
+
+or for shared builds
+
+```
+$ TEST_CERTS_DIR=test/certs util/shlib_wrap.sh test/ssl_test \
+  test/ssl-tests/01-simple.conf
+```
+
+Note that the test expectations sometimes depend on the Configure settings. For
+example, the negotiated protocol depends on the set of available (enabled)
+protocols: a build with `enable-ssl3` has different test expectations than a
+build with `no-ssl3`.
+
+The Perl test harness automatically generates expected outputs, so users who
+just run `make test` do not need any extra steps.
+
+However, when running a test manually, keep in mind that the repository version
+of the generated `test/ssl-tests/*.conf` correspond to expected outputs in with
+the default Configure options. To run `ssl_test` manually from the command line
+in a build with a different configuration, you may need to generate the right
+`*.conf` file from the `*.conf.in` input first.

test/build.info

@@ -14,7 +14,8 @@ PROGRAMS=\
         danetest heartbeat_test p5_crpt2_test \
         constant_time_test verify_extra_test clienthellotest \
         packettest asynctest secmemtest srptest memleaktest \
-        dtlsv1listentest ct_test threadstest afalgtest d2i_test
+        dtlsv1listentest ct_test threadstest afalgtest d2i_test \
+        ssl_test_ctx_test ssl_test
 
 SOURCE[aborttest]=aborttest.c
 INCLUDE[aborttest]={- rel2abs(catdir($builddir,"../include")) -} ../include
@@ -224,4 +225,14 @@ SOURCE[d2i_test]=d2i_test.c testutil.c
 INCLUDE[d2i_test]={- rel2abs(catdir($builddir,"../include")) -} .. ../include
 DEPEND[d2i_test]=../libcrypto
 
+SOURCE[ssl_test_ctx_test]=ssl_test_ctx_test.c ssl_test_ctx.c testutil.c
+INCLUDE[ssl_test_ctx_test]={- rel2abs(catdir($builddir,"../include")) -} .. ../include
+DEPEND[ssl_test_ctx_test]=../libcrypto
+
+SOURCE[ssl_test]=ssl_test.c ssl_test_ctx.c testutil.c handshake_helper.c
+INCLUDE[ssl_test]={- rel2abs(catdir($builddir,"../include")) -} .. ../include
+DEPEND[ssl_test]=../libcrypto ../libssl
+
 INCLUDE[testutil.o]=..
+INCLUDE[ssl_test_ctx.o]={- rel2abs(catdir($builddir,"../include")) -} ../include
+INCLUDE[handshake_helper.o]={- rel2abs(catdir($builddir,"../include")) -} ../include

test/generate_ssl_tests.pl

+#! /usr/bin/perl
+# -*- mode: perl; -*-
+
+## SSL testcase generator
+
+use strict;
+use warnings;
+
+use File::Basename;
+use File::Spec::Functions;
+
+use OpenSSL::Test qw/srctop_dir srctop_file/;
+use OpenSSL::Test::Utils;
+
+# This block needs to run before 'use lib srctop_dir' directives.
+BEGIN {
+    OpenSSL::Test::setup("no_test_here");
+}
+
+use lib srctop_dir("util");  # for with_fallback
+use lib srctop_dir("test", "ssl-tests");  # for ssltests_base
+
+use with_fallback qw(Text::Template);
+
+use vars qw/@ISA/;
+push (@ISA, qw/Text::Template/);
+
+use ssltests_base;
+
+sub print_templates {
+    my $source = srctop_file("test", "ssl_test.tmpl");
+    my $template = Text::Template->new(TYPE => 'FILE', SOURCE => $source);
+
+    print "# Generated with generate_ssl_tests.pl\n\n";
+
+    my $num = scalar @ssltests::tests;
+
+    # Add the implicit base configuration.
+    foreach my $test (@ssltests::tests) {
+        $test->{"server"} = { (%ssltests::base_server, %{$test->{"server"}}) };
+        $test->{"client"} = { (%ssltests::base_client, %{$test->{"client"}}) };
+    }
+
+    # ssl_test expects to find a
+    #
+    # num_tests = n
+    #
+    # directive in the file. It'll then look for configuration directives
+    # for n tests, that each look like this:
+    #
+    # test-n = test-section
+    #
+    # [test-section]
+    # (SSL modules for client and server configuration go here.)
+    #
+    # [test-n]
+    # (Test configuration goes here.)
+    print "num_tests = $num\n\n";
+
+    # The conf module locations must come before everything else, because
+    # they look like
+    #
+    # test-n = test-section
+    #
+    # and you can't mix and match them with sections.
+    my $idx = 0;
+
+    foreach my $test (@ssltests::tests) {
+        my $testname = "${idx}-" . $test->{'name'};
+        print "test-$idx = $testname\n";
+        $idx++;
+    }
+
+    $idx = 0;
+
+    foreach my $test (@ssltests::tests) {
+        my $testname = "${idx}-" . $test->{'name'};
+        my $text = $template->fill_in(
+            HASH => [{ idx => $idx, testname => $testname } , $test],
+            DELIMITERS => [ "{-", "-}" ]);
+        print "# ===========================================================\n\n";
+        print "$text\n";
+        $idx++;
+    }
+}
+
+# Shamelessly copied from Configure.
+sub read_config {
+    my $fname = shift;
+    open(INPUT, "< $fname")
+	or die "Can't open input file '$fname'!\n";
+    local $/ = undef;
+    my $content = <INPUT>;
+    close(INPUT);
+    eval $content;
+    warn $@ if $@;
+}
+
+my $input_file = shift;
+# Reads the tests into ssltests::tests.
+read_config($input_file);
+print_templates();
+
+1;

test/handshake_helper.c

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#include <string.h>
+
+#include <openssl/bio.h>
+#include <openssl/ssl.h>
+
+#include "handshake_helper.h"
+
+/*
+ * Since there appears to be no way to extract the sent/received alert
+ * from the SSL object directly, we use the info callback and stash
+ * the result in ex_data.
+ */
+typedef struct handshake_ex_data {
+    int alert_sent;
+    int alert_received;
+} HANDSHAKE_EX_DATA;
+
+static int ex_data_idx;
+
+static void info_callback(const SSL *s, int where, int ret)
+{
+    if (where & SSL_CB_ALERT) {
+        HANDSHAKE_EX_DATA *ex_data =
+            (HANDSHAKE_EX_DATA*)(SSL_get_ex_data(s, ex_data_idx));
+        if (where & SSL_CB_WRITE) {
+            ex_data->alert_sent = ret;
+        } else {
+            ex_data->alert_received = ret;
+        }
+    }
+}
+
+typedef enum {
+    PEER_SUCCESS,
+    PEER_RETRY,
+    PEER_ERROR
+} peer_status_t;
+
+static peer_status_t do_handshake_step(SSL *ssl)
+{
+    int ret;
+
+    ret = SSL_do_handshake(ssl);
+
+    if (ret == 1) {
+        return PEER_SUCCESS;
+    } else if (ret == 0) {
+        return PEER_ERROR;
+    } else {
+        int error = SSL_get_error(ssl, ret);
+        /* Memory bios should never block with SSL_ERROR_WANT_WRITE. */
+        if (error == SSL_ERROR_WANT_READ)
+            return PEER_RETRY;
+        else
+            return PEER_ERROR;
+    }
+}
+
+typedef enum {
+    /* Both parties succeeded. */
+    HANDSHAKE_SUCCESS,
+    /* Client errored. */
+    CLIENT_ERROR,
+    /* Server errored. */
+    SERVER_ERROR,
+    /* Peers are in inconsistent state. */
+    INTERNAL_ERROR,
+    /* One or both peers not done. */
+    HANDSHAKE_RETRY
+} handshake_status_t;
+
+/*
+ * Determine the handshake outcome.
+ * last_status: the status of the peer to have acted last.
+ * previous_status: the status of the peer that didn't act last.
+ * client_spoke_last: 1 if the client went last.
+ */
+static handshake_status_t handshake_status(peer_status_t last_status,
+                                           peer_status_t previous_status,
+                                           int client_spoke_last)
+{
+    switch (last_status) {
+    case PEER_SUCCESS:
+        switch (previous_status) {
+        case PEER_SUCCESS:
+            /* Both succeeded. */
+            return HANDSHAKE_SUCCESS;
+        case PEER_RETRY:
+            /* Let the first peer finish. */
+            return HANDSHAKE_RETRY;
+        case PEER_ERROR:
+            /*
+             * Second peer succeeded despite the fact that the first peer
+             * already errored. This shouldn't happen.
+             */
+            return INTERNAL_ERROR;
+        }
+
+    case PEER_RETRY:
+        if (previous_status == PEER_RETRY) {
+            /* Neither peer is done. */
+            return HANDSHAKE_RETRY;
+        } else {
+            /*
+             * Deadlock: second peer is waiting for more input while first
+             * peer thinks they're done (no more input is coming).
+             */
+            return INTERNAL_ERROR;
+        }
+    case PEER_ERROR:
+        switch (previous_status) {
+        case PEER_SUCCESS:
+            /*
+             * First peer succeeded but second peer errored.
+             * TODO(emilia): we should be able to continue here (with some
+             * application data?) to ensure the first peer receives the
+             * alert / close_notify.
+             */
+            return client_spoke_last ? CLIENT_ERROR : SERVER_ERROR;
+        case PEER_RETRY:
+            /* We errored; let the peer finish. */
+            return HANDSHAKE_RETRY;
+        case PEER_ERROR:
+            /* Both peers errored. Return the one that errored first. */
+            return client_spoke_last ? SERVER_ERROR : CLIENT_ERROR;
+        }
+    }
+    /* Control should never reach here. */
+    return INTERNAL_ERROR;
+}
+
+HANDSHAKE_RESULT do_handshake(SSL_CTX *server_ctx, SSL_CTX *client_ctx)
+{
+    SSL *server, *client;
+    BIO *client_to_server, *server_to_client;
+    HANDSHAKE_EX_DATA server_ex_data, client_ex_data;
+    HANDSHAKE_RESULT ret;
+    int client_turn = 1;
+    peer_status_t client_status = PEER_RETRY, server_status = PEER_RETRY;
+    handshake_status_t status = HANDSHAKE_RETRY;
+
+    server = SSL_new(server_ctx);
+    client = SSL_new(client_ctx);
+    OPENSSL_assert(server != NULL && client != NULL);
+
+    memset(&server_ex_data, 0, sizeof(server_ex_data));
+    memset(&client_ex_data, 0, sizeof(client_ex_data));
+    memset(&ret, 0, sizeof(ret));
+    ret.result = SSL_TEST_INTERNAL_ERROR;
+
+    client_to_server = BIO_new(BIO_s_mem());
+    server_to_client = BIO_new(BIO_s_mem());
+
+    OPENSSL_assert(client_to_server != NULL && server_to_client != NULL);
+
+    /* Non-blocking bio. */
+    BIO_set_nbio(client_to_server, 1);
+    BIO_set_nbio(server_to_client, 1);
+
+    SSL_set_connect_state(client);
+    SSL_set_accept_state(server);
+
+    /* The bios are now owned by the SSL object. */
+    SSL_set_bio(client, server_to_client, client_to_server);
+    OPENSSL_assert(BIO_up_ref(server_to_client) > 0);
+    OPENSSL_assert(BIO_up_ref(client_to_server) > 0);
+    SSL_set_bio(server, client_to_server, server_to_client);
+
+    ex_data_idx = SSL_get_ex_new_index(0, "ex data", NULL, NULL, NULL);
+    OPENSSL_assert(ex_data_idx >= 0);
+
+    OPENSSL_assert(SSL_set_ex_data(server, ex_data_idx,
+                                   &server_ex_data) == 1);
+    OPENSSL_assert(SSL_set_ex_data(client, ex_data_idx,
+                                   &client_ex_data) == 1);
+
+    SSL_set_info_callback(server, &info_callback);
+    SSL_set_info_callback(client, &info_callback);
+
+    /*
+     * Half-duplex handshake loop.
+     * Client and server speak to each other synchronously in the same process.
+     * We use non-blocking BIOs, so whenever one peer blocks for read, it
+     * returns PEER_RETRY to indicate that it's the other peer's turn to write.
+     * The handshake succeeds once both peers have succeeded. If one peer
+     * errors out, we also let the other peer retry (and presumably fail).
+     */
+    for(;;) {
+        if (client_turn) {
+            client_status = do_handshake_step(client);
+            status = handshake_status(client_status, server_status,
+                                      1 /* client went last */);
+        } else {
+            server_status = do_handshake_step(server);
+            status = handshake_status(server_status, client_status,
+                                      0 /* server went last */);
+        }
+
+        switch (status) {
+        case HANDSHAKE_SUCCESS:
+            ret.result = SSL_TEST_SUCCESS;
+            goto err;
+        case CLIENT_ERROR:
+            ret.result = SSL_TEST_CLIENT_FAIL;
+            goto err;
+        case SERVER_ERROR:
+            ret.result = SSL_TEST_SERVER_FAIL;
+            goto err;
+        case INTERNAL_ERROR:
+            ret.result = SSL_TEST_INTERNAL_ERROR;
+            goto err;
+        case HANDSHAKE_RETRY:
+            /* Continue. */
+            client_turn ^= 1;
+            break;
+        }
+    }
+ err:
+    ret.server_alert_sent = server_ex_data.alert_sent;
+    ret.server_alert_received = client_ex_data.alert_received;
+    ret.client_alert_sent = client_ex_data.alert_sent;
+    ret.client_alert_received = server_ex_data.alert_received;
+    ret.server_protocol = SSL_version(server);
+    ret.client_protocol = SSL_version(client);
+
+    SSL_free(server);
+    SSL_free(client);
+    return ret;
+}

test/handshake_helper.h

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#ifndef HEADER_HANDSHAKE_HELPER_H
+#define HEADER_HANDSHAKE_HELPER_H
+
+#include "ssl_test_ctx.h"
+
+typedef struct handshake_result {
+    ssl_test_result_t result;
+    /* These alerts are in the 2-byte format returned by the info_callback. */
+    /* Alert sent by the client; 0 if no alert. */
+    int client_alert_sent;
+    /* Alert received by the server; 0 if no alert. */
+    int client_alert_received;
+    /* Alert sent by the server; 0 if no alert. */
+    int server_alert_sent;
+    /* Alert received by the client; 0 if no alert. */
+    int server_alert_received;
+    /* Negotiated protocol. On success, these should always match. */
+    int server_protocol;
+    int client_protocol;
+} HANDSHAKE_RESULT;
+
+/* Do a handshake and report some information about the result. */
+HANDSHAKE_RESULT do_handshake(SSL_CTX *server_ctx, SSL_CTX *client_ctx);
+
+#endif  /* HEADER_HANDSHAKE_HELPER_H */

test/recipes/80-test_ssl.t

@@ -66,10 +66,13 @@ my $P2intermediate="tmp_intP2.ss";
 my $server_sess="server.ss";
 my $client_sess="client.ss";
 
+# ssltest.c is deprecated in favour of the new framework in ssl_test.c
+# If you're adding tests here, you probably want to convert them to the
+# new format in ssl_test.c and add recipes to 80-test_ssl_new.t instead.
 plan tests =>
     1				# For testss
     + 1				# For ssltest -test_cipherlist
-    + 15			# For the first testssl
+    + 14			# For the first testssl
     + 16			# For the first testsslproxy
     + 16			# For the second testsslproxy
     ;
@@ -712,55 +715,6 @@ sub testssl {
 	}
     };
 
-    subtest 'TLS Version min/max tests' => sub {
-        my @protos;
-        push(@protos, "ssl3") unless $no_ssl3;
-        push(@protos, "tls1") unless $no_tls1;
-        push(@protos, "tls1.1") unless $no_tls1_1;
-        push(@protos, "tls1.2") unless $no_tls1_2;
-        my @minprotos = (undef, @protos);
-        my @maxprotos = (@protos, undef);
-        my @shdprotos = (@protos, $protos[$#protos]);
-        my $n = ((@protos+2) * (@protos+3))/2 - 2;
-        my $ntests = $n * $n;
-	plan tests => $ntests;
-      SKIP: {
-        skip "TLS disabled", 1 if $ntests == 1;
-
-        my $should;
-        for (my $smin = 0; $smin < @minprotos; ++$smin) {
-        for (my $smax = $smin ? $smin - 1 : 0; $smax < @maxprotos; ++$smax) {
-        for (my $cmin = 0; $cmin < @minprotos; ++$cmin) {
-        for (my $cmax = $cmin ? $cmin - 1 : 0; $cmax < @maxprotos; ++$cmax) {
-            if ($cmax < $smin-1) {
-                $should = "fail-server";
-            } elsif ($smax < $cmin-1) {
-                $should = "fail-client";
-            } elsif ($cmax > $smax) {
-                $should = $shdprotos[$smax];
-            } else {
-                $should = $shdprotos[$cmax];
-            }
-
-            my @args = @ssltest;
-            push(@args, "-should_negotiate", $should);
-            push(@args, "-server_min_proto", $minprotos[$smin])
-                if (defined($minprotos[$smin]));
-            push(@args, "-server_max_proto", $maxprotos[$smax])
-                if (defined($maxprotos[$smax]));
-            push(@args, "-client_min_proto", $minprotos[$cmin])
-                if (defined($minprotos[$cmin]));
-            push(@args, "-client_max_proto", $maxprotos[$cmax])
-                if (defined($maxprotos[$cmax]));
-            my $ok = run(test[@args]);
-            if (! $ok) {
-                print STDERR "\nsmin=$smin, smax=$smax, cmin=$cmin, cmax=$cmax\n";
-                print STDERR "\nFailed: @args\n";
-            }
-            ok($ok);
-        }}}}}
-    };
-
     subtest 'DTLS Version min/max tests' => sub {
         my @protos;
         push(@protos, "dtls1") unless ($no_dtls1 || $no_dtls);

test/recipes/80-test_ssl_new.t

+#! /usr/bin/perl
+
+use strict;
+use warnings;
+
+use File::Basename;
+use File::Compare qw/compare_text/;
+
+use OpenSSL::Test qw/:DEFAULT srctop_dir srctop_file/;
+use OpenSSL::Test::Utils qw/disabled alldisabled available_protocols/;
+
+setup("test_ssl_new");
+
+$ENV{TEST_CERTS_DIR} = srctop_dir("test", "certs");
+
+my @conf_srcs =  glob(srctop_file("test", "ssl-tests", "*.conf"));
+my @conf_files = map {basename($_)} @conf_srcs;
+
+# 02-protocol-version.conf test results depend on the configuration of enabled
+# protocols. We only verify generated sources in the default configuration.
+my $is_default = (disabled("ssl3") && !disabled("tls1") &&
+                  !disabled("tls1_1") && !disabled("tls1_2"));
+
+my %conf_dependent_tests = ("02-protocol-version.conf" => 1);
+
+foreach my $conf (@conf_files) {
+    subtest "Test configuration $conf" => sub {
+        test_conf($conf, $conf_dependent_tests{$conf} ?  0 : 1);
+    }
+}
+
+# We hard-code the number of tests to double-check that the globbing above
+# finds all files as expected.
+plan tests => 2;  # = scalar @conf_files
+
+sub test_conf {
+    plan tests => 3;
+
+    my ($conf, $check_source) = @_;
+
+    my $conf_file = srctop_file("test", "ssl-tests", $conf);
+    my $tmp_file = "${conf}.$$.tmp";
+    my $run_test = 1;
+
+  SKIP: {
+      # "Test" 1. Generate the source.
+      my $input_file = $conf_file . ".in";
+
+      skip 'failure', 2 unless
+        ok(run(perltest(["generate_ssl_tests.pl", $input_file],
+                        stdout => $tmp_file)),
+           "Getting output from generate_ssl_tests.pl.");
+
+    SKIP: {
+        # Test 2. Compare against existing output in test/ssl_tests.conf.
+        skip "Skipping generated source test for $conf", 1
+          if !$check_source;
+
+        $run_test = is(cmp_text($tmp_file, $conf_file), 0,
+                       "Comparing generated sources.");
+      }
+
+      # Test 3. Run the test.
+      my $no_tls = alldisabled(available_protocols("tls"));
+      skip "No TLS tests available; skipping tests", 1 if $no_tls;
+      skip "Stale sources; skipping tests", 1 if !$run_test;
+
+      ok(run(test(["ssl_test", $tmp_file])), "running ssl_test $conf");
+    }
+
+    unlink glob $tmp_file;
+}
+
+sub cmp_text {
+    return compare_text(@_, sub {
+        $_[0] =~ s/\R//g;
+        $_[1] =~ s/\R//g;
+        return $_[0] ne $_[1];
+    });
+}

test/recipes/80-test_ssl_test_ctx.t

+#! /usr/bin/perl
+
+use strict;
+use warnings;
+
+use OpenSSL::Test qw/:DEFAULT srctop_file/;
+
+setup("test_ssl_test_ctx");
+
+plan tests => 1;
+ok(run(test(["ssl_test_ctx_test", srctop_file("test", "ssl_test_ctx_test.conf")])),
+   "running ssl_test_ctx_test ssl_test_ctx_test.conf");

test/ssl-tests/01-simple.conf

+# Generated with generate_ssl_tests.pl
+
+num_tests = 2
+
+test-0 = 0-default
+test-1 = 1-verify-cert
+# ===========================================================
+
+[0-default]
+ssl_conf = 0-default-ssl
+
+[0-default-ssl]
+server = 0-default-server
+client = 0-default-client
+
+[0-default-server]
+Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
+CipherString = DEFAULT
+PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
+
+
+[0-default-client]
+CipherString = DEFAULT
+VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
+VerifyMode = Peer
+
+
+[test-0]
+ExpectedResult = Success
+
+
+# ===========================================================
+
+[1-verify-cert]
+ssl_conf = 1-verify-cert-ssl
+
+[1-verify-cert-ssl]
+server = 1-verify-cert-server
+client = 1-verify-cert-client
+
+[1-verify-cert-server]
+Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
+CipherString = DEFAULT
+PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
+
+
+[1-verify-cert-client]
+CipherString = DEFAULT
+VerifyMode = Peer
+
+
+[test-1]
+ClientAlert = UnknownCA
+ExpectedResult = ClientFail
+
+

test/ssl-tests/01-simple.conf.in

+# -*- mode: perl; -*-
+
+## SSL test configurations
+
+package ssltests;
+
+our @tests = (
+    {
+        name => "default",
+        server => { },
+        client => { },
+        test   => { "ExpectedResult" => "Success" },
+    },
+
+    {
+        name => "verify-cert",
+        server => { },
+        client => {
+            # Don't set up the client root file.
+            "VerifyCAFile" => undef,
+        },
+        test   => {
+          "ExpectedResult" => "ClientFail",
+          "ClientAlert" => "UnknownCA",
+        },
+    },
+);

test/ssl-tests/02-protocol-version.conf.in

+# -*- mode: perl; -*-
+
+## Test version negotiation
+
+package ssltests;
+
+use List::Util qw/max min/;
+
+use OpenSSL::Test;
+use OpenSSL::Test::Utils qw/anydisabled alldisabled/;
+setup("no_test_here");
+
+my @protocols = ("SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2");
+# undef stands for "no limit".
+my @min_protocols = (undef, "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2");
+my @max_protocols = ("SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", undef);
+
+my @is_disabled = anydisabled("ssl3", "tls1", "tls1_1", "tls1_2");
+
+my $min_enabled; my $max_enabled;
+
+# Protocol configuration works in cascades, i.e.,
+# $no_tls1_1 disables TLSv1.1 and below.
+#
+# $min_enabled and $max_enabled will be correct if there is at least one
+# protocol enabled.
+foreach my $i (0..$#protocols) {
+    if (!$is_disabled[$i]) {
+        $min_enabled = $i;
+        last;
+    }
+}
+
+foreach my $i (0..$#protocols) {
+    if (!$is_disabled[$i]) {
+        $max_enabled = $i;
+    }
+}
+
+our @tests = ();
+
+sub generate_tests() {
+    foreach my $c_min (0..$#min_protocols) {
+        my $c_max_min = $c_min == 0 ? 0 : $c_min - 1;
+        foreach my $c_max ($c_max_min..$#max_protocols) {
+            foreach my $s_min (0..$#min_protocols) {
+                my $s_max_min = $s_min == 0 ? 0 : $s_min - 1;
+                foreach my $s_max ($s_max_min..$#max_protocols) {
+                    my ($result, $protocol) =
+                        expected_result($c_min, $c_max, $s_min, $s_max);
+                    push @tests, {
+                        "name" => "version-negotiation",
+                        "client" => {
+                            "MinProtocol" => $min_protocols[$c_min],
+                            "MaxProtocol" => $max_protocols[$c_max],
+                        },
+                        "server" => {
+                            "MinProtocol" => $min_protocols[$s_min],
+                            "MaxProtocol" => $max_protocols[$s_max],
+                        },
+                        "test" => {
+                            "ExpectedResult" => $result,
+                            "Protocol" => $protocol
+                        }
+                    };
+                }
+            }
+        }
+    }
+}
+
+sub expected_result {
+    my $no_tls = alldisabled("ssl3", "tls1", "tls1_1", "tls1_2");
+    if ($no_tls) {
+        return ("InternalError", undef);
+    }
+
+    my ($c_min, $c_max, $s_min, $s_max) = @_;
+
+    # Adjust for "undef" (no limit).
+    $c_min = $c_min == 0 ? 0 : $c_min - 1;
+    $c_max = $c_max == scalar(@max_protocols) - 1 ? $c_max - 1 : $c_max;
+    $s_min = $s_min == 0 ? 0 : $s_min - 1;
+    $s_max = $s_max == scalar(@max_protocols) - 1 ? $s_max - 1 : $s_max;
+
+    # We now have at least one protocol enabled, so $min_enabled and
+    # $max_enabled are well-defined.
+    $c_min = max $c_min, $min_enabled;
+    $s_min = max $s_min, $min_enabled;
+    $c_max = min $c_max, $max_enabled;
+    $s_max = min $s_max, $max_enabled;
+
+    if ($c_min > $c_max) {
+        # Client should fail to even send a hello.
+        # This results in an internal error since the server will be
+        # waiting for input that never arrives.
+        return ("InternalError", undef);
+    } elsif ($s_min > $s_max) {
+        # Server has no protocols, should always fail.
+        return ("ServerFail", undef);
+    } elsif ($s_min > $c_max) {
+        # Server doesn't support the client range.
+        return ("ServerFail", undef);
+    } elsif ($c_min > $s_max) {
+        # Server will try with a version that is lower than the lowest
+        # supported client version.
+        return ("ClientFail", undef);
+    } else {
+        # Server and client ranges overlap.
+        my $max_common = $s_max < $c_max ? $s_max : $c_max;
+        return ("Success", $protocols[$max_common]);
+    }
+}
+
+generate_tests();

test/ssl-tests/ssltests_base.pm

+# -*- mode: perl; -*-
+
+## SSL test configurations
+
+package ssltests;
+
+our %base_server = (
+    "Certificate" => "\${ENV::TEST_CERTS_DIR}/servercert.pem",
+    "PrivateKey"  => "\${ENV::TEST_CERTS_DIR}/serverkey.pem",
+    "CipherString" => "DEFAULT",
+);
+
+our %base_client = (
+    "VerifyCAFile" => "\${ENV::TEST_CERTS_DIR}/rootcert.pem",
+    "VerifyMode" => "Peer",
+    "CipherString" => "DEFAULT",
+);

test/ssl_test.c

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#include <stdio.h>
+
+#include <openssl/conf.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+#include "handshake_helper.h"
+#include "ssl_test_ctx.h"
+#include "testutil.h"
+
+static CONF *conf = NULL;
+
+/* Currently the section names are of the form test-<number>, e.g. test-15. */
+#define MAX_TESTCASE_NAME_LENGTH 100
+
+typedef struct ssl_test_ctx_test_fixture {
+    const char *test_case_name;
+    char test_app[MAX_TESTCASE_NAME_LENGTH];
+} SSL_TEST_FIXTURE;
+
+static SSL_TEST_FIXTURE set_up(const char *const test_case_name)
+{
+    SSL_TEST_FIXTURE fixture;
+    fixture.test_case_name = test_case_name;
+    return fixture;
+}
+
+static const char *print_alert(int alert)
+{
+    return alert ? SSL_alert_desc_string_long(alert) : "no alert";
+}
+
+static int check_result(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.result != test_ctx->expected_result) {
+        fprintf(stderr, "ExpectedResult mismatch: expected %s, got %s.\n",
+                ssl_test_result_t_name(test_ctx->expected_result),
+                ssl_test_result_t_name(result.result));
+        return 0;
+    }
+    return 1;
+}
+
+static int check_alerts(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.client_alert_sent != result.client_alert_received) {
+        fprintf(stderr, "Client sent alert %s but server received %s\n.",
+                print_alert(result.client_alert_sent),
+                print_alert(result.client_alert_received));
+        /*
+         * We can't bail here because the peer doesn't always get far enough
+         * to process a received alert. Specifically, in protocol version
+         * negotiation tests, we have the following scenario.
+         * Client supports TLS v1.2 only; Server supports TLS v1.1.
+         * Client proposes TLS v1.2; server responds with 1.1;
+         * Client now sends a protocol alert, using TLS v1.2 in the header.
+         * The server, however, rejects the alert because of version mismatch
+         * in the record layer; therefore, the server appears to never
+         * receive the alert.
+         */
+        /* return 0; */
+    }
+
+    if (result.server_alert_sent != result.server_alert_received) {
+        fprintf(stderr, "Server sent alert %s but client received %s\n.",
+                print_alert(result.server_alert_sent),
+                print_alert(result.server_alert_received));
+        /* return 0; */
+    }
+
+    /* Tolerate an alert if one wasn't explicitly specified in the test. */
+    if (test_ctx->client_alert
+        /*
+         * The info callback alert value is computed as
+         * (s->s3->send_alert[0] << 8) | s->s3->send_alert[1]
+         * where the low byte is the alert code and the high byte is other stuff.
+         */
+        && (result.client_alert_sent & 0xff) != test_ctx->client_alert) {
+        fprintf(stderr, "ClientAlert mismatch: expected %s, got %s.\n",
+                print_alert(test_ctx->client_alert),
+                print_alert(result.client_alert_sent));
+        return 0;
+    }
+
+    if (test_ctx->server_alert
+        && (result.server_alert_sent & 0xff) != test_ctx->server_alert) {
+        fprintf(stderr, "ServerAlert mismatch: expected %s, got %s.\n",
+                print_alert(test_ctx->server_alert),
+                print_alert(result.server_alert_sent));
+        return 0;
+    }
+
+    return 1;
+}
+
+static int check_protocol(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    if (result.client_protocol != result.server_protocol) {
+        fprintf(stderr, "Client has protocol %s but server has %s\n.",
+                ssl_protocol_name(result.client_protocol),
+                ssl_protocol_name(result.server_protocol));
+        return 0;
+    }
+
+    if (test_ctx->protocol) {
+        if (result.client_protocol != test_ctx->protocol) {
+            fprintf(stderr, "Protocol mismatch: expected %s, got %s.\n",
+                    ssl_protocol_name(test_ctx->protocol),
+                    ssl_protocol_name(result.client_protocol));
+            return 0;
+        }
+    }
+    return 1;
+}
+
+/*
+ * This could be further simplified by constructing an expected
+ * HANDSHAKE_RESULT, and implementing comparison methods for
+ * its fields.
+ */
+static int check_test(HANDSHAKE_RESULT result, SSL_TEST_CTX *test_ctx)
+{
+    int ret = 1;
+    ret &= check_result(result, test_ctx);
+    ret &= check_alerts(result, test_ctx);
+    if (result.result == SSL_TEST_SUCCESS)
+        ret &= check_protocol(result, test_ctx);
+    return ret;
+}
+
+static int execute_test(SSL_TEST_FIXTURE fixture)
+{
+    /* TODO(emilia): this is confusing. Flip to return 1 on success. */
+    int ret = 1;
+    SSL_CTX *server_ctx = NULL, *client_ctx = NULL;
+    SSL_TEST_CTX *test_ctx = NULL;
+    HANDSHAKE_RESULT result;
+
+    server_ctx = SSL_CTX_new(TLS_server_method());
+    client_ctx = SSL_CTX_new(TLS_client_method());
+    OPENSSL_assert(server_ctx != NULL && client_ctx != NULL);
+
+    OPENSSL_assert(CONF_modules_load(conf, fixture.test_app, 0) > 0);
+
+    if (!SSL_CTX_config(server_ctx, "server")
+       || !SSL_CTX_config(client_ctx, "client")) {
+        goto err;
+    }
+
+    test_ctx = SSL_TEST_CTX_create(conf, fixture.test_app);
+    if (test_ctx == NULL)
+        goto err;
+
+    result = do_handshake(server_ctx, client_ctx);
+
+    if (check_test(result, test_ctx))
+        ret = 0;
+
+err:
+    CONF_modules_unload(0);
+    SSL_CTX_free(server_ctx);
+    SSL_CTX_free(client_ctx);
+    SSL_TEST_CTX_free(test_ctx);
+    if (ret != 0)
+        ERR_print_errors_fp(stderr);
+    return ret;
+}
+
+static void tear_down(SSL_TEST_FIXTURE fixture)
+{
+}
+
+#define SETUP_SSL_TEST_FIXTURE()                        \
+    SETUP_TEST_FIXTURE(SSL_TEST_FIXTURE, set_up)
+#define EXECUTE_SSL_TEST()             \
+    EXECUTE_TEST(execute_test, tear_down)
+
+static int test_handshake(int idx)
+{
+    SETUP_SSL_TEST_FIXTURE();
+    snprintf(fixture.test_app, sizeof(fixture.test_app),
+             "test-%d", idx);
+    EXECUTE_SSL_TEST();
+}
+
+int main(int argc, char **argv)
+{
+    int result = 0;
+    long num_tests;
+
+    if (argc != 2)
+        return 1;
+
+    conf = NCONF_new(NULL);
+    OPENSSL_assert(conf != NULL);
+
+    /* argv[1] should point to the test conf file */
+    OPENSSL_assert(NCONF_load(conf, argv[1], NULL) > 0);
+
+    OPENSSL_assert(NCONF_get_number_e(conf, NULL, "num_tests", &num_tests));
+
+    ADD_ALL_TESTS(test_handshake, (int)(num_tests));
+    result = run_tests(argv[0]);
+
+    CONF_modules_free();
+    return result;
+}

test/ssl_test.tmpl

+[{-$testname-}]
+ssl_conf = {-$testname-}-ssl
+
+[{-$testname-}-ssl]
+server = {-$testname-}-server
+client = {-$testname-}-client
+
+[{-$testname-}-server]
+{-
+    foreach my $key (sort keys %server) {
+        $OUT .= qq{$key} . " = " . qq{$server{$key}\n} if defined $server{$key};
+    }
+-}
+
+[{-$testname-}-client]
+{-
+    foreach my $key (sort keys %client) {
+        $OUT .= qq{$key} . " = " . qq{$client{$key}\n} if defined $client{$key};
+    }
+-}
+
+[test-{-$idx-}]
+{-
+    foreach my $key (sort keys %test) {
+        $OUT .= qq{$key} ." = " . qq{$test{$key}\n} if defined $test{$key};
+    }
+-}

test/ssl_test_ctx.c

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#include <string.h>
+
+#include <openssl/e_os2.h>
+#include <openssl/crypto.h>
+
+#include "e_os.h"
+#include "ssl_test_ctx.h"
+
+/* True enums and other test configuration values that map to an int. */
+typedef struct {
+    const char *name;
+    int value;
+} test_enum;
+
+
+__owur static int parse_enum(const test_enum *enums, size_t num_enums,
+                             int *value, const char *name)
+{
+    size_t i;
+    for (i = 0; i < num_enums; i++) {
+        if (strcmp(enums[i].name, name) == 0) {
+            *value = enums[i].value;
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static const char *enum_name(const test_enum *enums, size_t num_enums,
+                             int value)
+{
+    size_t i;
+    for (i = 0; i < num_enums; i++) {
+        if (enums[i].value == value) {
+            return enums[i].name;
+        }
+    }
+    return "InvalidValue";
+}
+
+
+/*******************/
+/* ExpectedResult. */
+/*******************/
+
+static const test_enum ssl_test_results[] = {
+    {"Success", SSL_TEST_SUCCESS},
+    {"ServerFail", SSL_TEST_SERVER_FAIL},
+    {"ClientFail", SSL_TEST_CLIENT_FAIL},
+    {"InternalError", SSL_TEST_INTERNAL_ERROR},
+};
+
+__owur static int parse_expected_result(SSL_TEST_CTX *test_ctx, const char *value)
+{
+    int ret_value;
+    if (!parse_enum(ssl_test_results, OSSL_NELEM(ssl_test_results),
+                    &ret_value, value)) {
+        return 0;
+    }
+    test_ctx->expected_result = ret_value;
+    return 1;
+}
+
+const char *ssl_test_result_t_name(ssl_test_result_t result)
+{
+    return enum_name(ssl_test_results, OSSL_NELEM(ssl_test_results), result);
+}
+
+/******************************/
+/* ClientAlert / ServerAlert. */
+/******************************/
+
+static const test_enum ssl_alerts[] = {
+    {"UnknownCA", SSL_AD_UNKNOWN_CA},
+};
+
+__owur static int parse_alert(int *alert, const char *value)
+{
+    return parse_enum(ssl_alerts, OSSL_NELEM(ssl_alerts), alert, value);
+}
+
+__owur static int parse_client_alert(SSL_TEST_CTX *test_ctx, const char *value)
+{
+    return parse_alert(&test_ctx->client_alert, value);
+}
+
+__owur static int parse_server_alert(SSL_TEST_CTX *test_ctx, const char *value)
+{
+    return parse_alert(&test_ctx->server_alert, value);
+}
+
+const char *ssl_alert_name(int alert)
+{
+    return enum_name(ssl_alerts, OSSL_NELEM(ssl_alerts), alert);
+}
+
+/************/
+/* Protocol */
+/************/
+
+static const test_enum ssl_protocols[] = {
+     {"TLSv1.2", TLS1_2_VERSION},
+     {"TLSv1.1", TLS1_1_VERSION},
+     {"TLSv1", TLS1_VERSION},
+     {"SSLv3", SSL3_VERSION},
+};
+
+__owur static int parse_protocol(SSL_TEST_CTX *test_ctx, const char *value)
+{
+    return parse_enum(ssl_protocols, OSSL_NELEM(ssl_protocols),
+                      &test_ctx->protocol, value);
+}
+
+const char *ssl_protocol_name(int protocol)
+{
+    return enum_name(ssl_protocols, OSSL_NELEM(ssl_protocols), protocol);
+}
+
+
+/*************************************************************/
+/* Known test options and their corresponding parse methods. */
+/*************************************************************/
+
+typedef struct {
+    const char *name;
+    int (*parse)(SSL_TEST_CTX *test_ctx, const char *value);
+} ssl_test_ctx_option;
+
+static const ssl_test_ctx_option ssl_test_ctx_options[] = {
+    { "ExpectedResult", &parse_expected_result },
+    { "ClientAlert", &parse_client_alert },
+    { "ServerAlert", &parse_server_alert },
+    { "Protocol", &parse_protocol },
+};
+
+
+/*
+ * Since these methods are used to create tests, we use OPENSSL_assert liberally
+ * for malloc failures and other internal errors.
+ */
+SSL_TEST_CTX *SSL_TEST_CTX_new()
+{
+    SSL_TEST_CTX *ret;
+    ret = OPENSSL_zalloc(sizeof(*ret));
+    OPENSSL_assert(ret != NULL);
+    ret->expected_result = SSL_TEST_SUCCESS;
+    return ret;
+}
+
+void SSL_TEST_CTX_free(SSL_TEST_CTX *ctx)
+{
+    OPENSSL_free(ctx);
+}
+
+SSL_TEST_CTX *SSL_TEST_CTX_create(const CONF *conf, const char *test_section)
+{
+    STACK_OF(CONF_VALUE) *sk_conf;
+    SSL_TEST_CTX *ctx;
+    int i;
+    size_t j;
+
+    sk_conf = NCONF_get_section(conf, test_section);
+    OPENSSL_assert(sk_conf != NULL);
+
+    ctx = SSL_TEST_CTX_new();
+    OPENSSL_assert(ctx != NULL);
+
+    for (i = 0; i < sk_CONF_VALUE_num(sk_conf); i++) {
+        int found = 0;
+        const CONF_VALUE *option = sk_CONF_VALUE_value(sk_conf, i);
+        for (j = 0; j < OSSL_NELEM(ssl_test_ctx_options); j++) {
+            if (strcmp(option->name, ssl_test_ctx_options[j].name) == 0) {
+                if (!ssl_test_ctx_options[j].parse(ctx, option->value)) {
+                    fprintf(stderr, "Bad value %s for option %s\n",
+                            option->value, option->name);
+                    goto err;
+                }
+                found = 1;
+                break;
+            }
+        }
+        if (!found) {
+            fprintf(stderr, "Unknown test option: %s\n", option->name);
+            goto err;
+        }
+    }
+
+    goto done;
+
+ err:
+    SSL_TEST_CTX_free(ctx);
+    ctx = NULL;
+ done:
+    return ctx;
+}

test/ssl_test_ctx.h

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+#ifndef HEADER_SSL_TEST_CTX_H
+#define HEADER_SSL_TEST_CTX_H
+
+#include <openssl/conf.h>
+#include <openssl/ssl.h>
+
+typedef enum {
+    SSL_TEST_SUCCESS,  /* Default */
+    SSL_TEST_SERVER_FAIL,
+    SSL_TEST_CLIENT_FAIL,
+    SSL_TEST_INTERNAL_ERROR
+} ssl_test_result_t;
+
+typedef struct ssl_test_ctx {
+    /* Test expectations. */
+    /* Defaults to SUCCESS. */
+    ssl_test_result_t expected_result;
+    /* Alerts. 0 if no expectation. */
+    /* See ssl.h for alert codes. */
+    /* Alert sent by the client / received by the server. */
+    int client_alert;
+    /* Alert sent by the server / received by the client. */
+    int server_alert;
+    /* Negotiated protocol version. 0 if no expectation. */
+    /* See ssl.h for protocol versions. */
+    int protocol;
+} SSL_TEST_CTX;
+
+const char *ssl_test_result_t_name(ssl_test_result_t result);
+const char *ssl_alert_name(int alert);
+const char *ssl_protocol_name(int protocol);
+
+/*
+ * Load the test case context from |conf|.
+ * See test/README.ssl_test for details on the conf file format.
+ */
+SSL_TEST_CTX *SSL_TEST_CTX_create(const CONF *conf, const char *test_section);
+
+SSL_TEST_CTX *SSL_TEST_CTX_new(void);
+
+void SSL_TEST_CTX_free(SSL_TEST_CTX *ctx);
+
+#endif  /* HEADER_SSL_TEST_CTX_H */

test/ssl_test_ctx_test.c

+/*
+ * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL licenses, (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * https://www.openssl.org/source/license.html
+ * or in the file LICENSE in the source distribution.
+ */
+
+/*
+ * Ideally, CONF should offer standard parsing methods and cover them
+ * in tests. But since we have no CONF tests, we use a custom test for now.
+ */
+
+#include <stdio.h>
+
+#include "e_os.h"
+#include "ssl_test_ctx.h"
+#include "testutil.h"
+#include <openssl/e_os2.h>
+#include <openssl/err.h>
+#include <openssl/conf.h>
+#include <openssl/ssl.h>
+
+static CONF *conf = NULL;
+
+typedef struct ssl_test_ctx_test_fixture {
+    const char *test_case_name;
+    const char *test_section;
+    /* Expected parsed configuration. */
+    SSL_TEST_CTX *expected_ctx;
+} SSL_TEST_CTX_TEST_FIXTURE;
+
+/* Returns 1 if the contexts are equal, 0 otherwise. */
+static int SSL_TEST_CTX_equal(SSL_TEST_CTX *ctx, SSL_TEST_CTX *ctx2)
+{
+    if (ctx->expected_result != ctx2->expected_result) {
+        fprintf(stderr, "ExpectedResult mismatch: %s vs %s.\n",
+                ssl_test_result_t_name(ctx->expected_result),
+                ssl_test_result_t_name(ctx2->expected_result));
+        return 0;
+    }
+    if (ctx->client_alert != ctx2->client_alert) {
+        fprintf(stderr, "ClientAlert mismatch: %s vs %s.\n",
+                ssl_alert_name(ctx->expected_result),
+                ssl_alert_name(ctx2->expected_result));
+        return 0;
+    }
+    if (ctx->server_alert != ctx2->server_alert) {
+        fprintf(stderr, "ServerAlert mismatch: %s vs %s.\n",
+                ssl_alert_name(ctx->expected_result),
+                ssl_alert_name(ctx2->expected_result));
+        return 0;
+    }
+    if (ctx->protocol != ctx2->protocol) {
+        fprintf(stderr, "ClientAlert mismatch: %s vs %s.\n",
+                ssl_protocol_name(ctx->expected_result),
+                ssl_protocol_name(ctx2->expected_result));
+        return 0;
+    }
+
+    return 1;
+}
+
+static SSL_TEST_CTX_TEST_FIXTURE set_up(const char *const test_case_name)
+{
+    SSL_TEST_CTX_TEST_FIXTURE fixture;
+    fixture.test_case_name = test_case_name;
+    fixture.expected_ctx = SSL_TEST_CTX_new();
+    OPENSSL_assert(fixture.expected_ctx != NULL);
+    return fixture;
+}
+
+static int execute_test(SSL_TEST_CTX_TEST_FIXTURE fixture)
+{
+    int ret = 1;
+
+    SSL_TEST_CTX *ctx = SSL_TEST_CTX_create(conf, fixture.test_section);
+
+    if (ctx == NULL) {
+        fprintf(stderr, "Failed to parse good configuration %s.\n",
+                fixture.test_section);
+        goto err;
+    }
+
+    if (!SSL_TEST_CTX_equal(ctx, fixture.expected_ctx))
+        goto err;
+
+    ret = 0;
+ err:
+    SSL_TEST_CTX_free(ctx);
+    return ret;
+}
+
+static int execute_failure_test(SSL_TEST_CTX_TEST_FIXTURE fixture)
+{
+    SSL_TEST_CTX *ctx = SSL_TEST_CTX_create(conf, fixture.test_section);
+
+    if (ctx != NULL) {
+        fprintf(stderr, "Parsing bad configuration %s succeeded.\n",
+                fixture.test_section);
+        SSL_TEST_CTX_free(ctx);
+        return 1;
+    }
+
+    return 0;
+}
+
+static void tear_down(SSL_TEST_CTX_TEST_FIXTURE fixture)
+{
+    SSL_TEST_CTX_free(fixture.expected_ctx);
+    ERR_print_errors_fp(stderr);
+}
+
+#define SETUP_SSL_TEST_CTX_TEST_FIXTURE()                       \
+    SETUP_TEST_FIXTURE(SSL_TEST_CTX_TEST_FIXTURE, set_up)
+#define EXECUTE_SSL_TEST_CTX_TEST()             \
+    EXECUTE_TEST(execute_test, tear_down)
+#define EXECUTE_SSL_TEST_CTX_FAILURE_TEST()             \
+    EXECUTE_TEST(execute_failure_test, tear_down)
+
+static int test_empty_configuration()
+{
+    SETUP_SSL_TEST_CTX_TEST_FIXTURE();
+    fixture.test_section = "ssltest_default";
+    fixture.expected_ctx->expected_result = SSL_TEST_SUCCESS;
+    EXECUTE_SSL_TEST_CTX_TEST();
+}
+
+static int test_good_configuration()
+{
+    SETUP_SSL_TEST_CTX_TEST_FIXTURE();
+    fixture.test_section = "ssltest_good";
+    fixture.expected_ctx->expected_result = SSL_TEST_SERVER_FAIL;
+    fixture.expected_ctx->client_alert = SSL_AD_UNKNOWN_CA;
+    fixture.expected_ctx->server_alert = 0;  /* No alert. */
+    fixture.expected_ctx->protocol = TLS1_1_VERSION;
+    EXECUTE_SSL_TEST_CTX_TEST();
+}
+
+static const char *bad_configurations[] = {
+    "ssltest_unknown_option",
+    "ssltest_unknown_expected_result",
+    "ssltest_unknown_alert",
+    "ssltest_unknown_protocol",
+};
+
+static int test_bad_configuration(int idx)
+{
+        SETUP_SSL_TEST_CTX_TEST_FIXTURE();
+        fixture.test_section = bad_configurations[idx];
+        EXECUTE_SSL_TEST_CTX_FAILURE_TEST();
+}
+
+int main(int argc, char **argv)
+{
+    int result = 0;
+
+    if (argc != 2)
+        return 1;
+
+    conf = NCONF_new(NULL);
+    OPENSSL_assert(conf != NULL);
+
+    /* argv[1] should point to test/ssl_test_ctx_test.conf */
+    OPENSSL_assert(NCONF_load(conf, argv[1], NULL) > 0);
+
+
+    ADD_TEST(test_empty_configuration);
+    ADD_TEST(test_good_configuration);
+    ADD_ALL_TESTS(test_bad_configuration, OSSL_NELEM(bad_configurations));
+
+    result = run_tests(argv[0]);
+
+    NCONF_free(conf);
+
+    return result;
+}

test/ssl_test_ctx_test.conf

+[ssltest_default]
+
+[ssltest_good]
+ExpectedResult = ServerFail
+ClientAlert = UnknownCA
+Protocol = TLSv1.1
+
+[ssltest_unknown_option]
+UnknownOption = Foo
+
+[ssltest_unknown_expected_result]
+ExpectedResult = Foo
+
+[ssltest_unknown_alert]
+ServerAlert = Foo
+
+[ssltest_unknown_protocol]
+Protocol = Foo

test/testutil.c

@@ -68,37 +68,70 @@
 typedef struct test_info {
     const char *test_case_name;
     int (*test_fn) ();
+    int (*param_test_fn)(int idx);
+    int num;
 } TEST_INFO;
 
 static TEST_INFO all_tests[1024];
 static int num_tests = 0;
+/*
+ * A parameterised tests runs a loop of test cases.
+ * |num_test_cases| counts the total number of test cases
+ * across all tests.
+ */
+static int num_test_cases = 0;
 
 void add_test(const char *test_case_name, int (*test_fn) ())
 {
     assert(num_tests != OSSL_NELEM(all_tests));
     all_tests[num_tests].test_case_name = test_case_name;
     all_tests[num_tests].test_fn = test_fn;
+    all_tests[num_tests].num = -1;
+    ++num_test_cases;
+    ++num_tests;
+}
+
+void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
+                   int num)
+{
+    assert(num_tests != OSSL_NELEM(all_tests));
+    all_tests[num_tests].test_case_name = test_case_name;
+    all_tests[num_tests].param_test_fn = test_fn;
+    all_tests[num_tests].num = num;
     ++num_tests;
+    num_test_cases += num;
 }
 
 int run_tests(const char *test_prog_name)
 {
     int num_failed = 0;
-    int i = 0;
 
-    printf("%s: %d test case%s\n", test_prog_name, num_tests,
-           num_tests == 1 ? "" : "s");
+    int i, j;
+
+    printf("%s: %d test case%s\n", test_prog_name, num_test_cases,
+           num_test_cases == 1 ? "" : "s");
+
     for (i = 0; i != num_tests; ++i) {
-        if (all_tests[i].test_fn()) {
-            printf("** %s failed **\n--------\n",
-                   all_tests[i].test_case_name);
-            ++num_failed;
+        if (all_tests[i].num == -1) {
+            if (all_tests[i].test_fn()) {
+                printf("** %s failed **\n--------\n",
+                       all_tests[i].test_case_name);
+                ++num_failed;
+            }
+        } else {
+            for (j = 0; j < all_tests[i].num; j++) {
+                if (all_tests[i].param_test_fn(j)) {
+                    printf("** %s failed test %d\n--------\n",
+                           all_tests[i].test_case_name, j);
+                    ++num_failed;
+                }
+            }
         }
     }
 
     if (num_failed != 0) {
         printf("%s: %d test%s failed (out of %d)\n", test_prog_name,
-               num_failed, num_failed != 1 ? "s" : "", num_tests);
+               num_failed, num_failed != 1 ? "s" : "", num_test_cases);
         return EXIT_FAILURE;
     }
     printf("  All tests passed.\n");

test/testutil.h

@@ -90,8 +90,8 @@
  *      }
  */
 # define SETUP_TEST_FIXTURE(TEST_FIXTURE_TYPE, set_up)\
-        TEST_FIXTURE_TYPE fixture = set_up(TEST_CASE_NAME);\
-        int result = 0
+    TEST_FIXTURE_TYPE fixture = set_up(TEST_CASE_NAME); \
+    int result = 0
 
 # define EXECUTE_TEST(execute_func, tear_down)\
         if (execute_func(fixture) != 0) result = 1;\
@@ -120,7 +120,16 @@
  * returned from run_tests() should be used as the return value for main().
  */
 # define ADD_TEST(test_function) add_test(#test_function, test_function)
+
+/*
+ * Simple parameterized tests. Adds a test_function(idx) test for each
+ * 0 <= idx < num.
+ */
+# define ADD_ALL_TESTS(test_function, num) \
+  add_all_tests(#test_function, test_function, num)
+
 void add_test(const char *test_case_name, int (*test_fn) ());
+void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num);
 int run_tests(const char *test_prog_name);
 
 #endif                          /* HEADER_TESTUTIL_H */