Refactor the async wait fd logic

Implementation experience has shown that the original plan for async wait
fds was too simplistic. Originally the async logic created a pipe internally
and user/engine code could then get access to it via API calls. It is more
flexible if the engine is able to create its own fd and provide it to the
async code.

Another issue is that there can be a lot of churn in the fd value within
the context of (say) a single SSL connection leading to continually adding
and removing fds from (say) epoll. It is better if we can provide some
stability of the fd value across a whole SSL connection. This is
problematic because an engine has no concept of an SSL connection.

This commit refactors things to introduce an ASYNC_WAIT_CTX which acts as a
proxy for an SSL connection down at the engine layer.
Reviewed-by: NRichard Levitte <levitte@openssl.org>

apps/apps.c

@@ -2638,15 +2638,27 @@ BIO *bio_open_default_quiet(const char *filename, char mode, int format)
 
 void wait_for_async(SSL *s)
 {
-    int width, fd;
+    int width = 0;
     fd_set asyncfds;
+    OSSL_ASYNC_FD *fds;
+    size_t numfds;
 
-    fd = SSL_get_async_wait_fd(s);
-    if (fd < 0)
+    if (!SSL_get_all_async_fds(s, NULL, &numfds))
+        return;
+    if (numfds == 0)
         return;
+    fds = OPENSSL_malloc(sizeof(OSSL_ASYNC_FD) * numfds);
+    if (!SSL_get_all_async_fds(s, fds, &numfds)) {
+        OPENSSL_free(fds);
+    }
 
-    width = fd + 1;
     FD_ZERO(&asyncfds);
-    openssl_fdset(fd, &asyncfds);
+    while (numfds > 0) {
+        if (width <= (int)*fds)
+            width = (int)*fds + 1;
+        openssl_fdset((int)*fds, &asyncfds);
+        numfds--;
+        fds++;
+    }
     select(width, (void *)&asyncfds, NULL, NULL, NULL);
 }

crypto/async/Makefile.in

@@ -17,8 +17,8 @@ TEST=
 APPS=
 
 LIB=$(TOP)/libcrypto.a
-LIBSRC=async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
-LIBOBJ=async.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
+LIBSRC=async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+LIBOBJ=async.o async_wait.o async_err.o arch/async_posix.o arch/async_win.o arch/async_null.o
 
 SRC= $(LIBSRC)
 

crypto/async/arch/async_null.c

@@ -55,26 +55,6 @@
 
 #ifdef ASYNC_NULL
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    return -1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    return 0;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    return -1;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    return -1;
-}
-
 int async_global_init(void)
 {
     return 0;

crypto/async/arch/async_posix.c

@@ -103,36 +103,4 @@ void async_fibre_free(async_fibre *fibre)
     fibre->fibre.uc_stack.ss_sp = NULL;
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (pipe(pipefds) == 0)
-        return 1;
-
-    return 0;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (close(fd) != 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    if (write(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    if (read(fd, buf, 1) > 0)
-        return 1;
-
-    return 0;
-}
-
 #endif

crypto/async/arch/async_win.c

@@ -127,42 +127,6 @@ VOID CALLBACK async_start_func_win(PVOID unused)
     async_start_func();
 }
 
-int async_pipe(OSSL_ASYNC_FD *pipefds)
-{
-    if (CreatePipe(&pipefds[0], &pipefds[1], NULL, 256) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_close_fd(OSSL_ASYNC_FD fd)
-{
-    if (CloseHandle(fd) == 0)
-        return 0;
-
-    return 1;
-}
-
-int async_write1(OSSL_ASYNC_FD fd, const void *buf)
-{
-    DWORD numwritten = 0;
-
-    if (WriteFile(fd, buf, 1, &numwritten, NULL) && numwritten == 1)
-        return 1;
-
-    return 0;
-}
-
-int async_read1(OSSL_ASYNC_FD fd, void *buf)
-{
-    DWORD numread = 0;
-
-    if (ReadFile(fd, buf, 1, &numread, NULL) && numread == 1)
-        return 1;
-
-    return 0;
-}
-
 async_pool *async_get_pool(void)
 {
     return (async_pool *)TlsGetValue(asyncwinpool);

crypto/async/async.c

@@ -119,26 +119,14 @@ static int async_ctx_free(void)
 static ASYNC_JOB *async_job_new(void)
 {
     ASYNC_JOB *job = NULL;
-    OSSL_ASYNC_FD pipefds[2];
 
-    job = OPENSSL_malloc(sizeof (ASYNC_JOB));
+    job = OPENSSL_zalloc(sizeof (ASYNC_JOB));
     if (job == NULL) {
         ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
 
-    if (!async_pipe(pipefds)) {
-        OPENSSL_free(job);
-        ASYNCerr(ASYNC_F_ASYNC_JOB_NEW, ASYNC_R_CANNOT_CREATE_WAIT_PIPE);
-        return NULL;
-    }
-
-    job->wake_set = 0;
-    job->wait_fd = pipefds[0];
-    job->wake_fd = pipefds[1];
-
     job->status = ASYNC_JOB_RUNNING;
-    job->funcargs = NULL;
 
     return job;
 }
@@ -148,8 +136,6 @@ static void async_job_free(ASYNC_JOB *job)
     if (job != NULL) {
         OPENSSL_free(job->funcargs);
         async_fibre_free(&job->fibrectx);
-        async_close_fd(job->wait_fd);
-        async_close_fd(job->wake_fd);
         OPENSSL_free(job);
     }
 }
@@ -219,8 +205,8 @@ void async_start_func(void)
     }
 }
 
-int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                         void *args, size_t size)
+int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret,
+                    int (*func)(void *), void *args, size_t size)
 {
     async_ctx *ctx = async_get_ctx();
     if (ctx == NULL)
@@ -237,6 +223,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         if (ctx->currjob != NULL) {
             if (ctx->currjob->status == ASYNC_JOB_STOPPING) {
                 *ret = ctx->currjob->ret;
+                ctx->currjob->waitctx = NULL;
                 async_release_job(ctx->currjob);
                 ctx->currjob = NULL;
                 *job = NULL;
@@ -289,6 +276,7 @@ int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
         }
 
         ctx->currjob->func = func;
+        ctx->currjob->waitctx = wctx;
         if (!async_fibre_swapcontext(&ctx->dispatcher,
                 &ctx->currjob->fibrectx, 1)) {
             ASYNCerr(ASYNC_F_ASYNC_START_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
@@ -303,7 +291,6 @@ err:
     return ASYNC_ERR;
 }
 
-
 int ASYNC_pause_job(void)
 {
     ASYNC_JOB *job;
@@ -327,6 +314,8 @@ int ASYNC_pause_job(void)
         ASYNCerr(ASYNC_F_ASYNC_PAUSE_JOB, ASYNC_R_FAILED_TO_SWAP_CONTEXT);
         return 0;
     }
+    /* Reset counts of added and deleted fds */
+    async_wait_ctx_reset_counts(job->waitctx);
 
     return 1;
 }
@@ -441,28 +430,9 @@ ASYNC_JOB *ASYNC_get_current_job(void)
     return ctx->currjob;
 }
 
-OSSL_ASYNC_FD ASYNC_get_wait_fd(ASYNC_JOB *job)
-{
-    return job->wait_fd;
-}
-
-void ASYNC_wake(ASYNC_JOB *job)
-{
-    char dummy = 0;
-
-    if (job->wake_set)
-        return;
-    async_write1(job->wake_fd, &dummy);
-    job->wake_set = 1;
-}
-
-void ASYNC_clear_wake(ASYNC_JOB *job)
+ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job)
 {
-    char dummy = 0;
-    if (!job->wake_set)
-        return;
-    async_read1(job->wait_fd, &dummy);
-    job->wake_set = 0;
+    return job->waitctx;
 }
 
 void ASYNC_block_pause(void)

crypto/async/async_locl.h

@@ -81,9 +81,23 @@ struct async_job_st {
     void *funcargs;
     int ret;
     int status;
-    int wake_set;
-    OSSL_ASYNC_FD wait_fd;
-    OSSL_ASYNC_FD wake_fd;
+    ASYNC_WAIT_CTX *waitctx;
+};
+
+struct fd_lookup_st {
+    const void *key;
+    OSSL_ASYNC_FD fd;
+    void *custom_data;
+    void (*cleanup)(ASYNC_WAIT_CTX *, const void *, OSSL_ASYNC_FD, void *);
+    int add;
+    int del;
+    struct fd_lookup_st *next;
+};
+
+struct async_wait_ctx_st {
+    struct fd_lookup_st *fds;
+    size_t numadd;
+    size_t numdel;
 };
 
 DEFINE_STACK_OF(ASYNC_JOB)
@@ -98,7 +112,6 @@ int async_global_init(void);
 void async_local_cleanup(void);
 void async_global_cleanup(void);
 void async_start_func(void);
-int async_pipe(OSSL_ASYNC_FD *pipefds);
-int async_close_fd(OSSL_ASYNC_FD fd);
-int async_write1(OSSL_ASYNC_FD fd, const void *buf);
-int async_read1(OSSL_ASYNC_FD fd, void *buf);
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx);
+

crypto/async/async_wait.c

+/*
+ * Written by Matt Caswell (matt@openssl.org) for the OpenSSL project.
+ */
+/* ====================================================================
+ * Copyright (c) 2016 The OpenSSL Project.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. All advertising materials mentioning features or use of this
+ *    software must display the following acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
+ *
+ * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For written permission, please contact
+ *    licensing@OpenSSL.org.
+ *
+ * 5. Products derived from this software may not be called "OpenSSL"
+ *    nor may "OpenSSL" appear in their names without prior written
+ *    permission of the OpenSSL Project.
+ *
+ * 6. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by the OpenSSL Project
+ *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
+ * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ * ====================================================================
+ */
+
+/* This must be the first #include file */
+#include "async_locl.h"
+
+#include <openssl/err.h>
+
+ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void)
+{
+    return OPENSSL_zalloc(sizeof(ASYNC_WAIT_CTX));
+}
+
+void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr;
+
+    if (ctx == NULL)
+        return;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has already been deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->cleanup != NULL)
+            curr->cleanup(ctx, curr->key, curr->fd, curr->custom_data);
+        curr = curr->next;
+    }
+
+    OPENSSL_free(ctx);
+}
+int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                               OSSL_ASYNC_FD fd, void *custom_data,
+                               void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *))
+{
+    struct fd_lookup_st *fdlookup;
+
+    fdlookup = OPENSSL_zalloc(sizeof *fdlookup);
+    if (fdlookup == NULL)
+        return 0;
+
+    fdlookup->key = key;
+    fdlookup->fd = fd;
+    fdlookup->custom_data = custom_data;
+    fdlookup->cleanup = cleanup;
+    fdlookup->add = 1;
+    fdlookup->next = ctx->fds;
+    ctx->fds = fdlookup;
+    ctx->numadd++;
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                          OSSL_ASYNC_FD *fd, void **custom_data)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            *fd = curr->fd;
+            *custom_data = curr->custom_data;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                               size_t *numfds)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    *numfds = 0;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (fd != NULL) {
+            *fd = curr->fd;
+            fd++;
+        }
+        (*numfds)++;
+        curr = curr->next;
+    }
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                   size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                   size_t *numdelfds)
+{
+    struct fd_lookup_st *curr;
+
+    *numaddfds = ctx->numadd;
+    *numdelfds = ctx->numdel;
+    if (addfd == NULL && delfd == NULL)
+        return 1;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        /* We ignore fds that have been marked as both added and deleted */
+        if (curr->del && !curr->add && (delfd != NULL)) {
+            *delfd = curr->fd;
+            delfd++;
+        }
+        if (curr->add && !curr->del && (addfd != NULL)) {
+            *addfd = curr->fd;
+            addfd++;
+        }
+        curr = curr->next;
+    }
+
+    return 1;
+}
+
+int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key)
+{
+    struct fd_lookup_st *curr;
+
+    curr = ctx->fds;
+    while (curr != NULL) {
+        if (curr->del) {
+            /* This one has been marked deleted already so do nothing */
+            curr = curr->next;
+            continue;
+        }
+        if (curr->key == key) {
+            /*
+             * Mark it as deleted. We don't call cleanup if explicitly asked
+             * to clear an fd. We assume the caller is going to do that
+             */
+            curr->del = 1;
+            ctx->numdel++;
+            return 1;
+        }
+        curr = curr->next;
+    }
+    return 0;
+}
+
+void async_wait_ctx_reset_counts(ASYNC_WAIT_CTX *ctx)
+{
+    struct fd_lookup_st *curr, *prev = NULL;
+
+    ctx->numadd = 0;
+    ctx->numdel = 0;
+
+    curr = ctx->fds;
+
+    while (curr != NULL) {
+        if (curr->del) {
+            if (prev == NULL)
+                ctx->fds = curr->next;
+            else
+                prev->next = curr->next;
+            OPENSSL_free(curr);
+            if (prev == NULL)
+                curr = ctx->fds;
+            else
+                curr = prev->next;
+            continue;
+        }
+        if (curr->add) {
+            curr->add = 0;
+        }
+        prev = curr;
+        curr = curr->next;
+    }
+}

crypto/async/build.info

 LIBS=../../libcrypto
 SOURCE[../../libcrypto]=\
-        async.c async_err.c arch/async_posix.c arch/async_win.c arch/async_null.c
+        async.c async_wait.c async_err.c arch/async_posix.c arch/async_win.c \
+        arch/async_null.c

doc/crypto/ASYNC_WAIT_CTX_new.pod

+=pod
+
+=head1 NAME
+
+ASYNC_WAIT_CTX_new, ASYNC_WAIT_CTX_free, ASYNC_WAIT_CTX_set_wait_fd,
+ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds, ASYNC_WAIT_CTX_clear_fd - functions to manage
+waiting for asynchronous jobs to complete
+
+=head1 SYNOPSIS
+
+ #include <openssl/async.h>
+
+ ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void);
+ void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx);
+ int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                                OSSL_ASYNC_FD fd,
+                                void *custom_data,
+                                void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *));
+ int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                           OSSL_ASYNC_FD *fd, void **custom_data);
+ int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                                size_t *numfds);
+ int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                    size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                    size_t *numdelfds);
+ int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key);
+
+
+=head1 DESCRIPTION
+
+For an overview of how asynchronous operations are implemented in OpenSSL see
+L<ASYNC_start_job(3)>. An ASYNC_WAIT_CTX object represents an asynchronous
+"session", i.e. a related set of crypto operations. For example in SSL terms
+this would have a one-to-one correspondence with an SSL connection.
+
+Application code must create an ASYNC_WAIT_CTX using the ASYNC_WAIT_CTX_new()
+function prior to calling ASYNC_start_job() (see L<ASYNC_start_job(3)>). When
+the job is started it is associated with the ASYNC_WAIT_CTX for the duration of
+that job. An ASYNC_WAIT_CTX should only be used for one ASYNC_JOB at any one
+time, but can be reused after an ASYNC_JOB has finished for a subsequent
+ASYNC_JOB. When the session is complete (e.g. the SSL connection is closed),
+application code cleans up with ASYNC_WAIT_CTX_free().
+
+ASYNC_WAIT_CTXs can have "wait" file descriptors associated with them. Calling
+ASYNC_WAIT_CTX_get_all_fds() and passing in a pointer to an ASYNC_WAIT_CTX in
+the B<ctx> parameter will return the wait file descriptors associated with that
+job in B<*fd>. The number of file descriptors returned will be stored in
+B<*numfds>. It is the caller's responsibility to ensure that sufficient memory
+has been allocated in B<*fd> to receive all the file descriptors. Calling
+ASYNC_WAIT_CTX_get_all_fds() with a NULL B<fd> value will return no file
+descriptors but will still populate B<*numfds>. Therefore application code is
+typically expected to call this function twice: once to get the number of fds,
+and then again when sufficient memory has been allocated. If only one
+asynchronous engine is being used then noramlly this call will only ever return
+one fd. If multiple asynchronous engines are being used then more could be
+returned.
+
+The function ASYNC_WAIT_CTX_fds_have_changed() can be used to detect if any fds
+have changed since the last call time ASYNC_start_job() returned an ASYNC_PAUSE
+result (or since the ASYNC_WAIT_CTX was created if no ASYNC_PAUSE result has
+been received). The B<numaddfds> and B<numdelfds> parameters will be populated
+with the number of fds added or deleted respectively. B<*addfd> and B<*delfd>
+will be populated with the list of added and deleted fds respectively. Similarly
+to ASYNC_WAIT_CTX_get_all_fds() either of these can be NULL, but if they are not
+NULL then the caller is responsible for ensuring sufficient memory is allocated.
+
+Implementors of async aware code (e.g. engines) are encouraged to return a
+stable fd for the lifetime of the ASYNC_WAIT_CTX in order to reduce the "churn"
+of regularly changing fds - although no guarantees of this are provided to
+applications.
+
+Applications can wait for the file descriptor to be ready for "read" using a
+system function call such as select or poll (being ready for "read" indicates
+that the job should be resumed). If no file descriptor is made available then an
+application will have to periodically "poll" the job by attempting to restart it
+to see if it is ready to continue.
+
+Async aware code (e.g. engines) can get the current ASYNC_WAIT_CTX from the job
+via L<ASYNC_get_async_wait_ctx(3)> and provide a file descriptor to use for
+waiting on by calling ASYNC_WAIT_CTX_set_wait_fd(). Typically this would be done
+by an engine immediately prior to calling ASYNC_pause_job() and not by end user
+code. An existing association with a file descriptor can be obtained using
+ASYNC_WAIT_CTX_get_fd() and cleared using ASYNC_WAIT_CTX_clear_fd(). Both of
+these functions requires a B<key> value which is unique to the async aware code.
+This could be any unique value but a good candidate might be the B<ENGINE *> for
+the engine. The B<custom_data> parameter can be any value, and will be returned
+in a subsequent call to ASYNC_WAIT_CTX_get_fd(). The
+ASYNC_WAIT_CTX_set_wait_fd() function also expects a pointer to a "cleanup"
+routine. This can be NULL but if provided will automatically get called when the
+ASYNC_WAIT_CTX is freed, and gives the engine the opportunity to close the fd or
+any other resources.
+
+An example of typical usage might be an async capable engine. User code would
+initiate cryptographic operations. The engine would initiate those operations
+asynchronously and then call ASYNC_WAIT_CTX_set_wait_fd() followed by
+ASYNC_pause_job() to return control to the user code. The user code can then
+perform other tasks or wait for the job to be ready by calling "select" or other
+similar function on the wait file descriptor. The engine can signal to the user
+code that the job should be resumed by making the wait file descriptor
+"readable". Once resumed the engine should clear the wake signal on the wait
+file descriptor.
+
+=head1 RETURN VALUES
+
+ASYNC_WAIT_CTX_new() returns a pointer to the newly allocated ASYNC_WAIT_CTX or
+NULL on error.
+
+ASYNC_WAIT_CTX_set_wait_fd, ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds and ASYNC_WAIT_CTX_clear_fd all return 1 on
+success or 0 on error.
+
+=head1 SEE ALSO
+
+L<crypto(3)>, L<ASYNC_start_job(3)>
+
+=head1 HISTORY
+
+ASYNC_WAIT_CTX_new, ASYNC_WAIT_CTX_free, ASYNC_WAIT_CTX_set_wait_fd,
+ASYNC_WAIT_CTX_get_fd, ASYNC_WAIT_CTX_get_all_fds,
+ASYNC_WAIT_CTX_get_changed_fds, ASYNC_WAIT_CTX_clear_fd were first added to
+OpenSSL 1.1.0.
+
+=cut

doc/crypto/ASYNC_start_job.pod

@@ -2,29 +2,24 @@
 
 =head1 NAME
 
-ASYNC_init, ASYNC_cleanup, ASYNC_init_thread, ASYNC_cleanup_thread,
-ASYNC_start_job, ASYNC_pause_job, ASYNC_in_job, ASYNC_get_wait_fd,
-ASYNC_get_current_job, ASYNC_wake, ASYNC_clear_wake, ASYNC_block_pause,
-ASYNC_unblock_pause - asynchronous job management functions
+ASYNC_init_thread, ASYNC_cleanup_thread, ASYNC_start_job, ASYNC_pause_job,
+ASYNC_in_job, ASYNC_get_wait_fd, ASYNC_set_wait_fd, ASYNC_clear_wait_fd,
+ASYNC_get_current_job, ASYNC_block_pause, ASYNC_unblock_pause - asynchronous job
+management functions
 
 =head1 SYNOPSIS
 
  #include <openssl/async.h>
 
- int ASYNC_init(int init_thread, size_t max_size, size_t init_size);
- void ASYNC_cleanup(int cleanupthread);
-
  int ASYNC_init_thread(size_t max_size, size_t init_size);
  void ASYNC_cleanup_thread(void);
 
- int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                     void *args, size_t size);
+ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *ctx, int *ret,
+                     int (*func)(void *), void *args, size_t size);
  int ASYNC_pause_job(void);
 
- int ASYNC_get_wait_fd(ASYNC_JOB *job);
  ASYNC_JOB *ASYNC_get_current_job(void);
- void ASYNC_wake(ASYNC_JOB *job);
- void ASYNC_clear_wake(ASYNC_JOB *job);
+ ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job);
  void ASYNC_block_pause(void);
  void ASYNC_unblock_pause(void);
 
@@ -38,19 +33,14 @@ subsequent event indicates that the job can be resumed.
 The creation of an ASYNC_JOB is a relatively expensive operation. Therefore, for
 efficiency reasons, jobs can be created up front and reused many times. They are
 held in a pool until they are needed, at which point they are removed from the
-pool, used, and then returned to the pool when the job completes. Before using
-any of the asynchronous job functions, user code should first call
-ASYNC_init(). If the user application is multi-threaded, then
-ASYNC_init_thread() should be called for each thread that will initiate
-asynchronous jobs. If the B<init_thread> parameter to ASYNC_init() is non-zero
-then ASYNC_init_thread is automatically called for the current thread. Before
-user code exits it should free up resources for each thread that was initialised
-using ASYNC_cleanup_thread(). No asynchronous jobs must be outstanding for the thread
-when ASYNC_cleanup_thread() is called. Failing to ensure this will result in memory
-leaks. Additionally an application should call ASYNC_cleanup() when all
-asynchronous work is complete across all threads. If B<cleanupthread> is
-non-zero then ASYNC_cleanup_thread() is automatically called for the current
-thread.
+pool, used, and then returned to the pool when the job completes. If the user
+application is multi-threaded, then ASYNC_init_thread() may be called for each
+thread that will initiate asynchronous jobs. Before
+user code exits per-thread resources need to be cleaned up. This will normally
+occur automatically (see L<OPENSSL_init_crypto(3)>) but may be explicitly
+initiated by using ASYNC_cleanup_thread(). No asynchronous jobs must be
+outstanding for the thread when ASYNC_cleanup_thread() is called. Failing to
+ensure this will result in memory leaks.
 
 The B<max_size> argument limits the number of ASYNC_JOBs that will be held in
 the pool. If B<max_size> is set to 0 then no upper limit is set. When an
@@ -60,16 +50,16 @@ pool does not exceed B<max_size>. When the pool is first initialised
 B<init_size> ASYNC_JOBs will be created immediately. If ASYNC_init_thread() is
 not called before the pool is first used then it will be called automatically
 with a B<max_size> of 0 (no upper limit) and an B<init_size> of 0 (no ASYNC_JOBs
-created up front). If a pool is created in this way it must still be cleaned up
-with an explicit call to ASYNC_cleanup_thread().
+created up front).
 
 An asynchronous job is started by calling the ASYNC_start_job() function.
-Initially B<*job> should be NULL. B<ret> should point to a location where the
-return value of the asynchronous function should be stored on completion of the
-job. B<func> represents the function that should be started asynchronously. The
-data pointed to by B<args> and of size B<size> will be copied and then passed as
-an argument to B<func> when the job starts. ASYNC_start_job will return one of
-the following values:
+Initially B<*job> should be NULL. B<ctx> should point to an ASYNC_WAIT_CTX
+object created through the L<ASYNC_WAIT_CTX_new(3)> function. B<ret> should
+point to a location where the return value of the asynchronous function should
+be stored on completion of the job. B<func> represents the function that should
+be started asynchronously. The data pointed to by B<args> and of size B<size>
+will be copied and then passed as an argument to B<func> when the job starts.
+ASYNC_start_job will return one of the following values:
 
 =over 4
 
@@ -114,23 +104,23 @@ B<*job> parameter will resume execution from the ASYNC_pause_job() call. If
 ASYNC_pause_job() is called whilst not within the context of a job then no
 action is taken and ASYNC_pause_job() returns immediately.
 
-Every ASYNC_JOB has a "wait" file descriptor associated with it. Calling
-ASYNC_get_wait_fd() and passing in a pointer to an ASYNC_JOB in the B<job>
-parameter will return the wait file descriptor associated with that job. This
-file descriptor can be used to signal that the job should be resumed.
-Applications can wait for the file descriptor to be ready for "read" using a
-system function call such as select or poll (being ready for "read" indicates
-that the job should be resumed). Applications can signal that a job is ready to
-resume using ASYNC_wake() or clear an existing signal using ASYNC_clear_wake().
+ASYNC_get_wait_ctx() can be used to get a pointer to the ASYNC_WAIT_CTX
+for the B<job>. ASYNC_WAIT_CTXs can have a "wait" file descriptor associated
+with them. Applications can wait for the file descriptor to be ready for "read"
+using a system function call such as select or poll (being ready for "read"
+indicates that the job should be resumed). If no file descriptor is made
+available then an application will have to priodically "poll" the job by
+attempting to restart it to see if it is ready to continue.
 
 An example of typical usage might be an async capable engine. User code would
 initiate cryptographic operations. The engine would initiate those operations
-asynchronously and then call ASYNC_pause_job() to return control to the user
-code. The user code can then perform other tasks or wait for the job to be ready
-by calling "select" or other similar function on the wait file descriptor. The
-engine can signal to the user code that the job should be resumed using
-ASYNC_wake(). Once resumed the engine would clear the wake signal by calling
-ASYNC_clear_wake().
+asynchronously and then call L<ASYNC_WAIT_CTX_set_wait_fd(3)> followed by
+ASYNC_pause_job() to return control to the user code. The user code can then
+perform other tasks or wait for the job to be ready by calling "select" or other
+similar function on the wait file descriptor. The engine can signal to the user
+code that the job should be resumed by making the wait file descriptor
+"readable". Once resumed the engine should clear the wake signal on the wait
+file descriptor.
 
 The ASYNC_block_pause() function will prevent the currently active job from
 pausing. The block will remain in place until a subsequent call to
@@ -149,7 +139,7 @@ occur.
 
 =head1 RETURN VALUES
 
-ASYNC_init and ASYNC_init_thread return 1 on success or 0 otherwise.
+ASYNC_init_thread returns 1 on success or 0 otherwise.
 
 ASYNC_start_job returns one of ASYNC_ERR, ASYNC_NO_JOBS, ASYNC_PAUSE or
 ASYNC_FINISH as described above.
@@ -158,23 +148,39 @@ ASYNC_pause_job returns 0 if an error occurred or 1 on success. If called when
 not within the context of an ASYNC_JOB then this is counted as success so 1 is
 returned.
 
-ASYNC_get_wait_fd returns the "wait" file descriptor associated with the
-ASYNC_JOB provided as an argument.
-
 ASYNC_get_current_job returns a pointer to the currently executing ASYNC_JOB or
 NULL if not within the context of a job.
 
+ASYNC_get_wait_ctx() returns a pointer to the ASYNC_WAIT_CTX for the job.
+
 =head1 EXAMPLE
 
 The following example demonstrates how to use most of the core async APIs:
 
  #include <stdio.h>
+ #include <unistd.h>
  #include <openssl/async.h>
+ #include <openssl/crypto.h>
+
+ #define WAIT_SIGNAL_CHAR   'X'
+
+ int unique = 0;
+
+ void cleanup(ASYNC_WAIT_CTX *ctx, const void *key, OSSL_ASYNC_FD r, void *vw)
+ {
+     OSSL_ASYNC_FD *w = (OSSL_ASYNC_FD *)vw;
+     close(r);
+     close(*w);
+     OPENSSL_free(w);
+ }
 
  int jobfunc(void *arg)
  {
      ASYNC_JOB *currjob;
      unsigned char *msg;
+     int pipefds[2] = {0, 0};
+     OSSL_ASYNC_FD *wptr;
+     char buf = WAIT_SIGNAL_CHAR;
 
      currjob = ASYNC_get_current_job();
      if (currjob != NULL) {
@@ -187,19 +193,32 @@ The following example demonstrates how to use most of the core async APIs:
      msg = (unsigned char *)arg;
      printf("Passed in message is: %s\n", msg);
 
+     if (pipe(pipefds) != 0) {
+         printf("Failed to create pipe\n");
+         return 0;
+     }
+     wptr = OPENSSL_malloc(sizeof(OSSL_ASYNC_FD));
+     if (wptr == NULL) {
+         printf("Failed to malloc\n");
+         return 0;
+     }
+     *wptr = pipefds[1];
+     ASYNC_WAIT_CTX_set_wait_fd(ASYNC_get_wait_ctx(currjob), &unique,
+                                pipefds[0], wptr, cleanup);
+
      /*
       * Normally some external event would cause this to happen at some
       * later point - but we do it here for demo purposes, i.e.
       * immediately signalling that the job is ready to be woken up after
       * we return to main via ASYNC_pause_job().
       */
-     ASYNC_wake(currjob);
+     write(pipefds[1], &buf, 1);
 
      /* Return control back to main */
      ASYNC_pause_job();
 
      /* Clear the wake signal */
-     ASYNC_clear_wake(currjob);
+     read(pipefds[0], &buf, 1);
 
      printf ("Resumed the job after a pause\n");
 
@@ -209,23 +228,23 @@ The following example demonstrates how to use most of the core async APIs:
  int main(void)
  {
      ASYNC_JOB *job = NULL;
-     int ret, waitfd;
+     ASYNC_WAIT_CTX *ctx = NULL;
+     int ret;
+     OSSL_ASYNC_FD waitfd;
      fd_set waitfdset;
+     size_t numfds;
      unsigned char msg[13] = "Hello world!";
 
-     /*
-      * We're only expecting 1 job to be used here so we're only creating
-      * a pool of 1
-      */
-     if (!ASYNC_init(1, 1, 1)) {
-         printf("Error creating pool\n");
-         goto end;
-     }
-
      printf("Starting...\n");
 
+     ctx = ASYNC_WAIT_CTX_new();
+     if (ctx == NULL) {
+         printf("Failed to create ASYNC_WAIT_CTX\n");
+         abort();
+     }
+
      for (;;) {
-         switch(ASYNC_start_job(&job, &ret, jobfunc, msg, sizeof(msg))) {
+         switch(ASYNC_start_job(&job, ctx, &ret, jobfunc, msg, sizeof(msg))) {
          case ASYNC_ERR:
          case ASYNC_NO_JOBS:
                  printf("An error occurred\n");
@@ -240,15 +259,21 @@ The following example demonstrates how to use most of the core async APIs:
 
          /* Wait for the job to be woken */
          printf("Waiting for the job to be woken up\n");
-         waitfd = ASYNC_get_wait_fd(job);
+        
+         if (!ASYNC_WAIT_CTX_get_all_fds(ctx, NULL, &numfds)
+                 || numfds > 1) {
+             printf("Unexpected number of fds\n");
+             abort();
+         }
+         ASYNC_WAIT_CTX_get_all_fds(ctx, &waitfd, &numfds);
          FD_ZERO(&waitfdset);
          FD_SET(waitfd, &waitfdset);
          select(waitfd + 1, &waitfdset, NULL, NULL, NULL);
      }
 
  end:
+     ASYNC_WAIT_CTX_free(ctx);
      printf("Finishing\n");
-     ASYNC_cleanup(1);
 
      return 0;
  }

doc/ssl/SSL_get_all_async_fds.pod

+=pod
+
+=head1 NAME
+
+SSL_waiting_for_async, SSL_get_all_async_fds, SSL_get_changed_async_fds - manage
+asynchronous operations
+
+=head1 SYNOPSIS
+
+ #include <openssl/ssl.h>
+
+ int SSL_waiting_for_async(SSL *s);
+ int SSL_get_all_async_fds(SSL *s, OSSL_ASYNC_FD *fd, size_t *numfds);
+ int SSL_get_changed_async_fds(SSL *s, OSSL_ASYNC_FD *addfd, size_t *numaddfds,
+                               OSSL_ASYNC_FD *delfd, size_t *numdelfds);
+
+=head1 DESCRIPTION
+
+SSL_waiting_for_async() determines whether an SSL connection is currently
+waiting for asynchronous operations to complete (see the SSL_MODE_ASYNC mode in
+L<SSL_CTX_set_mode(3)>).
+
+SSL_get_all_async_fds() returns a list of file descriptor which can be used in a
+call to select() or poll() to determine whether the current asynchronous
+operation has completed or not. A completed operation will result in data
+appearing as "read ready" on the file descriptor (no actual data should be read
+from the file descriptor). This function should only be called if the SSL object
+is currently waiting for asynchronous work to complete (i.e.
+SSL_ERROR_WANT_ASYNC has been received - see L<SSL_get_error(3)>). Typically the
+list will only contain one file descriptor. However if multiple asynchronous
+capable engines are in use then more than one is possible. The number of file
+descriptors returned is stored in B<*numfds> and the file descriptors themselves
+are in B<*fds>. The B<fds> parameter may be NULL in which case no file
+descriptors are returned but B<*numfds> is still populated. It is the callers
+responsibility to ensure sufficient memory is allocated at B<*fds> so typically
+this function is called twice (once with a NULL B<fds> parameter and once
+without).
+
+SSL_get_changed_async_fds() returns a list of the asynchronous file descriptors
+that have been added and a list that have been deleted since the last
+SSL_ERROR_WANT_ASYNC was received (or since the SSL object was created if no
+SSL_ERROR_WANT_ASYNC has been received). Similar to SSL_get_all_async_fds() it
+is the callers responsibility to ensure that B<*addfd> and B<*delfd> have
+sufficient memory allocated, although they may be NULL. The number of added fds
+and the number of deleted fds are stored in B<*numaddfds> and B<*numdelfds>
+respectively.
+
+=head1 RETURN VALUES
+
+SSL_waiting_for_async() will return 1 if the current SSL operation is waiting
+for an async operation to complete and 0 otherwise.
+
+SSL_get_all_async_fds() and SSL_get_changed_async_fds() return 1 on success or
+0 on error.
+
+=head1 SEE ALSO
+
+L<SSL_get_error(3)>, L<SSL_CTX_set_mode(3)>
+
+=head1 HISTORY
+
+SSL_waiting_for_async(), SSL_get_all_async_fds() and SSL_get_changed_async_fds()
+were first added to OpenSSL 1.1.0.
+
+=cut

doc/ssl/SSL_get_async_wait_fd.pod

-=pod
-
-=head1 NAME
-
-SSL_waiting_for_async, SSL_get_async_wait_fd - manage asynchronous operations
-
-=head1 SYNOPSIS
-
- #include <openssl/ssl.h>
-
- int SSL_waiting_for_async(SSL *s);
- int SSL_get_async_wait_fd(SSL *s);
-
-=head1 DESCRIPTION
-
-SSL_waiting_for_async() determines whether an SSL connection is currently
-waiting for asynchronous operations to complete (see the SSL_MODE_ASYNC mode in
-L<SSL_CTX_set_mode(3)>).
-
-SSL_get_async_wait_fd() returns a file descriptor which can be used in a call to
-select() or poll() to determine whether the current asynchronous operation has
-completed or not. A completed operation will result in data appearing as
-"read ready" on the file descriptor (no actual data should be read from the
-file descriptor). This function should only be called if the SSL object is
-currently waiting for asynchronous work to complete (i.e. SSL_ERROR_WANT_ASYNC
-has been received - see L<SSL_get_error(3)>).
-
-=head1 RETURN VALUES
-
-SSL_waiting_for_async() will return 1 if the current SSL operation is waiting
-for an async operation to complete and 0 otherwise.
-
-SSL_get_async_wait_fd() will return a file descriptor that can be used in a call
-to select() or poll() to wait for asynchronous work to complete, or -1 on error.
-
-=head1 SEE ALSO
-
-L<SSL_get_error(3)>, L<SSL_CTX_set_mode(3)>
-
-=head1 HISTORY
-
-SSL_waiting_for_async() and SSL_get_async_wait_fd() were first added to
-OpenSSL 1.1.0
-
-=cut

engines/e_dasync.c

@@ -61,6 +61,16 @@
 #include <openssl/bn.h>
 #include <openssl/crypto.h>
 
+#if (defined(OPENSSL_SYS_UNIX) || defined(OPENSSL_SYS_CYGWIN)) && defined(OPENSSL_THREADS)
+# undef ASYNC_POSIX
+# define ASYNC_POSIX
+# include <unistd.h>
+#elif defined(_WIN32)
+# undef ASYNC_WIN
+# define ASYNC_WIN
+# include <windows.h>
+#endif
+
 #define DASYNC_LIB_NAME "DASYNC"
 #include "e_dasync_err.c"
 
@@ -261,26 +271,87 @@ static int dasync_digests(ENGINE *e, const EVP_MD **digest,
     return ok;
 }
 
+static void wait_cleanup(ASYNC_WAIT_CTX *ctx, const void *key,
+                         OSSL_ASYNC_FD readfd, void *pvwritefd)
+{
+    OSSL_ASYNC_FD *pwritefd = (OSSL_ASYNC_FD *)pvwritefd;
+#if defined(ASYNC_WIN)
+    CloseHandle(readfd);
+    CloseHandle(*pwritefd);
+#elif defined(ASYNC_POSIX)
+    close(readfd);
+    close(*pwritefd);
+#endif
+    OPENSSL_free(pwritefd);
+}
+
+#define DUMMY_CHAR 'X'
+
 static void dummy_pause_job(void) {
     ASYNC_JOB *job;
+    ASYNC_WAIT_CTX *waitctx;
+    OSSL_ASYNC_FD pipefds[2] = {0, 0};
+    OSSL_ASYNC_FD *writefd;
+#if defined(ASYNC_WIN)
+    DWORD numwritten, numread;
+    char buf = DUMMY_CHAR;
+#elif defined(ASYNC_POSIX)
+    char buf = DUMMY_CHAR;
+#endif
 
     if ((job = ASYNC_get_current_job()) == NULL)
         return;
 
+    waitctx = ASYNC_get_wait_ctx(job);
+
+    if (ASYNC_WAIT_CTX_get_fd(waitctx, engine_dasync_id, &pipefds[0],
+                              (void **)&writefd)) {
+        pipefds[1] = *writefd;
+    } else {
+        writefd = OPENSSL_malloc(sizeof(*writefd));
+        if (writefd == NULL)
+            return;
+#if defined(ASYNC_WIN)
+        if (CreatePipe(&pipefds[0], &pipefds[1], NULL, 256) == 0) {
+            OPENSSL_free(writefd);
+            return;
+        }
+#elif defined(ASYNC_POSIX)
+        if (pipe(pipefds) != 0) {
+            OPENSSL_free(writefd);
+            return;
+        }
+#endif
+        *writefd = pipefds[1];
+
+        if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_dasync_id, pipefds[0],
+                                       writefd, wait_cleanup)) {
+            wait_cleanup(waitctx, engine_dasync_id, pipefds[0], writefd);
+            return;
+        }
+    }
     /*
      * In the Dummy async engine we are cheating. We signal that the job
      * is complete by waking it before the call to ASYNC_pause_job(). A real
      * async engine would only wake when the job was actually complete
      */
-    ASYNC_wake(job);
+#if defined(ASYNC_WIN)
+    WriteFile(pipefds[1], &buf, 1, &numwritten, NULL);
+#elif defined(ASYNC_POSIX)
+    write(pipefds[1], &buf, 1);
+#endif
 
     /* Ignore errors - we carry on anyway */
     ASYNC_pause_job();
 
-    ASYNC_clear_wake(job);
+    /* Clear the wake signal */
+#if defined(ASYNC_WIN)
+    ReadFile(pipefds[0], &buf, 1, &numread, NULL);
+#elif defined(ASYNC_POSIX)
+    read(pipefds[0], &buf, 1);
+#endif
 }
 
-
 /*
  * SHA1 implementation. At the moment we just defer to the standard
  * implementation

include/openssl/async.h

@@ -57,9 +57,11 @@
 
 #if defined(_WIN32)
 #include <windows.h>
-#define OSSL_ASYNC_FD   HANDLE
+#define OSSL_ASYNC_FD       HANDLE
+#define OSSL_BAD_ASYNC_FD   INVALID_HANDLE_VALUE
 #else
-#define OSSL_ASYNC_FD   int
+#define OSSL_ASYNC_FD       int
+#define OSSL_BAD_ASYNC_FD   -1
 #endif
 
 
@@ -68,6 +70,7 @@ extern "C" {
 # endif
 
 typedef struct async_job_st ASYNC_JOB;
+typedef struct async_wait_ctx_st ASYNC_WAIT_CTX;
 
 #define ASYNC_ERR      0
 #define ASYNC_NO_JOBS  1
@@ -77,14 +80,28 @@ typedef struct async_job_st ASYNC_JOB;
 int ASYNC_init_thread(size_t max_size, size_t init_size);
 void ASYNC_cleanup_thread(void);
 
-int ASYNC_start_job(ASYNC_JOB **job, int *ret, int (*func)(void *),
-                         void *args, size_t size);
+ASYNC_WAIT_CTX *ASYNC_WAIT_CTX_new(void);
+void ASYNC_WAIT_CTX_free(ASYNC_WAIT_CTX *ctx);
+int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                               OSSL_ASYNC_FD fd,
+                               void *custom_data,
+                               void (*cleanup)(ASYNC_WAIT_CTX *, const void *,
+                                               OSSL_ASYNC_FD, void *));
+int ASYNC_WAIT_CTX_get_fd(ASYNC_WAIT_CTX *ctx, const void *key,
+                        OSSL_ASYNC_FD *fd, void **custom_data);
+int ASYNC_WAIT_CTX_get_all_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *fd,
+                               size_t *numfds);
+int ASYNC_WAIT_CTX_get_changed_fds(ASYNC_WAIT_CTX *ctx, OSSL_ASYNC_FD *addfd,
+                                   size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                   size_t *numdelfds);
+int ASYNC_WAIT_CTX_clear_fd(ASYNC_WAIT_CTX *ctx, const void *key);
+
+int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *ctx, int *ret,
+                    int (*func)(void *), void *args, size_t size);
 int ASYNC_pause_job(void);
 
-OSSL_ASYNC_FD ASYNC_get_wait_fd(ASYNC_JOB *job);
 ASYNC_JOB *ASYNC_get_current_job(void);
-void ASYNC_wake(ASYNC_JOB *job);
-void ASYNC_clear_wake(ASYNC_JOB *job);
+ASYNC_WAIT_CTX *ASYNC_get_wait_ctx(ASYNC_JOB *job);
 void ASYNC_block_pause(void);
 void ASYNC_unblock_pause(void);
 

include/openssl/ssl.h

@@ -155,6 +155,7 @@
 # endif
 # include <openssl/pem.h>
 # include <openssl/hmac.h>
+# include <openssl/async.h>
 
 # include <openssl/safestack.h>
 # include <openssl/symhacks.h>
@@ -1599,7 +1600,10 @@ __owur char *SSL_get_srp_userinfo(SSL *s);
 void SSL_certs_clear(SSL *s);
 void SSL_free(SSL *ssl);
 __owur int SSL_waiting_for_async(SSL *s);
-__owur int SSL_get_async_wait_fd(SSL *s);
+__owur int SSL_get_all_async_fds(SSL *s, OSSL_ASYNC_FD *fds, size_t *numfds);
+__owur int SSL_get_changed_async_fds(SSL *s, OSSL_ASYNC_FD *addfd,
+                                     size_t *numaddfds, OSSL_ASYNC_FD *delfd,
+                                     size_t *numdelfds);
 __owur int SSL_accept(SSL *ssl);
 __owur int SSL_connect(SSL *ssl);
 __owur int SSL_read(SSL *ssl, void *buf, int num);

ssl/ssl_lib.c

@@ -1055,6 +1055,8 @@ void SSL_free(SSL *s)
 
     SSL_CTX_free(s->ctx);
 
+    ASYNC_WAIT_CTX_free(s->waitctx);
+
 #if !defined(OPENSSL_NO_NEXTPROTONEG)
     OPENSSL_free(s->next_proto_negotiated);
 #endif
@@ -1399,12 +1401,24 @@ int SSL_waiting_for_async(SSL *s)
     return 0;
 }
 
-int SSL_get_async_wait_fd(SSL *s)
+int SSL_get_all_async_fds(SSL *s, OSSL_ASYNC_FD *fds, size_t *numfds)
 {
-    if (!s->job)
-        return -1;
+    ASYNC_WAIT_CTX *ctx = s->waitctx;
+
+    if (ctx == NULL)
+        return 0;
+    return ASYNC_WAIT_CTX_get_all_fds(ctx, fds, numfds);
+}
 
-    return ASYNC_get_wait_fd(s->job);
+int SSL_get_changed_async_fds(SSL *s, OSSL_ASYNC_FD *addfd, size_t *numaddfds,
+                              OSSL_ASYNC_FD *delfd, size_t *numdelfds)
+{
+    ASYNC_WAIT_CTX *ctx = s->waitctx;
+
+    if (ctx == NULL)
+        return 0;
+    return ASYNC_WAIT_CTX_get_changed_fds(ctx, addfd, numaddfds, delfd,
+                                          numdelfds);
 }
 
 int SSL_accept(SSL *s)
@@ -1435,7 +1449,12 @@ long SSL_get_default_timeout(const SSL *s)
 static int ssl_start_async_job(SSL *s, struct ssl_async_args *args,
                           int (*func)(void *)) {
     int ret;
-    switch(ASYNC_start_job(&s->job, &ret, func, args,
+    if (s->waitctx == NULL) {
+        s->waitctx = ASYNC_WAIT_CTX_new();
+        if (s->waitctx == NULL)
+            return -1;
+    }
+    switch(ASYNC_start_job(&s->job, s->waitctx, &ret, func, args,
         sizeof(struct ssl_async_args))) {
     case ASYNC_ERR:
         s->rwstate = SSL_NOTHING;

ssl/ssl_locl.h

@@ -1175,6 +1175,7 @@ struct ssl_st {
 
     /* Async Job info */
     ASYNC_JOB *job;
+    ASYNC_WAIT_CTX *waitctx;
 };
 
 

test/asynctest.c

@@ -103,12 +103,24 @@ static int save_current(void *args)
     return 1;
 }
 
-static int wake(void *args)
+#define MAGIC_WAIT_FD   ((OSSL_ASYNC_FD)99)
+static int waitfd(void *args)
 {
+    ASYNC_JOB *job;
+    ASYNC_WAIT_CTX *waitctx;
     ASYNC_pause_job();
-    ASYNC_wake(ASYNC_get_current_job());
+    job = ASYNC_get_current_job();
+    if (job == NULL)
+        return 0;
+    waitctx = ASYNC_get_wait_ctx(job);
+    if (waitctx == NULL)
+        return 0;
+    if(!ASYNC_WAIT_CTX_set_wait_fd(waitctx, waitctx, MAGIC_WAIT_FD, NULL, NULL))
+        return 0;
     ASYNC_pause_job();
-    ASYNC_clear_wake(ASYNC_get_current_job());
+
+    if (!ASYNC_WAIT_CTX_clear_fd(waitctx, waitctx))
+        return 0;
 
     return 1;
 }
@@ -127,30 +139,34 @@ static int test_ASYNC_init_thread()
 {
     ASYNC_JOB *job1 = NULL, *job2 = NULL, *job3 = NULL;
     int funcret1, funcret2, funcret3;
+    ASYNC_WAIT_CTX *waitctx;
 
     if (       !ASYNC_init_thread(2, 0)
-            || ASYNC_start_job(&job1, &funcret1, only_pause, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job1, waitctx, &funcret1, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job2, &funcret2, only_pause, NULL, 0)
+            || ASYNC_start_job(&job2, waitctx, &funcret2, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_NO_JOBS
-            || ASYNC_start_job(&job1, &funcret1, only_pause, NULL, 0)
+            || ASYNC_start_job(&job1, waitctx, &funcret1, only_pause, NULL, 0)
                 != ASYNC_FINISH
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job2, &funcret2, only_pause, NULL, 0)
+            || ASYNC_start_job(&job2, waitctx, &funcret2, only_pause, NULL, 0)
                 != ASYNC_FINISH
-            || ASYNC_start_job(&job3, &funcret3, only_pause, NULL, 0)
+            || ASYNC_start_job(&job3, waitctx, &funcret3, only_pause, NULL, 0)
                 != ASYNC_FINISH
             || funcret1 != 1
             || funcret2 != 1
             || funcret3 != 1) {
         fprintf(stderr, "test_ASYNC_init_thread() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -159,20 +175,26 @@ static int test_ASYNC_start_job()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     ctr = 0;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_PAUSE
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, add_two, NULL, 0)
+               != ASYNC_PAUSE
             || ctr != 1
-            || ASYNC_start_job(&job, &funcret, add_two, NULL, 0) != ASYNC_FINISH
+            || ASYNC_start_job(&job, waitctx, &funcret, add_two, NULL, 0)
+               != ASYNC_FINISH
             || ctr != 2
             || funcret != 2) {
         fprintf(stderr, "test_ASYNC_start_job() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -181,74 +203,83 @@ static int test_ASYNC_get_current_job()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     currjob = NULL;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, save_current, NULL, 0)
                 != ASYNC_PAUSE
             || currjob != job
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || ASYNC_start_job(&job, waitctx, &funcret, save_current, NULL, 0)
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_current_job() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
 
-static int hasdata(OSSL_ASYNC_FD fd)
-{
-#ifdef ASYNC_POSIX
-    fd_set checkfds;
-    struct timeval tv;
-    FD_ZERO(&checkfds);
-    openssl_fdset(fd, &checkfds);
-    memset(&tv, 0, sizeof tv);
-    if (select(fd + 1, (void *)&checkfds, NULL, NULL, &tv) < 0)
-        return -1;
-    if (FD_ISSET(fd, &checkfds))
-        return 1;
-    return 0;
-#else
-    DWORD avail = 0;
-
-    if (PeekNamedPipe(fd, NULL, 0, NULL, &avail, NULL) && avail > 0)
-        return 1;
-
-    return 0;
-#endif
-}
-
-static int test_ASYNC_get_wait_fd()
+static int test_ASYNC_WAIT_CTX_get_all_fds()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
-    OSSL_ASYNC_FD fd;
+    ASYNC_WAIT_CTX *waitctx;
+    OSSL_ASYNC_FD fd = OSSL_BAD_ASYNC_FD, delfd = OSSL_BAD_ASYNC_FD;
+    size_t numfds, numdelfds;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, wake, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+               /* On first run we're not expecting any wait fds */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_PAUSE
-            || (fd = ASYNC_get_wait_fd(job)) < 0
-            || hasdata(fd) != 0
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                               &numdelfds)
+            || numfds != 0
+            || numdelfds != 0
+               /* On second run we're expecting one added fd */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_PAUSE
-            || hasdata(fd) != 1
-            || (ASYNC_clear_wake(job), 0)
-            || hasdata(fd) != 0
-            || (ASYNC_wake(job), 0)
-            || hasdata(fd) != 1
-            || ASYNC_start_job(&job, &funcret, save_current, NULL, 0)
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 1
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, &fd, &numfds)
+            || fd != MAGIC_WAIT_FD
+            || (fd = OSSL_BAD_ASYNC_FD, 0) /* Assign to something else */
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                              &numdelfds)
+            || numfds != 1
+            || numdelfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, &fd, &numfds, NULL,
+                                               &numdelfds)
+            || fd != MAGIC_WAIT_FD
+               /* On final run we expect one deleted fd */
+            || ASYNC_start_job(&job, waitctx, &funcret, waitfd, NULL, 0)
                 != ASYNC_FINISH
+            || !ASYNC_WAIT_CTX_get_all_fds(waitctx, NULL, &numfds)
+            || numfds != 0
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, NULL,
+                                               &numdelfds)
+            || numfds != 0
+            || numdelfds != 1
+            || !ASYNC_WAIT_CTX_get_changed_fds(waitctx, NULL, &numfds, &delfd,
+                                               &numdelfds)
+            || delfd != MAGIC_WAIT_FD
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_get_wait_fd() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -257,18 +288,22 @@ static int test_ASYNC_block_pause()
 {
     ASYNC_JOB *job = NULL;
     int funcret;
+    ASYNC_WAIT_CTX *waitctx;
 
     if (       !ASYNC_init_thread(1, 0)
-            || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
+            || (waitctx = ASYNC_WAIT_CTX_new()) == NULL
+            || ASYNC_start_job(&job, waitctx, &funcret, blockpause, NULL, 0)
                 != ASYNC_PAUSE
-            || ASYNC_start_job(&job, &funcret, blockpause, NULL, 0)
+            || ASYNC_start_job(&job, waitctx, &funcret, blockpause, NULL, 0)
                 != ASYNC_FINISH
             || funcret != 1) {
         fprintf(stderr, "test_ASYNC_block_pause() failed\n");
+        ASYNC_WAIT_CTX_free(waitctx);
         ASYNC_cleanup_thread();
         return 0;
     }
 
+    ASYNC_WAIT_CTX_free(waitctx);
     ASYNC_cleanup_thread();
     return 1;
 }
@@ -287,7 +322,7 @@ int main(int argc, char **argv)
     if (       !test_ASYNC_init_thread()
             || !test_ASYNC_start_job()
             || !test_ASYNC_get_current_job()
-            || !test_ASYNC_get_wait_fd()
+            || !test_ASYNC_WAIT_CTX_get_all_fds()
             || !test_ASYNC_block_pause()) {
         return 1;
     }

util/libeay.num

@@ -4515,10 +4515,10 @@ ASYNC_pause_job                         5013	1_1_0	EXIST::FUNCTION:
 ASYNC_start_job                         5014	1_1_0	EXIST::FUNCTION:
 ASYNC_init_thread                       5015	1_1_0	EXIST::FUNCTION:
 ASYNC_cleanup_thread                    5016	1_1_0	EXIST::FUNCTION:
-ASYNC_wake                              5017	1_1_0	EXIST::FUNCTION:
-ASYNC_clear_wake                        5018	1_1_0	EXIST::FUNCTION:
+ASYNC_wake                              5017	1_1_0	NOEXIST::FUNCTION:
+ASYNC_clear_wake                        5018	1_1_0	NOEXIST::FUNCTION:
 ASYNC_get_current_job                   5019	1_1_0	EXIST::FUNCTION:
-ASYNC_get_wait_fd                       5020	1_1_0	EXIST::FUNCTION:
+ASYNC_get_wait_fd                       5020	1_1_0	NOEXIST::FUNCTION:
 ERR_load_ASYNC_strings                  5021	1_1_0	EXIST::FUNCTION:
 ASYNC_unblock_pause                     5022	1_1_0	EXIST::FUNCTION:
 ASYNC_block_pause                       5023	1_1_0	EXIST::FUNCTION:
@@ -4758,3 +4758,11 @@ SCT_set_version                         5261	1_1_0	EXIST::FUNCTION:
 SCT_LIST_free                           5262	1_1_0	EXIST::FUNCTION:
 SCT_get_log_entry_type                  5263	1_1_0	EXIST::FUNCTION:
 EC_KEY_can_sign                         5264	1_1_0	EXIST::FUNCTION:EC
+ASYNC_WAIT_CTX_free                     5265	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_get_all_fds              5266	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_get_fd                   5267	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_clear_fd                 5268	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_get_changed_fds          5269	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_set_wait_fd              5270	1_1_0	EXIST::FUNCTION:
+ASYNC_WAIT_CTX_new                      5271	1_1_0	EXIST::FUNCTION:
+ASYNC_get_wait_ctx                      5272	1_1_0	EXIST::FUNCTION:

util/ssleay.num

@@ -391,7 +391,7 @@ SSL_get_state                           446	1_1_0	EXIST::FUNCTION:
 SSL_set_default_passwd_cb               447	1_1_0	EXIST::FUNCTION:
 SSL_set_default_passwd_cb_userdata      448	1_1_0	EXIST::FUNCTION:
 SSL_waiting_for_async                   449	1_1_0	EXIST::FUNCTION:
-SSL_get_async_wait_fd                   450	1_1_0	EXIST::FUNCTION:
+SSL_get_all_async_fds                   450	1_1_0	EXIST::FUNCTION:
 SSL_add_ssl_module                      451	1_1_0	EXIST::FUNCTION:
 SSL_CTX_config                          452	1_1_0	EXIST::FUNCTION:
 SSL_config                              453	1_1_0	EXIST::FUNCTION:
@@ -417,3 +417,4 @@ SSL_CTX_up_ref                          472	1_1_0	EXIST::FUNCTION:
 DTLSv1_listen                           473	1_1_0	EXIST::FUNCTION:
 SSL_get0_verified_chain                 474	1_1_0	EXIST::FUNCTION:
 OPENSSL_init_ssl                        475	1_1_0	EXIST::FUNCTION:
+SSL_get_changed_async_fds               476	1_1_0	EXIST::FUNCTION: