Add documentation for the new custom extensions API

Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3139)

doc/man3/SSL_extension_supported.pod

@@ -3,6 +3,7 @@
 =head1 NAME
 
 SSL_extension_supported,
+SSL_CTX_add_custom_ext,
 SSL_CTX_add_client_custom_ext, SSL_CTX_add_server_custom_ext,
 custom_ext_add_cb, custom_ext_free_cb, custom_ext_parse_cb
 - custom TLS extension handling
@@ -11,19 +12,29 @@ custom_ext_add_cb, custom_ext_free_cb, custom_ext_parse_cb
 
  #include <openssl/ssl.h>
 
- int SSL_CTX_add_client_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
-                                   custom_ext_add_cb add_cb,
-                                   custom_ext_free_cb free_cb, void *add_arg,
-                                   custom_ext_parse_cb parse_cb,
-                                   void *parse_arg);
-
- int SSL_CTX_add_server_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
-                                   custom_ext_add_cb add_cb,
-                                   custom_ext_free_cb free_cb, void *add_arg,
-                                   custom_ext_parse_cb parse_cb,
-                                   void *parse_arg);
-
- int SSL_extension_supported(unsigned int ext_type);
+ typedef int (*custom_ext_add_cb_ex) (SSL *s, unsigned int ext_type,
+                                      unsigned int context,
+                                      const unsigned char **out,
+                                      size_t *outlen, X509 *x, size_t chainidx,
+                                      int *al, void *add_arg);
+
+ typedef void (*custom_ext_free_cb_ex) (SSL *s, unsigned int ext_type,
+                                        unsigned int context,
+                                        const unsigned char *out,
+                                        void *add_arg);
+
+ typedef int (*custom_ext_parse_cb_ex) (SSL *s, unsigned int ext_type,
+                                        unsigned int context,
+                                        const unsigned char *in,
+                                        size_t inlen, X509 *x, size_t chainidx,
+                                        int *al, void *parse_arg);
+
+ int SSL_CTX_add_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
+                            unsigned int context,
+                            custom_ext_add_cb_ex add_cb,
+                            custom_ext_free_cb_ex free_cb,
+                            void *add_arg,
+                            custom_ext_parse_cb_ex parse_cb, void *parse_arg);
 
  typedef int (*custom_ext_add_cb)(SSL *s, unsigned int ext_type,
                                   const unsigned char **out,
@@ -39,18 +50,45 @@ custom_ext_add_cb, custom_ext_free_cb, custom_ext_parse_cb
                                     size_t inlen, int *al,
                                     void *parse_arg);
 
+ int SSL_CTX_add_client_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
+                                   custom_ext_add_cb add_cb,
+                                   custom_ext_free_cb free_cb, void *add_arg,
+                                   custom_ext_parse_cb parse_cb,
+                                   void *parse_arg);
 
-=head1 DESCRIPTION
+ int SSL_CTX_add_server_custom_ext(SSL_CTX *ctx, unsigned int ext_type,
+                                   custom_ext_add_cb add_cb,
+                                   custom_ext_free_cb free_cb, void *add_arg,
+                                   custom_ext_parse_cb parse_cb,
+                                   void *parse_arg);
 
-SSL_CTX_add_client_custom_ext() adds a custom extension for a TLS client
-with extension type B<ext_type> and callbacks B<add_cb>, B<free_cb> and
-B<parse_cb>.
+ int SSL_extension_supported(unsigned int ext_type);
 
-SSL_CTX_add_server_custom_ext() adds a custom extension for a TLS server
-with extension type B<ext_type> and callbacks B<add_cb>, B<free_cb> and
-B<parse_cb>.
+=head1 DESCRIPTION
 
-In both cases the extension type must not be handled by OpenSSL internally
+SSL_CTX_add_custom_ext() adds a custom extension for a (D)TLS client or server
+for all supported protocol versions with extension type B<ext_type> and
+callbacks B<add_cb>, B<free_cb> and B<parse_cb> (see the
+L</EXTENSION CALLBACKS> section below). The B<context> value determines
+which messages and under what conditions the extension will be added/parsed (see
+the L</EXTENSION CONTEXTS> section below).
+
+SSL_CTX_add_client_custom_ext() adds a custom extension for a (D)TLS client with
+extension type B<ext_type> and callbacks B<add_cb>, B<free_cb> and B<parse_cb>.
+This function is similar to SSL_CTX_add_custom_ext() except it only applies
+to clients, uses the older style of callbacks, and implicitly sets the
+B<context> value to:
+
+ SSL_EXT_TLS1_2_AND_BELOW_ONLY | SSL_EXT_CLIENT_HELLO
+ | SSL_EXT_TLS1_2_SERVER_HELLO | SSL_EXT_IGNORE_ON_RESUMPTION
+
+SSL_CTX_add_server_custom_ext() adds a custom extension for a (D)TLS server with
+extension type B<ext_type> and callbacks B<add_cb>, B<free_cb> and
+B<parse_cb>. This function is similar to SSL_CTX_add_custom_ext() except it
+only applies to servers, uses the older style of callbacks, and implicitly sets
+the B<context> value to the same as for SSL_CTX_add_client_custom_ext() above.
+
+In all cases the extension type must not be handled by OpenSSL internally
 or an error occurs.
 
 SSL_extension_supported() returns 1 if the extension B<ext_type> is handled
@@ -59,9 +97,11 @@ internally by OpenSSL and 0 otherwise.
 =head1 EXTENSION CALLBACKS
 
 The callback B<add_cb> is called to send custom extension data to be
-included in ClientHello for TLS clients or ServerHello for servers. The
-B<ext_type> parameter is set to the extension type which will be added and
-B<add_arg> to the value set when the extension handler was added.
+included in various TLS messages. The B<ext_type> parameter is set to the
+extension type which will be added and B<add_arg> to the value set when the
+extension handler was added. When using the new style callbacks the B<context>
+parameter will indicate which message is currently being constructed e.g. for
+the ClientHello it will be set to B<SSL_EXT_CLIENT_HELLO>.
 
 If the application wishes to include the extension B<ext_type> it should
 set B<*out> to the extension data, set B<*outlen> to the length of the
@@ -72,16 +112,25 @@ If the B<add_cb> does not wish to include the extension it must return 0.
 If B<add_cb> returns -1 a fatal handshake error occurs using the TLS
 alert value specified in B<*al>.
 
-For clients (but not servers) if B<add_cb> is set to NULL a zero length
-extension is added for B<ext_type>.
+When constructing the ClientHello if B<add_cb> is set to NULL a zero length
+extension is added for B<ext_type>. For all other messages if B<add_cb> is set
+to NULL then no extension is added.
+
+When constructing a Certificate message the callback will be called for each
+certificate in the message. The B<x> parameter will indicate the
+current certificate and the B<chainidx> parameter will indicate the position
+of the certificate in the message. The first certificate is always the end
+entity certificate and has a B<chainidx> value of 0.
 
-For clients every registered B<add_cb> is always called to see if the
-application wishes to add an extension to ClientHello.
+For all messages except the ServerHello and EncryptedExtensions every
+registered B<add_cb> is always called to see if the application wishes to add an
+extension (as long as all requirements of the specified B<context> are met).
 
-For servers every registered B<add_cb> is called once if and only if the
-corresponding extension was received in ClientHello to see if the application
-wishes to add the extension to ServerHello. That is, if no corresponding extension
-was received in ClientHello then B<add_cb> will not be called.
+For the ServerHello and EncryptedExtension messages every registered B<add_cb>
+is called once if and only if the requirements of the specified B<context> are
+met and the corresponding extension was received in the ClientHello. That is, if
+no corresponding extension was received in the ClientHello then B<add_cb> will
+not be called.
 
 If an extension is added (that is B<add_cb> returns 1) B<free_cb> is called
 (if it is set) with the value of B<out> set by the add callback. It can be
@@ -89,12 +138,19 @@ used to free up any dynamic extension data set by B<add_cb>. Since B<out> is
 constant (to permit use of constant data in B<add_cb>) applications may need to
 cast away const to free the data.
 
-The callback B<parse_cb> receives data for TLS extensions. For TLS clients
-the extension data will come from ServerHello and for TLS servers it will
-come from ClientHello.
+The callback B<parse_cb> receives data for TLS extensions. The callback is only
+called if the extension is present and relevant for the context (see
+L</EXTENSION CONTEXTS> below).
 
 The extension data consists of B<inlen> bytes in the buffer B<in> for the
-extension B<extension_type>.
+extension B<ext_type>.
+
+If the message being parsed is a TLSv1.3 compatible Certificate message then
+B<parse_cb> will be called for each certificate contained within the message.
+The B<x> parameter will indicate the current certificate and the B<chainidx>
+parameter will indicate the position of the certificate in the message. The
+first certificate is always the end entity certificate and has a B<chainidx>
+value of 0.
 
 If the B<parse_cb> considers the extension data acceptable it must return
 1. If it returns 0 or a negative value a fatal handshake error occurs
@@ -103,6 +159,87 @@ using the TLS alert value specified in B<*al>.
 The buffer B<in> is a temporary internal buffer which will not be valid after
 the callback returns.
 
+=head1 EXTENSION CONTEXTS
+
+An extension context defines which messages and under which conditions an
+extension should be added or expected. The context is built up by performing
+a bitwise OR of multiple pre-defined values together. The valid context values
+are:
+
+=over 4
+
+=item SSL_EXT_TLS_ONLY
+
+The extension is only allowed in TLS
+
+=item SSL_EXT_DTLS_ONLY
+
+The extension is only allowed in DTLS
+
+=item SSL_EXT_TLS_IMPLEMENTATION_ONLY
+
+The extension is allowed in DTLS, but there is only a TLS implementation
+available (so it is ignored in DTLS).
+
+=item SSL_EXT_SSL3_ALLOWED
+
+Extensions are not typically defined for SSLv3. Setting this value will allow
+the extension in SSLv3. Applications will not typically need to use this.
+
+=item SSL_EXT_TLS1_2_AND_BELOW_ONLY
+
+The extension is only defined for (D)TLSv1.2 and below. Servers will ignore this
+extension if it is present in the ClientHello and TLSv1.3 is negotiated.
+
+=item SSL_EXT_TLS1_3_ONLY
+
+The extension is only defined for TLS1.3 and above. Servers will ignore this
+extension if it is present in the ClientHello and TLSv1.2 or below is
+negotiated.
+
+=item SSL_EXT_IGNORE_ON_RESUMPTION
+
+The extension will be ignored during parsing if a previous session is being
+successfully resumed.
+
+=item SSL_EXT_CLIENT_HELLO
+
+The extension may be present in the ClientHello message.
+
+=item SSL_EXT_TLS1_2_SERVER_HELLO
+
+The extension may be present in a TLSv1.2 or below compatible ServerHello
+message.
+
+=item SSL_EXT_TLS1_3_SERVER_HELLO
+
+The extension may be present in a TLSv1.3 compatible ServerHello message.
+
+=item SSL_EXT_TLS1_3_ENCRYPTED_EXTENSIONS
+
+The extension may be present in an EncryptedExtensions message.
+
+=item SSL_EXT_TLS1_3_HELLO_RETRY_REQUEST
+
+The extension may be present in a HelloRetryRequest message.
+
+=item SSL_EXT_TLS1_3_CERTIFICATE
+
+The extension may be present in a TLSv1.3 compatible Certificate message.
+
+=item SSL_EXT_TLS1_3_NEW_SESSION_TICKET
+
+The extension may be present in a TLSv1.3 compatible NewSessionTicket message.
+
+=item SSL_EXT_TLS1_3_CERTIFICATE_REQUEST
+
+The extension may be present in a TLSv1.3 compatible CertificateRequest message.
+
+=back
+
+The context must include at least one message value (otherwise the extension
+will never be used).
+
 =head1 NOTES
 
 The B<add_arg> and B<parse_arg> parameters can be set to arbitrary values
@@ -115,27 +252,33 @@ RFC5246 et al. It is B<not> a NID.
 
 If the same custom extension type is received multiple times a fatal
 B<decode_error> alert is sent and the handshake aborts. If a custom extension
-is received in ServerHello which was not sent in ClientHello a fatal
-B<unsupported_extension> alert is sent and the handshake is aborted. The
-ServerHello B<add_cb> callback is only called if the corresponding extension
-was received in ClientHello. This is compliant with the TLS specifications.
-This behaviour ensures that each callback is called at most once and that
-an application can never send unsolicited extensions.
+is received in a ServerHello/EncryptedExtensions message which was not sent in
+the ClientHello a fatal B<unsupported_extension> alert is sent and the
+handshake is aborted. The ServerHello/EncryptedExtensions B<add_cb> callback is
+only called if the corresponding extension was received in the ClientHello. This
+is compliant with the TLS specifications. This behaviour ensures that each
+callback is called at most once and that an application can never send
+unsolicited extensions.
 
 =head1 RETURN VALUES
 
-SSL_CTX_add_client_custom_ext() and SSL_CTX_add_server_custom_ext() return 1 for
-success and 0 for failure. A failure can occur if an attempt is made to
-add the same B<ext_type> more than once, if an attempt is made to use an
-extension type handled internally by OpenSSL or if an internal error occurs
-(for example a memory allocation failure).
+SSL_CTX_add_custom_ext(), SSL_CTX_add_client_custom_ext() and
+SSL_CTX_add_server_custom_ext() return 1 for success and 0 for failure. A
+failure can occur if an attempt is made to add the same B<ext_type> more than
+once, if an attempt is made to use an extension type handled internally by
+OpenSSL or if an internal error occurs (for example a memory allocation
+failure).
 
 SSL_extension_supported() returns 1 if the extension B<ext_type> is handled
 internally by OpenSSL and 0 otherwise.
 
+=head1 HISTORY
+
+The function SSL_CTX_add_custom_ext() was added in OpenSSL version 1.1.1.
+
 =head1 COPYRIGHT
 
-Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
+Copyright 2014-2017 The OpenSSL Project Authors. All Rights Reserved.
 
 Licensed under the OpenSSL license (the "License").  You may not use
 this file except in compliance with the License.  You can obtain a copy