Don't use an enum in the return type for a public API function

We use an int instead. That means SSL_key_update() also should use an int.
Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2609)

doc/man3/SSL_key_update.pod

@@ -13,17 +13,8 @@ SSL_renegotiate_pending
 
  #include <openssl/ssl.h>
 
- /* TLSv1.3 KeyUpdate message types */
- typedef enum {
-     /* -1 used so that this is an invalid value for the on-the-wire protocol */
-     SSL_KEY_UPDATE_NONE = -1,
-     /* Values as defined for the on-the-wire protocol */
-     SSL_KEY_UPDATE_NOT_REQUESTED = 0,
-     SSL_KEY_UPDATE_REQUESTED = 1
- } SSL_KEY_UPDATE;
-
- int SSL_key_update(SSL *s, SSL_KEY_UPDATE updatetype);
- SSL_KEY_UPDATE SSL_get_key_update_type(SSL *s);
+ int SSL_key_update(SSL *s, int updatetype);
+ int SSL_get_key_update_type(SSL *s);
 
  int SSL_renegotiate(SSL *s);
  int SSL_renegotiate_abbreviated(SSL *s);

doc/man7/ssl.pod

@@ -552,7 +552,7 @@ fresh handle for each connection.
 
 =item void (*B<SSL_get_info_callback>(const SSL *ssl);)()
 
-=item SSL_KEY_UPDATE B<SSL_get_key_update_type>(SSL *s);
+=item int B<SSL_get_key_update_type>(SSL *s);
 
 =item STACK *B<SSL_get_peer_cert_chain>(const SSL *ssl);
 
@@ -602,7 +602,7 @@ fresh handle for each connection.
 
 =item int B<SSL_is_init_finished>(SSL *ssl);
 
-=item int B<SSL_key_update>(SSL *s, SSL_KEY_UPDATE updatetype);
+=item int B<SSL_key_update>(SSL *s, int updatetype);
 
 =item STACK *B<SSL_load_client_CA_file>(const char *file);
 

include/openssl/ssl.h

@@ -826,13 +826,11 @@ DEFINE_STACK_OF(SSL_COMP)
 DEPRECATEDIN_1_1_0(void SSL_set_debug(SSL *s, int debug))
 
 /* TLSv1.3 KeyUpdate message types */
-typedef enum {
-    /* -1 used so that this is an invalid value for the on-the-wire protocol */
-    SSL_KEY_UPDATE_NONE = -1,
-    /* Values as defined for the on-the-wire protocol */
-    SSL_KEY_UPDATE_NOT_REQUESTED = 0,
-    SSL_KEY_UPDATE_REQUESTED = 1
-} SSL_KEY_UPDATE;
+/* -1 used so that this is an invalid value for the on-the-wire protocol */
+#define SSL_KEY_UPDATE_NONE             -1
+/* Values as defined for the on-the-wire protocol */
+#define SSL_KEY_UPDATE_NOT_REQUESTED     0
+#define SSL_KEY_UPDATE_REQUESTED         1
 
 /*
  * The valid handshake states (one for each type message sent and one for each
@@ -1662,8 +1660,8 @@ __owur STACK_OF(SSL_CIPHER) *SSL_get_client_ciphers(const SSL *s);
 __owur STACK_OF(SSL_CIPHER) *SSL_get1_supported_ciphers(SSL *s);
 
 __owur int SSL_do_handshake(SSL *s);
-int SSL_key_update(SSL *s, SSL_KEY_UPDATE updatetype);
-SSL_KEY_UPDATE SSL_get_key_update_type(SSL *s);
+int SSL_key_update(SSL *s, int updatetype);
+int SSL_get_key_update_type(SSL *s);
 int SSL_renegotiate(SSL *s);
 int SSL_renegotiate_abbreviated(SSL *s);
 __owur int SSL_renegotiate_pending(SSL *s);

ssl/ssl_lib.c

@@ -1718,7 +1718,7 @@ int SSL_shutdown(SSL *s)
     }
 }
 
-int SSL_key_update(SSL *s, SSL_KEY_UPDATE updatetype)
+int SSL_key_update(SSL *s, int updatetype)
 {
     /*
      * TODO(TLS1.3): How will applications know whether TLSv1.3+ has been
@@ -1746,7 +1746,7 @@ int SSL_key_update(SSL *s, SSL_KEY_UPDATE updatetype)
     return 1;
 }
 
-SSL_KEY_UPDATE SSL_get_key_update_type(SSL *s)
+int SSL_get_key_update_type(SSL *s)
 {
     return s->key_update;
 }

ssl/ssl_locl.h

@@ -1177,7 +1177,7 @@ struct ssl_st {
      */
     int renegotiate;
     /* If sending a KeyUpdate is pending */
-    SSL_KEY_UPDATE key_update;
+    int key_update;
 # ifndef OPENSSL_NO_SRP
     /* ctx for SRP authentication */
     SRP_CTX srp_ctx;

test/ssl_test_ctx.h

@@ -124,7 +124,7 @@ typedef struct {
     /* Maximum send fragment size. */
     int max_fragment_size;
     /* KeyUpdate type */
-    SSL_KEY_UPDATE key_update_type;
+    int key_update_type;
 
     /*
      * Extra server/client configurations. Per-handshake.