ssl/statem/*.c: address "enum mixed with another type" warnings.

This is actually not all warnings, only return values.
Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4297)

ssl/statem/extensions_clnt.c

@@ -851,7 +851,7 @@ EXT_RETURN tls_construct_ctos_padding(SSL *s, WPACKET *pkt,
         if (!WPACKET_put_bytes_u16(pkt, TLSEXT_TYPE_padding)
                 || !WPACKET_sub_allocate_bytes_u16(pkt, hlen, &padbytes)) {
             SSLerr(SSL_F_TLS_CONSTRUCT_CTOS_PADDING, ERR_R_INTERNAL_ERROR);
-            return 0;
+            return EXT_RETURN_FAIL;
         }
         memset(padbytes, 0, hlen);
     }
@@ -1052,7 +1052,7 @@ EXT_RETURN tls_construct_ctos_psk(SSL *s, WPACKET *pkt, unsigned int context,
  err:
     return ret;
 #else
-    return 1;
+    return EXT_RETURN_SENT;
 #endif
 }
 

ssl/statem/statem_clnt.c

@@ -1658,7 +1658,8 @@ static MSG_PROCESS_RETURN tls_process_hello_retry_request(SSL *s, PACKET *pkt)
 
 MSG_PROCESS_RETURN tls_process_server_certificate(SSL *s, PACKET *pkt)
 {
-    int al, i, ret = MSG_PROCESS_ERROR;
+    int al, i;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     unsigned long cert_list_len, cert_len;
     X509 *x = NULL;
     const unsigned char *certstart, *certbytes;
@@ -2314,7 +2315,7 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
 
 MSG_PROCESS_RETURN tls_process_certificate_request(SSL *s, PACKET *pkt)
 {
-    int ret = MSG_PROCESS_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     int al = SSL_AD_DECODE_ERROR;
     size_t i;
 
@@ -3246,7 +3247,7 @@ WORK_STATE tls_prepare_client_certificate(SSL *s, WORK_STATE wst)
             if (i == 0) {
                 ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                 ossl_statem_set_error(s);
-                return 0;
+                return WORK_ERROR;
             }
             s->rwstate = SSL_NOTHING;
         }
@@ -3292,7 +3293,7 @@ WORK_STATE tls_prepare_client_certificate(SSL *s, WORK_STATE wst)
                 if (!ssl3_digest_cached_records(s, 0)) {
                     ssl3_send_alert(s, SSL3_AL_FATAL, SSL_AD_INTERNAL_ERROR);
                     ossl_statem_set_error(s);
-                    return 0;
+                    return WORK_ERROR;
                 }
             }
         }

ssl/statem/statem_lib.c

@@ -316,7 +316,8 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
 #ifndef OPENSSL_NO_GOST
     unsigned char *gost_data = NULL;
 #endif
-    int al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
+    int al = SSL_AD_INTERNAL_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     int j;
     unsigned int len;
     X509 *peer;

ssl/statem/statem_srvr.c

@@ -1353,7 +1353,7 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL *s, PACKET *pkt)
              */
             if (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE) {
                 if (clienthello->dtls_cookie_len == 0)
-                    return 1;
+                    return MSG_PROCESS_FINISHED_READING;
             }
         }
 
@@ -3251,7 +3251,8 @@ WORK_STATE tls_post_process_client_key_exchange(SSL *s, WORK_STATE wst)
 
 MSG_PROCESS_RETURN tls_process_client_certificate(SSL *s, PACKET *pkt)
 {
-    int i, al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
+    int i, al = SSL_AD_INTERNAL_ERROR;
+    MSG_PROCESS_RETURN ret = MSG_PROCESS_ERROR;
     X509 *x = NULL;
     unsigned long l, llen;
     const unsigned char *certstart, *certbytes;