Convert some libssl local functions to size_t

Reviewed-by: NRich Salz <rsalz@openssl.org>

ssl/s3_enc.c

@@ -407,7 +407,8 @@ int ssl3_digest_cached_records(SSL *s, int keep)
     return 1;
 }
 
-size_t ssl3_final_finish_mac(SSL *s, const char *sender, int len, unsigned char *p)
+size_t ssl3_final_finish_mac(SSL *s, const char *sender, size_t len,
+                             unsigned char *p)
 {
     int ret;
     EVP_MD_CTX *ctx = NULL;

ssl/s3_lib.c

@@ -3941,7 +3941,7 @@ long ssl_get_algorithm2(SSL *s)
  * Fill a ClientRandom or ServerRandom field of length len. Returns <= 0 on
  * failure, 1 on success.
  */
-int ssl_fill_hello_random(SSL *s, int server, unsigned char *result, int len)
+int ssl_fill_hello_random(SSL *s, int server, unsigned char *result, size_t len)
 {
     int send_time = 0;
 

ssl/ssl_lib.c

@@ -65,7 +65,7 @@ SSL3_ENC_METHOD ssl3_undef_enc_method = {
     (int (*)(SSL *, unsigned char *, unsigned char *, size_t, size_t *))
         ssl_undefined_function,
     (int (*)(SSL *, int))ssl_undefined_function,
-    (size_t (*)(SSL *, const char *, int, unsigned char *))
+    (size_t (*)(SSL *, const char *, size_t, unsigned char *))
         ssl_undefined_function,
     0,                          /* finish_mac_length */
     NULL,                       /* client_finished_label */

ssl/ssl_locl.h

@@ -1572,7 +1572,7 @@ typedef struct ssl3_enc_method {
     int (*generate_master_secret) (SSL *, unsigned char *, unsigned char *,
                                    size_t, size_t *);
     int (*change_cipher_state) (SSL *, int);
-    size_t (*final_finish_mac) (SSL *, const char *, int, unsigned char *);
+    size_t (*final_finish_mac) (SSL *, const char *, size_t, unsigned char *);
     int finish_mac_length;
     const char *client_finished_label;
     int client_finished_label_len;
@@ -1862,7 +1862,7 @@ __owur int ssl_verify_alarm_type(long type);
 void ssl_sort_cipher_list(void);
 void ssl_load_ciphers(void);
 __owur int ssl_fill_hello_random(SSL *s, int server, unsigned char *field,
-                                 int len);
+                                 size_t len);
 __owur int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen,
                                       int free_pms);
 __owur EVP_PKEY *ssl_generate_pkey(EVP_PKEY *pm);
@@ -1887,7 +1887,7 @@ __owur const SSL_CIPHER *ssl3_get_cipher(unsigned int u);
 int ssl3_renegotiate(SSL *ssl);
 int ssl3_renegotiate_check(SSL *ssl);
 __owur int ssl3_dispatch_alert(SSL *s);
-__owur size_t ssl3_final_finish_mac(SSL *s, const char *sender, int slen,
+__owur size_t ssl3_final_finish_mac(SSL *s, const char *sender, size_t slen,
                                     unsigned char *p);
 __owur int ssl3_finish_mac(SSL *s, const unsigned char *buf, size_t len);
 void ssl3_free_digest_list(SSL *s);
@@ -1983,7 +1983,7 @@ void ssl_free_wbio_buffer(SSL *s);
 
 __owur int tls1_change_cipher_state(SSL *s, int which);
 __owur int tls1_setup_key_block(SSL *s);
-__owur size_t tls1_final_finish_mac(SSL *s, const char *str, int slen,
+__owur size_t tls1_final_finish_mac(SSL *s, const char *str, size_t slen,
                                     unsigned char *p);
 __owur int tls1_generate_master_secret(SSL *s, unsigned char *out,
                                        unsigned char *p, size_t len,

ssl/t1_enc.c

@@ -43,19 +43,18 @@
 
 /* seed1 through seed5 are concatenated */
 static int tls1_PRF(SSL *s,
-                    const void *seed1, int seed1_len,
-                    const void *seed2, int seed2_len,
-                    const void *seed3, int seed3_len,
-                    const void *seed4, int seed4_len,
-                    const void *seed5, int seed5_len,
-                    const unsigned char *sec, int slen,
-                    unsigned char *out, int olen)
+                    const void *seed1, size_t seed1_len,
+                    const void *seed2, size_t seed2_len,
+                    const void *seed3, size_t seed3_len,
+                    const void *seed4, size_t seed4_len,
+                    const void *seed5, size_t seed5_len,
+                    const unsigned char *sec, size_t slen,
+                    unsigned char *out, size_t olen)
 {
     const EVP_MD *md = ssl_prf_md(s);
     EVP_PKEY_CTX *pctx = NULL;
 
     int ret = 0;
-    size_t outlen = olen;
 
     if (md == NULL) {
         /* Should never happen */
@@ -79,7 +78,7 @@ static int tls1_PRF(SSL *s,
     if (EVP_PKEY_CTX_add1_tls1_prf_seed(pctx, seed5, seed5_len) <= 0)
         goto err;
 
-    if (EVP_PKEY_derive(pctx, out, &outlen) <= 0)
+    if (EVP_PKEY_derive(pctx, out, &olen) <= 0)
         goto err;
     ret = 1;
 
@@ -88,7 +87,6 @@ static int tls1_PRF(SSL *s,
     return ret;
 }
 
-/* TODO(size_t): convert me */
 static int tls1_generate_key_block(SSL *s, unsigned char *km, size_t num)
 {
     int ret;
@@ -451,7 +449,8 @@ int tls1_setup_key_block(SSL *s)
     return (ret);
 }
 
-size_t tls1_final_finish_mac(SSL *s, const char *str, int slen, unsigned char *out)
+size_t tls1_final_finish_mac(SSL *s, const char *str, size_t slen,
+                             unsigned char *out)
 {
     size_t hashlen;
     unsigned char hash[EVP_MAX_MD_SIZE];