HTTP client: Allow streaming of response data (with possibly indefinite length)

Also clean up max_resp_len and add OSSL_HTTP_REQ_CTX_get_resp_len().
Reviewed-by: NTomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15053)

crypto/http/http_client.c

@@ -62,8 +62,8 @@ struct ossl_http_req_ctx_st {
     int expect_asn1;            /* Response must be ASN.1-encoded */
     unsigned char *pos;         /* Current position sending data */
     long len_to_send;           /* Number of bytes still to send */
-    unsigned long resp_len;     /* Length of response */
-    size_t max_resp_len;        /* Maximum length of response */
+    size_t resp_len;            /* Length of response */
+    size_t max_resp_len;        /* Maximum length of response, or 0 */
     int keep_alive;             /* Persistent conn. 0=no, 1=prefer, 2=require */
     time_t max_time;            /* Maximum end time of current transfer, or 0 */
     time_t max_total_time;      /* Maximum end time of total transfer, or 0 */
@@ -72,19 +72,20 @@ struct ossl_http_req_ctx_st {
 
 /* HTTP states */
 
-#define OHS_NOREAD          0x1000 /* If set no reading should be performed */
-#define OHS_ERROR           (0 | OHS_NOREAD) /* Error condition */
-#define OHS_FIRSTLINE       1 /* First line of response being read */
-#define OHS_HEADERS         2 /* MIME headers of response being read */
-#define OHS_REDIRECT        3 /* MIME headers being read, expecting Location */
-#define OHS_ASN1_HEADER     4 /* HTTP initial header (tag+length) being read */
-#define OHS_CONTENT         5 /* HTTP content octets being read */
+#define OHS_NOREAD         0x1000 /* If set no reading should be performed */
+#define OHS_ERROR          (0 | OHS_NOREAD) /* Error condition */
 #define OHS_ADD_HEADERS    (1 | OHS_NOREAD) /* Adding header lines to request */
 #define OHS_WRITE_INIT     (2 | OHS_NOREAD) /* 1st call: ready to start send */
 #define OHS_WRITE_HDR      (3 | OHS_NOREAD) /* Request header being sent */
 #define OHS_WRITE_REQ      (4 | OHS_NOREAD) /* Request contents being sent */
 #define OHS_FLUSH          (5 | OHS_NOREAD) /* Request being flushed */
-#define OHS_DONE           (6 | OHS_NOREAD) /* Completed */
+#define OHS_FIRSTLINE       1 /* First line of response being read */
+#define OHS_HEADERS         2 /* MIME headers of response being read */
+#define OHS_REDIRECT        3 /* MIME headers being read, expecting Location */
+#define OHS_ASN1_HEADER     4 /* ASN1 sequence header (tag+length) being read */
+#define OHS_ASN1_CONTENT    5 /* ASN1 content octets being read */
+#define OHS_ASN1_DONE      (6 | OHS_NOREAD) /* ASN1 content read completed */
+#define OHS_STREAM         (7 | OHS_NOREAD) /* HTTP content stream to be read */
 
 /* Low-level HTTP API implementation */
 
@@ -334,14 +335,12 @@ static OSSL_HTTP_REQ_CTX *http_req_ctx_new(int free_wbio, BIO *wbio, BIO *rbio,
                                            void *arg, int use_ssl,
                                            const char *proxy,
                                            const char *server, const char *port,
-                                           int buf_size, unsigned long max_len,
-                                           int overall_timeout)
+                                           int buf_size, int overall_timeout)
 {
     OSSL_HTTP_REQ_CTX *rctx = OSSL_HTTP_REQ_CTX_new(wbio, rbio, buf_size);
 
     if (rctx == NULL)
         return NULL;
-    OSSL_HTTP_REQ_CTX_set_max_response_length(rctx, max_len);
     rctx->free_wbio = free_wbio;
     rctx->upd_fn = bio_update_fn;
     rctx->upd_arg = arg;
@@ -371,7 +370,7 @@ static OSSL_HTTP_REQ_CTX *http_req_ctx_new(int free_wbio, BIO *wbio, BIO *rbio,
 
 static int parse_http_line1(char *line, int *found_keep_alive)
 {
-    int retcode;
+    int i, retcode;
     char *code, *reason, *end;
 
     if (strncmp(line, HTTP_PREFIX_VERSION, HTTP_VERSION_PATT_LEN) != 0)
@@ -440,18 +439,22 @@ static int parse_http_line1(char *line, int *found_keep_alive)
     }
 
  err:
-    ERR_raise_data(ERR_LIB_HTTP, HTTP_R_HEADER_PARSE_ERROR, "%.40s", line);
+    i = 0;
+    while (i < 60 && ossl_isprint(line[i]))
+        i++;
+    line[i] = '\0';
+    ERR_raise_data(ERR_LIB_HTTP, HTTP_R_HEADER_PARSE_ERROR, "content=%s", line);
     return 0;
 }
 
-static int check_set_resp_len(OSSL_HTTP_REQ_CTX *rctx, unsigned long len)
+static int check_set_resp_len(OSSL_HTTP_REQ_CTX *rctx, size_t len)
 {
-    if (len > rctx->max_resp_len)
+    if (rctx->max_resp_len != 0 && len > rctx->max_resp_len)
         ERR_raise_data(ERR_LIB_HTTP, HTTP_R_MAX_RESP_LEN_EXCEEDED,
-                       "length=%lu, max=%lu", len, rctx->max_resp_len);
+                       "length=%zu, max=%zu", len, rctx->max_resp_len);
     if (rctx->resp_len != 0 && rctx->resp_len != len)
         ERR_raise_data(ERR_LIB_HTTP, HTTP_R_INCONSISTENT_CONTENT_LENGTH,
-                       "ASN.1 length=%lu, Content-Length=%lu",
+                       "ASN.1 length=%zu, Content-Length=%zu",
                        len, rctx->resp_len);
     rctx->resp_len = len;
     return 1;
@@ -465,7 +468,7 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
 {
     int i, found_expected_ct = 0, found_keep_alive = 0;
     long n;
-    unsigned long resp_len;
+    size_t resp_len;
     const unsigned char *p;
     char *key, *value, *line_end = NULL;
 
@@ -481,7 +484,10 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
     rctx->redirection_url = NULL;
  next_io:
     if ((rctx->state & OHS_NOREAD) == 0) {
-        n = BIO_read(rctx->rbio, rctx->readbuf, rctx->readbuflen);
+        if (rctx->expect_asn1)
+            n = BIO_read(rctx->rbio, rctx->readbuf, rctx->readbuflen);
+        else
+            n = BIO_gets(rctx->rbio, (char *)rctx->readbuf, rctx->readbuflen);
         if (n <= 0) {
             if (BIO_should_retry(rctx->rbio))
                 return -1;
@@ -655,7 +661,7 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
                     found_keep_alive = 0;
             }
             if (strcasecmp(key, "Content-Length") == 0) {
-                resp_len = strtoul(value, &line_end, 10);
+                resp_len = (size_t)strtoul(value, &line_end, 10);
                 if (line_end == value || *line_end != '\0') {
                     ERR_raise_data(ERR_LIB_HTTP,
                                    HTTP_R_ERROR_PARSING_CONTENT_LENGTH,
@@ -697,8 +703,8 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
         }
 
         if (!rctx->expect_asn1) {
-            rctx->state = OHS_CONTENT;
-            goto content;
+            rctx->state = OHS_STREAM;
+            return 1;
         }
 
         rctx->state = OHS_ASN1_HEADER;
@@ -747,17 +753,16 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
         if (!check_set_resp_len(rctx, resp_len))
             return 0;
 
- content:
-        rctx->state = OHS_CONTENT;
+        rctx->state = OHS_ASN1_CONTENT;
 
         /* Fall thru */
-    case OHS_CONTENT:
+    case OHS_ASN1_CONTENT:
     default:
         n = BIO_get_mem_data(rctx->mem, NULL);
-        if (n < (long)rctx->resp_len /* may be 0 if no Content-Length or ASN.1 */)
+        if (n < 0 || (size_t)n < rctx->resp_len)
             goto next_io;
 
-        rctx->state = OHS_DONE;
+        rctx->state = OHS_ASN1_DONE;
         return 1;
     }
 }
@@ -840,7 +845,7 @@ BIO *OSSL_HTTP_REQ_CTX_exchange(OSSL_HTTP_REQ_CTX *rctx)
         }
         return NULL;
     }
-    return rctx->mem;
+    return rctx->state == OHS_STREAM ? rctx->rbio : rctx->mem;
 }
 
 int OSSL_HTTP_is_alive(const OSSL_HTTP_REQ_CTX *rctx)
@@ -855,8 +860,7 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
                                   const char *proxy, const char *no_proxy,
                                   int use_ssl, BIO *bio, BIO *rbio,
                                   OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                  int buf_size, unsigned long max_resp_len,
-                                  int overall_timeout)
+                                  int buf_size, int overall_timeout)
 {
     BIO *cbio; /* == bio if supplied, used as connection BIO if rbio is NULL */
     OSSL_HTTP_REQ_CTX *rctx = NULL;
@@ -926,7 +930,7 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
 
     rctx = http_req_ctx_new(bio == NULL, cbio, rbio != NULL ? rbio : cbio,
                             bio_update_fn, arg, use_ssl, proxy, server, port,
-                            buf_size, max_resp_len, overall_timeout);
+                            buf_size, overall_timeout);
 
  end:
     if (rctx != NULL)
@@ -942,7 +946,7 @@ int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
                           const STACK_OF(CONF_VALUE) *headers,
                           const char *content_type, BIO *req,
                           const char *expected_content_type, int expect_asn1,
-                          int timeout, int keep_alive)
+                          size_t max_resp_len, int timeout, int keep_alive)
 {
     int use_http_proxy;
 
@@ -955,6 +959,7 @@ int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
         ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_INVALID_ARGUMENT);
         return 0;
     }
+    rctx->max_resp_len = max_resp_len; /* allows for 0: indefinite */
 
     return OSSL_HTTP_REQ_CTX_set_request_line(rctx, req != NULL,
                                               use_http_proxy ? rctx->server
@@ -1077,13 +1082,13 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
 
         rctx = OSSL_HTTP_open(host, port, proxy, no_proxy,
                               use_ssl, bio, rbio, bio_update_fn, arg,
-                              buf_size, max_resp_len, timeout);
+                              buf_size, timeout);
     new_rpath:
         if (rctx != NULL) {
             if (!OSSL_HTTP_set_request(rctx, path, headers,
                                        NULL /* content_type */,
                                        NULL /* req */,
-                                       expected_ct, expect_asn1,
+                                       expected_ct, expect_asn1, max_resp_len,
                                        -1 /* use same max time (timeout) */,
                                        0 /* no keep_alive */))
                 OSSL_HTTP_REQ_CTX_free(rctx);
@@ -1130,7 +1135,7 @@ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
                         int buf_size, const STACK_OF(CONF_VALUE) *headers,
                         const char *content_type, BIO *req,
                         const char *expected_ct, int expect_asn1,
-                        unsigned long max_resp_len, int timeout, int keep_alive)
+                        size_t max_resp_len, int timeout, int keep_alive)
 {
     OSSL_HTTP_REQ_CTX *rctx = prctx == NULL ? NULL : *prctx;
     BIO *resp = NULL;
@@ -1138,13 +1143,13 @@ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
     if (rctx == NULL) {
         rctx = OSSL_HTTP_open(server, port, proxy, no_proxy,
                               use_ssl, bio, rbio, bio_update_fn, arg,
-                              buf_size, max_resp_len, timeout);
+                              buf_size, timeout);
         timeout = -1; /* Already set during opening the connection */
     }
     if (rctx != NULL) {
         if (OSSL_HTTP_set_request(rctx, path, headers, content_type, req,
                                   expected_ct, expect_asn1,
-                                  timeout, keep_alive))
+                                  max_resp_len, timeout, keep_alive))
             resp = OSSL_HTTP_exchange(rctx, NULL);
         if (resp == NULL || !OSSL_HTTP_is_alive(rctx)) {
             if (!OSSL_HTTP_close(rctx, resp != NULL)) {

doc/man3/OSSL_HTTP_REQ_CTX.pod

@@ -87,16 +87,19 @@ For example, to add a C<Host> header for C<example.com> you would call:
  OSSL_HTTP_REQ_CTX_add1_header(ctx, "Host", "example.com");
 
 OSSL_HTTP_REQ_CTX_set_expected() optionally sets in I<rctx> some expectations
-of the HTTP client.
+of the HTTP client on the response.
 Due to the structure of an HTTP request, if the I<keep_alive> argument is
 nonzero the function must be used before calling OSSL_HTTP_REQ_CTX_set1_req().
 If the I<content_type> parameter
 is not NULL then the client will check that the given content type string
 is included in the HTTP header of the response and return an error if not.
 If the I<asn1> parameter is nonzero a structure in ASN.1 encoding will be
-expected as the response content.  This means that an ASN.1 sequence header
-is required and the its length field is checked.
+expected as the response content and input streaming is disabled.  This means
+that an ASN.1 sequence header is required, its length field is checked, and
+OSSL_HTTP_REQ_CTX_get0_mem_bio() should be used to get the buffered response.
 Else any form of input is allowed without length checks, which is the default.
+In this case the BIO given as I<rbio> argument to OSSL_HTTP_REQ_CTX_new() should
+be used directly to read the response contents, which may support streaming.
 If the I<timeout> parameter is > 0 this indicates the maximum number of seconds
 the subsequent HTTP transfer (sending the request and receiving a response)
 is allowed to take.
@@ -121,9 +124,6 @@ All of this ends up in the internal memory B<BIO>.
 OSSL_HTTP_REQ_CTX_nbio() attempts to send the request prepared in I<rctx>
 and to gather the response via HTTP, using the I<wbio> and I<rbio>
 that were given when calling OSSL_HTTP_REQ_CTX_new().
-If successful, the contents of the internal memory B<BIO> contains
-the contents of the HTTP response, without the response headers.
-This does not support streaming.
 The function may need to be called again if its result is -1, which indicates
 L<BIO_should_retry(3)>.  In such a case it is advisable to sleep a little in
 between, using L<BIO_wait(3)> on the read BIO to prevent a busy loop.

doc/man3/OSSL_HTTP_transfer.pod

@@ -30,7 +30,7 @@ OSSL_HTTP_close
                            const STACK_OF(CONF_VALUE) *headers,
                            const char *content_type, BIO *req,
                            const char *expected_content_type, int expect_asn1,
-                           int timeout, int keep_alive);
+                           size_t max_resp_len, int timeout, int keep_alive);
  BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);
  BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                     BIO *bio, BIO *rbio,
@@ -126,8 +126,6 @@ The I<buf_size> parameter specifies the response header maximum line length.
 A value <= 0 indicates that
 the B<HTTP_DEFAULT_MAX_LINE_LENGTH> of 4KiB should be used.
 This length is also used as the number of content bytes that are read at a time.
-The I<max_resp_len> specifies the maximum allowed response content length.
-The value 0 indicates B<HTTP_DEFAULT_MAX_RESP_LEN>, which currently is 100 KiB.
 
 If the I<overall_timeout> parameter is > 0 this indicates the maximum number of
 seconds the overall HTTP transfer (i.e., connection setup if needed,
@@ -161,6 +159,8 @@ is not NULL then the client will check that the given content type string
 is included in the HTTP header of the response and return an error if not.
 If the I<expect_asn1> parameter is nonzero,
 a structure in ASN.1 encoding will be expected as response content.
+The I<max_resp_len> parameter specifies the maximum allowed
+response content length, where the value 0 indicates no limit.
 If the I<timeout> parameter is > 0 this indicates the maximum number of seconds
 the subsequent HTTP transfer (sending the request and receiving a response)
 is allowed to take.
@@ -234,8 +234,9 @@ OSSL_HTTP_proxy_connect() and OSSL_HTTP_set_request()
 return 1 on success, 0 on error.
 
 On success, OSSL_HTTP_exchange(), OSSL_HTTP_get(), and OSSL_HTTP_transfer()
-return a memory BIO containing the data received.
-This must be freed by the caller.
+return a memory BIO containing the data received if an ASN.1-encoded response
+is expected, else a BIO that may support streaming.
+The BIO must be freed by the caller.
 On failure, they return NULL.
 Failure conditions include connection/transfer timeout, parse errors, etc.
 

include/openssl/http.h

@@ -65,8 +65,7 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
                                   const char *proxy, const char *no_proxy,
                                   int use_ssl, BIO *bio, BIO *rbio,
                                   OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                  int buf_size, unsigned long max_resp_len,
-                                  int overall_timeout);
+                                  int buf_size, int overall_timeout);
 int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
                             const char *proxyuser, const char *proxypass,
                             int timeout, BIO *bio_err, const char *prog);
@@ -74,14 +73,14 @@ int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
                           const STACK_OF(CONF_VALUE) *headers,
                           const char *content_type, BIO *req,
                           const char *expected_content_type, int expect_asn1,
-                          int timeout, int keep_alive);
+                          size_t max_resp_len, int timeout, int keep_alive);
 BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);
 BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                    BIO *bio, BIO *rbio,
                    OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
                    int buf_size, const STACK_OF(CONF_VALUE) *headers,
                    const char *expected_content_type, int expect_asn1,
-                   unsigned long max_resp_len, int timeout);
+                   size_t max_resp_len, int timeout);
 BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
                         const char *server, const char *port,
                         const char *path, int use_ssl,
@@ -91,7 +90,7 @@ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
                         int buf_size, const STACK_OF(CONF_VALUE) *headers,
                         const char *content_type, BIO *req,
                         const char *expected_content_type, int expect_asn1,
-                        unsigned long max_resp_len, int timeout, int keep_alive);
+                        size_t max_resp_len, int timeout, int keep_alive);
 int OSSL_HTTP_close(OSSL_HTTP_REQ_CTX *rctx, int ok);
 
 /* Auxiliary functions */