HttpClient: Add cookie support (cookie jar) (#6216)

* Support concatenation of headers (as in https://github.com/esp8266/Arduino/commit/1de0c341b55ba8c0993fd3d2e0c5696935578751#diff-977435a9cc4619fa0b8b995085f6ae683485cf563722756bab57108b362da316 for ESP8266, fixes https://github.com/espressif/arduino-esp32/issues/4069)

* Add support for receiving, storing and sending cookies (cookie jar)

* Cookie support: Respect `secure` attribute when sending a request

* Fix missing `_secure` flag

* Comment out support concatenation of headers (not needed anymore when using cookie jar)

libraries/HTTPClient/src/HTTPClient.cpp

@@ -39,6 +39,9 @@
 
 #include "HTTPClient.h"
 
+/// Cookie jar support
+#include <time.h>
+
 #ifdef HTTPCLIENT_1_1_COMPATIBLE
 class TransportTraits
 {
@@ -157,6 +160,7 @@ bool HTTPClient::begin(WiFiClient &client, String url) {
     }
 
     _port = (protocol == "https" ? 443 : 80);
+    _secure = (protocol == "https");
     return beginInternal(url, protocol.c_str());
 }
 
@@ -187,6 +191,7 @@ bool HTTPClient::begin(WiFiClient &client, String host, uint16_t port, String ur
     _port = port;
     _uri = uri;
     _protocol = (https ? "https" : "http");
+    _secure = https;
     return true;
 }
 
@@ -603,6 +608,12 @@ int HTTPClient::sendRequest(const char * type, uint8_t * payload, size_t size)
             addHeader(F("Content-Length"), String(size));
         }
 
+        // add cookies to header, if present
+        String cookie_string;
+        if(generateCookieString(&cookie_string)) {
+            addHeader("Cookie", cookie_string);
+        }
+
         // send Header
         if(!sendHeader(type)) {
             return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
@@ -706,6 +717,12 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size)
         addHeader("Content-Length", String(size));
     }
 
+    // add cookies to header, if present
+    String cookie_string;
+    if(generateCookieString(&cookie_string)) {
+        addHeader("Cookie", cookie_string);
+    }
+
     // send Header
     if(!sendHeader(type)) {
         return returnError(HTTPC_ERROR_SEND_HEADER_FAILED);
@@ -1222,6 +1239,7 @@ int HTTPClient::handleHeaderResponse()
     _transferEncoding = HTTPC_TE_IDENTITY;
     unsigned long lastDataTime = millis();
     bool firstLine = true;
+    String date;
 
     while(connected()) {
         size_t len = _client->available();
@@ -1234,7 +1252,7 @@ int HTTPClient::handleHeaderResponse()
             log_v("RX: '%s'", headerLine.c_str());
 
             if(firstLine) {
-		firstLine = false;
+		        firstLine = false;
                 if(_canReuse && headerLine.startsWith("HTTP/1.")) {
                     _canReuse = (headerLine[sizeof "HTTP/1." - 1] != '0');
                 }
@@ -1245,6 +1263,10 @@ int HTTPClient::handleHeaderResponse()
                 String headerValue = headerLine.substring(headerLine.indexOf(':') + 1);
                 headerValue.trim();
 
+                if(headerName.equalsIgnoreCase("Date")) {
+                    date = headerValue;
+                }
+
                 if(headerName.equalsIgnoreCase("Content-Length")) {
                     _size = headerValue.toInt();
                 }
@@ -1263,12 +1285,24 @@ int HTTPClient::handleHeaderResponse()
                     _location = headerValue;
                 }
 
-                for(size_t i = 0; i < _headerKeysCount; i++) {
-                    if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
+                if (headerName.equalsIgnoreCase("Set-Cookie")) {
+                    setCookie(date, headerValue);
+                }
+
+                for (size_t i = 0; i < _headerKeysCount; i++) {
+                    if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
+                        // Uncomment the following lines if you need to add support for multiple headers with the same key:
+                        // if (!_currentHeaders[i].value.isEmpty()) {
+                        //     // Existing value, append this one with a comma
+                        //     _currentHeaders[i].value += ',';
+                        //     _currentHeaders[i].value += headerValue;
+                        // } else {
                         _currentHeaders[i].value = headerValue;
-                        break;
+                        // }
+                        break; // We found a match, stop looking
                     }
                 }
+
             }
 
             if(headerLine == "") {
@@ -1491,3 +1525,164 @@ const String &HTTPClient::getLocation(void)
 {
     return _location;
 }
+
+void HTTPClient::setCookieJar(CookieJar* cookieJar)
+{
+    _cookieJar = cookieJar;
+}
+
+void HTTPClient::resetCookieJar()
+{
+    _cookieJar = nullptr;
+}
+
+void HTTPClient::clearAllCookies()
+{
+    if (_cookieJar) _cookieJar->clear();
+}
+
+void HTTPClient::setCookie(String date, String headerValue)
+{
+    #define HTTP_TIME_PATTERN "%a, %d %b %Y %H:%M:%S"
+
+    Cookie cookie;
+    String value;
+    int pos1, pos2;
+
+    headerValue.toLowerCase();
+
+    struct tm tm;
+    strptime(date.c_str(), HTTP_TIME_PATTERN, &tm);
+    cookie.date = mktime(&tm);
+
+    pos1 = headerValue.indexOf('=');
+    pos2 = headerValue.indexOf(';');
+
+    if (pos1 >= 0 && pos2 > pos1){
+        cookie.name = headerValue.substring(0, pos1);
+        cookie.value = headerValue.substring(pos1 + 1, pos2);
+    } else {
+        return;     // invalid cookie header
+    }
+
+    // expires
+    if (headerValue.indexOf("expires=") >= 0){
+        pos1 = headerValue.indexOf("expires=") + strlen("expires=");
+        pos2 = headerValue.indexOf(';', pos1);
+
+        if (pos2 > pos1)
+            value = headerValue.substring(pos1, pos2);
+        else
+            value = headerValue.substring(pos1);
+    
+        strptime(value.c_str(), HTTP_TIME_PATTERN, &tm);
+        cookie.expires.date = mktime(&tm);
+        cookie.expires.valid = true;
+    }
+
+    // max-age
+    if (headerValue.indexOf("max-age=") >= 0){
+        pos1 = headerValue.indexOf("max-age=") + strlen("max-age=");
+        pos2 = headerValue.indexOf(';', pos1);
+
+        if (pos2 > pos1)
+            value = headerValue.substring(pos1, pos2);
+        else
+            value = headerValue.substring(pos1);
+    
+        cookie.max_age.duration = value.toInt();
+        cookie.max_age.valid = true;
+    }
+
+    // domain
+    if (headerValue.indexOf("domain=") >= 0){
+        pos1 = headerValue.indexOf("domain=") + strlen("domain=");
+        pos2 = headerValue.indexOf(';', pos1);
+
+        if (pos2 > pos1)
+            value = headerValue.substring(pos1, pos2);
+        else
+            value = headerValue.substring(pos1);
+
+        if (value.startsWith(".")) value.remove(0, 1);
+
+        if (_host.indexOf(value) >= 0) {
+            cookie.domain = value;
+        } else {
+            return;     // server tries to set a cookie on a different domain; ignore it
+        }
+    } else {
+        pos1 = _host.lastIndexOf('.', _host.lastIndexOf('.') - 1);
+        if (pos1 >= 0)
+            cookie.domain = _host.substring(pos1 + 1);
+        else
+            cookie.domain = _host;
+    }
+
+    // path
+    if (headerValue.indexOf("path=") >= 0){
+        pos1 = headerValue.indexOf("path=") + strlen("path=");
+        pos2 = headerValue.indexOf(';', pos1);
+
+        if (pos2 > pos1)
+            cookie.path = headerValue.substring(pos1, pos2);
+        else
+            cookie.path = headerValue.substring(pos1);
+    }
+
+    // HttpOnly
+    cookie.http_only = (headerValue.indexOf("httponly") >= 0);
+
+    // secure
+    cookie.secure = (headerValue.indexOf("secure") >= 0);
+
+    // overwrite or delete cookie in/from cookie jar
+    time_t now_local = time(NULL);
+    time_t now_gmt = mktime(gmtime(&now_local));
+    
+    bool found = false;
+
+	for (auto c = _cookieJar->begin(); c != _cookieJar->end(); ++c) {
+        if (c->domain == cookie.domain && c->name == cookie.name) {
+            // when evaluating, max-age takes precedence over expires if both are defined
+            if (cookie.max_age.valid && ((cookie.date + cookie.max_age.duration) < now_gmt || cookie.max_age.duration <= 0)
+            || (!cookie.max_age.valid && cookie.expires.valid && cookie.expires.date < now_gmt)) {
+                _cookieJar->erase(c);
+                c--;
+            } else {
+                *c = cookie;
+            }
+            found = true;
+        }
+    }
+
+    // add cookie to jar
+    if (!found && !(cookie.max_age.valid && cookie.max_age.duration <= 0)) 
+        _cookieJar->push_back(cookie);
+
+}
+
+bool HTTPClient::generateCookieString(String *cookieString)
+{
+    time_t now_local = time(NULL);
+    time_t now_gmt = mktime(gmtime(&now_local));
+
+    *cookieString = "";
+    bool found = false;
+
+ 	for (auto c = _cookieJar->begin(); c != _cookieJar->end(); ++c) {
+        if (c->max_age.valid && ((c->date + c->max_age.duration) < now_gmt) || (!c->max_age.valid && c->expires.valid && c->expires.date < now_gmt)) {
+            _cookieJar->erase(c);
+            c--;
+        } else if (_host.indexOf(c->domain) >= 0 && (!c->secure || _secure) ) {
+            if (*cookieString == "")
+                *cookieString = c->name + "=" + c->value;
+            else
+                *cookieString += " ;" + c->name + "=" + c->value;
+            found = true;
+        }
+    }
+    return found;
+}
+
+

libraries/HTTPClient/src/HTTPClient.h

@@ -36,6 +36,9 @@
 #include <WiFiClient.h>
 #include <WiFiClientSecure.h>
 
+/// Cookie jar support
+#include <vector>
+
 #define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000)
 
 /// HTTP client errors
@@ -144,6 +147,28 @@ class TransportTraits;
 typedef std::unique_ptr<TransportTraits> TransportTraitsPtr;
 #endif
 
+// cookie jar support
+typedef struct  {
+    String host;       // host which tries to set the cookie
+    time_t date;       // timestamp of the response that set the cookie
+    String name;
+    String value;
+    String domain;
+    String path = "";
+    struct {
+        time_t date = 0;
+        bool valid = false;
+    } expires;
+    struct {
+        time_t duration = 0;
+        bool valid = false;
+    } max_age;
+    bool http_only = false;
+    bool secure = false;
+} Cookie;
+typedef std::vector<Cookie> CookieJar;
+
+
 class HTTPClient
 {
 public:
@@ -217,6 +242,11 @@ public:
 
     static String errorToString(int error);
 
+    /// Cookie jar support
+    void setCookieJar(CookieJar* cookieJar);
+    void resetCookieJar();
+    void clearAllCookies();
+
 protected:
     struct RequestArgument {
         String key;
@@ -232,6 +262,9 @@ protected:
     int handleHeaderResponse();
     int writeToStreamDataBlock(Stream * stream, int len);
 
+    /// Cookie jar support
+    void setCookie(String date, String headerValue);
+    bool generateCookieString(String *cookieString);
 
 #ifdef HTTPCLIENT_1_1_COMPATIBLE
     TransportTraitsPtr _transportTraits;
@@ -267,6 +300,10 @@ protected:
     uint16_t _redirectLimit = 10;
     String _location;
     transferEncoding_t _transferEncoding = HTTPC_TE_IDENTITY;
+
+    /// Cookie jar support
+    CookieJar* _cookieJar = nullptr;
+
 };