Merge pull request #22323 from abyss7/poco-http2

Follow up to #19516

src/Core/Settings.h

@@ -228,6 +228,7 @@ class IColumn;
     M(Seconds, http_connection_timeout, DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT, "HTTP connection timeout.", 0) \
     M(Seconds, http_send_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP send timeout", 0) \
     M(Seconds, http_receive_timeout, DEFAULT_HTTP_READ_BUFFER_TIMEOUT, "HTTP receive timeout", 0) \
+    M(UInt64, http_max_uri_size, 16384, "Maximum URI length of HTTP request", 0) \
     M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \
     M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \
     M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \
@@ -546,7 +547,7 @@ struct Settings : public BaseSettings<SettingsTraits>
 {
     /// For initialization from empty initializer-list to be "value initialization", not "aggregate initialization" in C++14.
     /// http://en.cppreference.com/w/cpp/language/aggregate_initialization
-    Settings() {}
+    Settings() = default;
 
     /** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).
      * The profile can also be set using the `set` functions, like the profile setting.

src/Server/HTTP/HTMLForm.cpp

@@ -71,23 +71,6 @@ HTMLForm::HTMLForm(const Poco::URI & uri) : field_limit(DFL_FIELD_LIMIT), value_
 }
 
 
-void HTMLForm::setEncoding(const std::string & encoding_)
-{
-    encoding = encoding_;
-}
-
-
-void HTMLForm::addPart(const std::string & name, Poco::Net::PartSource * source)
-{
-    poco_check_ptr(source);
-
-    Part part;
-    part.name = name;
-    part.source = std::unique_ptr<Poco::Net::PartSource>(source);
-    parts.push_back(std::move(part));
-}
-
-
 void HTMLForm::load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler)
 {
     clear();
@@ -126,36 +109,12 @@ void HTMLForm::load(const Poco::Net::HTTPRequest & request, ReadBuffer & request
 }
 
 
-void HTMLForm::load(const Poco::Net::HTTPRequest & request)
-{
-    NullPartHandler nah;
-    EmptyReadBuffer nis;
-    load(request, nis, nah);
-}
-
-
-void HTMLForm::read(ReadBuffer & in, PartHandler & handler)
-{
-    if (encoding == ENCODING_URL)
-        readQuery(in);
-    else
-        readMultipart(in, handler);
-}
-
-
 void HTMLForm::read(ReadBuffer & in)
 {
     readQuery(in);
 }
 
 
-void HTMLForm::read(const std::string & queryString)
-{
-    ReadBufferFromString istr(queryString);
-    readQuery(istr);
-}
-
-
 void HTMLForm::readQuery(ReadBuffer & in)
 {
     size_t fields = 0;
@@ -269,22 +228,6 @@ void HTMLForm::readMultipart(ReadBuffer & in_, PartHandler & handler)
 }
 
 
-void HTMLForm::setFieldLimit(int limit)
-{
-    poco_assert(limit >= 0);
-
-    field_limit = limit;
-}
-
-
-void HTMLForm::setValueLengthLimit(int limit)
-{
-    poco_assert(limit >= 0);
-
-    value_length_limit = limit;
-}
-
-
 HTMLForm::MultipartReadBuffer::MultipartReadBuffer(ReadBuffer & in_, const std::string & boundary_)
     : ReadBuffer(nullptr, 0), in(in_), boundary("--" + boundary_)
 {

src/Server/HTTP/HTMLForm.h

@@ -52,24 +52,6 @@ public:
         return (it != end()) ? DB::parse<T>(it->second) : default_value;
     }
 
-    template <typename T>
-    T getParsed(const std::string & key)
-    {
-        return DB::parse<T>(get(key));
-    }
-
-    /// Sets the encoding used for posting the form.
-    /// Encoding must be either "application/x-www-form-urlencoded" (which is the default) or "multipart/form-data".
-    void setEncoding(const std::string & encoding);
-
-    /// Returns the encoding used for posting the form.
-    const std::string & getEncoding() const { return encoding; }
-
-    /// Adds an part/attachment (file upload) to the form.
-    /// The form takes ownership of the PartSource and deletes it when it is no longer needed.
-    /// The part will only be sent if the encoding set for the form is "multipart/form-data"
-    void addPart(const std::string & name, Poco::Net::PartSource * pSource);
-
     /// Reads the form data from the given HTTP request.
     /// Uploaded files are passed to the given PartHandler.
     void load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody, PartHandler & handler);
@@ -78,41 +60,10 @@ public:
     /// Uploaded files are silently discarded.
     void load(const Poco::Net::HTTPRequest & request, ReadBuffer & requestBody);
 
-    /// Reads the form data from the given HTTP request.
-    /// The request must be a GET request and the form data must be in the query string (URL encoded).
-    /// For POST requests, you must use one of the overloads taking an additional input stream for the request body.
-    void load(const Poco::Net::HTTPRequest & request);
-
-    /// Reads the form data from the given input stream.
-    /// The form data read from the stream must be in the encoding specified for the form.
-    /// Note that read() does not clear the form before reading the new values.
-    void read(ReadBuffer & in, PartHandler & handler);
-
     /// Reads the URL-encoded form data from the given input stream.
     /// Note that read() does not clear the form before reading the new values.
     void read(ReadBuffer & in);
 
-    /// Reads the form data from the given HTTP query string.
-    /// Note that read() does not clear the form before reading the new values.
-    void read(const std::string & queryString);
-
-    /// Returns the MIME boundary used for writing multipart form data.
-    const std::string & getBoundary() const { return boundary; }
-
-    /// Returns the maximum number of header fields allowed.
-    /// See setFieldLimit() for more information.
-    int getFieldLimit() const { return field_limit; }
-
-    /// Sets the maximum number of header fields allowed. This limit is used to defend certain kinds of denial-of-service attacks.
-    /// Specify 0 for unlimited (not recommended). The default limit is 100.
-    void setFieldLimit(int limit);
-
-    /// Sets the maximum size for form field values stored as strings.
-    void setValueLengthLimit(int limit);
-
-    /// Returns the maximum size for form field values stored as strings.
-    int getValueLengthLimit() const { return value_length_limit; }
-
     static const std::string ENCODING_URL; /// "application/x-www-form-urlencoded"
     static const std::string ENCODING_MULTIPART; /// "multipart/form-data"
     static const int UNKNOWN_CONTENT_LENGTH;

src/Server/HTTP/HTTPServer.cpp

@@ -8,9 +8,9 @@ namespace DB
 HTTPServer::HTTPServer(
     const Context & context,
     HTTPRequestHandlerFactoryPtr factory_,
-    UInt16 portNumber,
+    UInt16 port_number,
     Poco::Net::HTTPServerParams::Ptr params)
-    : TCPServer(new HTTPServerConnectionFactory(context, params, factory_), portNumber, params), factory(factory_)
+    : TCPServer(new HTTPServerConnectionFactory(context, params, factory_), port_number, params), factory(factory_)
 {
 }
 
@@ -26,10 +26,10 @@ HTTPServer::HTTPServer(
 HTTPServer::HTTPServer(
     const Context & context,
     HTTPRequestHandlerFactoryPtr factory_,
-    Poco::ThreadPool & threadPool,
+    Poco::ThreadPool & thread_pool,
     const Poco::Net::ServerSocket & socket,
     Poco::Net::HTTPServerParams::Ptr params)
-    : TCPServer(new HTTPServerConnectionFactory(context, params, factory_), threadPool, socket, params), factory(factory_)
+    : TCPServer(new HTTPServerConnectionFactory(context, params, factory_), thread_pool, socket, params), factory(factory_)
 {
 }
 

src/Server/HTTP/HTTPServer.h

@@ -19,7 +19,7 @@ public:
     explicit HTTPServer(
         const Context & context,
         HTTPRequestHandlerFactoryPtr factory,
-        UInt16 portNumber = 80,
+        UInt16 port_number = 80,
         Poco::Net::HTTPServerParams::Ptr params = new Poco::Net::HTTPServerParams);
 
     HTTPServer(
@@ -31,13 +31,13 @@ public:
     HTTPServer(
         const Context & context,
         HTTPRequestHandlerFactoryPtr factory,
-        Poco::ThreadPool & threadPool,
+        Poco::ThreadPool & thread_pool,
         const Poco::Net::ServerSocket & socket,
         Poco::Net::HTTPServerParams::Ptr params);
 
     ~HTTPServer() override;
 
-    void stopAll(bool abortCurrent = false);
+    void stopAll(bool abort_current = false);
 
 private:
     HTTPRequestHandlerFactoryPtr factory;

src/Server/HTTP/HTTPServerConnection.cpp

@@ -67,15 +67,15 @@ void HTTPServerConnection::run()
                 }
             }
         }
-        catch (Poco::Net::NoMessageException &)
+        catch (const Poco::Net::NoMessageException &)
         {
             break;
         }
-        catch (Poco::Net::MessageException &)
+        catch (const Poco::Net::MessageException &)
         {
             sendErrorResponse(session, Poco::Net::HTTPResponse::HTTP_BAD_REQUEST);
         }
-        catch (Poco::Exception &)
+        catch (const Poco::Exception &)
         {
             if (session.networkException())
             {
@@ -98,31 +98,4 @@ void HTTPServerConnection::sendErrorResponse(Poco::Net::HTTPServerSession & sess
     session.setKeepAlive(false);
 }
 
-void HTTPServerConnection::onServerStopped(const bool & abortCurrent)
-{
-    stopped = true;
-    if (abortCurrent)
-    {
-        try
-        {
-            socket().shutdown();
-        }
-        catch (...)
-        {
-        }
-    }
-    else
-    {
-        std::unique_lock<std::mutex> lock(mutex);
-
-        try
-        {
-            socket().shutdown();
-        }
-        catch (...)
-        {
-        }
-    }
-}
-
 }

src/Server/HTTP/HTTPServerConnection.h

@@ -23,7 +23,6 @@ public:
 
 protected:
     static void sendErrorResponse(Poco::Net::HTTPServerSession & session, Poco::Net::HTTPResponse::HTTPStatus status);
-    void onServerStopped(const bool & abortCurrent);
 
 private:
     Context context;

src/Server/HTTP/HTTPServerRequest.cpp

@@ -15,8 +15,8 @@
 
 namespace DB
 {
-
 HTTPServerRequest::HTTPServerRequest(const Context & context, HTTPServerResponse & response, Poco::Net::HTTPServerSession & session)
+    : max_uri_size(context.getSettingsRef().http_max_uri_size)
 {
     response.attachRequest(this);
 
@@ -92,10 +92,10 @@ void HTTPServerRequest::readRequest(ReadBuffer & in)
 
     skipWhitespaceIfAny(in);
 
-    while (in.read(ch) && !Poco::Ascii::isSpace(ch) && uri.size() <= MAX_URI_LENGTH)
+    while (in.read(ch) && !Poco::Ascii::isSpace(ch) && uri.size() <= max_uri_size)
         uri += ch;
 
-    if (uri.size() > MAX_URI_LENGTH)
+    if (uri.size() > max_uri_size)
         throw Poco::Net::MessageException("HTTP request URI invalid or too long");
 
     skipWhitespaceIfAny(in);

src/Server/HTTP/HTTPServerRequest.h

@@ -43,11 +43,12 @@ private:
         MAX_NAME_LENGTH = 256,
         MAX_VALUE_LENGTH = 8192,
         MAX_METHOD_LENGTH = 32,
-        MAX_URI_LENGTH = 16384,
         MAX_VERSION_LENGTH = 8,
         MAX_FIELDS_NUMBER = 100,
     };
 
+    const size_t max_uri_size;
+
     std::unique_ptr<ReadBuffer> stream;
     Poco::Net::SocketImpl * socket;
     Poco::Net::SocketAddress client_address;

src/Server/HTTP/HTTPServerResponse.cpp

@@ -94,32 +94,6 @@ std::pair<std::shared_ptr<std::ostream>, std::shared_ptr<std::ostream>> HTTPServ
     return std::make_pair(header_stream, stream);
 }
 
-void HTTPServerResponse::sendFile(const std::string & path, const std::string & mediaType)
-{
-    poco_assert(!stream);
-
-    Poco::File f(path);
-    Poco::Timestamp date_time = f.getLastModified();
-    Poco::File::FileSize length = f.getSize();
-    set("Last-Modified", Poco::DateTimeFormatter::format(date_time, Poco::DateTimeFormat::HTTP_FORMAT));
-    setContentLength64(length);
-    setContentType(mediaType);
-    setChunkedTransferEncoding(false);
-
-    Poco::FileInputStream istr(path);
-    if (istr.good())
-    {
-        stream = std::make_shared<Poco::Net::HTTPHeaderOutputStream>(session);
-        write(*stream);
-        if (request && request->getMethod() != HTTPRequest::HTTP_HEAD)
-        {
-            Poco::StreamCopier::copyStream(istr, *stream);
-        }
-    }
-    else
-        throw Poco::OpenFileException(path);
-}
-
 void HTTPServerResponse::sendBuffer(const void * buffer, std::size_t length)
 {
     poco_assert(!stream);
@@ -135,20 +109,6 @@ void HTTPServerResponse::sendBuffer(const void * buffer, std::size_t length)
     }
 }
 
-void HTTPServerResponse::redirect(const std::string & uri, HTTPStatus status)
-{
-    poco_assert(!stream);
-
-    setContentLength(0);
-    setChunkedTransferEncoding(false);
-
-    setStatusAndReason(status);
-    set("Location", uri);
-
-    stream = std::make_shared<Poco::Net::HTTPHeaderOutputStream>(session);
-    write(*stream);
-}
-
 void HTTPServerResponse::requireAuthentication(const std::string & realm)
 {
     poco_assert(!stream);

src/Server/HTTP/HTTPServerResponse.h

@@ -36,17 +36,6 @@ public:
     /// or redirect() has been called.
     std::pair<std::shared_ptr<std::ostream>, std::shared_ptr<std::ostream>> beginSend(); /// TODO: use some WriteBuffer implementation here.
 
-    /// Sends the response header to the client, followed
-    /// by the content of the given file.
-    ///
-    /// Must not be called after send(), sendBuffer()
-    /// or redirect() has been called.
-    ///
-    /// Throws a FileNotFoundException if the file
-    /// cannot be found, or an OpenFileException if
-    /// the file cannot be opened.
-    void sendFile(const std::string & path, const std::string & mediaType);
-
     /// Sends the response header to the client, followed
     /// by the contents of the given buffer.
     ///
@@ -61,16 +50,6 @@ public:
     /// or redirect() has been called.
     void sendBuffer(const void * pBuffer, std::size_t length); /// FIXME: do we need this one?
 
-    /// Sets the status code, which must be one of
-    /// HTTP_MOVED_PERMANENTLY (301), HTTP_FOUND (302),
-    /// or HTTP_SEE_OTHER (303),
-    /// and sets the "Location" header field
-    /// to the given URI, which according to
-    /// the HTTP specification, must be absolute.
-    ///
-    /// Must not be called after send() has been called.
-    void redirect(const std::string & uri, Poco::Net::HTTPResponse::HTTPStatus status = Poco::Net::HTTPResponse::HTTP_FOUND);
-
     void requireAuthentication(const std::string & realm);
     /// Sets the status code to 401 (Unauthorized)
     /// and sets the "WWW-Authenticate" header field
@@ -83,7 +62,7 @@ public:
 
 private:
     Poco::Net::HTTPServerSession & session;
-    HTTPServerRequest * request;
+    HTTPServerRequest * request = nullptr;
     std::shared_ptr<std::ostream> stream;
     std::shared_ptr<std::ostream> header_stream;
 };

src/Server/HTTP/README.md

+# Notice
+
+The source code located in this folder is based on some files from the POCO project, from here `contrib/poco/Net/src`.

tests/queries/0_stateless/01753_max_uri_size.reference

+1

tests/queries/0_stateless/01753_max_uri_size.sh

+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# NOTE: since 'max_uri_size' doesn't affect the request itself, this test hardly depends on the default value of this setting (16Kb).
+
+LONG_REQUEST=$(python3 -c "print('&max_uri_size=1'*2000, end='')")  # ~30K
+
+${CLICKHOUSE_CURL} -sSv "${CLICKHOUSE_URL}${LONG_REQUEST}&query=SELECT+1" 2>&1 | grep -Fc "HTTP/1.1 400 Bad Request"