diff --git a/dbms/DataStreams/HTTPInputStreams.cpp b/dbms/DataStreams/HTTPInputStreams.cpp deleted file mode 100644 index e760069a0409a4f47f5bf28043ad0d4ad74c6125..0000000000000000000000000000000000000000 --- a/dbms/DataStreams/HTTPInputStreams.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include - -#include - -#include -#include -#include -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int UNKNOWN_COMPRESSION_METHOD; -} - -HTTPInputStreams::HTTPInputStreams(Context & context, HTTPServerRequest & request, HTMLForm & from) - : in(plainBuffer(request)) - , in_maybe_compressed(compressedBuffer(request, in)) - , in_maybe_internal_compressed(internalCompressedBuffer(from, in_maybe_compressed)) -{ - /// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on, - /// checksums of client data compressed with internal algorithm are not checked. - if (context.getSettingsRef().http_native_compression_disable_checksumming_on_decompress) - { - if (CompressedReadBuffer * compressed_buffer = typeid_cast(in_maybe_internal_compressed.get())) - compressed_buffer->disableChecksumming(); - } -} - -std::unique_ptr HTTPInputStreams::plainBuffer(HTTPServerRequest & request) const -{ - return std::make_unique(request.stream()); -} - -std::unique_ptr HTTPInputStreams::compressedBuffer(HTTPServerRequest & request, std::unique_ptr & plain_buffer) const -{ - /// Request body can be compressed using algorithm specified in the Content-Encoding header. - String http_compressed_method = request.get("Content-Encoding", ""); - - if (!http_compressed_method.empty()) - { - if (http_compressed_method == "gzip") - return std::make_unique(std::move(plain_buffer), CompressionMethod::Gzip); - else if (http_compressed_method == "deflate") - return std::make_unique(std::move(plain_buffer), CompressionMethod::Zlib); -#if USE_BROTLI - else if (http_compressed_method == "br") - return std::make_unique(std::move(plain_buffer)); -#endif - else - throw Exception("Unknown Content-Encoding of HTTP request: " + http_compressed_method, ErrorCodes::UNKNOWN_COMPRESSION_METHOD); - } - - return std::move(plain_buffer); -} - -std::unique_ptr HTTPInputStreams::internalCompressedBuffer( - HTMLForm ¶ms, std::unique_ptr &http_maybe_encoding_buffer) const -{ - /// The data can also be compressed using incompatible internal algorithm. This is indicated by - /// 'decompress' query parameter. - std::unique_ptr in_post_maybe_compressed; - if (params.getParsed("decompress", false)) - return std::make_unique(*http_maybe_encoding_buffer); - - return std::move(http_maybe_encoding_buffer); -} - -} diff --git a/dbms/DataStreams/HTTPInputStreams.h b/dbms/DataStreams/HTTPInputStreams.h deleted file mode 100644 index f2325da676f98587ed2fc24e562f4210410d3b03..0000000000000000000000000000000000000000 --- a/dbms/DataStreams/HTTPInputStreams.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -using HTTPServerRequest = Poco::Net::HTTPServerRequest; - -struct HTTPInputStreams -{ - using ReadBufferUniquePtr = std::unique_ptr; - - ReadBufferUniquePtr in; - ReadBufferUniquePtr in_maybe_compressed; - ReadBufferUniquePtr in_maybe_internal_compressed; - - HTTPInputStreams(Context & context, HTTPServerRequest & request, HTMLForm & from); - - ReadBufferUniquePtr plainBuffer(HTTPServerRequest & request) const; - ReadBufferUniquePtr compressedBuffer(HTTPServerRequest & request, ReadBufferUniquePtr & plain_buffer) const; - ReadBufferUniquePtr internalCompressedBuffer(HTMLForm & params, ReadBufferUniquePtr & http_maybe_encoding_buffer) const; -}; - -} diff --git a/dbms/DataStreams/HTTPOutputStreams.cpp b/dbms/DataStreams/HTTPOutputStreams.cpp deleted file mode 100644 index 31f4929bef1f2b0a0356e2ef8c973accb78792f8..0000000000000000000000000000000000000000 --- a/dbms/DataStreams/HTTPOutputStreams.cpp +++ /dev/null @@ -1,213 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "HTTPOutputStreams.h" - - -namespace DB -{ - -namespace -{ - inline void listeningProgress(Context & context, ProgressCallback listener) - { - auto prev = context.getProgressCallback(); - context.setProgressCallback([prev, listener] (const Progress & progress) - { - if (prev) - prev(progress); - - listener(progress); - }); - } - - inline ProgressCallback cancelListener(Context & context, Poco::Net::StreamSocket & socket) - { - /// Assume that at the point this method is called no one is reading data from the socket any more. - /// True for read-only queries. - return [&context, &socket](const Progress &) - { - try - { - char b; - int status = socket.receiveBytes(&b, 1, MSG_DONTWAIT | MSG_PEEK); - if (status == 0) - context.killCurrentQuery(); - } - catch (Poco::TimeoutException &) - { - } - catch (...) - { - context.killCurrentQuery(); - } - }; - } -} - -HTTPOutputStreams::HTTPOutputStreams(HTTPResponseBufferPtr & raw_out, bool internal_compress) - : out(raw_out) - , out_maybe_compressed(createMaybeCompressionOut(internal_compress, out)) - , out_maybe_delayed_and_compressed(out_maybe_compressed) -{ -} - -HTTPOutputStreams::HTTPOutputStreams(HTTPResponseBufferPtr & raw_out, Context & context, HTTPServerRequest & request, HTMLForm & form) - : out(raw_out) - , out_maybe_compressed(createMaybeCompressionOut(form.getParsed("compress", false), out)) - , out_maybe_delayed_and_compressed(createMaybeDelayedAndCompressionOut(context, form, out_maybe_compressed)) -{ - Settings & settings = context.getSettingsRef(); - - /// HTTP response compression is turned on only if the client signalled that they support it - /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on. - out->setCompression(out->getCompression() && settings.enable_http_compression); - if (out->getCompression()) - out->setCompressionLevel(settings.http_zlib_compression_level); - - out->setSendProgressInterval(settings.http_headers_progress_interval_ms); - - /// Add CORS header if 'add_http_cors_header' setting is turned on and the client passed Origin header. - out->addHeaderCORS(settings.add_http_cors_header && !request.get("Origin", "").empty()); - - /// While still no data has been sent, we will report about query execution progress by sending HTTP headers. - if (settings.send_progress_in_http_headers) - listeningProgress(context, [this] (const Progress & progress) { out->onProgress(progress); }); - - if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close) - { - Poco::Net::StreamSocket & socket = dynamic_cast(request).socket(); - - listeningProgress(context, cancelListener(context, socket)); - } -} - -WriteBufferPtr HTTPOutputStreams::createMaybeCompressionOut(bool compression, HTTPResponseBufferPtr & out_) -{ - /// Client can pass a 'compress' flag in the query string. In this case the query result is - /// compressed using internal algorithm. This is not reflected in HTTP headers. - return compression ? std::make_shared(*out_) : WriteBufferPtr(out_); -} - -WriteBufferPtr HTTPOutputStreams::createMaybeDelayedAndCompressionOut(Context & context, HTMLForm & form, WriteBufferPtr & out_) -{ - /// If it is specified, the whole result will be buffered. - /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. - bool buffer_until_eof = form.getParsed("wait_end_of_query", false); - - /// At least, we should postpone sending of first buffer_size result bytes - size_t buffer_size_total = std::max(form.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); - - size_t buffer_size_memory = (buffer_size_total > DBMS_DEFAULT_BUFFER_SIZE) ? buffer_size_total : 0; - - if (buffer_size_memory > 0 || buffer_until_eof) - { - CascadeWriteBuffer::WriteBufferPtrs cascade_buffer1; - CascadeWriteBuffer::WriteBufferConstructors cascade_buffer2; - - if (buffer_size_memory > 0) - cascade_buffer1.emplace_back(std::make_shared(buffer_size_memory)); - - if (buffer_until_eof) - { - std::string tmp_path_template = context.getTemporaryPath() + "http_buffers/"; - - auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) - { - return WriteBufferFromTemporaryFile::create(tmp_path_template); - }; - - cascade_buffer2.emplace_back(std::move(create_tmp_disk_buffer)); - } - else - { - auto push_memory_buffer_and_continue = [next_buffer = out_] (const WriteBufferPtr & prev_buf) - { - auto prev_memory_buffer = typeid_cast(prev_buf.get()); - if (!prev_memory_buffer) - throw Exception("Expected MemoryWriteBuffer", ErrorCodes::LOGICAL_ERROR); - - auto rdbuf = prev_memory_buffer->tryGetReadBuffer(); - copyData(*rdbuf , *next_buffer); - - return next_buffer; - }; - - cascade_buffer2.emplace_back(push_memory_buffer_and_continue); - } - - return std::make_shared(std::move(cascade_buffer1), std::move(cascade_buffer2)); - } - - return out_; -} - -HTTPOutputStreams::~HTTPOutputStreams() -{ - /// This could be a broken HTTP Request - /// Because it does not call finalize or writes some data to output stream after call finalize - /// In this case we need to clean up its broken state to ensure that they are not sent to the client - - /// For delayed stream, we destory CascadeBuffer and without sending any data to client. - if (out_maybe_delayed_and_compressed != out_maybe_compressed) - out_maybe_delayed_and_compressed.reset(); - - if (out->count() == out->offset()) - { - /// If buffer has data and server never sends data to client - /// no need to send that data - out_maybe_compressed->position() = out_maybe_compressed->buffer().begin(); - out->position() = out->buffer().begin(); - } -} - -void HTTPOutputStreams::finalize() const -{ - if (out_maybe_delayed_and_compressed != out_maybe_compressed) - { - /// TODO: set Content-Length if possible - std::vector write_buffers; - std::vector read_buffers; - std::vector read_buffers_raw_ptr; - - auto cascade_buffer = typeid_cast(out_maybe_delayed_and_compressed.get()); - if (!cascade_buffer) - throw Exception("Expected CascadeWriteBuffer", ErrorCodes::LOGICAL_ERROR); - - cascade_buffer->getResultBuffers(write_buffers); - - if (write_buffers.empty()) - throw Exception("At least one buffer is expected to overwrite result into HTTP response", ErrorCodes::LOGICAL_ERROR); - - for (auto & write_buf : write_buffers) - { - IReadableWriteBuffer * write_buf_concrete; - ReadBufferPtr reread_buf; - - if (write_buf - && (write_buf_concrete = dynamic_cast(write_buf.get())) - && (reread_buf = write_buf_concrete->tryGetReadBuffer())) - { - read_buffers.emplace_back(reread_buf); - read_buffers_raw_ptr.emplace_back(reread_buf.get()); - } - } - - ConcatReadBuffer concat_read_buffer(read_buffers_raw_ptr); - copyData(concat_read_buffer, *out_maybe_compressed); - } - - /// Send HTTP headers with code 200 if no exception happened and the data is still not sent to the client. - out_maybe_compressed->next(); - out->next(); - out->finalize(); -} - -} diff --git a/dbms/DataStreams/HTTPOutputStreams.h b/dbms/DataStreams/HTTPOutputStreams.h deleted file mode 100644 index fba122ea29484fe53c41f0156509e0181f977f6e..0000000000000000000000000000000000000000 --- a/dbms/DataStreams/HTTPOutputStreams.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -using HTTPServerRequest = Poco::Net::HTTPServerRequest; -using HTTPServerResponse = Poco::Net::HTTPServerResponse; -using HTTPResponseBufferPtr = std::shared_ptr; - -/* Raw data - * ↓ - * CascadeWriteBuffer out_maybe_delayed_and_compressed (optional) - * ↓ (forwards data if an overflow is occur or explicitly via pushDelayedResults) - * CompressedWriteBuffer out_maybe_compressed (optional) - * ↓ - * WriteBufferFromHTTPServerResponse out - */ -struct HTTPOutputStreams -{ - HTTPResponseBufferPtr out; - /// Points to 'out' or to CompressedWriteBuffer(*out), depending on settings. - std::shared_ptr out_maybe_compressed; - /// Points to 'out' or to CompressedWriteBuffer(*out) or to CascadeWriteBuffer. - std::shared_ptr out_maybe_delayed_and_compressed; - - ~HTTPOutputStreams(); - - void finalize() const; - - WriteBufferPtr createMaybeDelayedAndCompressionOut(Context & context, HTMLForm & form, WriteBufferPtr & out_); - - WriteBufferPtr createMaybeCompressionOut(bool compression, std::shared_ptr & out_); - - HTTPOutputStreams(HTTPResponseBufferPtr & raw_out, bool internal_compress); - - HTTPOutputStreams(HTTPResponseBufferPtr & raw_out, Context & context, HTTPServerRequest & request, HTMLForm & form); -}; - -using HTTPOutputStreamsPtr = std::unique_ptr; - -} diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 064cda4847a967f228fdc45c8cac0919474b8de8..a252310cc93b3ee3d1edc5508cce7b8dfb5f6c90 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -4,21 +4,12 @@ set(CLICKHOUSE_SERVER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusMetricsWriter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/PrometheusRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/StaticRequestHandler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandlerFactory.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPQueryRequestHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPPingRequestHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPRootRequestHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPSessionContextHolder.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPExceptionHandler.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.cpp ) set(CLICKHOUSE_SERVER_SOURCES diff --git a/programs/server/HTTPHandler.cpp b/programs/server/HTTPHandler.cpp index 593a156ca5270cf8bf3dc630eb17b4186875040c..98093e94c7940700e9e9202c2c4478e368916022 100644 --- a/programs/server/HTTPHandler.cpp +++ b/programs/server/HTTPHandler.cpp @@ -1,5 +1,8 @@ #include "HTTPHandler.h" +#include "HTTPHandlerFactory.h" +#include "HTTPHandlerRequestFilter.h" + #include #include #include @@ -7,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +36,7 @@ #include #include #include +#include #include #include @@ -78,6 +83,7 @@ namespace ErrorCodes extern const int UNKNOWN_FORMAT; extern const int UNKNOWN_DATABASE_ENGINE; extern const int UNKNOWN_TYPE_OF_QUERY; + extern const int NO_ELEMENTS_IN_CONFIG; extern const int QUERY_IS_TOO_LARGE; @@ -90,7 +96,6 @@ namespace ErrorCodes extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; - extern const int UNKNOW_QUERY_EXECUTOR; } @@ -118,8 +123,7 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti exception_code == ErrorCodes::INCORRECT_DATA || exception_code == ErrorCodes::TYPE_MISMATCH) return HTTPResponse::HTTP_BAD_REQUEST; - else if (exception_code == ErrorCodes::UNKNOW_QUERY_EXECUTOR || - exception_code == ErrorCodes::UNKNOWN_TABLE || + else if (exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION || exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || exception_code == ErrorCodes::UNKNOWN_TYPE || @@ -172,44 +176,67 @@ static std::chrono::steady_clock::duration parseSessionTimeout( } -HTTPHandler::HTTPHandler(IServer & server_) - : server(server_), log(&Logger::get("HTTPHandler")) +void HTTPHandler::pushDelayedResults(Output & used_output) { - server_display_name = server.config().getString("display_name", getFQDNOrHostName()); -} + std::vector write_buffers; + std::vector read_buffers; + std::vector read_buffers_raw_ptr; + auto cascade_buffer = typeid_cast(used_output.out_maybe_delayed_and_compressed.get()); + if (!cascade_buffer) + throw Exception("Expected CascadeWriteBuffer", ErrorCodes::LOGICAL_ERROR); -HTTPHandler::SessionContextHolder::~SessionContextHolder() -{ - if (session_context) - session_context->releaseSession(session_id, session_timeout); -} - + cascade_buffer->getResultBuffers(write_buffers); -HTTPHandler::SessionContextHolder::SessionContextHolder(Context & query_context_, HTTPRequest & request, HTMLForm & params) - : query_context(query_context_) -{ - authentication(request, params); + if (write_buffers.empty()) + throw Exception("At least one buffer is expected to overwrite result into HTTP response", ErrorCodes::LOGICAL_ERROR); + for (auto & write_buf : write_buffers) { - session_id = params.get("session_id", ""); + IReadableWriteBuffer * write_buf_concrete; + ReadBufferPtr reread_buf; - if (!session_id.empty()) + if (write_buf + && (write_buf_concrete = dynamic_cast(write_buf.get())) + && (reread_buf = write_buf_concrete->tryGetReadBuffer())) { - session_timeout = parseSessionTimeout(query_context.getConfigRef(), params); - session_context = query_context.acquireSession(session_id, session_timeout, params.check("session_check", "1")); - - query_context = *session_context; - query_context.setSessionContext(*session_context); + read_buffers.emplace_back(reread_buf); + read_buffers_raw_ptr.emplace_back(reread_buf.get()); } } + + ConcatReadBuffer concat_read_buffer(read_buffers_raw_ptr); + copyData(concat_read_buffer, *used_output.out_maybe_compressed); +} + + +HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) + : server(server_) + , log(&Logger::get(name)) +{ + server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } -void HTTPHandler::SessionContextHolder::authentication(HTTPServerRequest & request, HTMLForm & params) + +void HTTPHandler::processQuery( + Poco::Net::HTTPServerRequest & request, + HTMLForm & params, + Poco::Net::HTTPServerResponse & response, + Output & used_output) { - auto user = request.get("X-ClickHouse-User", ""); - auto password = request.get("X-ClickHouse-Key", ""); - auto quota_key = request.get("X-ClickHouse-Quota", ""); + Context context = server.context(); + + CurrentThread::QueryScope query_scope(context); + + LOG_TRACE(log, "Request URI: " << request.getURI()); + + std::istream & istr = request.stream(); + + /// The user and password can be passed by headers (similar to X-Auth-*), + /// which is used by load balancers to pass authentication information. + std::string user = request.get("X-ClickHouse-User", ""); + std::string password = request.get("X-ClickHouse-Key", ""); + std::string quota_key = request.get("X-ClickHouse-Quota", ""); if (user.empty() && password.empty() && quota_key.empty()) { @@ -243,26 +270,331 @@ void HTTPHandler::SessionContextHolder::authentication(HTTPServerRequest & reque } std::string query_id = params.get("query_id", ""); - query_context.setUser(user, password, request.clientAddress(), quota_key); - query_context.setCurrentQueryId(query_id); -} + context.setUser(user, password, request.clientAddress(), quota_key); + context.setCurrentQueryId(query_id); -void HTTPHandler::processQuery(Context & context, HTTPRequest & request, HTMLForm & params, HTTPResponse & response, HTTPResponseBufferPtr & response_out) -{ - const auto & name_with_custom_executor = context.getCustomExecutor(request, params); - LOG_TRACE(log, "Using '" << name_with_custom_executor.first << "' CustomExecutor to execute URI: " << request.getURI()); + /// The user could specify session identifier and session timeout. + /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. + + std::shared_ptr session; + String session_id; + std::chrono::steady_clock::duration session_timeout; + bool session_is_set = params.has("session_id"); + const auto & config = server.config(); + + if (session_is_set) + { + session_id = params.get("session_id"); + session_timeout = parseSessionTimeout(config, params); + std::string session_check = params.get("session_check", ""); + + session = context.acquireNamedSession(session_id, session_timeout, session_check == "1"); + + context = session->context; + context.setSessionContext(session->context); + } + + SCOPE_EXIT({ + if (session) + session->release(); + }); + + /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). + String http_response_compression_methods = request.get("Accept-Encoding", ""); + CompressionMethod http_response_compression_method = CompressionMethod::None; + + if (!http_response_compression_methods.empty()) + { + /// If client supports brotli - it's preferred. + /// Both gzip and deflate are supported. If the client supports both, gzip is preferred. + /// NOTE parsing of the list of methods is slightly incorrect. + + if (std::string::npos != http_response_compression_methods.find("br")) + http_response_compression_method = CompressionMethod::Brotli; + else if (std::string::npos != http_response_compression_methods.find("gzip")) + http_response_compression_method = CompressionMethod::Gzip; + else if (std::string::npos != http_response_compression_methods.find("deflate")) + http_response_compression_method = CompressionMethod::Zlib; + } + + bool client_supports_http_compression = http_response_compression_method != CompressionMethod::None; + + /// Client can pass a 'compress' flag in the query string. In this case the query result is + /// compressed using internal algorithm. This is not reflected in HTTP headers. + bool internal_compression = params.getParsed("compress", false); + + /// At least, we should postpone sending of first buffer_size result bytes + size_t buffer_size_total = std::max( + params.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); + + /// If it is specified, the whole result will be buffered. + /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. + bool buffer_until_eof = params.getParsed("wait_end_of_query", false); + + size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE; + size_t buffer_size_memory = (buffer_size_total > buffer_size_http) ? buffer_size_total : 0; + + unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", 10); + + used_output.out = std::make_shared( + request, response, keep_alive_timeout, client_supports_http_compression, http_response_compression_method); + + if (internal_compression) + used_output.out_maybe_compressed = std::make_shared(*used_output.out); + else + used_output.out_maybe_compressed = used_output.out; + + if (buffer_size_memory > 0 || buffer_until_eof) + { + CascadeWriteBuffer::WriteBufferPtrs cascade_buffer1; + CascadeWriteBuffer::WriteBufferConstructors cascade_buffer2; + + if (buffer_size_memory > 0) + cascade_buffer1.emplace_back(std::make_shared(buffer_size_memory)); + + if (buffer_until_eof) + { + const std::string tmp_path(context.getTemporaryVolume()->getNextDisk()->getPath()); + const std::string tmp_path_template(tmp_path + "http_buffers/"); + + auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) + { + return WriteBufferFromTemporaryFile::create(tmp_path_template); + }; + + cascade_buffer2.emplace_back(std::move(create_tmp_disk_buffer)); + } + else + { + auto push_memory_buffer_and_continue = [next_buffer = used_output.out_maybe_compressed] (const WriteBufferPtr & prev_buf) + { + auto prev_memory_buffer = typeid_cast(prev_buf.get()); + if (!prev_memory_buffer) + throw Exception("Expected MemoryWriteBuffer", ErrorCodes::LOGICAL_ERROR); + + auto rdbuf = prev_memory_buffer->tryGetReadBuffer(); + copyData(*rdbuf , *next_buffer); + + return next_buffer; + }; + + cascade_buffer2.emplace_back(push_memory_buffer_and_continue); + } + + used_output.out_maybe_delayed_and_compressed = std::make_shared( + std::move(cascade_buffer1), std::move(cascade_buffer2)); + } + else + { + used_output.out_maybe_delayed_and_compressed = used_output.out_maybe_compressed; + } + + std::unique_ptr in_post_raw = std::make_unique(istr); + + /// Request body can be compressed using algorithm specified in the Content-Encoding header. + String http_request_compression_method_str = request.get("Content-Encoding", ""); + std::unique_ptr in_post = wrapReadBufferWithCompressionMethod( + std::make_unique(istr), chooseCompressionMethod({}, http_request_compression_method_str)); + + /// The data can also be compressed using incompatible internal algorithm. This is indicated by + /// 'decompress' query parameter. + std::unique_ptr in_post_maybe_compressed; + bool in_post_compressed = false; + if (params.getParsed("decompress", false)) + { + in_post_maybe_compressed = std::make_unique(*in_post); + in_post_compressed = true; + } + else + in_post_maybe_compressed = std::move(in_post); + + std::unique_ptr in; + + static const NameSet reserved_param_names{"compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", + "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"}; + + Names reserved_param_suffixes; + + auto param_could_be_skipped = [&] (const String & name) + { + if (reserved_param_names.count(name)) + return true; + + for (const String & suffix : reserved_param_suffixes) + { + if (endsWith(name, suffix)) + return true; + } + + return false; + }; + + /// Settings can be overridden in the query. + /// Some parameters (database, default_format, everything used in the code above) do not + /// belong to the Settings class. + + /// 'readonly' setting values mean: + /// readonly = 0 - any query is allowed, client can change any setting. + /// readonly = 1 - only readonly queries are allowed, client can't change settings. + /// readonly = 2 - only readonly queries are allowed, client can change any setting except 'readonly'. + + /// In theory if initially readonly = 0, the client can change any setting and then set readonly + /// to some other value. + const auto & settings = context.getSettingsRef(); + + /// Only readonly queries are allowed for HTTP GET requests. + if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_GET) + { + if (settings.readonly == 0) + context.setSetting("readonly", 2); + } + + bool has_external_data = startsWith(request.getContentType(), "multipart/form-data"); + + if (has_external_data) + { + /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters. + reserved_param_suffixes.reserve(3); + /// It is a bug and ambiguity with `date_time_input_format` and `low_cardinality_allow_in_native_format` formats/settings. + reserved_param_suffixes.emplace_back("_format"); + reserved_param_suffixes.emplace_back("_types"); + reserved_param_suffixes.emplace_back("_structure"); + } + + SettingsChanges settings_changes; + for (const auto & [key, value] : params) + { + if (key == "database") + { + context.setCurrentDatabase(value); + } + else if (key == "default_format") + { + context.setDefaultFormat(value); + } + else if (param_could_be_skipped(key)) + { + } + else + { + /// Other than query parameters are treated as settings. + if (!customizeQueryParam(context, key, value)) + settings_changes.push_back({key, value}); + } + } + + /// For external data we also want settings + context.checkSettingsConstraints(settings_changes); + context.applySettingsChanges(settings_changes); + + const auto & query = getQuery(request, params, context); + std::unique_ptr in_param = std::make_unique(query); + in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); + + /// HTTP response compression is turned on only if the client signalled that they support it + /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on. + used_output.out->setCompression(client_supports_http_compression && settings.enable_http_compression); + if (client_supports_http_compression) + used_output.out->setCompressionLevel(settings.http_zlib_compression_level); + + used_output.out->setSendProgressInterval(settings.http_headers_progress_interval_ms); + + /// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on, + /// checksums of client data compressed with internal algorithm are not checked. + if (in_post_compressed && settings.http_native_compression_disable_checksumming_on_decompress) + static_cast(*in_post_maybe_compressed).disableChecksumming(); + + /// Add CORS header if 'add_http_cors_header' setting is turned on and the client passed + /// Origin header. + used_output.out->addHeaderCORS(settings.add_http_cors_header && !request.get("Origin", "").empty()); + + ClientInfo & client_info = context.getClientInfo(); + client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + client_info.interface = ClientInfo::Interface::HTTP; + + /// Query sent through HTTP interface is initial. + client_info.initial_user = client_info.current_user; + client_info.initial_query_id = client_info.current_query_id; + client_info.initial_address = client_info.current_address; + + ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; + if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_GET) + http_method = ClientInfo::HTTPMethod::GET; + else if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_POST) + http_method = ClientInfo::HTTPMethod::POST; + + client_info.http_method = http_method; + client_info.http_user_agent = request.get("User-Agent", ""); + + auto append_callback = [&context] (ProgressCallback callback) + { + auto prev = context.getProgressCallback(); + + context.setProgressCallback([prev, callback] (const Progress & progress) + { + if (prev) + prev(progress); + + callback(progress); + }); + }; - ExtractorClientInfo{context.getClientInfo()}.extract(request); - ExtractorContextChange{context, name_with_custom_executor.second}.extract(request, params); + /// While still no data has been sent, we will report about query execution progress by sending HTTP headers. + if (settings.send_progress_in_http_headers) + append_callback([&used_output] (const Progress & progress) { used_output.out->onProgress(progress); }); - HTTPInputStreams input_streams{context, request, params}; - HTTPOutputStreams output_streams = HTTPOutputStreams(response_out, context, request, params); - name_with_custom_executor.second->executeQuery(context, request, response, params, input_streams, output_streams); + if (settings.readonly > 0 && settings.cancel_http_readonly_queries_on_client_close) + { + Poco::Net::StreamSocket & socket = dynamic_cast(request).socket(); + + append_callback([&context, &socket](const Progress &) + { + /// Assume that at the point this method is called no one is reading data from the socket any more. + /// True for read-only queries. + try + { + char b; + //FIXME looks like MSG_DONTWAIT is useless because of POCO_BROKEN_TIMEOUTS + int status = socket.receiveBytes(&b, 1, MSG_DONTWAIT | MSG_PEEK); + if (status == 0) + context.killCurrentQuery(); + } + catch (Poco::TimeoutException &) + { + } + catch (...) + { + context.killCurrentQuery(); + } + }); + } + + customizeContext(context); + + executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, + [&response] (const String & current_query_id, const String & content_type, const String & format, const String & timezone) + { + response.setContentType(content_type); + response.add("X-ClickHouse-Query-Id", current_query_id); + response.add("X-ClickHouse-Format", format); + response.add("X-ClickHouse-Timezone", timezone); + } + ); + + if (used_output.hasDelayed()) + { + /// TODO: set Content-Length if possible + pushDelayedResults(used_output); + } + + /// Send HTTP headers with code 200 if no exception happened and the data is still not sent to + /// the client. + used_output.out->finalize(); } -void HTTPHandler::trySendExceptionToClient( - const std::string & message, int exception_code, HTTPRequest & request, - HTTPResponse & response, HTTPResponseBufferPtr response_out, bool compression) +void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_code, + Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response, + Output & used_output) { try { @@ -270,14 +602,19 @@ void HTTPHandler::trySendExceptionToClient( /// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body /// to avoid reading part of the current request body in the next request. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && response.getKeepAlive() - && !request.stream().eof() && exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED) + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST + && response.getKeepAlive() + && !request.stream().eof() + && exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED) { request.stream().ignore(std::numeric_limits::max()); } - if (exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD || - exception_code == ErrorCodes::REQUIRED_PASSWORD) + bool auth_fail = exception_code == ErrorCodes::UNKNOWN_USER || + exception_code == ErrorCodes::WRONG_PASSWORD || + exception_code == ErrorCodes::REQUIRED_PASSWORD; + + if (auth_fail) { response.requireAuthentication("ClickHouse server HTTP API"); } @@ -286,16 +623,36 @@ void HTTPHandler::trySendExceptionToClient( response.setStatusAndReason(exceptionCodeToHTTPStatus(exception_code)); } - if (!response_out && !response.sent()) - response.send() << message << std::endl; - else + if (!response.sent() && !used_output.out_maybe_compressed) + { + /// If nothing was sent yet and we don't even know if we must compress the response. + response.send() << s << std::endl; + } + else if (used_output.out_maybe_compressed) { - HTTPOutputStreams output_streams(response_out, compression); + /// Destroy CascadeBuffer to actualize buffers' positions and reset extra references + if (used_output.hasDelayed()) + used_output.out_maybe_delayed_and_compressed.reset(); + + /// Send the error message into already used (and possibly compressed) stream. + /// Note that the error message will possibly be sent after some data. + /// Also HTTP code 200 could have already been sent. + + /// If buffer has data, and that data wasn't sent yet, then no need to send that data + bool data_sent = used_output.out->count() != used_output.out->offset(); - writeString(message, *output_streams.out_maybe_compressed); - writeChar('\n', *output_streams.out_maybe_compressed); + if (!data_sent) + { + used_output.out_maybe_compressed->position() = used_output.out_maybe_compressed->buffer().begin(); + used_output.out->position() = used_output.out->buffer().begin(); + } - output_streams.finalize(); + writeString(s, *used_output.out_maybe_compressed); + writeChar('\n', *used_output.out_maybe_compressed); + + used_output.out_maybe_compressed->next(); + used_output.out->next(); + used_output.out->finalize(); } } catch (...) @@ -304,41 +661,37 @@ void HTTPHandler::trySendExceptionToClient( } } + void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { setThreadName("HTTPHandler"); ThreadStatus thread_status; + Output used_output; + /// In case of exception, send stack trace to client. - HTTPResponseBufferPtr response_out; - bool with_stacktrace = false, internal_compression = false; + bool with_stacktrace = false; try { - response_out = createResponseOut(request, response); - response.set("Content-Type", "text/plain; charset=UTF-8"); + response.setContentType("text/plain; charset=UTF-8"); response.set("X-ClickHouse-Server-Display-Name", server_display_name); - /// For keep-alive to work. if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); HTMLForm params(request); with_stacktrace = params.getParsed("stacktrace", false); - internal_compression = params.getParsed("compress", false); /// Workaround. Poco does not detect 411 Length Required case. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && !request.hasContentLength()) - throw Exception("The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", ErrorCodes::HTTP_LENGTH_REQUIRED); - + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && + !request.hasContentLength()) { - Context query_context = server.context(); - CurrentThread::QueryScope query_scope(query_context); - - SessionContextHolder holder{query_context, request, params}; - processQuery(holder.query_context, request, params, response, response_out); - LOG_INFO(log, "Done processing query"); + throw Exception("The Transfer-Encoding is not chunked and there is no Content-Length header for POST request", ErrorCodes::HTTP_LENGTH_REQUIRED); } + + processQuery(request, params, response, used_output); + LOG_INFO(log, "Done processing query"); } catch (...) { @@ -347,36 +700,103 @@ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne /** If exception is received from remote server, then stack trace is embedded in message. * If exception is thrown on local server, then stack trace is in separate field. */ - int exception_code = getCurrentExceptionCode(); std::string exception_message = getCurrentExceptionMessage(with_stacktrace, true); - trySendExceptionToClient(exception_message, exception_code, request, response, response_out, internal_compression); + int exception_code = getCurrentExceptionCode(); + + trySendExceptionToClient(exception_message, exception_code, request, response, used_output); } } -HTTPResponseBufferPtr HTTPHandler::createResponseOut(HTTPServerRequest & request, HTTPServerResponse & response) +DynamicQueryHandler::DynamicQueryHandler(IServer & server_, const std::string & param_name_) + : HTTPHandler(server_, "DynamicQueryHandler"), param_name(param_name_) { - size_t keep_alive = server.config().getUInt("keep_alive_timeout", 10); - /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). - String http_response_compression_methods = request.get("Accept-Encoding", ""); +} - if (!http_response_compression_methods.empty()) +bool DynamicQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value) +{ + if (key == param_name) + return true; /// do nothing + + if (startsWith(key, "param_")) { - /// Both gzip and deflate are supported. If the client supports both, gzip is preferred. - /// NOTE parsing of the list of methods is slightly incorrect. - if (std::string::npos != http_response_compression_methods.find("gzip")) - return std::make_shared( - request, response, keep_alive, true, CompressionMethod::Gzip, DBMS_DEFAULT_BUFFER_SIZE); - else if (std::string::npos != http_response_compression_methods.find("deflate")) - return std::make_shared( - request, response, keep_alive, true, CompressionMethod::Zlib, DBMS_DEFAULT_BUFFER_SIZE); -#if USE_BROTLI - else if (http_response_compression_methods == "br") - return std::make_shared( - request, response, keep_alive, true, CompressionMethod::Brotli, DBMS_DEFAULT_BUFFER_SIZE); -#endif + /// Save name and values of substitution in dictionary. + const String parameter_name = key.substr(strlen("param_")); + context.setQueryParameter(parameter_name, value); + return true; } - return std::make_shared(request, response, keep_alive, false, CompressionMethod{}, DBMS_DEFAULT_BUFFER_SIZE); + return false; } +std::string DynamicQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) +{ + + if (likely(!startsWith(request.getContentType(), "multipart/form-data"))) + { + /// Part of the query can be passed in the 'query' parameter and the rest in the request body + /// (http method need not necessarily be POST). In this case the entire query consists of the + /// contents of the 'query' parameter, a line break and the request body. + std::string query_param = params.get(param_name, ""); + return query_param.empty() ? query_param : query_param + "\n"; + } + + /// Support for "external data for query processing". + /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope. + ExternalTablesHandler handler(context, params); + params.load(request, request.stream(), handler); + + std::string full_query; + /// Params are of both form params POST and uri (GET params) + for (const auto & it : params) + if (it.first == param_name) + full_query += it.second; + + return full_query; +} + +PredefineQueryHandler::PredefineQueryHandler(IServer & server, const NameSet & receive_params_, const std::string & predefine_query_) + : HTTPHandler(server, "PredefineQueryHandler"), receive_params(receive_params_), predefine_query(predefine_query_) +{ +} + +bool PredefineQueryHandler::customizeQueryParam(Context & context, const std::string & key, const std::string & value) +{ + if (receive_params.count(key)) + { + context.setQueryParameter(key, value); + return true; + } + + return false; +} + +std::string PredefineQueryHandler::getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) +{ + if (unlikely(startsWith(request.getContentType(), "multipart/form-data"))) + { + /// Support for "external data for query processing". + ExternalTablesHandler handler(context, params); + params.load(request, request.stream(), handler); + } + + return predefine_query; +} + +Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix) +{ + const auto & query_param_name = server.config().getString(config_prefix + ".handler.query_param_name", "query"); + return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory(server, query_param_name), server.config(), config_prefix); +} + + +Poco::Net::HTTPRequestHandlerFactory * createPredefineHandlerFactory(IServer & server, const std::string & config_prefix) +{ + if (!server.config().has(config_prefix + ".handler.query")) + throw Exception("There is no path '" + config_prefix + ".handler.query" + "' in configuration file.", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + + const auto & predefine_query = server.config().getString(config_prefix + ".handler.query"); + + return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory( + server, analyzeReceiveQueryParams(predefine_query), predefine_query), server.config(), config_prefix); } +} \ No newline at end of file diff --git a/programs/server/HTTPHandler.h b/programs/server/HTTPHandler.h index 1faa0efe9070ce130f6af0d643a298f801c82997..f5139b6a9cc14b9303737cdb7c9b1652962f0ccc 100644 --- a/programs/server/HTTPHandler.h +++ b/programs/server/HTTPHandler.h @@ -7,8 +7,6 @@ #include #include -#include - namespace CurrentMetrics { @@ -26,26 +24,39 @@ class WriteBufferFromHTTPServerResponse; class HTTPHandler : public Poco::Net::HTTPRequestHandler { public: - explicit HTTPHandler(IServer & server_); + explicit HTTPHandler(IServer & server_, const std::string & name); void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; -private: - using HTTPRequest = Poco::Net::HTTPServerRequest; - using HTTPResponse = Poco::Net::HTTPServerResponse; + /// This method is called right before the query execution. + virtual void customizeContext(Context & /* context */) {} - struct SessionContextHolder - { - ~SessionContextHolder(); + virtual bool customizeQueryParam(Context & context, const std::string & key, const std::string & value) = 0; - void authentication(HTTPServerRequest & request, HTMLForm & params); + virtual std::string getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) = 0; - SessionContextHolder(Context & query_context_, HTTPRequest & request, HTMLForm & params); - - String session_id; - Context & query_context; - std::shared_ptr session_context = nullptr; - std::chrono::steady_clock::duration session_timeout; +private: + struct Output + { + /* Raw data + * ↓ + * CascadeWriteBuffer out_maybe_delayed_and_compressed (optional) + * ↓ (forwards data if an overflow is occur or explicitly via pushDelayedResults) + * CompressedWriteBuffer out_maybe_compressed (optional) + * ↓ + * WriteBufferFromHTTPServerResponse out + */ + + std::shared_ptr out; + /// Points to 'out' or to CompressedWriteBuffer(*out), depending on settings. + std::shared_ptr out_maybe_compressed; + /// Points to 'out' or to CompressedWriteBuffer(*out) or to CascadeWriteBuffer. + std::shared_ptr out_maybe_delayed_and_compressed; + + inline bool hasDelayed() const + { + return out_maybe_delayed_and_compressed != out_maybe_compressed; + } }; IServer & server; @@ -56,16 +67,46 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; - size_t getKeepAliveTimeout() { return server.config().getUInt("keep_alive_timeout", 10); } + /// Also initializes 'used_output'. + void processQuery( + Poco::Net::HTTPServerRequest & request, + HTMLForm & params, + Poco::Net::HTTPServerResponse & response, + Output & used_output); - HTTPResponseBufferPtr createResponseOut(HTTPServerRequest & request, HTTPServerResponse & response); + void trySendExceptionToClient( + const std::string & s, + int exception_code, + Poco::Net::HTTPServerRequest & request, + Poco::Net::HTTPServerResponse & response, + Output & used_output); - void processQuery(Context & context, HTTPRequest & request, HTMLForm & params, HTTPResponse & response, HTTPResponseBufferPtr & response_out); + static void pushDelayedResults(Output & used_output); +}; - void trySendExceptionToClient( - const std::string & message, int exception_code, HTTPRequest & request, - HTTPResponse & response, HTTPResponseBufferPtr response_out, bool compression); +class DynamicQueryHandler : public HTTPHandler +{ +private: + std::string param_name; +public: + explicit DynamicQueryHandler(IServer & server_, const std::string & param_name_ = "query"); + + std::string getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) override; + + bool customizeQueryParam(Context &context, const std::string &key, const std::string &value) override; +}; + +class PredefineQueryHandler : public HTTPHandler +{ +private: + NameSet receive_params; + std::string predefine_query; +public: + explicit PredefineQueryHandler(IServer & server, const NameSet & receive_params, const std::string & predefine_query_); + + std::string getQuery(Poco::Net::HTTPServerRequest & request, HTMLForm & params, Context & context) override; + bool customizeQueryParam(Context &context, const std::string &key, const std::string &value) override; }; } diff --git a/programs/server/HTTPHandlerFactory.cpp b/programs/server/HTTPHandlerFactory.cpp index b2c07075d54b7afe96031254ffe8f5062bb7b3c2..8dee745d0e34a175ee4357fcb022458948075b70 100644 --- a/programs/server/HTTPHandlerFactory.cpp +++ b/programs/server/HTTPHandlerFactory.cpp @@ -1,9 +1,21 @@ #include "HTTPHandlerFactory.h" +#include +#include +#include +#include + +#include "HTTPHandler.h" #include "NotFoundHandler.h" -#include "HTTPRequestHandler/HTTPRootRequestHandler.h" -#include "HTTPRequestHandler/HTTPPingRequestHandler.h" -#include "HTTPRequestHandler/HTTPReplicasStatusRequestHandler.h" +#include "StaticRequestHandler.h" +#include "ReplicasStatusHandler.h" +#include "InterserverIOHTTPHandler.h" + +#if USE_RE2_ST +#include +#else +#define re2_st re2 +#endif namespace DB @@ -11,130 +23,128 @@ namespace DB namespace ErrorCodes { - extern const int SYNTAX_ERROR; - extern const int UNKNOWN_HTTP_HANDLER_TYPE; - extern const int EMPTY_HTTP_HANDLER_IN_CONFIG; + extern const int CANNOT_COMPILE_REGEXP; + extern const int NO_ELEMENTS_IN_CONFIG; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; } -InterserverIOHTTPHandlerFactory::InterserverIOHTTPHandlerFactory(IServer & server_, const std::string & name_) +HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_) : server(server_), log(&Logger::get(name_)), name(name_) { } -Poco::Net::HTTPRequestHandler * InterserverIOHTTPHandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) +Poco::Net::HTTPRequestHandler * HTTPRequestHandlerFactoryMain::createRequestHandler(const Poco::Net::HTTPServerRequest & request) // override { LOG_TRACE(log, "HTTP Request for " << name << ". " - << "Method: " << request.getMethod() - << ", Address: " << request.clientAddress().toString() - << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") - << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) - << ", Content Type: " << request.getContentType() - << ", Transfer Encoding: " << request.getTransferEncoding()); - - const auto & uri = request.getURI(); - - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) - { - if (uri == "/") - return new HTTPRootRequestHandler(server); - if (uri == "/ping") - return new HTTPPingRequestHandler(server); - else if (startsWith(uri, "/replicas_status")) - return new HTTPReplicasStatusRequestHandler(server.context()); - } - - if (uri.find('?') != std::string::npos || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + << "Method: " + << request.getMethod() + << ", Address: " + << request.clientAddress().toString() + << ", User-Agent: " + << (request.has("User-Agent") ? request.get("User-Agent") : "none") + << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) + << ", Content Type: " << request.getContentType() + << ", Transfer Encoding: " << request.getTransferEncoding()); + + for (auto & handler_factory : child_handler_factories) { - return new InterserverIOHTTPHandler(server); + auto handler = handler_factory->createRequestHandler(request); + if (handler != nullptr) + return handler; } - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) { - return new NotFoundHandler( - "Use / or /ping for health checks.\n" - "Or /replicas_status for more sophisticated health checks.\n" - "Send queries from your program with POST method or GET /?query=...\n\n" - " Use clickhouse-client:\n\n" - " For interactive data analysis:\n" - " clickhouse-client\n\n" - " For batch query processing:\n" - " clickhouse-client --query='SELECT 1' > result\n" - " clickhouse-client < query > result" - ); + return new NotFoundHandler; } return nullptr; } -HTTPHandlerFactory::HTTPHandlerFactory(IServer & server_, const std::string & name_) - : server(server_), log(&Logger::get(name_)), name(name_) +HTTPRequestHandlerFactoryMain::~HTTPRequestHandlerFactoryMain() { - updateHTTPHandlersCreator(server.config()); + while (!child_handler_factories.empty()) + delete child_handler_factories.back(), child_handler_factories.pop_back(); +} - if (handlers_creator.empty()) - throw Exception("The HTTPHandlers does not exist in the config.xml", ErrorCodes::EMPTY_HTTP_HANDLER_IN_CONFIG); +HTTPRequestHandlerFactoryMain::TThis * HTTPRequestHandlerFactoryMain::addHandler(Poco::Net::HTTPRequestHandlerFactory * child_factory) +{ + child_handler_factories.emplace_back(child_factory); + return this; } -Poco::Net::HTTPRequestHandler * HTTPHandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) +static inline auto createHandlersFactoryFromConfig(IServer & server, const std::string & name, const String & prefix) { - LOG_TRACE(log, "HTTP Request for " << name << ". " - << "Method: " << request.getMethod() - << ", Address: " << request.clientAddress().toString() - << ", User-Agent: " << (request.has("User-Agent") ? request.get("User-Agent") : "none") - << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) - << ", Content Type: " << request.getContentType() - << ", Transfer Encoding: " << request.getTransferEncoding()); - - for (const auto & [matcher, creator] : handlers_creator) + auto main_handler_factory = new HTTPRequestHandlerFactoryMain(server, name); + + Poco::Util::AbstractConfiguration::Keys keys; + server.config().keys(prefix, keys); + + for (const auto & key : keys) { - if (matcher(request)) - return creator(); + if (!startsWith(key, "routing_rule")) + throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'routing_rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", ""); + + if (handler_type == "static") + main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix)); + else if (handler_type == "dynamic_query_handler") + main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix)); + else if (handler_type == "predefine_query_handler") + main_handler_factory->addHandler(createPredefineHandlerFactory(server, prefix)); + else + throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'routing_rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } - return new NotFoundHandler(no_handler_description); + return main_handler_factory; } -HTTPHandlerMatcher createRootHandlerMatcher(IServer &, const String &); -HTTPHandlerMatcher createPingHandlerMatcher(IServer &, const String &); -HTTPHandlerMatcher createDynamicQueryHandlerMatcher(IServer &, const String &); -HTTPHandlerMatcher createReplicasStatusHandlerMatcher(IServer &, const String &); -HTTPHandlerMatcher createPredefinedQueryHandlerMatcher(IServer &, const String &); +static const auto ping_response_expression = "Ok.\n"; +static const auto root_response_expression = "config://http_server_default_response"; -HTTPHandlerCreator createRootHandlerCreator(IServer &, const String &); -HTTPHandlerCreator createPingHandlerCreator(IServer &, const String &); -HTTPHandlerCreator createDynamicQueryHandlerCreator(IServer &, const String &); -HTTPHandlerCreator createReplicasStatusHandlerCreator(IServer &, const String &); -HTTPHandlerCreator createPredefinedQueryHandlerCreator(IServer &, const String &); - -void HTTPHandlerFactory::updateHTTPHandlersCreator(Poco::Util::AbstractConfiguration & configuration, const String & key) +static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IServer & server, const std::string & name) { - Poco::Util::AbstractConfiguration::Keys http_handlers_item_key; - configuration.keys(key, http_handlers_item_key); - - handlers_creator.reserve(http_handlers_item_key.size()); - for (const auto & http_handler_type_name : http_handlers_item_key) + if (server.config().has("routing_rules")) + return createHandlersFactoryFromConfig(server, name, "routing_rules"); + else { - if (http_handler_type_name.find('.') != String::npos) - throw Exception("HTTPHandler type name with dots are not supported: '" + http_handler_type_name + "'", ErrorCodes::SYNTAX_ERROR); - - const auto & handler_key = key + "." + http_handler_type_name; - - if (startsWith(http_handler_type_name, "root_handler")) - handlers_creator.push_back({createRootHandlerMatcher(server, handler_key), createRootHandlerCreator(server, handler_key)}); - else if (startsWith(http_handler_type_name, "ping_handler")) - handlers_creator.push_back({createPingHandlerMatcher(server, handler_key), createPingHandlerCreator(server, handler_key)}); - else if (startsWith(http_handler_type_name, "dynamic_query_handler")) - handlers_creator.push_back({createDynamicQueryHandlerMatcher(server, handler_key), createDynamicQueryHandlerCreator(server, handler_key)}); - else if (startsWith(http_handler_type_name, "predefined_query_handler")) - handlers_creator.push_back({createPredefinedQueryHandlerMatcher(server, handler_key), createPredefinedQueryHandlerCreator(server, handler_key)}); - else if (startsWith(http_handler_type_name, "replicas_status_handler")) - handlers_creator.push_back({createReplicasStatusHandlerMatcher(server, handler_key), createReplicasStatusHandlerCreator(server, handler_key)}); - else if (http_handler_type_name == "no_handler_description") - no_handler_description = configuration.getString(key + ".no_handler_description"); - else - throw Exception("Unknown HTTPHandler type name: " + http_handler_type_name, ErrorCodes::UNKNOWN_HTTP_HANDLER_TYPE); + return (new HTTPRequestHandlerFactoryMain(server, name)) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server, root_response_expression)) + ->attachStrictPath("/")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server, ping_response_expression)) + ->attachStrictPath("/ping")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server)) + ->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server, "query"))->allowPostAndGetParamsRequest()); + /// TODO: +// if (configuration.has("prometheus") && configuration.getInt("prometheus.port", 0) == 0) +// handler_factory->addHandler(async_metrics); } } +static inline Poco::Net::HTTPRequestHandlerFactory * createInterserverHTTPHandlerFactory(IServer & server, const std::string & name) +{ + return (new HTTPRequestHandlerFactoryMain(server, name)) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server, root_response_expression)) + ->attachStrictPath("/")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server, ping_response_expression)) + ->attachStrictPath("/ping")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server)) + ->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest()) + ->addHandler((new RoutingRuleHTTPHandlerFactory(server))->allowPostAndGetParamsRequest()); +} + +Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, const std::string & name) +{ + if (name == "HTTPHandler-factory" || name == "HTTPSHandler-factory") + return createHTTPHandlerFactory(server, name); + else if (name == "InterserverIOHTTPHandler-factory" || name == "InterserverIOHTTPSHandler-factory") + return createInterserverHTTPHandlerFactory(server, name); + + throw Exception("LOGICAL ERROR: Unknown HTTP handler factory name.", ErrorCodes::LOGICAL_ERROR); } + +} \ No newline at end of file diff --git a/programs/server/HTTPHandlerFactory.h b/programs/server/HTTPHandlerFactory.h index 5add7b619af06620d0ac232a9de52c6ecbc070e2..15f15db9768c2516dead1ddecc7f8d005060011e 100644 --- a/programs/server/HTTPHandlerFactory.h +++ b/programs/server/HTTPHandlerFactory.h @@ -1,51 +1,113 @@ #pragma once +#include "IServer.h" +#include +#include #include #include #include -#include -#include -#include "IServer.h" -#include "InterserverIOHTTPHandler.h" - namespace DB { -class InterserverIOHTTPHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +/// Handle request using child handlers +class HTTPRequestHandlerFactoryMain : public Poco::Net::HTTPRequestHandlerFactory { -public: - InterserverIOHTTPHandlerFactory(IServer & server_, const std::string & name_); - - Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; - private: + using TThis = HTTPRequestHandlerFactoryMain; + IServer & server; Logger * log; std::string name; -}; -using HTTPHandlerCreator = std::function; -using HTTPHandlerMatcher = std::function; -using HTTPHandlerMatcherAndCreator = std::pair; -using HTTPHandlersMatcherAndCreator = std::vector; + std::vector child_handler_factories; +public: + + ~HTTPRequestHandlerFactoryMain(); -class HTTPHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory + HTTPRequestHandlerFactoryMain(IServer & server_, const std::string & name_); + + TThis * addHandler(Poco::Net::HTTPRequestHandlerFactory * child_factory); + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; +}; + +template +class RoutingRuleHTTPHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory { public: - HTTPHandlerFactory(IServer & server_, const std::string & name_); + using TThis = RoutingRuleHTTPHandlerFactory; + using Filter = std::function; - Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; + template + RoutingRuleHTTPHandlerFactory(TArgs &&... args) + { + creator = [args = std::tuple(std::forward(args) ...)]() + { + return std::apply([&](auto && ... endpoint_args) + { + return new TEndpoint(std::forward(endpoint_args)...); + }, std::move(args)); + }; + } - void updateHTTPHandlersCreator(Poco::Util::AbstractConfiguration & configuration, const String & key = "http_handlers"); + TThis * addFilter(Filter cur_filter) + { + Filter prev_filter = filter; + filter = [prev_filter, cur_filter](const auto & request) + { + return prev_filter ? prev_filter(request) && cur_filter(request) : cur_filter(request); + }; -private: - IServer & server; - Logger * log; - std::string name; + return this; + } + + TThis * attachStrictPath(const String & strict_path) + { + return addFilter([strict_path](const auto & request) { return request.getURI() == strict_path; }); + } + + TThis * attachNonStrictPath(const String & non_strict_path) + { + return addFilter([non_strict_path](const auto & request) { return startsWith(request.getURI(), non_strict_path); }); + } - String no_handler_description; - HTTPHandlersMatcherAndCreator handlers_creator; + /// Handle GET or HEAD endpoint on specified path + TThis * allowGetAndHeadRequest() + { + return addFilter([](const auto & request) + { + return request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD; + }); + } + + /// Handle POST or GET with params + TThis * allowPostAndGetParamsRequest() + { + return addFilter([](const auto & request) + { + return request.getURI().find('?') != std::string::npos + || request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST; + }); + } + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override + { + return filter(request) ? creator() : nullptr; + } + +private: + Filter filter; + std::function creator; }; +Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, const std::string & name); + +Poco::Net::HTTPRequestHandlerFactory * createStaticHandlerFactory(IServer & server, const std::string & config_prefix); + +Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & server, const std::string & config_prefix); + +Poco::Net::HTTPRequestHandlerFactory * createPredefineHandlerFactory(IServer & server, const std::string & config_prefix); + } diff --git a/programs/server/HTTPHandlerRequestFilter.h b/programs/server/HTTPHandlerRequestFilter.h new file mode 100644 index 0000000000000000000000000000000000000000..5f794884239324e17bbec5e32d773d857bec9fba --- /dev/null +++ b/programs/server/HTTPHandlerRequestFilter.h @@ -0,0 +1,98 @@ +#pragma once + +#include "HTTPHandlerFactory.h" + +#include +#include +#include +#include + +#include + +#if USE_RE2_ST +#include +#else +#define re2_st re2 +#endif + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_COMPILE_REGEXP; +} + +static inline std::string uriPathGetter(const Poco::Net::HTTPServerRequest & request) +{ + const auto & uri = request.getURI(); + const auto & end = find_first_symbols<'?'>(uri.data(), uri.data() + uri.size()); + + return std::string(uri.data(), end - uri.data()); +} + +static inline std::function headerGetter(const std::string & header_name) +{ + return [header_name](const Poco::Net::HTTPServerRequest & request) { return request.get(header_name, ""); }; +} + +static inline auto methodsExpressionFilter(const std::string &methods_expression) +{ + Poco::StringTokenizer tokenizer(Poco::toUpper(Poco::trim(methods_expression)), ","); + return [methods = std::vector(tokenizer.begin(), tokenizer.end())](const Poco::Net::HTTPServerRequest & request) + { + return std::count(methods.begin(), methods.end(), request.getMethod()); + }; +} + +template +static inline auto regularExpressionFilter(const std::string & regular_expression, const GetFunction & get) +{ + auto compiled_regex = std::make_shared(regular_expression); + + if (!compiled_regex->ok()) + throw Exception("cannot compile re2: " + regular_expression + " for routing_rule, error: " + compiled_regex->error() + + ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP); + + return std::make_pair(compiled_regex, [get = std::move(get), compiled_regex](const Poco::Net::HTTPServerRequest & request) + { + const auto & test_content = get(request); + int num_captures = compiled_regex->NumberOfCapturingGroups() + 1; + + re2_st::StringPiece matches[num_captures]; + re2_st::StringPiece input(test_content.data(), test_content.size()); + return compiled_regex->Match(input, 0, test_content.size(), re2_st::RE2::Anchor::ANCHOR_BOTH, matches, num_captures); + }); +} + +template +static inline std::function expressionFilter(const std::string & expression, const GetFunction & get) +{ + if (startsWith(expression, "regex:")) + return regularExpressionFilter(expression, get).second; + + return [expression, get = std::move(get)](const Poco::Net::HTTPServerRequest & request) { return get(request) == expression; }; +} + +template +static inline Poco::Net::HTTPRequestHandlerFactory * addFiltersFromConfig( + RoutingRuleHTTPHandlerFactory * factory, Poco::Util::AbstractConfiguration & config, const std::string & prefix) +{ + Poco::Util::AbstractConfiguration::Keys filters_type; + config.keys(prefix, filters_type); + + for (const auto & filter_type : filters_type) + { + if (filter_type == "handler") + continue; /// Skip handler config + else if (filter_type == "method") + factory->addFilter(methodsExpressionFilter(config.getString(prefix + "." + filter_type))); + else + factory->addFilter(expressionFilter(config.getString(prefix + "." + filter_type), filter_type == "url" + ? uriPathGetter : headerGetter(filter_type))); + } + + return factory; +} + +} diff --git a/programs/server/HTTPRequestHandler/ExtractorClientInfo.h b/programs/server/HTTPRequestHandler/ExtractorClientInfo.h deleted file mode 100644 index 76840f3d6827eb514c08cda6f6c4560e24d13117..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/ExtractorClientInfo.h +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include -#include - -namespace DB -{ - -class ExtractorClientInfo -{ -public: - ExtractorClientInfo(ClientInfo & info_) : client_info(info_) {} - - void extract(Poco::Net::HTTPServerRequest & request) - { - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::HTTP; - - /// Query sent through HTTP interface is initial. - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - - ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; - if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_GET) - http_method = ClientInfo::HTTPMethod::GET; - else if (request.getMethod() == Poco::Net::HTTPServerRequest::HTTP_POST) - http_method = ClientInfo::HTTPMethod::POST; - - client_info.http_method = http_method; - client_info.http_user_agent = request.get("User-Agent", ""); - } - -private: - ClientInfo & client_info; -}; - -} diff --git a/programs/server/HTTPRequestHandler/ExtractorContextChange.h b/programs/server/HTTPRequestHandler/ExtractorContextChange.h deleted file mode 100644 index 1d4b4e5f58acde1086c3142a42c762cc7c0d73f1..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/ExtractorContextChange.h +++ /dev/null @@ -1,116 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace DB -{ - -class ExtractorContextChange -{ -public: - ExtractorContextChange(Context & context_, bool settings_may_in_post_) : context(context_), settings_may_in_post(settings_may_in_post_) {} - - static const NameSet & getReservedParamNames() - { - static const NameSet reserved_param_names{ - "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check" - }; - - return reserved_param_names; - } - - static std::function reservedParamSuffixesFilter(bool reserved) - { - if (!reserved) - return [&](const String &) { return false; }; - - /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters. - /// It is a bug and ambiguity with `date_time_input_format` and `low_cardinality_allow_in_native_format` formats/settings. - return [&](const String & param_name) - { - if (endsWith(param_name, "_format")) - return true; - else if (endsWith(param_name, "_types")) - return true; - else if (endsWith(param_name, "_structure")) - return true; - - return false; - }; - } - - void extract(Poco::Net::HTTPServerRequest & request, HTMLForm & params) - { - bool is_multipart_data = startsWith(request.getContentType().data(), "multipart/form-data"); - - /// Settings can be overridden in the query. - /// Some parameters (database, default_format, everything used in the code above) do not - /// belong to the Settings class. - becomeReadonlyIfNeeded(request); - changeSettingsFromParams(params, reservedParamSuffixesFilter(is_multipart_data)); - - if (is_multipart_data || settings_may_in_post) - { - ExternalTablesHandler handler(context, params); - params.load(request, request.stream(), handler); - - /// We use the `Post Request Body Settings` to override the `Qeruy String Param settings` - if (settings_may_in_post) - changeSettingsFromParams(params, reservedParamSuffixesFilter(is_multipart_data)); - } - } - -private: - Context & context; - bool settings_may_in_post; - - /// 'readonly' setting values mean: - /// readonly = 0 - any query is allowed, client can change any setting. - /// readonly = 1 - only readonly queries are allowed, client can't change settings. - /// readonly = 2 - only readonly queries are allowed, client can change any setting except 'readonly'. - - /// In theory if initially readonly = 0, the client can change any setting and then set readonly - /// to some other value. - /// Only readonly queries are allowed for HTTP GET requests. - void becomeReadonlyIfNeeded(Poco::Net::HTTPServerRequest & request) - { - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) - { - Settings & settings = context.getSettingsRef(); - - if (settings.readonly == 0) - settings.readonly = 2; - } - } - - - void changeSettingsFromParams(HTMLForm & params, const std::function & reserved_param_suffixes) - { - SettingsChanges settings_changes; - const auto & reserved_param_names = getReservedParamNames(); - - for (const auto & [name, value] : params) - { - if (name == "database") - context.setCurrentDatabase(value); - else if (name == "default_format") - context.setDefaultFormat(value); - else if (!reserved_param_names.count(name) && !reserved_param_suffixes(name)) - { - if (Settings::findIndex(name) != Settings::npos) - settings_changes.push_back({name, value}); - } - } - - /// For external data we also want settings - context.checkSettingsConstraints(settings_changes); - context.applySettingsChanges(settings_changes); - } -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPExceptionHandler.cpp b/programs/server/HTTPRequestHandler/HTTPExceptionHandler.cpp deleted file mode 100644 index 8bfa351aab2e7b50ad9c4515b55dc0d33defd136..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPExceptionHandler.cpp +++ /dev/null @@ -1,154 +0,0 @@ -#include "HTTPExceptionHandler.h" - -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int READONLY; - extern const int UNKNOWN_COMPRESSION_METHOD; - - extern const int CANNOT_PARSE_TEXT; - extern const int CANNOT_PARSE_ESCAPE_SEQUENCE; - extern const int CANNOT_PARSE_QUOTED_STRING; - extern const int CANNOT_PARSE_DATE; - extern const int CANNOT_PARSE_DATETIME; - extern const int CANNOT_PARSE_NUMBER; - extern const int CANNOT_OPEN_FILE; - - extern const int UNKNOWN_ELEMENT_IN_AST; - extern const int UNKNOWN_TYPE_OF_AST_NODE; - extern const int TOO_DEEP_AST; - extern const int TOO_BIG_AST; - extern const int UNEXPECTED_AST_STRUCTURE; - - extern const int SYNTAX_ERROR; - - extern const int INCORRECT_DATA; - extern const int TYPE_MISMATCH; - - extern const int UNKNOWN_TABLE; - extern const int UNKNOWN_FUNCTION; - extern const int UNKNOWN_IDENTIFIER; - extern const int UNKNOWN_TYPE; - extern const int UNKNOWN_STORAGE; - extern const int UNKNOWN_DATABASE; - extern const int UNKNOWN_SETTING; - extern const int UNKNOWN_DIRECTION_OF_SORTING; - extern const int UNKNOWN_AGGREGATE_FUNCTION; - extern const int UNKNOWN_FORMAT; - extern const int UNKNOWN_DATABASE_ENGINE; - extern const int UNKNOWN_TYPE_OF_QUERY; - - extern const int QUERY_IS_TOO_LARGE; - - extern const int NOT_IMPLEMENTED; - extern const int SOCKET_TIMEOUT; - - extern const int UNKNOWN_USER; - extern const int WRONG_PASSWORD; - extern const int REQUIRED_PASSWORD; - - extern const int INVALID_SESSION_TIMEOUT; - extern const int HTTP_LENGTH_REQUIRED; -} - -static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int exception_code) -{ - using namespace Poco::Net; - - if (exception_code == ErrorCodes::REQUIRED_PASSWORD) - return HTTPResponse::HTTP_UNAUTHORIZED; - else if (exception_code == ErrorCodes::CANNOT_PARSE_TEXT || - exception_code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE || - exception_code == ErrorCodes::CANNOT_PARSE_QUOTED_STRING || - exception_code == ErrorCodes::CANNOT_PARSE_DATE || - exception_code == ErrorCodes::CANNOT_PARSE_DATETIME || - exception_code == ErrorCodes::CANNOT_PARSE_NUMBER || - - exception_code == ErrorCodes::UNKNOWN_ELEMENT_IN_AST || - exception_code == ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE || - exception_code == ErrorCodes::TOO_DEEP_AST || - exception_code == ErrorCodes::TOO_BIG_AST || - exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE || - - exception_code == ErrorCodes::SYNTAX_ERROR || - - exception_code == ErrorCodes::INCORRECT_DATA || - exception_code == ErrorCodes::TYPE_MISMATCH) - return HTTPResponse::HTTP_BAD_REQUEST; - else if (exception_code == ErrorCodes::UNKNOWN_TABLE || - exception_code == ErrorCodes::UNKNOWN_FUNCTION || - exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || - exception_code == ErrorCodes::UNKNOWN_TYPE || - exception_code == ErrorCodes::UNKNOWN_STORAGE || - exception_code == ErrorCodes::UNKNOWN_DATABASE || - exception_code == ErrorCodes::UNKNOWN_SETTING || - exception_code == ErrorCodes::UNKNOWN_DIRECTION_OF_SORTING || - exception_code == ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION || - exception_code == ErrorCodes::UNKNOWN_FORMAT || - exception_code == ErrorCodes::UNKNOWN_DATABASE_ENGINE || - - exception_code == ErrorCodes::UNKNOWN_TYPE_OF_QUERY) - return HTTPResponse::HTTP_NOT_FOUND; - else if (exception_code == ErrorCodes::QUERY_IS_TOO_LARGE) - return HTTPResponse::HTTP_REQUESTENTITYTOOLARGE; - else if (exception_code == ErrorCodes::NOT_IMPLEMENTED) - return HTTPResponse::HTTP_NOT_IMPLEMENTED; - else if (exception_code == ErrorCodes::SOCKET_TIMEOUT || - exception_code == ErrorCodes::CANNOT_OPEN_FILE) - return HTTPResponse::HTTP_SERVICE_UNAVAILABLE; - else if (exception_code == ErrorCodes::HTTP_LENGTH_REQUIRED) - return HTTPResponse::HTTP_LENGTH_REQUIRED; - - return HTTPResponse::HTTP_INTERNAL_SERVER_ERROR; -} - - -void HTTPExceptionHandler::handle( - const std::string & message, int exception_code, Poco::Net::HTTPServerRequest & request, - Poco::Net::HTTPServerResponse & response, std::shared_ptr response_out, - bool compression, Poco::Logger * log) -{ - try - { - /// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body - /// to avoid reading part of the current request body in the next request. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && response.getKeepAlive() - && !request.stream().eof() && exception_code != ErrorCodes::HTTP_LENGTH_REQUIRED) - { - request.stream().ignore(std::numeric_limits::max()); - } - - if (exception_code == ErrorCodes::UNKNOWN_USER || exception_code == ErrorCodes::WRONG_PASSWORD || - exception_code == ErrorCodes::REQUIRED_PASSWORD) - { - response.requireAuthentication("ClickHouse server HTTP API"); - } - else - { - response.setStatusAndReason(exceptionCodeToHTTPStatus(exception_code)); - } - - if (!response_out && !response.sent()) - response.send() << message << std::endl; - else - { - HTTPOutputStreams output_streams(response_out, compression); - - writeString(message, *output_streams.out_maybe_compressed); - writeChar('\n', *output_streams.out_maybe_compressed); - - output_streams.finalize(); - } - } - catch (...) - { - tryLogCurrentException(log, "Cannot send exception to client"); - } -} - -} diff --git a/programs/server/HTTPRequestHandler/HTTPExceptionHandler.h b/programs/server/HTTPRequestHandler/HTTPExceptionHandler.h deleted file mode 100644 index d2a5383ed4a1353977fee0ec583d63e2faee1ddd..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPExceptionHandler.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace DB -{ - -class HTTPExceptionHandler -{ -public: - static void handle(const std::string & message, int exception_code, Poco::Net::HTTPServerRequest & request, - Poco::Net::HTTPServerResponse & response, std::shared_ptr response_out, bool compression, - Poco::Logger * log); -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.cpp b/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.cpp deleted file mode 100644 index 292f98ba0eb145fad45efabe929893287015ea2f..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "HTTPPingRequestHandler.h" - -#include - -#include - -#include -#include - -namespace DB -{ - -void HTTPPingRequestHandler::handleRequest(Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) -{ - try - { - const auto & config = server.config(); - setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - - const char * data = "Ok.\n"; - response.sendBuffer(data, strlen(data)); - } - catch (...) - { - tryLogCurrentException("HTTPPingRequestHandler"); - } -} - -HTTPHandlerMatcher createPingHandlerMatcher(IServer & server, const String & key) -{ - const auto & path = server.config().getString(key, "/ping"); - - return [&, path = path](const Poco::Net::HTTPServerRequest & request) - { - return (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) && - request.getURI() == path; - }; -} - -HTTPHandlerCreator createPingHandlerCreator(IServer & server, const String &) -{ - return [&]() { return new HTTPPingRequestHandler(server); }; -} - -} diff --git a/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.h b/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.h deleted file mode 100644 index 7d524dd638d2d8bbfed968f6c660fccd67a76547..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPPingRequestHandler.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "../IServer.h" -#include "../HTTPHandlerFactory.h" - -#include - - -namespace DB -{ - -/// Response with "Ok.\n". Used for availability checks. -class HTTPPingRequestHandler : public Poco::Net::HTTPRequestHandler -{ -public: - explicit HTTPPingRequestHandler(const IServer & server_) : server(server_) - { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - const IServer & server; -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.cpp b/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.cpp deleted file mode 100644 index 723ff85abe3bade0cd2f00cd96bf1f095d869c24..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.cpp +++ /dev/null @@ -1,175 +0,0 @@ -#include "HTTPQueryRequestHandler.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ExtractorClientInfo.h" -#include "ExtractorContextChange.h" -#include "HTTPQueryRequestHandlerMatcherAndCreator.h" -#include "HTTPSessionContextHolder.h" -#include "HTTPExceptionHandler.h" - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int HTTP_LENGTH_REQUIRED; -} - -template -HTTPQueryRequestHandler::HTTPQueryRequestHandler(const IServer & server_, const QueryParamExtractor & extractor_) - : server(server_), log(&Logger::get("HTTPQueryRequestHandler")), extractor(extractor_) -{ - server_display_name = server.config().getString("display_name", getFQDNOrHostName()); -} - -template -void HTTPQueryRequestHandler::processQuery( - Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params, - Poco::Net::HTTPServerResponse & response, HTTPResponseBufferPtr & response_out) -{ - ExtractorClientInfo{context.getClientInfo()}.extract(request); - ExtractorContextChange{context, extractor.loadSettingsFromPost()}.extract(request, params); - - HTTPInputStreams input_streams{context, request, params}; - HTTPOutputStreams output_streams = HTTPOutputStreams(response_out, context, request, params); - - const auto & queries = extractor.extract(context, request, params); - - for (const auto & [execute_query, not_touch_post] : queries) - { - ReadBufferPtr temp_query_buf; - ReadBufferPtr execute_query_buf = std::make_shared(execute_query); - - if (not_touch_post && !startsWith(request.getContentType().data(), "multipart/form-data")) - { - temp_query_buf = execute_query_buf; /// we create a temporary reference for not to be destroyed - execute_query_buf = std::make_unique(*temp_query_buf, *input_streams.in_maybe_internal_compressed); - } - - executeQuery( - *execute_query_buf, *output_streams.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, - context, [&response] (const String & content_type) { response.setContentType(content_type); }, - [&response] (const String & current_query_id) { response.add("X-ClickHouse-Query-Id", current_query_id); } - ); - } - - /// Send HTTP headers with code 200 if no exception happened and the data is still not sent to the client. - output_streams.finalize(); -} - -template -void HTTPQueryRequestHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) -{ - setThreadName("HTTPHandler"); - ThreadStatus thread_status; - - /// In case of exception, send stack trace to client. - HTTPResponseBufferPtr response_out; - bool with_stacktrace = false, internal_compression = false; - - try - { - response_out = createResponseOut(request, response); - response.set("Content-Type", "text/plain; charset=UTF-8"); - response.set("X-ClickHouse-Server-Display-Name", server_display_name); - - /// For keep-alive to work. - if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) - response.setChunkedTransferEncoding(true); - - HTMLForm params(request); - with_stacktrace = params.getParsed("stacktrace", false); - internal_compression = params.getParsed("compress", false); - - /// Workaround. Poco does not detect 411 Length Required case. - if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST && !request.getChunkedTransferEncoding() && !request.hasContentLength()) - throw Exception("There is neither Transfer-Encoding header nor Content-Length header", ErrorCodes::HTTP_LENGTH_REQUIRED); - - { - Context query_context = server.context(); - CurrentThread::QueryScope query_scope(query_context); - - HTTPSessionContextHolder holder{query_context, request, params}; - processQuery(holder.query_context, request, params, response, response_out); - LOG_INFO(log, "Done processing query"); - } - } - catch (...) - { - tryLogCurrentException(log); - - /** If exception is received from remote server, then stack trace is embedded in message. - * If exception is thrown on local server, then stack trace is in separate field. - */ - int exception_code = getCurrentExceptionCode(); - std::string exception_message = getCurrentExceptionMessage(with_stacktrace, true); - HTTPExceptionHandler::handle(exception_message, exception_code, request, response, response_out, internal_compression, log); - } -} - -template -HTTPResponseBufferPtr HTTPQueryRequestHandler::createResponseOut(HTTPServerRequest & request, HTTPServerResponse & response) -{ - size_t keep_alive = server.config().getUInt("keep_alive_timeout", 10); - /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). - String http_response_compression_methods = request.get("Accept-Encoding", ""); - - if (!http_response_compression_methods.empty()) - { - /// Both gzip and deflate are supported. If the client supports both, gzip is preferred. - /// NOTE parsing of the list of methods is slightly incorrect. - if (std::string::npos != http_response_compression_methods.find("gzip")) - return std::make_shared(request, response, keep_alive, true, CompressionMethod::Gzip); - else if (std::string::npos != http_response_compression_methods.find("deflate")) - return std::make_shared(request, response, keep_alive, true, CompressionMethod::Zlib); -#if USE_BROTLI - else if (http_response_compression_methods == "br") - return std::make_shared(request, response, keep_alive, true, CompressionMethod::Brotli); -#endif - } - - return std::make_shared(request, response, keep_alive, false, CompressionMethod{}); -} - - -template class HTTPQueryRequestHandler; -template class HTTPQueryRequestHandler; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.h b/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.h deleted file mode 100644 index 5303b0cda89f53b47d7012c85e62c9b18241bf4d..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandler.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include "../IServer.h" - -#include - -#include -#include - -#include - - -namespace CurrentMetrics -{ - extern const Metric HTTPConnection; -} - -namespace Poco { class Logger; } - -namespace DB -{ - -template -class HTTPQueryRequestHandler : public Poco::Net::HTTPRequestHandler -{ -public: - explicit HTTPQueryRequestHandler(const IServer & server_, const QueryParamExtractor & extractor_); - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - const IServer & server; - Poco::Logger * log; - QueryParamExtractor extractor; - - /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. - String server_display_name; - - CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; - - HTTPResponseBufferPtr createResponseOut(HTTPServerRequest & request, HTTPServerResponse & response); - - void processQuery( - Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params, - Poco::Net::HTTPServerResponse & response, HTTPResponseBufferPtr & response_out); -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.cpp b/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.cpp deleted file mode 100644 index 81f25f0bd08c62518747f0b02f7481e8f2767c2d..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.cpp +++ /dev/null @@ -1,339 +0,0 @@ -#include "HTTPQueryRequestHandlerMatcherAndCreator.h" - -#include "../HTTPHandlerFactory.h" -#include "HTTPQueryRequestHandler.h" -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int EMPTY_PREDEFINED_QUERY; - extern const int CANNOT_COMPILE_REGEXP; - extern const int UNKNOWN_QUERY_PARAMETER; - extern const int DUPLICATE_CAPTURE_QUERY_PARAM; - extern const int ILLEGAL_HTTP_HANDLER_PARAM_NAME; - extern const int TOO_MANY_INSERT_QUERY_WITH_PREDEFINED_QUERY; -} - -ExtractorDynamicQueryParameters::ExtractorDynamicQueryParameters( - Poco::Util::AbstractConfiguration & configuration, const String & key, const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_) - : url_regex(url_regex_), headers_regex(headers_regex_) -{ - dynamic_param_name = configuration.getString(key + ".query_param_name", "query"); - - NameSet extracted_names; - - if (url_regex) - { - for (const auto & [capturing_name, capturing_index] : url_regex->NamedCapturingGroups()) - { - if (startsWith(capturing_name, "param_")) - { - if (extracted_names.count(capturing_name)) - throw Exception("Duplicate capture query parameter '" + capturing_name + "'", ErrorCodes::DUPLICATE_CAPTURE_QUERY_PARAM); - - extracted_names.emplace(capturing_name); - extract_from_url[capturing_name] = capturing_index; - } - } - } - - if (!headers_regex.empty()) - { - for (const auto & [header_name, header_regex] : headers_regex) - { - for (const auto & [capturing_name, capturing_index] : header_regex->NamedCapturingGroups()) - { - if (startsWith(capturing_name, "param_")) - { - if (extracted_names.count(capturing_name)) - throw Exception("Duplicate capture query parameter '" + capturing_name + "'", ErrorCodes::DUPLICATE_CAPTURE_QUERY_PARAM); - - extracted_names.emplace(capturing_name); - extract_from_headers[header_name][capturing_name] = capturing_index; - } - } - } - } -} - -template -void extractParamWithRegex(Context & context, const RegexRule & regex, const std::map & extract_params, const String & value) -{ - if (value.empty()) - return; - - int num_captures = regex->NumberOfCapturingGroups() + 1; - - re2_st::StringPiece matches[num_captures]; - re2_st::StringPiece input(value.data(), value.size()); - - if (regex->Match(input, 0, value.size(), re2_st::RE2::Anchor::UNANCHORED, matches, num_captures)) - { - for (const auto & [capturing_name, capturing_index] : extract_params) - { - const auto & capturing_value = matches[capturing_index]; - - if (capturing_value.data()) - { - String param_name = capturing_name; - if constexpr (remove_prefix_for_param) - { - const static size_t prefix_size = strlen("param_"); - param_name = capturing_name.substr(prefix_size); - } - - context.setQueryParameter(param_name, String(capturing_value.data(), capturing_value.size())); - } - } - } -} - -ExtractRes ExtractorDynamicQueryParameters::extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params) -{ - if (!extract_from_url.empty()) - extractParamWithRegex(context, url_regex, extract_from_url, Poco::URI{request.getURI()}.getPath()); - - - if (!extract_from_headers.empty()) - for (const auto & [header_name, extract_params] : extract_from_headers) - extractParamWithRegex(context, headers_regex.at(header_name), extract_params, request.get(header_name, "")); - - String extracted_query_from_params; - const static size_t prefix_size = strlen("param_"); - - for (const auto & [param_name, param_value] : params) - { - if (param_name == dynamic_param_name) - extracted_query_from_params += param_value; - else if (startsWith(param_name, "param_")) - context.setQueryParameter(param_name.substr(prefix_size), param_value); - } - - if (!extracted_query_from_params.empty()) - extracted_query_from_params += "\n"; - - return {{extracted_query_from_params, true}}; -} - -ExtractorPredefinedQueryParameters::ExtractorPredefinedQueryParameters( - Poco::Util::AbstractConfiguration & configuration, const String & key, const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_) - : url_regex(url_regex_), headers_regex(headers_regex_) -{ - Poco::Util::AbstractConfiguration::Keys queries_key; - configuration.keys(key + ".queries", queries_key); - - if (queries_key.empty()) - throw Exception("There must be at least one predefined query in the predefined HTTPHandler.", ErrorCodes::EMPTY_PREDEFINED_QUERY); - - for (const auto & query_key : queries_key) - { - const auto & predefine_query = configuration.getString(key + ".queries." + query_key); - - const char * query_begin = predefine_query.data(); - const char * query_end = predefine_query.data() + predefine_query.size(); - - ParserQuery parser(query_end, false); - ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "", 0); - QueryParameterVisitor{queries_names}.visit(extract_query_ast); - - bool is_insert_query = extract_query_ast->as(); - - if (has_insert_query && is_insert_query) - throw Exception("Too many insert queries in predefined queries.", ErrorCodes::TOO_MANY_INSERT_QUERY_WITH_PREDEFINED_QUERY); - - has_insert_query |= is_insert_query; - predefine_queries.push_back({predefine_query, is_insert_query}); - } - - const auto & reserved_params_name = ExtractorContextChange::getReservedParamNames(); - for (const auto & predefine_query_name : queries_names) - { - if (Settings::findIndex(predefine_query_name) != Settings::npos || reserved_params_name.count(predefine_query_name)) - throw Exception("Illegal http_handler param name '" + predefine_query_name + - "', Because it's reserved name or Settings name", ErrorCodes::ILLEGAL_HTTP_HANDLER_PARAM_NAME); - } - - NameSet extracted_names; - - if (url_regex) - { - for (const auto & [capturing_name, capturing_index] : url_regex->NamedCapturingGroups()) - { - if (queries_names.count(capturing_name)) - { - if (extracted_names.count(capturing_name)) - throw Exception("Duplicate capture query parameter '" + capturing_name + "'", ErrorCodes::DUPLICATE_CAPTURE_QUERY_PARAM); - - extracted_names.emplace(capturing_name); - extract_from_url[capturing_name] = capturing_index; - } - } - } - - if (!headers_regex.empty()) - { - for (const auto & [header_name, header_regex] : headers_regex) - { - for (const auto & [capturing_name, capturing_index] : header_regex->NamedCapturingGroups()) - { - if (queries_names.count(capturing_name)) - { - if (extracted_names.count(capturing_name)) - throw Exception("Duplicate capture query parameter '" + capturing_name + "'", ErrorCodes::DUPLICATE_CAPTURE_QUERY_PARAM); - - extracted_names.emplace(capturing_name); - extract_from_headers[header_name][capturing_name] = capturing_index; - } - } - } - } -} - -ExtractRes ExtractorPredefinedQueryParameters::extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params) -{ - if (!extract_from_url.empty()) - extractParamWithRegex(context, url_regex, extract_from_url, Poco::URI{request.getURI()}.getPath()); - - if (!extract_from_headers.empty()) - for (const auto & [header_name, extract_params] : extract_from_headers) - extractParamWithRegex(context, headers_regex.at(header_name), extract_params, request.get(header_name, "")); - - for (const auto & param : params) - if (queries_names.count(param.first)) - context.setQueryParameter(param.first, param.second); - - return predefine_queries; -} - -RegexRule HTTPQueryRequestHandlerMatcherAndCreator::createRegexRule(Poco::Util::AbstractConfiguration & configuration, const String & key) -{ - if (!configuration.has(key)) - return {}; - - const auto & regex_str = configuration.getString(key); - const auto & url_regex_rule = std::make_shared(regex_str); - - if (!url_regex_rule->ok()) - throw Exception("cannot compile re2: " + regex_str + " for HTTPHandler url, error: " + url_regex_rule->error() + - ". Look at https://github.com/google/re2/wiki/Syntax for reference.", ErrorCodes::CANNOT_COMPILE_REGEXP); - - return url_regex_rule; -} - -HeadersRegexRule HTTPQueryRequestHandlerMatcherAndCreator::createHeadersRegexRule(Poco::Util::AbstractConfiguration & configuration, const String & key) -{ - if (!configuration.has(key)) - return {}; - - Poco::Util::AbstractConfiguration::Keys headers_names; - configuration.keys(key, headers_names); - - HeadersRegexRule headers_regex_rule; - for (const auto & header_name : headers_names) - { - if (headers_regex_rule.count(header_name)) - throw Exception("Duplicate header match declaration '" + header_name + "'", ErrorCodes::LOGICAL_ERROR); - - headers_regex_rule[header_name] = createRegexRule(configuration, key + "." + header_name); - } - - return headers_regex_rule; -} - -size_t findFirstMissingMatchPos(const re2_st::RE2 & regex_rule, const String & match_content) -{ - int num_captures = regex_rule.NumberOfCapturingGroups() + 1; - - re2_st::StringPiece matches[num_captures]; - re2_st::StringPiece input(match_content.data(), match_content.size()); - if (regex_rule.Match(input, 0, match_content.size(), re2_st::RE2::Anchor::UNANCHORED, matches, num_captures)) - return matches[0].size(); - - return size_t(0); -} - -HTTPHandlerMatcher HTTPQueryRequestHandlerMatcherAndCreator::createHandlerMatcher( - const String & method, const RegexRule & url_rule, const HeadersRegexRule & headers_rule) -{ - return [method = Poco::toLower(method), url_rule = url_rule, headers_rule = headers_rule](const Poco::Net::HTTPServerRequest & request) - { - if (!method.empty() && Poco::toLower(request.getMethod()) != method) - return false; - - if (url_rule) - { - Poco::URI uri(request.getURI()); - const auto & request_uri = uri.getPath(); - size_t first_missing_pos = findFirstMissingMatchPos(*url_rule, request_uri); - - const char * url_end = request_uri.data() + request_uri.size(); - const char * first_missing = request_uri.data() + first_missing_pos; - - if (first_missing != url_end && *first_missing == '/') - ++first_missing; - - if (first_missing != url_end && *first_missing != '?') - return false; /// Not full matched - } - - if (!headers_rule.empty()) - { - for (const auto & [header_name, header_rule] : headers_rule) - { - if (!request.has(header_name)) - return false; - - const String & header_value = request.get(header_name); - if (header_value.size() != findFirstMissingMatchPos(*header_rule, header_value)) - return false; - } - } - - return true; - }; -} - -HTTPHandlerMatcher createDynamicQueryHandlerMatcher(IServer & server, const String & key) -{ - return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule(server.config(), key, - HTTPQueryRequestHandlerMatcherAndCreator::createHandlerMatcher); -} - - -HTTPHandlerMatcher createPredefinedQueryHandlerMatcher(IServer & server, const String & key) -{ - return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule(server.config(), key, - HTTPQueryRequestHandlerMatcherAndCreator::createHandlerMatcher); -} - -HTTPHandlerCreator createDynamicQueryHandlerCreator(IServer & server, const String & key) -{ - return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule( - server.config(), key, [&](const String &, const RegexRule & url_rule, const HeadersRegexRule & headers_rule) - { - const auto & extract = std::make_shared(server.config(), key, url_rule, headers_rule); - - return [&, query_extract = extract]() - { - return new HTTPQueryRequestHandler(server, *query_extract); - }; - }); -} - -HTTPHandlerCreator createPredefinedQueryHandlerCreator(IServer & server, const String & key) -{ - return HTTPQueryRequestHandlerMatcherAndCreator::invokeWithParsedRegexRule( - server.config(), key, [&](const String &, const RegexRule & url_rule, const HeadersRegexRule & headers_rule) - { - const auto & extract = std::make_shared(server.config(), key, url_rule, headers_rule); - - return [&, query_extract = extract]() - { - return new HTTPQueryRequestHandler(server, *query_extract); - }; - }); -} -} diff --git a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.h b/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.h deleted file mode 100644 index 0d7bc8356bbc5266683446b0ed723795f1bda76a..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPQueryRequestHandlerMatcherAndCreator.h +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "ExtractorContextChange.h" -#include "../HTTPHandlerFactory.h" - -#if USE_RE2_ST -#include -#else -#define re2_st re2 -#endif - -namespace DB -{ - -using RegexRule = std::shared_ptr; -using HeadersRegexRule = std::map; -using ExtractRes = std::vector>; - -class ExtractorDynamicQueryParameters -{ -public: - ExtractorDynamicQueryParameters( - Poco::Util::AbstractConfiguration & configuration, const String & key, - const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_ - ); - - bool loadSettingsFromPost() const { return false; } - - ExtractRes extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params); - -private: - const RegexRule url_regex; - const HeadersRegexRule headers_regex; - - String dynamic_param_name; - std::map extract_from_url; - std::map> extract_from_headers; -}; - -class ExtractorPredefinedQueryParameters -{ -public: - ExtractorPredefinedQueryParameters( - Poco::Util::AbstractConfiguration & configuration, const String & key, - const RegexRule & url_regex_, const HeadersRegexRule & headers_regex_ - ); - - bool loadSettingsFromPost() const { return !has_insert_query; } - - ExtractRes extract(Context & context, Poco::Net::HTTPServerRequest & request, HTMLForm & params); - -private: - const RegexRule url_regex; - const HeadersRegexRule headers_regex; - - NameSet queries_names; - bool has_insert_query{false}; - ExtractRes predefine_queries; - std::map extract_from_url; - std::map> extract_from_headers; -}; - -class HTTPQueryRequestHandlerMatcherAndCreator -{ -public: - template - static auto invokeWithParsedRegexRule(Poco::Util::AbstractConfiguration & configuration, const String & key, const NestedFunction & fun) - { - return fun(configuration.getString(key + ".method", ""), createRegexRule(configuration, key + ".url"), - createHeadersRegexRule(configuration, key + ".headers")); - } - - static HTTPHandlerMatcher createHandlerMatcher(const String & method, const RegexRule & url_rule, const HeadersRegexRule & headers_rule); - -private: - static RegexRule createRegexRule(Poco::Util::AbstractConfiguration & configuration, const String & key); - - static HeadersRegexRule createHeadersRegexRule(Poco::Util::AbstractConfiguration & configuration, const String & key); -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.cpp b/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.cpp deleted file mode 100644 index ea70abbcc6fa4f060a022ac1159661cbab47eb25..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include "HTTPReplicasStatusRequestHandler.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - - -namespace DB -{ - - -HTTPReplicasStatusRequestHandler::HTTPReplicasStatusRequestHandler(Context & context_) - : context(context_) -{ -} - -void HTTPReplicasStatusRequestHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) -{ - try - { - HTMLForm params(request); - - /// Even if lag is small, output detailed information about the lag. - bool verbose = params.get("verbose", "") == "1"; - - const MergeTreeSettings & settings = context.getMergeTreeSettings(); - - bool ok = true; - std::stringstream message; - - auto databases = context.getDatabases(); - - /// Iterate through all the replicated tables. - for (const auto & db : databases) - { - /// Lazy database can not contain replicated tables - if (db.second->getEngineName() == "Lazy") - continue; - - for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) - { - auto & table = iterator->table(); - StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); - - if (!table_replicated) - continue; - - time_t absolute_delay = 0; - time_t relative_delay = 0; - - table_replicated->getReplicaDelays(absolute_delay, relative_delay); - - if ((settings.min_absolute_delay_to_close && absolute_delay >= static_cast(settings.min_absolute_delay_to_close)) - || (settings.min_relative_delay_to_close && relative_delay >= static_cast(settings.min_relative_delay_to_close))) - ok = false; - - message << backQuoteIfNeed(db.first) << "." << backQuoteIfNeed(iterator->name()) - << ":\tAbsolute delay: " << absolute_delay << ". Relative delay: " << relative_delay << ".\n"; - } - } - - const auto & config = context.getConfigRef(); - setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - - if (ok && !verbose) - { - const char * data = "Ok.\n"; - response.sendBuffer(data, strlen(data)); - } - else - { - response.send() << message.rdbuf(); - } - } - catch (...) - { - tryLogCurrentException("HTTPReplicasStatusRequestHandler"); - - try - { - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); - - if (!response.sent()) - { - /// We have not sent anything yet and we don't even know if we need to compress response. - response.send() << getCurrentExceptionMessage(false) << std::endl; - } - } - catch (...) - { - LOG_ERROR((&Logger::get("HTTPReplicasStatusRequestHandler")), "Cannot send exception to client"); - } - } -} - -HTTPHandlerMatcher createReplicasStatusHandlerMatcher(IServer & server, const String & key) -{ - const auto & prefix = server.config().getString(key, "/replicas_status"); - - return [&, prefix = prefix](const Poco::Net::HTTPServerRequest & request) - { - return (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) && - startsWith(request.getURI(), prefix); - }; -} - -HTTPHandlerCreator createReplicasStatusHandlerCreator(IServer & server, const String &) -{ - return [&]() { return new HTTPReplicasStatusRequestHandler(server.context()); }; -} - -} diff --git a/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.h b/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.h deleted file mode 100644 index 277e59eb02c3f6d0772d36e35b4e75c238e2b3f6..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPReplicasStatusRequestHandler.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include "../HTTPHandlerFactory.h" -#include - - -namespace DB -{ - -class Context; - -/// Replies "Ok.\n" if all replicas on this server don't lag too much. Otherwise output lag information. -class HTTPReplicasStatusRequestHandler : public Poco::Net::HTTPRequestHandler -{ -public: - explicit HTTPReplicasStatusRequestHandler(Context & context_); - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - Context & context; -}; - - -} diff --git a/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.cpp b/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.cpp deleted file mode 100644 index 032b51d5b7a7e761bcbd0d9b74f0d27357c1a160..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "HTTPRootRequestHandler.h" - -#include - -#include - -#include -#include - -namespace DB -{ - -void HTTPRootRequestHandler::handleRequest(Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) -{ - try - { - const auto & config = server.config(); - setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - - response.setContentType("text/html; charset=UTF-8"); - - const std::string data = config.getString("http_server_default_response", "Ok.\n"); - response.sendBuffer(data.data(), data.size()); - } - catch (...) - { - tryLogCurrentException("HTTPRootRequestHandler"); - } -} - -HTTPHandlerMatcher createRootHandlerMatcher(IServer &, const String &) -{ - return [&](const Poco::Net::HTTPServerRequest & request) -> bool - { - return (request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET || request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD) - && request.getURI() == "/"; - }; -} - -HTTPHandlerCreator createRootHandlerCreator(IServer & server, const String &) -{ - return [&]() { return new HTTPRootRequestHandler(server); }; -} - -} diff --git a/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.h b/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.h deleted file mode 100644 index 136f3292385e29e1e33aef45c9f51409a77c662f..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPRootRequestHandler.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "../IServer.h" -#include "../HTTPHandlerFactory.h" - -#include - - -namespace DB -{ - -/// Response with custom string. Can be used for browser. -class HTTPRootRequestHandler : public Poco::Net::HTTPRequestHandler -{ -public: - explicit HTTPRootRequestHandler(const IServer & server_) : server(server_) - { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - const IServer & server; -}; - -} diff --git a/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.cpp b/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.cpp deleted file mode 100644 index 840044634aa32c90b2f5faa9282fdf5a0da4cf52..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include "HTTPSessionContextHolder.h" -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int REQUIRED_PASSWORD; - extern const int INVALID_SESSION_TIMEOUT; -} - -static std::chrono::steady_clock::duration parseSessionTimeout( - const Poco::Util::AbstractConfiguration & config, - const HTMLForm & params) -{ - unsigned session_timeout = config.getInt("default_session_timeout", 60); - - if (params.has("session_timeout")) - { - unsigned max_session_timeout = config.getUInt("max_session_timeout", 3600); - std::string session_timeout_str = params.get("session_timeout"); - - ReadBufferFromString buf(session_timeout_str); - if (!tryReadIntText(session_timeout, buf) || !buf.eof()) - throw Exception("Invalid session timeout: '" + session_timeout_str + "'", ErrorCodes::INVALID_SESSION_TIMEOUT); - - if (session_timeout > max_session_timeout) - throw Exception("Session timeout '" + session_timeout_str + "' is larger than max_session_timeout: " + toString(max_session_timeout) - + ". Maximum session timeout could be modified in configuration file.", - ErrorCodes::INVALID_SESSION_TIMEOUT); - } - - return std::chrono::seconds(session_timeout); -} - -HTTPSessionContextHolder::~HTTPSessionContextHolder() -{ - if (session_context) - session_context->releaseSession(session_id, session_timeout); -} - -void HTTPSessionContextHolder::authentication(Poco::Net::HTTPServerRequest & request, HTMLForm & params) -{ - auto user = request.get("X-ClickHouse-User", ""); - auto password = request.get("X-ClickHouse-Key", ""); - auto quota_key = request.get("X-ClickHouse-Quota", ""); - - if (user.empty() && password.empty() && quota_key.empty()) - { - /// User name and password can be passed using query parameters - /// or using HTTP Basic auth (both methods are insecure). - if (request.hasCredentials()) - { - Poco::Net::HTTPBasicCredentials credentials(request); - - user = credentials.getUsername(); - password = credentials.getPassword(); - } - else - { - user = params.get("user", "default"); - password = params.get("password", ""); - } - - quota_key = params.get("quota_key", ""); - } - else - { - /// It is prohibited to mix different authorization schemes. - if (request.hasCredentials() - || params.has("user") - || params.has("password") - || params.has("quota_key")) - { - throw Exception("Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously", ErrorCodes::REQUIRED_PASSWORD); - } - } - - std::string query_id = params.get("query_id", ""); - query_context.setUser(user, password, request.clientAddress(), quota_key); - query_context.setCurrentQueryId(query_id); -} - -HTTPSessionContextHolder::HTTPSessionContextHolder(Context & query_context_, Poco::Net::HTTPServerRequest & request, HTMLForm & params) - : query_context(query_context_) -{ - authentication(request, params); - - { - session_id = params.get("session_id", ""); - - if (!session_id.empty()) - { - session_timeout = parseSessionTimeout(query_context.getConfigRef(), params); - session_context = query_context.acquireSession(session_id, session_timeout, params.check("session_check", "1")); - - query_context = *session_context; - query_context.setSessionContext(*session_context); - } - } -} - -} diff --git a/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.h b/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.h deleted file mode 100644 index deadd3910ad98b780f8ef88ba201ee79f10dcda2..0000000000000000000000000000000000000000 --- a/programs/server/HTTPRequestHandler/HTTPSessionContextHolder.h +++ /dev/null @@ -1,25 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace DB -{ - -/// Manage the lifetime of the session context. -struct HTTPSessionContextHolder -{ - ~HTTPSessionContextHolder(); - - void authentication(Poco::Net::HTTPServerRequest & request, HTMLForm & params); - - HTTPSessionContextHolder(Context & query_context_, Poco::Net::HTTPServerRequest & request, HTMLForm & params); - - String session_id; - Context & query_context; - std::shared_ptr session_context = nullptr; - std::chrono::steady_clock::duration session_timeout; -}; - -} diff --git a/programs/server/NotFoundHandler.cpp b/programs/server/NotFoundHandler.cpp index 26df1a1b7d4a71a5603654ceea99c67f73a31ef6..f9b51719b0dbeae2a16fa5798cea8a8516dfbc3d 100644 --- a/programs/server/NotFoundHandler.cpp +++ b/programs/server/NotFoundHandler.cpp @@ -17,14 +17,16 @@ void NotFoundHandler::handleRequest( try { response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_NOT_FOUND); - - std::stringstream output_description; - output_description << "There is no handle " << request.getURI() << "\n\n"; - - if (!no_handler_description.empty()) - output_description << no_handler_description << "\n"; - - response.send() << output_description.str(); + response.send() << "There is no handle " << request.getURI() << "\n\n" + << "Use / or /ping for health checks.\n" + << "Or /replicas_status for more sophisticated health checks.\n\n" + << "Send queries from your program with POST method or GET /?query=...\n\n" + << "Use clickhouse-client:\n\n" + << "For interactive data analysis:\n" + << " clickhouse-client\n\n" + << "For batch query processing:\n" + << " clickhouse-client --query='SELECT 1' > result\n" + << " clickhouse-client < query > result\n"; } catch (...) { @@ -32,4 +34,4 @@ void NotFoundHandler::handleRequest( } } -} +} \ No newline at end of file diff --git a/programs/server/NotFoundHandler.h b/programs/server/NotFoundHandler.h index caf527726c4e89c8c116a983c1a2503b72275120..0f76fe9c46ee3f7f746e102b05f9c6d7e80e3164 100644 --- a/programs/server/NotFoundHandler.h +++ b/programs/server/NotFoundHandler.h @@ -10,12 +10,9 @@ namespace DB class NotFoundHandler : public Poco::Net::HTTPRequestHandler { public: - NotFoundHandler(const std::string & no_handler_description_) : no_handler_description(no_handler_description_) {} - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; - -private: - const std::string no_handler_description; + void handleRequest( + Poco::Net::HTTPServerRequest & request, + Poco::Net::HTTPServerResponse & response) override; }; -} +} \ No newline at end of file diff --git a/programs/server/PingRequestHandler.cpp b/programs/server/PingRequestHandler.cpp deleted file mode 100644 index 141161ef45c335c91bd329d248ce75a97a7e7b62..0000000000000000000000000000000000000000 --- a/programs/server/PingRequestHandler.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "PingRequestHandler.h" - -#include - -#include - -#include -#include - -namespace DB -{ - -void PingRequestHandler::handleRequest( - Poco::Net::HTTPServerRequest &, - Poco::Net::HTTPServerResponse & response) -{ - try - { - const auto & config = server.config(); - setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - - const char * data = "Ok.\n"; - response.sendBuffer(data, strlen(data)); - } - catch (...) - { - tryLogCurrentException("PingRequestHandler"); - } -} - -} diff --git a/programs/server/PingRequestHandler.h b/programs/server/PingRequestHandler.h deleted file mode 100644 index 3728fb40adbe237dcb12f02ba4cdd00ac89b5fca..0000000000000000000000000000000000000000 --- a/programs/server/PingRequestHandler.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include "IServer.h" - -#include - - -namespace DB -{ - -/// Response with "Ok.\n". Used for availability checks. -class PingRequestHandler : public Poco::Net::HTTPRequestHandler -{ -private: - IServer & server; - -public: - explicit PingRequestHandler(IServer & server_) : server(server_) - { - } - - void handleRequest( - Poco::Net::HTTPServerRequest & request, - Poco::Net::HTTPServerResponse & response) override; -}; - -} diff --git a/programs/server/ReplicasStatusHandler.h b/programs/server/ReplicasStatusHandler.h index 479b013cfe8c0af1ddf7d7e8e79b29066fb1c7ba..a32f1ba905f1c50d6cfb218d8ac016287458145f 100644 --- a/programs/server/ReplicasStatusHandler.h +++ b/programs/server/ReplicasStatusHandler.h @@ -17,7 +17,7 @@ private: Context & context; public: - explicit ReplicasStatusHandler(IServer & server); + explicit ReplicasStatusHandler(IServer & server_); void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; }; diff --git a/programs/server/RootRequestHandler.cpp b/programs/server/RootRequestHandler.cpp deleted file mode 100644 index 86b57b632087fc1447b511e087c641fa59ac9988..0000000000000000000000000000000000000000 --- a/programs/server/RootRequestHandler.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "RootRequestHandler.h" - -#include - -#include - -#include -#include - -namespace DB -{ - -void RootRequestHandler::handleRequest( - Poco::Net::HTTPServerRequest &, - Poco::Net::HTTPServerResponse & response) -{ - try - { - const auto & config = server.config(); - setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - - response.setContentType("text/html; charset=UTF-8"); - - const std::string data = config.getString("http_server_default_response", "Ok.\n"); - response.sendBuffer(data.data(), data.size()); - } - catch (...) - { - tryLogCurrentException("RootRequestHandler"); - } -} - -} diff --git a/programs/server/RootRequestHandler.h b/programs/server/RootRequestHandler.h deleted file mode 100644 index 1b6d53eeeda453be808aa24f2698ab36f2332b2d..0000000000000000000000000000000000000000 --- a/programs/server/RootRequestHandler.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include "IServer.h" - -#include - - -namespace DB -{ - -/// Response with custom string. Can be used for browser. -class RootRequestHandler : public Poco::Net::HTTPRequestHandler -{ -private: - IServer & server; - -public: - explicit RootRequestHandler(IServer & server_) : server(server_) - { - } - - void handleRequest( - Poco::Net::HTTPServerRequest & request, - Poco::Net::HTTPServerResponse & response) override; -}; - -} diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3490ff6a445f7b331e70c7b80b63d58f0cf92c19..96a14dd3d8752ead1801ae581d8e60177a3d1367 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -767,9 +767,9 @@ int Server::main(const std::vector & /*args*/) auto address = socket_bind_listen(socket, listen_host, port); socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); - auto handler_factory = createDefaultHandlerFatory(*this, "HTTPHandler-factory"); - if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0) - handler_factory->addHandler(async_metrics); + auto handler_factory = createHandlerFactory(*this, "HTTPHandler-factory"); +// if (config().has("prometheus") && config().getInt("prometheus.port", 0) == 0) +// handler_factory->addHandler(async_metrics); servers.emplace_back(std::make_unique( handler_factory, @@ -789,7 +789,7 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - createDefaultHandlerFatory(*this, "HTTPSHandler-factory"), + createHandlerFactory(*this, "HTTPSHandler-factory"), server_pool, socket, http_params)); @@ -847,7 +847,7 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - createDefaultHandlerFatory(*this, "InterserverIOHTTPHandler-factory"), + createHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), server_pool, socket, http_params)); @@ -863,7 +863,7 @@ int Server::main(const std::vector & /*args*/) socket.setReceiveTimeout(settings.http_receive_timeout); socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(std::make_unique( - createDefaultHandlerFatory(*this, "InterserverIOHTTPHandler-factory"), + createHandlerFactory(*this, "InterserverIOHTTPSHandler-factory"), server_pool, socket, http_params)); @@ -892,22 +892,22 @@ int Server::main(const std::vector & /*args*/) }); /// Prometheus (if defined and not setup yet with http_port) - create_server("prometheus.port", [&](UInt16 port) - { - Poco::Net::ServerSocket socket; - auto address = socket_bind_listen(socket, listen_host, port); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); - auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory"); - handler_factory->addHandler(async_metrics); - servers.emplace_back(std::make_unique( - handler_factory, - server_pool, - socket, - http_params)); - - LOG_INFO(log, "Listening for Prometheus: http://" + address.toString()); - }); +// create_server("prometheus.port", [&](UInt16 port) +// { +// Poco::Net::ServerSocket socket; +// auto address = socket_bind_listen(socket, listen_host, port); +// socket.setReceiveTimeout(settings.http_receive_timeout); +// socket.setSendTimeout(settings.http_send_timeout); +// auto handler_factory = new HTTPRequestHandlerFactoryMain(*this, "PrometheusHandler-factory"); +// handler_factory->addHandler(async_metrics); +// servers.emplace_back(std::make_unique( +// handler_factory, +// server_pool, +// socket, +// http_params)); +// +// LOG_INFO(log, "Listening for Prometheus: http://" + address.toString()); +// }); } if (servers.empty()) diff --git a/programs/server/StaticRequestHandler.cpp b/programs/server/StaticRequestHandler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..730060dea2e2b921b94ca1d2cabb089386abfe43 --- /dev/null +++ b/programs/server/StaticRequestHandler.cpp @@ -0,0 +1,86 @@ +#include "StaticRequestHandler.h" + +#include "HTTPHandlerFactory.h" +#include "HTTPHandlerRequestFilter.h" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_FILE_NAME; + extern const int INVALID_CONFIG_PARAMETER; +} + +void StaticRequestHandler::handleRequest(Poco::Net::HTTPServerRequest &, Poco::Net::HTTPServerResponse & response) +{ + try + { + setResponseDefaultHeaders(response, server.config().getUInt("keep_alive_timeout", 10)); + + response.setContentType(content_type); + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTPStatus(status)); + + response.sendBuffer(response_content.data(), response_content.size()); + } + catch (...) + { + tryLogCurrentException("StaticRequestHandler"); + } +} + +StaticRequestHandler::StaticRequestHandler(IServer & server_, const String & expression, int status_, const String & content_type_) + : server(server_), status(status_), content_type(content_type_) +{ + static const String file_prefix = "file://"; + static const String config_prefix = "config://"; + + if (startsWith(expression, file_prefix)) + { + std::string config_dir = Poco::Path(server.context().getPath()).parent().toString(); + const std::string & file_path = config_dir + expression.substr(file_prefix.size(), expression.size() - file_prefix.size()); + + if (!Poco::File(file_path).exists()) + throw Exception("Invalid file name for static HTTPHandler." + file_path, ErrorCodes::INCORRECT_FILE_NAME); + + WriteBufferFromOwnString out; + ReadBufferFromFile in(file_path); + copyData(in, out); + response_content = out.str(); + } + else if (startsWith(expression, config_prefix)) + { + if (expression.size() <= config_prefix.size()) + throw Exception("Static routing rule handler must contain a complete configuration path, for example: config://config_key", + ErrorCodes::INVALID_CONFIG_PARAMETER); + + response_content = server.config().getString(expression.substr(config_prefix.size(), expression.size() - config_prefix.size()), "Ok.\n"); + } + else + response_content = expression; +} + +Poco::Net::HTTPRequestHandlerFactory * createStaticHandlerFactory(IServer & server, const std::string & config_prefix) +{ + const auto & status = server.config().getInt(config_prefix + ".handler.status", 200); + const auto & response_content = server.config().getRawString(config_prefix + ".handler.response_content", "Ok.\n"); + const auto & response_content_type = server.config().getString(config_prefix + ".handler.content_type", "text/plain; charset=UTF-8"); + + return addFiltersFromConfig(new RoutingRuleHTTPHandlerFactory( + server, response_content, status, response_content_type), server.config(), config_prefix); +} + +} diff --git a/programs/server/StaticRequestHandler.h b/programs/server/StaticRequestHandler.h new file mode 100644 index 0000000000000000000000000000000000000000..bfe7f7a3e7b6c1f0f6fd1ae570ef0240787180a3 --- /dev/null +++ b/programs/server/StaticRequestHandler.h @@ -0,0 +1,28 @@ +#pragma once + +#include "IServer.h" + +#include +#include + + +namespace DB +{ + +/// Response with custom string. Can be used for browser. +class StaticRequestHandler : public Poco::Net::HTTPRequestHandler +{ +private: + IServer & server; + + int status; + String content_type; + String response_content; + +public: + StaticRequestHandler(IServer & server, const String & expression, int status_ = 200, const String & content_type_ = "text/html; charset=UTF-8"); + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; +}; + +} diff --git a/src/Common/HTMLForm.h b/src/Common/HTMLForm.h index fa6f31bf5d225a49315ac206e55c2d6591e9cc37..2490d6131609e59d8cae1872b15a26b7b45b75a9 100644 --- a/src/Common/HTMLForm.h +++ b/src/Common/HTMLForm.h @@ -26,12 +26,6 @@ struct HTMLForm : public Poco::Net::HTMLForm readUrl(istr); } - template - bool check(const std::string & key, T check_value) - { - const auto & value = getParsed(key, T()); - return value == check_value; - } template T getParsed(const std::string & key, T default_value) diff --git a/dbms/Interpreters/QueryParameterVisitor.h b/src/Interpreters/QueryParameterVisitor.h similarity index 59% rename from dbms/Interpreters/QueryParameterVisitor.h rename to src/Interpreters/QueryParameterVisitor.h index d765aa00bea9943f48c1e1cb970bbe01bb438afd..d3e618058c000b4817225392794fd58a589ee65e 100644 --- a/dbms/Interpreters/QueryParameterVisitor.h +++ b/src/Interpreters/QueryParameterVisitor.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include namespace DB { @@ -32,4 +34,16 @@ private: } }; +NameSet analyzeReceiveQueryParams(const std::string & query) +{ + NameSet query_params; + const char * query_begin = query.data(); + const char * query_end = query.data() + query.size(); + + ParserQuery parser(query_end, false); + ASTPtr extract_query_ast = parseQuery(parser, query_begin, query_end, "analyzeReceiveQueryParams", 0, 0); + QueryParameterVisitor(query_params).visit(extract_query_ast); + return query_params; +} + }