From 5c8c9907876857eb9e8689bcaf9e1e4c49d37230 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 5 Jan 2020 01:59:08 +0300 Subject: [PATCH] Fixed error --- dbms/programs/server/HTTPHandler.cpp | 18 ++++----------- dbms/src/IO/CompressionMethod.cpp | 23 ++++++++++++++----- dbms/src/IO/CompressionMethod.h | 22 ++++++++++++++---- .../IO/WriteBufferFromHTTPServerResponse.cpp | 22 ++++++++---------- .../IO/WriteBufferFromHTTPServerResponse.h | 5 ++-- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/dbms/programs/server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp index 1d91432830..34a0093e9a 100644 --- a/dbms/programs/server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -298,30 +298,22 @@ void HTTPHandler::processQuery( /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); - bool client_supports_http_compression = false; - CompressionMethod http_response_compression_method {}; + CompressionMethod http_response_compression_method = CompressionMethod::None; if (!http_response_compression_methods.empty()) { /// Both gzip and deflate are supported. If the client supports both, gzip is preferred. /// NOTE parsing of the list of methods is slightly incorrect. if (std::string::npos != http_response_compression_methods.find("gzip")) - { - client_supports_http_compression = true; http_response_compression_method = CompressionMethod::Gzip; - } else if (std::string::npos != http_response_compression_methods.find("deflate")) - { - client_supports_http_compression = true; http_response_compression_method = CompressionMethod::Zlib; - } else if (http_response_compression_methods == "br") - { - client_supports_http_compression = true; http_response_compression_method = CompressionMethod::Brotli; - } } + bool client_supports_http_compression = http_response_compression_method != CompressionMethod::None; + /// Client can pass a 'compress' flag in the query string. In this case the query result is /// compressed using internal algorithm. This is not reflected in HTTP headers. bool internal_compression = params.getParsed("compress", false); @@ -340,8 +332,8 @@ void HTTPHandler::processQuery( unsigned keep_alive_timeout = config.getUInt("keep_alive_timeout", 10); used_output.out = std::make_shared( - request, response, keep_alive_timeout, - client_supports_http_compression, http_response_compression_method); + request, response, keep_alive_timeout, client_supports_http_compression, http_response_compression_method); + if (internal_compression) used_output.out_maybe_compressed = std::make_shared(*used_output.out); else diff --git a/dbms/src/IO/CompressionMethod.cpp b/dbms/src/IO/CompressionMethod.cpp index e1fe40bd77..20f1ea4430 100644 --- a/dbms/src/IO/CompressionMethod.cpp +++ b/dbms/src/IO/CompressionMethod.cpp @@ -58,13 +58,18 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s } -std::unique_ptr wrapReadBufferWithCompressionMethod(std::unique_ptr nested, CompressionMethod method) +std::unique_ptr wrapReadBufferWithCompressionMethod( + std::unique_ptr nested, + CompressionMethod method, + size_t buf_size, + char * existing_memory, + size_t alignment) { if (method == CompressionMethod::Gzip || method == CompressionMethod::Zlib) - return std::make_unique(std::move(nested), method); + return std::make_unique(std::move(nested), method, buf_size, existing_memory, alignment); #if USE_BROTLI if (method == CompressionMethod::Brotli) - return std::make_unique(std::move(nested)); + return std::make_unique(std::move(nested), buf_size, existing_memory, alignment); #endif if (method == CompressionMethod::None) @@ -74,14 +79,20 @@ std::unique_ptr wrapReadBufferWithCompressionMethod(std::unique_ptr< } -std::unique_ptr wrapWriteBufferWithCompressionMethod(std::unique_ptr nested, CompressionMethod method, int level) +std::unique_ptr wrapWriteBufferWithCompressionMethod( + std::unique_ptr nested, + CompressionMethod method, + int level, + size_t buf_size, + char * existing_memory, + size_t alignment) { if (method == DB::CompressionMethod::Gzip || method == CompressionMethod::Zlib) - return std::make_unique(std::move(nested), method, level); + return std::make_unique(std::move(nested), method, level, buf_size, existing_memory, alignment); #if USE_BROTLI if (method == DB::CompressionMethod::Brotli) - return std::make_unique(std::move(nested), level); + return std::make_unique(std::move(nested), level, buf_size, existing_memory, alignment); #endif if (method == CompressionMethod::None) diff --git a/dbms/src/IO/CompressionMethod.h b/dbms/src/IO/CompressionMethod.h index b84c7e2658..64c2ba3341 100644 --- a/dbms/src/IO/CompressionMethod.h +++ b/dbms/src/IO/CompressionMethod.h @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { @@ -17,14 +19,14 @@ class WriteBuffer; enum class CompressionMethod { + None, /// DEFLATE compression with gzip header and CRC32 checksum. /// This option corresponds to files produced by gzip(1) or HTTP Content-Encoding: gzip. Gzip, /// DEFLATE compression with zlib header and Adler32 checksum. /// This option corresponds to HTTP Content-Encoding: deflate. Zlib, - Brotli, - None + Brotli }; /// How the compression method is named in HTTP. @@ -37,7 +39,19 @@ std::string toContentEncodingName(CompressionMethod method); */ CompressionMethod chooseCompressionMethod(const std::string & path, const std::string & hint); -std::unique_ptr wrapReadBufferWithCompressionMethod(std::unique_ptr nested, CompressionMethod method); -std::unique_ptr wrapWriteBufferWithCompressionMethod(std::unique_ptr nested, CompressionMethod method, int level); +std::unique_ptr wrapReadBufferWithCompressionMethod( + std::unique_ptr nested, + CompressionMethod method, + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, + char * existing_memory = nullptr, + size_t alignment = 0); + +std::unique_ptr wrapWriteBufferWithCompressionMethod( + std::unique_ptr nested, + CompressionMethod method, + int level, + size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, + char * existing_memory = nullptr, + size_t alignment = 0); } diff --git a/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp index 0a7f574afa..270bad2ee2 100644 --- a/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp +++ b/dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp @@ -111,18 +111,19 @@ void WriteBufferFromHTTPServerResponse::nextImpl() *response_header_ostr << "Content-Encoding: " << content_encoding_name << "\r\n"; #else response.set("Content-Encoding", content_encoding_name); - response_body_ostr = &(response.send()); #endif - - out = wrapWriteBufferWithCompressionMethod(std::make_unique(*response_body_ostr), compression_method, compression_level); } - else - { + #if !defined(POCO_CLICKHOUSE_PATCH) - response_body_ostr = &(response.send()); + response_body_ostr = &(response.send()); #endif - out = std::make_unique(*response_body_ostr); - } + + out = wrapWriteBufferWithCompressionMethod( + std::make_unique(*response_body_ostr), + compress ? compression_method : CompressionMethod::None, + compression_level, + working_buffer.size(), + working_buffer.begin()); } finishSendHeaders(); @@ -134,9 +135,6 @@ void WriteBufferFromHTTPServerResponse::nextImpl() out->position() = position(); out->next(); } - - internal_buffer = out->internalBuffer(); - working_buffer = out->buffer(); } @@ -146,7 +144,7 @@ WriteBufferFromHTTPServerResponse::WriteBufferFromHTTPServerResponse( unsigned keep_alive_timeout_, bool compress_, CompressionMethod compression_method_) - : WriteBuffer(nullptr, 0) + : BufferWithOwnMemory(DBMS_DEFAULT_BUFFER_SIZE) , request(request_) , response(response_) , keep_alive_timeout(keep_alive_timeout_) diff --git a/dbms/src/IO/WriteBufferFromHTTPServerResponse.h b/dbms/src/IO/WriteBufferFromHTTPServerResponse.h index b3fb71e54a..f0b614c740 100644 --- a/dbms/src/IO/WriteBufferFromHTTPServerResponse.h +++ b/dbms/src/IO/WriteBufferFromHTTPServerResponse.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,7 @@ namespace DB /// Also this class write and flush special X-ClickHouse-Progress HTTP headers /// if no data was sent at the time of progress notification. /// This allows to implement progress bar in HTTP clients. -class WriteBufferFromHTTPServerResponse : public WriteBuffer +class WriteBufferFromHTTPServerResponse : public BufferWithOwnMemory { private: Poco::Net::HTTPServerRequest & request; @@ -90,7 +91,7 @@ public: Poco::Net::HTTPServerResponse & response_, unsigned keep_alive_timeout_, bool compress_ = false, /// If true - set Content-Encoding header and compress the result. - CompressionMethod compression_method_ = CompressionMethod::Gzip); + CompressionMethod compression_method_ = CompressionMethod::None); /// Writes progess in repeating HTTP headers. void onProgress(const Progress & progress); -- GitLab