Merge branch 'master' into bit-test-many-fuzz

.gitmodules

@@ -104,6 +104,10 @@
 [submodule "contrib/sparsehash-c11"]
 	path = contrib/sparsehash-c11
 	url = https://github.com/sparsehash/sparsehash-c11.git
+[submodule "contrib/grpc"]
+	path = contrib/grpc
+	url = https://github.com/grpc/grpc.git
+	branch = v1.25.0
 [submodule "contrib/aws"]
 	path = contrib/aws
 	url = https://github.com/aws/aws-sdk-cpp.git

CMakeLists.txt

@@ -341,6 +341,7 @@ endif()
 include (cmake/find/libxml2.cmake)
 include (cmake/find/brotli.cmake)
 include (cmake/find/protobuf.cmake)
+include (cmake/find/grpc.cmake)
 include (cmake/find/pdqsort.cmake)
 include (cmake/find/hdfs3.cmake) # uses protobuf
 include (cmake/find/s3.cmake)

cmake/find/grpc.cmake

+set(_PROTOBUF_PROTOC $<TARGET_FILE:protoc>)
+set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
+
+function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files")
+    return()
+  endif()
+
+  if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
+    foreach(FIL ${ARGN})
+      get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+      get_filename_component(ABS_PATH ${ABS_FIL} PATH)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  else()
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS)
+    foreach(DIR ${Protobuf_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h"
+      COMMAND  ${_PROTOBUF_PROTOC}
+      ARGS --grpc_out=${CMAKE_CURRENT_BINARY_DIR}
+           --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN_EXECUTABLE}
+           ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL}
+      COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}"
+      VERBATIM)
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()

cmake/find/h3.cmake

@@ -15,8 +15,9 @@ if (USE_INTERNAL_H3_LIBRARY)
     set (H3_LIBRARY h3)
     set (H3_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include)
 elseif (NOT MISSING_INTERNAL_H3_LIBRARY)
+    set (H3_INCLUDE_PATHS /usr/local/include/h3)
     find_library (H3_LIBRARY h3)
-    find_path (H3_INCLUDE_DIR NAMES h3/h3api.h PATHS ${H3_INCLUDE_PATHS})
+    find_path (H3_INCLUDE_DIR NAMES h3api.h PATHS ${H3_INCLUDE_PATHS})
 endif ()
 
 if (H3_LIBRARY AND H3_INCLUDE_DIR)

cmake/find/zlib.cmake

@@ -29,7 +29,7 @@ if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY)
     set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco
     set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf
     set (ZLIB_FOUND 1) # for poco
-    if (USE_STATIC_LIBRARIES AND DEFINED BUILD_SHARED_LIBS)
+    if (USE_STATIC_LIBRARIES)
         set (ZLIB_LIBRARIES zlibstatic CACHE INTERNAL "")
     else ()
         set (ZLIB_LIBRARIES zlib CACHE INTERNAL "")

cmake/tools.cmake

@@ -32,9 +32,8 @@ else ()
     find_program (GOLD_PATH NAMES "ld.gold" "gold")
 endif ()
 
-if (NOT OS_FREEBSD)
 # We prefer LLD linker over Gold or BFD.
-  if (NOT LINKER_NAME)
+if (NOT LINKER_NAME)
     if (LLD_PATH)
         if (COMPILER_GCC)
             # GCC driver requires one of supported linker names like "lld".
@@ -44,9 +43,9 @@ if (NOT OS_FREEBSD)
             set (LINKER_NAME ${LLD_PATH})
         endif ()
     endif ()
-  endif ()
+endif ()
 
-  if (NOT LINKER_NAME)
+if (NOT LINKER_NAME)
     if (GOLD_PATH)
         if (COMPILER_GCC)
             set (LINKER_NAME "gold")
@@ -54,7 +53,6 @@ if (NOT OS_FREEBSD)
             set (LINKER_NAME ${GOLD_PATH})
         endif ()
     endif ()
-  endif ()
 endif ()
 
 if (LINKER_NAME)

contrib/CMakeLists.txt

@@ -54,6 +54,7 @@ if (USE_INTERNAL_BTRIE_LIBRARY)
 endif ()
 
 if (USE_INTERNAL_ZLIB_LIBRARY)
+    unset (BUILD_SHARED_LIBS CACHE)
     set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "")
     set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "")
     set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP
@@ -69,14 +70,10 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
     add_subdirectory (${INTERNAL_ZLIB_NAME})
     # We should use same defines when including zlib.h as used when zlib compiled
     target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
-    if (TARGET zlibstatic)
-        target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
-    endif ()
+    target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
     if (ARCH_AMD64 OR ARCH_AARCH64)
-        target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK)
-        if (TARGET zlibstatic)
-            target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK)
-        endif ()
+       target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK)
+       target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK)
     endif ()
 endif ()
 
@@ -119,7 +116,7 @@ function(mysql_support)
     endif()
     if (USE_INTERNAL_ZLIB_LIBRARY)
         set(ZLIB_FOUND ON)
-        set(ZLIB_LIBRARY ${ZLIB_LIBRARIES})
+        set(ZLIB_LIBRARY zlibstatic)
         set(WITH_EXTERNAL_ZLIB ON)
     endif()
     add_subdirectory (mariadb-connector-c)
@@ -262,7 +259,6 @@ if (USE_EMBEDDED_COMPILER AND USE_INTERNAL_LLVM_LIBRARY)
     set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "")
     set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE STRING "")
     add_subdirectory (llvm/llvm)
-    target_include_directories(LLVMSupport SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
 endif ()
 
 if (USE_INTERNAL_LIBGSASL_LIBRARY)
@@ -340,4 +336,6 @@ if (USE_FASTOPS)
     add_subdirectory (fastops-cmake)
 endif()
 
+add_subdirectory(grpc-cmake)
+
 add_subdirectory(replxx-cmake)

contrib/arrow-cmake/CMakeLists.txt

@@ -348,7 +348,6 @@ if (ARROW_WITH_ZLIB)
 endif ()
 if (ARROW_WITH_ZSTD)
     target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY})
-    target_include_directories(${ARROW_LIBRARY} SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
 endif ()
 
 target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_INCLUDE_DIR})

contrib/boost-cmake/CMakeLists.txt

@@ -41,5 +41,4 @@ endif()
 if (USE_INTERNAL_AVRO_LIBRARY)
     add_boost_lib(iostreams)
     target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES})
-    target_include_directories(boost_iostreams_internal SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
 endif()

grpc @ c1d17652

+Subproject commit c1d176528fd8da9dd4066d16554bcd216d29033f

contrib/libxml2-cmake/CMakeLists.txt

@@ -59,4 +59,4 @@ endif()
 
 target_include_directories(libxml2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include)
 target_include_directories(libxml2 PUBLIC ${LIBXML2_SOURCE_DIR}/include)
-target_include_directories(libxml2 SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
+target_include_directories(libxml2 PRIVATE ${ZLIB_INCLUDE_DIR}/include)

contrib/replxx-cmake/CMakeLists.txt

@@ -45,10 +45,7 @@ if (ENABLE_REPLXX)
         endif ()
     endif ()
 
-    if (NOT (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9))
-        target_compile_options(replxx PUBLIC -Wno-documentation)
-    endif ()
-
+    target_compile_options(replxx PUBLIC -Wno-documentation)
     target_compile_definitions(replxx PUBLIC USE_REPLXX=1)
 
     message (STATUS "Using replxx")

dbms/CMakeLists.txt

@@ -176,12 +176,8 @@ elseif (COMPILER_GCC)
     add_cxx_compile_options(-Wsizeof-array-argument)
     # Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof
     add_cxx_compile_options(-Wsizeof-pointer-memaccess)
-
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9)
-        # Warn about overriding virtual functions that are not marked with the override keyword
-        add_cxx_compile_options(-Wsuggest-override)
-    endif ()
-
+    # Warn about overriding virtual functions that are not marked with the override keyword
+    add_cxx_compile_options(-Wsuggest-override)
     # Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type
     add_cxx_compile_options(-Wswitch-bool)
     # Warn if a self-comparison always evaluates to true or false

dbms/programs/server/Server.cpp

@@ -578,6 +578,25 @@ int Server::main(const std::vector<std::string> & /*args*/)
         global_context->initializeTraceCollector();
 #endif
 
+    /// Describe multiple reasons when query profiler cannot work.
+
+#if !USE_UNWIND
+    LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work without bundled unwind (stack unwinding) library.");
+#endif
+
+#if WITH_COVERAGE
+    LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they work extremely slow with test coverage.");
+#endif
+
+#if defined(SANITIZER)
+    LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they cannot work under sanitizers"
+        " when two different stack unwinding methods will interfere with each other.");
+#endif
+
+    if (!hasPHDRCache())
+        LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created"
+            " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe).");
+
     global_context->setCurrentDatabase(default_database);
 
     if (has_zookeeper && config().has("distributed_ddl"))

dbms/src/Functions/geoToH3.cpp

@@ -10,11 +10,7 @@
 #include <Common/typeid_cast.h>
 #include <ext/range.h>
 
-#if __has_include(<h3/h3api.h>)
-#    include <h3/h3api.h>
-#else
-#    include <h3api.h>
-#endif
+#include <h3api.h>
 
 
 namespace DB

dbms/src/Functions/h3EdgeAngle.cpp

@@ -7,11 +7,7 @@
 #    include <Common/typeid_cast.h>
 #    include <ext/range.h>
 
-#    if __has_include(<h3/h3api.h>)
-#        include <h3/h3api.h>
-#    else
-#        include <h3api.h>
-#    endif
+#    include <h3api.h>
 
 
 namespace DB

dbms/src/Functions/h3EdgeLengthM.cpp

@@ -8,13 +8,8 @@
 #    include <Common/typeid_cast.h>
 #    include <ext/range.h>
 
-#    if __has_include(<h3/h3api.h>)
-#        include <h3/h3api.h>
-#        include <h3/constants.h>
-#    else
-#        include <h3api.h>
-#        include <constants.h>
-#    endif
+#    include <h3api.h>
+#    include <constants.h>
 
 
 namespace DB

dbms/src/Functions/h3GetResolution.cpp

@@ -7,11 +7,7 @@
 #    include <Common/typeid_cast.h>
 #    include <ext/range.h>
 
-#    if __has_include(<h3/h3api.h>)
-#        include <h3/h3api.h>
-#    else
-#        include <h3api.h>
-#    endif
+#    include <h3api.h>
 
 
 namespace DB

dbms/src/Functions/h3IsValid.cpp

@@ -7,11 +7,7 @@
 #    include <Common/typeid_cast.h>
 #    include <ext/range.h>
 
-#    if __has_include(<h3/h3api.h>)
-#        include <h3/h3api.h>
-#    else
-#        include <h3api.h>
-#    endif
+#    include <h3api.h>
 
 
 namespace DB

dbms/src/Functions/h3kRing.cpp

@@ -11,11 +11,7 @@
 #    include <Common/typeid_cast.h>
 #    include <ext/range.h>
 
-#    if __has_include(<h3/h3api.h>)
-#        include <h3/h3api.h>
-#    else
-#        include <h3api.h>
-#    endif
+#    include <h3api.h>
 
 
 namespace DB

dbms/src/Interpreters/Cluster.cpp

@@ -72,7 +72,8 @@ bool Cluster::Address::isLocal(UInt16 clickhouse_port) const
 }
 
 
-Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix)
+Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_index_, UInt32 replica_index_) :
+    shard_index(shard_index_), replica_index(replica_index_)
 {
     host_name = config.getString(config_prefix + ".host");
     port = static_cast<UInt16>(config.getInt(config_prefix + ".port"));
@@ -137,12 +138,8 @@ std::pair<String, UInt16> Cluster::Address::fromString(const String & host_port_
 String Cluster::Address::toFullString() const
 {
     return
-        escapeForFileName(user) +
-        (password.empty() ? "" : (':' + escapeForFileName(password))) + '@' +
-        escapeForFileName(host_name) + ':' +
-        std::to_string(port) +
-        (default_database.empty() ? "" : ('#' + escapeForFileName(default_database)))
-        + ((secure == Protocol::Secure::Enable) ? "+secure" : "");
+        ((shard_index == 0) ? "" : "shard" + std::to_string(shard_index)) +
+        ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index));
 }
 
 Cluster::Address Cluster::Address::fromFullString(const String & full_string)
@@ -150,35 +147,55 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string)
     const char * address_begin = full_string.data();
     const char * address_end = address_begin + full_string.size();
 
-    Protocol::Secure secure = Protocol::Secure::Disable;
-    const char * secure_tag = "+secure";
-    if (endsWith(full_string, secure_tag))
+    const char * user_pw_end = strchr(full_string.data(), '@');
+
+    /// parsing with the new [shard{shard_index}[_replica{replica_index}]] format
+    if (!user_pw_end && startsWith(full_string, "shard"))
     {
-        address_end -= strlen(secure_tag);
-        secure = Protocol::Secure::Enable;
+        const char * underscore = strchr(full_string.data(), '_');
+
+        Address address;
+        address.shard_index = parse<UInt32>(address_begin + strlen("shard"));
+        address.replica_index = underscore ? parse<UInt32>(underscore + strlen("_replica")) : 0;
+
+        return address;
     }
+    else
+    {
+        /// parsing with the old user[:password]@host:port#default_database format
+        /// This format is appeared to be inconvenient for the following reasons:
+        /// - credentials are exposed in file name;
+        /// - the file name can be too long.
+
+        Protocol::Secure secure = Protocol::Secure::Disable;
+        const char * secure_tag = "+secure";
+        if (endsWith(full_string, secure_tag))
+        {
+            address_end -= strlen(secure_tag);
+            secure = Protocol::Secure::Enable;
+        }
 
-    const char * user_pw_end = strchr(full_string.data(), '@');
-    const char * colon = strchr(full_string.data(), ':');
-    if (!user_pw_end || !colon)
-        throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR);
-
-    const bool has_pw = colon < user_pw_end;
-    const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon;
-    if (!host_end)
-        throw Exception("Incorrect address '" + full_string + "', it does not contain port", ErrorCodes::SYNTAX_ERROR);
-
-    const char * has_db = strchr(full_string.data(), '#');
-    const char * port_end = has_db ? has_db : address_end;
-
-    Address address;
-    address.secure = secure;
-    address.port = parse<UInt16>(host_end + 1, port_end - (host_end + 1));
-    address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end));
-    address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end));
-    address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string();
-    address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string();
-    return address;
+        const char * colon = strchr(full_string.data(), ':');
+        if (!user_pw_end || !colon)
+            throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR);
+
+        const bool has_pw = colon < user_pw_end;
+        const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon;
+        if (!host_end)
+            throw Exception("Incorrect address '" + full_string + "', it does not contain port", ErrorCodes::SYNTAX_ERROR);
+
+        const char * has_db = strchr(full_string.data(), '#');
+        const char * port_end = has_db ? has_db : address_end;
+
+        Address address;
+        address.secure = secure;
+        address.port = parse<UInt16>(host_end + 1, port_end - (host_end + 1));
+        address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end));
+        address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end));
+        address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string();
+        address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string();
+        return address;
+    }
 }
 
 
@@ -309,7 +326,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting
 
                 if (startsWith(replica_key, "replica"))
                 {
-                    replica_addresses.emplace_back(config, partial_prefix + replica_key);
+                    replica_addresses.emplace_back(config, partial_prefix + replica_key, current_shard_num, current_replica_num);
                     ++current_replica_num;
 
                     if (!replica_addresses.back().is_local)

dbms/src/Interpreters/Cluster.h

@@ -57,6 +57,9 @@ public:
         UInt16 port;
         String user;
         String password;
+        UInt32 shard_index{}; /// shard serial number in configuration file, starting from 1.
+        UInt32 replica_index{}; /// replica serial number in this shard, starting from 1; zero means no replicas.
+
         /// This database is selected when no database is specified for Distributed table
         String default_database;
         /// The locality is determined at the initialization, and is not changed even if DNS is changed
@@ -67,7 +70,7 @@ public:
         Protocol::Secure secure = Protocol::Secure::Disable;
 
         Address() = default;
-        Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix);
+        Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_index_ = 0, UInt32 replica_index_ = 0);
         Address(const String & host_port_, const String & user_, const String & password_, UInt16 clickhouse_port, bool secure_ = false);
 
         /// Returns 'escaped_host_name:port'
@@ -80,8 +83,10 @@ public:
 
         static std::pair<String, UInt16> fromString(const String & host_port_string);
 
-        /// Retrurns escaped user:password@resolved_host_address:resolved_host_port#default_database
+        /// Returns escaped shard{shard_index}_replica{replica_index}
         String toFullString() const;
+
+        /// Returns address with only shard index and replica index or full address without shard index and replica index
         static Address fromFullString(const String & address_full_string);
 
         /// Returns resolved address if it does resolve.

dbms/src/Storages/Distributed/DirectoryMonitor.cpp

@@ -190,6 +190,12 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
         const auto & shards_info = cluster->getShardsInfo();
         const auto & shards_addresses = cluster->getShardsAddresses();
 
+        /// check new format shard{shard_index}_number{number_index}
+        if (address.shard_index != 0)
+        {
+            return shards_info[address.shard_index - 1].per_replica_pools[address.replica_index - 1];
+        }
+
         /// existing connections pool have a higher priority
         for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
         {
@@ -199,8 +205,15 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri
             {
                 const Cluster::Address & replica_address = replicas_addresses[replica_index];
 
-                if (address == replica_address)
+                if (address.user == replica_address.user &&
+                    address.password == replica_address.password &&
+                    address.host_name == replica_address.host_name &&
+                    address.port == replica_address.port &&
+                    address.default_database == replica_address.default_database &&
+                    address.secure == replica_address.secure)
+                {
                     return shards_info[shard_index].per_replica_pools[replica_index];
+                }
             }
         }
 

dbms/tests/integration/test_distributed_format/test.py

@@ -26,12 +26,12 @@ def test_single_file(started_cluster):
     node.query("create table distr_1 (x UInt64, s String) engine = Distributed('test_cluster', database, table)")
     node.query("insert into distr_1 values (1, 'a'), (2, 'bb'), (3, 'ccc')")
 
-    query = "select * from file('/var/lib/clickhouse/data/default/distr_1/default@not_existing:9000/1.bin', 'Distributed')"
+    query = "select * from file('/var/lib/clickhouse/data/default/distr_1/shard1_replica1/1.bin', 'Distributed')"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
 
     assert out == '1\ta\n2\tbb\n3\tccc\n'
 
-    query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_1/default@not_existing:9000/1.bin');" \
+    query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_1/shard1_replica1/1.bin');" \
             "select * from t"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
 
@@ -45,12 +45,12 @@ def test_two_files(started_cluster):
     node.query("insert into distr_2 values (0, '_'), (1, 'a')")
     node.query("insert into distr_2 values (2, 'bb'), (3, 'ccc')")
 
-    query = "select * from file('/var/lib/clickhouse/data/default/distr_2/default@not_existing:9000/{1,2,3,4}.bin', 'Distributed') order by x"
+    query = "select * from file('/var/lib/clickhouse/data/default/distr_2/shard1_replica1/{1,2,3,4}.bin', 'Distributed') order by x"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
 
     assert out == '0\t_\n1\ta\n2\tbb\n3\tccc\n'
 
-    query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_2/default@not_existing:9000/{1,2,3,4}.bin');" \
+    query = "create table t (dummy UInt32) engine = File('Distributed', '/var/lib/clickhouse/data/default/distr_2/shard1_replica1/{1,2,3,4}.bin');" \
             "select * from t order by x"
     out = node.exec_in_container(['/usr/bin/clickhouse', 'local', '--stacktrace', '-q', query])
 

dbms/tests/integration/test_distributed_storage_configuration/test.py

@@ -25,7 +25,7 @@ def _files_in_dist_mon(node, root, table):
         'bash',
         '-c',
         # `-maxdepth 1` to avoid /tmp/ subdirectory
-        'find /{root}/data/default/{table}/default@127%2E0%2E0%2E2:9000 -maxdepth 1 -type f | wc -l'.format(root=root, table=table)
+        'find /{root}/data/default/{table}/shard2_replica1 -maxdepth 1 -type f | wc -l'.format(root=root, table=table)
     ]).split('\n')[0])
 
 def test_different_versions(start_cluster):

dbms/tests/queries/0_stateless/00999_full_join_dup_keys_crash.reference

-1	2	0	0
 0	0	1	2
+1	2	0	0
 -
 0	0	1	2
 -
-1	2	0	0
 0	0	1	2
+1	2	0	0
 -
 0	0	1	2
 -
-1
 0
+1
 -
 0
 -
-1
 0
+1
 -
 0
 -
@@ -48,13 +48,13 @@
 -
 \N
 -
-\N
 1
+\N
 -
 1
 -
-\N
 1
+\N
 -
 1
 -

dbms/tests/queries/0_stateless/00999_full_join_dup_keys_crash.sql

 SET join_use_nulls = 0;
 
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
 
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
 
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
 
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
 
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
 
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
 
 SET join_use_nulls = 1;
 
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
 
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
-SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT * FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a, foo.b, bar.a, bar.b;
 SELECT '-';
 
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
 
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
-SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT foo.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY foo.a;
 SELECT '-';
 
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.a = bar.b) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
 
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo FULL JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';
-SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b);
+SELECT bar.a FROM (SELECT 1 AS a, 2 AS b) AS foo RIGHT JOIN (SELECT 1 AS a, 2 AS b) AS bar ON (foo.b = bar.a) AND (foo.b = bar.b) ORDER BY bar.a;
 SELECT '-';

debian/pbuilder-hooks/B90test-server

@@ -8,6 +8,11 @@ PACKAGE_INSTALL=${PACKAGE_INSTALL=1}
 TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1}
 
 if [ "${PACKAGE_INSTALL}" ]; then
+    #for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do
+    #    apt-get install -y --force-yes "$PKG" ||:
+    #    apt-get remove -y "$PKG" ||:
+    #done
+
     dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||:
     apt install -y -f --allow-downgrades ||:
     dpkg -l | grep clickhouse ||:
@@ -40,6 +45,8 @@ export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=9440}
 export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=8443}
 
 if [ "${TEST_CONNECT}" ]; then
+    sed -i 's/ssl_conf = ssl_sect/#ssl_conf = ssl_sect/g' /etc/ssl/openssl.cnf ||:
+
     [ "${TEST_PORT_RANDOM}" ] && echo "<yandex><http_port>${CLICKHOUSE_PORT_HTTP}</http_port><tcp_port>${CLICKHOUSE_PORT_TCP}</tcp_port><interserver_http_port>${CLICKHOUSE_PORT_INTERSERVER}</interserver_http_port></yandex>" > /etc/clickhouse-server/config.d/port.xml
 
     if  [ "${TEST_SSL}" ]; then

docs/en/development/build.md

-# How to Build ClickHouse Release Package
-
-## Install Git and Pbuilder
-
-```bash
-$ sudo apt-get update
-$ sudo apt-get install git pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring
-```
-
-## Checkout ClickHouse Sources
-
-```bash
-$ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git
-$ cd ClickHouse
-```
-
-## Run Release Script
-
-```bash
-$ ./release
-```
-
 # How to Build ClickHouse for Development
 
 The following tutorial is based on the Ubuntu Linux system.
 With appropriate changes, it should also work on any other Linux distribution.
 Supported platforms: x86_64 and AArch64. Support for Power9 is experimental.
 
-## Install Git, CMake and Ninja
+## Install Git, CMake, Python and Ninja
 
 ```bash
-$ sudo apt-get install git cmake ninja-build
+$ sudo apt-get install git cmake python ninja-build
 ```
 
 Or cmake3 instead of cmake on older systems.
@@ -66,12 +44,12 @@ $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git
 or
 ```bash
 $ git clone --recursive https://github.com/ClickHouse/ClickHouse.git
-$ cd ClickHouse
 ```
 
 ## Build ClickHouse
 
 ```bash
+$ cd ClickHouse
 $ mkdir build
 $ cd build
 $ cmake ..
@@ -83,4 +61,80 @@ To create an executable, run `ninja clickhouse`.
 This will create the `dbms/programs/clickhouse` executable, which can be used with `client` or `server` arguments.
 
 
+# How to Build ClickHouse on Any Linux
+
+The build requires the following componenets:
+
+- Git (is used only to checkout the sources, it's not needed for build)
+- CMake 3.10 or newer
+- Ninja (recommended) or Make
+- C++ compiler: gcc 9 or clang 8 or newer
+- Linker: lld or gold (the classic GNU ld won't work)
+- Python (is only used inside LLVM build and it is optional)
+
+If all the components are installed, you may build in the same way as the steps above.
+
+Example for Ubuntu Eoan:
+
+```
+sudo apt update
+sudo apt install git cmake ninja-build g++ python
+git clone --recursive https://github.com/ClickHouse/ClickHouse.git
+mkdir build && cd build
+cmake ../ClickHouse
+ninja
+```
+
+Example for OpenSUSE Tumbleweed:
+
+```
+sudo zypper install git cmake ninja gcc-c++ python lld
+git clone --recursive https://github.com/ClickHouse/ClickHouse.git
+mkdir build && cd build
+cmake ../ClickHouse
+ninja
+```
+
+Example for CentOS 8:
+
+```
+sudo yum update
+yum install git cmake make gcc-c++ python2
+git clone --recursive https://github.com/ClickHouse/ClickHouse.git
+cmake ../ClickHouse
+make
+```
+
+# You Don't Have to Build ClickHouse
+
+ClickHouse is available in pre-built binaries and packages. Binaries are portable and can be run on any Linux flavour.
+
+They are build for stable, prestable and testing releases as long as for every commit to master and for every pull request.
+
+To find the most fresh build from `master`, go to [commits page](https://github.com/ClickHouse/ClickHouse/commits/master), click on the first green check mark or red cross near commit, and click to the "Details" link right after "ClickHouse Build Check".
+
+
+# How to Build ClickHouse Debian Package
+
+## Install Git and Pbuilder
+
+```bash
+$ sudo apt-get update
+$ sudo apt-get install git python pbuilder debhelper lsb-release fakeroot sudo debian-archive-keyring debian-keyring
+```
+
+## Checkout ClickHouse Sources
+
+```bash
+$ git clone --recursive --branch master https://github.com/ClickHouse/ClickHouse.git
+$ cd ClickHouse
+```
+
+## Run Release Script
+
+```bash
+$ ./release
+```
+
+
 [Original article](https://clickhouse.tech/docs/en/development/build/) <!--hide-->

docs/en/query_language/functions/bit_functions.md

@@ -82,8 +82,11 @@ Returns result of [logical conjuction](https://en.wikipedia.org/wiki/Logical_con
 The conjuction for bitwise operations:
 
 0 AND 0 = 0
+
 0 AND 1 = 0
+
 1 AND 0 = 0
+
 1 AND 1 = 1
 
 **Syntax** 
@@ -144,8 +147,11 @@ Returns result of [logical disjunction](https://en.wikipedia.org/wiki/Logical_di
 The disjunction for bitwise operations:
 
 0 OR 0 = 0
+
 0 OR 1 = 1
+
 1 OR 0 = 1
+
 1 OR 1 = 1
 
 **Syntax** 

libs/libloggers/loggers/Loggers.cpp

@@ -138,7 +138,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
     if (config.getBool("logger.console", false)
         || (!config.hasProperty("logger.console") && !is_daemon && is_tty))
     {
-        bool color_enabled = config.getBool("logger.colored_console", false) && is_tty;
+        bool color_enabled = config.getBool("logger.color_terminal", true) && is_tty;
 
         Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter(this, OwnPatternFormatter::ADD_NOTHING, color_enabled);
         Poco::AutoPtr<DB::OwnFormattingChannel> log = new DB::OwnFormattingChannel(pf, new Poco::ConsoleChannel);

libs/libloggers/loggers/OwnPatternFormatter.cpp

@@ -2,40 +2,31 @@
 
 #include <functional>
 #include <optional>
+#include <sys/time.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
 #include <Common/HashTable/Hash.h>
 #include <Interpreters/InternalTextLogsQueue.h>
-#include <sys/time.h>
 #include <Common/CurrentThread.h>
 #include <common/getThreadId.h>
 #include "Loggers.h"
 
 
-static const char * setColor(UInt64 num)
+static std::string setColor(UInt64 num)
 {
-    /// ANSI escape sequences to set foreground font color in terminal.
+    /// Make a random RGB color that has constant brightness.
+    /// https://en.wikipedia.org/wiki/YCbCr
 
-    static constexpr auto num_colors = 12;
-    static const char * colors[num_colors] =
-    {
-        /// Black on black is meaningless
-        "\033[0;31m",
-        "\033[0;32m",
-        "\033[0;33m",
-        /// Low intense blue on black is too dark.
-        "\033[0;35m",
-        "\033[0;36m",
-        "\033[1;31m",
-        "\033[1;32m",
-        "\033[1;33m",
-        "\033[1;34m",
-        "\033[1;35m",
-        "\033[1;36m",
-        "\033[1m",  /// Not as white but just as high intense - for readability on white background.
-    };
+    UInt8 y = 128;
+    UInt8 cb = num % 256;
+    UInt8 cr = num / 256 % 256;
+
+    UInt8 r = std::max(0.0, std::min(255.0, y + 1.402 * (cr - 128)));
+    UInt8 g = std::max(0.0, std::min(255.0, y - 0.344136 * (cb - 128) - 0.714136 * (cr - 128)));
+    UInt8 b = std::max(0.0, std::min(255.0, y + 1.772 * (cb - 128)));
 
-    return colors[num % num_colors];
+    /// ANSI escape sequence to set 24-bit foreground font color in terminal.
+    return "\033[38;2;" + DB::toString(r) + ";" + DB::toString(g) + ";" + DB::toString(b) + "m";
 }
 
 static const char * setColorForLogPriority(int priority)
@@ -104,7 +95,7 @@ void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext,
 
     writeCString(" [ ", wb);
     if (color)
-        writeCString(setColor(intHash64(msg_ext.thread_id)), wb);
+        writeString(setColor(intHash64(msg_ext.thread_id)), wb);
     DB::writeIntText(msg_ext.thread_id, wb);
     if (color)
         writeCString(resetColor(), wb);
@@ -114,7 +105,7 @@ void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext,
     /// just to be convenient for various log parsers.
     writeCString("{", wb);
     if (color)
-        writeCString(setColor(std::hash<std::string>()(msg_ext.query_id)), wb);
+        writeString(setColor(std::hash<std::string>()(msg_ext.query_id)), wb);
     DB::writeString(msg_ext.query_id, wb);
     if (color)
         writeCString(resetColor(), wb);