Merge remote-tracking branch 'upstream/master' into query-poor-mans-profiler

e935cb08 · Nikita Lapkov · 3828684f · 99334d04 · e935cb08 · e935cb08
35 changed file
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -202,6 +202,9 @@ set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fn
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")

+set (CMAKE_EXE_LINKER_FLAGS              "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
+set (CMAKE_SHARED_LINKER_FLAGS           "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
+
 include (cmake/use_libcxx.cmake)
 include (cmake/find_unwind.cmake)

@@ -279,6 +282,7 @@ endif ()
 if (DEFAULT_LIBS)
    # Add default libs to all targets as the last dependency.
    set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
+    set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
 endif ()

 if (NOT MAKE_STATIC_LIBRARIES)
@@ -458,4 +462,13 @@ if (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
    add_default_dependencies(boost_program_options_internal)
    add_default_dependencies(boost_system_internal)
    add_default_dependencies(boost_regex_internal)
+    add_default_dependencies(roaring)
+    add_default_dependencies(btrie)
+    add_default_dependencies(cpuid)
+    add_default_dependencies(mysqlclient)
+    add_default_dependencies(zlib)
+    add_default_dependencies(thrift)
+    add_default_dependencies(brotli)
+    add_default_dependencies(libprotobuf)
+    add_default_dependencies(base64)
 endif ()
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@@ -114,6 +114,11 @@ add_headers_and_sources(dbms src/Storages/MergeTree)
 add_headers_and_sources(dbms src/Client)
 add_headers_and_sources(dbms src/Formats)

+if(USE_RDKAFKA)
+    add_headers_and_sources(dbms src/Storages/Kafka)
+endif()
+
+
 list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
 list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})

@@ -224,6 +229,13 @@ target_link_libraries(clickhouse_common_io
    roaring
 )

+if (USE_RDKAFKA)
+    target_link_libraries(dbms PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
+    if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
+        target_include_directories(dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
+    endif()
+endif()
+

 if(RE2_INCLUDE_DIR)
    target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
@@ -248,6 +260,8 @@ target_link_libraries (dbms
        PRIVATE
    clickhouse_parsers
    clickhouse_common_config
+    clickhouse_common_zookeeper
+    string_utils # FIXME: not sure if it's private
        PUBLIC
    clickhouse_common_io
        PRIVATE
@@ -257,6 +271,7 @@ target_link_libraries (dbms
        PRIVATE
    ${BTRIE_LIBRARIES}
    ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    ${Boost_FILESYSTEM_LIBRARY}
        PUBLIC
    ${Boost_SYSTEM_LIBRARY}
    Threads::Threads
@@ -320,10 +335,6 @@ if (USE_CAPNP)
    endif ()
 endif ()

-if (USE_RDKAFKA)
-    target_link_libraries (dbms PRIVATE clickhouse_storage_kafka)
-endif ()
-
 if (USE_PARQUET)
    target_link_libraries(dbms PRIVATE ${PARQUET_LIBRARY})
    if (NOT USE_INTERNAL_PARQUET_LIBRARY OR USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)

--- a/dbms/programs/benchmark/CMakeLists.txt
+++ b/dbms/programs/benchmark/CMakeLists.txt
 set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
-set(CLICKHOUSE_BENCHMARK_LINK PRIVATE clickhouse_aggregate_functions clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_BENCHMARK_LINK PRIVATE dbms clickhouse_aggregate_functions clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
 set(CLICKHOUSE_BENCHMARK_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 clickhouse_program_add(benchmark)

--- a/dbms/programs/client/CMakeLists.txt
+++ b/dbms/programs/client/CMakeLists.txt
@@ -3,7 +3,7 @@ set(CLICKHOUSE_CLIENT_SOURCES
    ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp
 )

-set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
 set(CLICKHOUSE_CLIENT_INCLUDE SYSTEM PRIVATE ${READLINE_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include)

 include(CheckSymbolExists)

--- a/dbms/programs/compressor/CMakeLists.txt
+++ b/dbms/programs/compressor/CMakeLists.txt
 # Also in utils

 set(CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
-set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE clickhouse_compression clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
 #set(CLICKHOUSE_COMPRESSOR_INCLUDE SYSTEM PRIVATE ...)

 clickhouse_program_add(compressor)
--- a/dbms/programs/copier/CMakeLists.txt
+++ b/dbms/programs/copier/CMakeLists.txt
 set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp)
-set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries PUBLIC daemon)
+set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_common_zookeeper clickhouse_parsers clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries string_utils PUBLIC daemon)
 set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 clickhouse_program_add(copier)
--- a/dbms/programs/extract-from-config/CMakeLists.txt
+++ b/dbms/programs/extract-from-config/CMakeLists.txt
 set(CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
-set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io clickhouse_common_zookeeper ${Boost_PROGRAM_OPTIONS_LIBRARY})
 #set(CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE SYSTEM PRIVATE ...)

 clickhouse_program_add(extract-from-config)
--- a/dbms/programs/local/CMakeLists.txt
+++ b/dbms/programs/local/CMakeLists.txt
 set(CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
-set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_storages_system clickhouse_dictionaries clickhouse_common_config clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
 #set(CLICKHOUSE_LOCAL_INCLUDE SYSTEM PRIVATE ...)

 clickhouse_program_add(local)

--- a/dbms/programs/odbc-bridge/CMakeLists.txt
+++ b/dbms/programs/odbc-bridge/CMakeLists.txt
@@ -11,7 +11,7 @@ set(CLICKHOUSE_ODBC_BRIDGE_SOURCES
    ${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp
 )

-set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE dbms clickhouse_common_io PUBLIC daemon)
+set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE dbms clickhouse_parsers PUBLIC daemon)
 set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)

 if (USE_POCO_SQLODBC)

--- a/dbms/programs/performance-test/CMakeLists.txt
+++ b/dbms/programs/performance-test/CMakeLists.txt
-set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES 
+set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES
  ${CMAKE_CURRENT_SOURCE_DIR}/JSONString.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/StopConditionsSet.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/TestStopConditions.cpp
@@ -12,7 +12,7 @@ set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES
  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestSuite.cpp
 )

-set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_config ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY})
 set(CLICKHOUSE_PERFORMANCE_TEST_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 clickhouse_program_add(performance-test)
--- a/dbms/programs/server/CMakeLists.txt
+++ b/dbms/programs/server/CMakeLists.txt
@@ -12,7 +12,7 @@ set(CLICKHOUSE_SERVER_SOURCES
    ${CMAKE_CURRENT_SOURCE_DIR}/MySQLHandlerFactory.cpp
   )

-set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
+set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_common_config clickhouse_common_zookeeper clickhouse_parsers string_utils PUBLIC daemon PRIVATE clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
 if (USE_POCO_NETSSL)
    set(CLICKHOUSE_SERVER_LINK ${CLICKHOUSE_SERVER_LINK} PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
 endif ()

--- a/dbms/programs/server/Server.cpp
+++ b/dbms/programs/server/Server.cpp
@@ -536,10 +536,18 @@ int Server::main(const std::vector<std::string> & /*args*/)
    if (!TaskStatsInfoGetter::checkPermissions())
    {
        LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled."
-                      " It could happen due to incorrect ClickHouse package installation."
-                      " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'."
-                      " Note that it will not work on 'nosuid' mounted filesystems."
-                      " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers.");
+            " It could happen due to incorrect ClickHouse package installation."
+            " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'."
+            " Note that it will not work on 'nosuid' mounted filesystems."
+            " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers.");
+    }
+
+    if (!hasLinuxCapability(CAP_SYS_NICE))
+    {
+        LOG_INFO(log, "It looks like the process has no CAP_SYS_NICE capability, the setting 'os_thread_nice' will have no effect."
+            " It could happen due to incorrect ClickHouse package installation."
+            " You could resolve the problem manually with 'sudo setcap cap_sys_nice=+ep /usr/bin/clickhouse'."
+            " Note that it will not work on 'nosuid' mounted filesystems.");
    }
 #else
    LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled.");

--- a/dbms/src/Common/ErrorCodes.cpp
+++ b/dbms/src/Common/ErrorCodes.cpp
@@ -433,6 +433,7 @@ namespace ErrorCodes
    extern const int UNKNOWN_QUERY_PARAMETER = 456;
    extern const int BAD_QUERY_PARAMETER = 457;
    extern const int CANNOT_UNLINK = 458;
+    extern const int CANNOT_SET_THREAD_PRIORITY = 459;

    extern const int KEEPER_EXCEPTION = 999;
    extern const int POCO_EXCEPTION = 1000;

--- a/dbms/src/Common/ThreadStatus.h
+++ b/dbms/src/Common/ThreadStatus.h
@@ -88,6 +88,8 @@ public:
    UInt32 thread_number = 0;
    /// Linux's PID (or TGID) (the same id is shown by ps util)
    Int32 os_thread_id = -1;
+    /// Also called "nice" value. If it was changed to non-zero (when attaching query) - will be reset to zero when query is detached.
+    Int32 os_thread_priority = 0;

    /// TODO: merge them into common entity
    ProfileEvents::Counters performance_counters{VariableContext::Thread};

--- a/dbms/src/Core/Settings.h
+++ b/dbms/src/Core/Settings.h
@@ -132,6 +132,7 @@ struct Settings : public SettingsCollection<Settings>
    M(SettingInt64, network_zstd_compression_level, 1, "Allows you to select the level of ZSTD compression.") \
    \
    M(SettingUInt64, priority, 0, "Priority of the query. 1 - the highest, higher value - lower priority; 0 - do not use priorities.") \
+    M(SettingInt64, os_thread_priority, 0, "If non zero - set corresponding 'nice' value for query processing threads. Can be used to adjust query priority for OS scheduler.") \
    \
    M(SettingBool, log_queries, 0, "Log requests and write the log to the system table.") \
    \

--- a/dbms/src/DataTypes/DataTypesDecimal.cpp
+++ b/dbms/src/DataTypes/DataTypesDecimal.cpp
@@ -241,7 +241,7 @@ static DataTypePtr create(const ASTPtr & arguments)
 }

 template <typename T>
-static DataTypePtr createExect(const ASTPtr & arguments)
+static DataTypePtr createExact(const ASTPtr & arguments)
 {
    if (!arguments || arguments->children.size() != 1)
        throw Exception("Decimal data type family must have exactly two arguments: precision and scale",
@@ -260,9 +260,9 @@ static DataTypePtr createExect(const ASTPtr & arguments)

 void registerDataTypeDecimal(DataTypeFactory & factory)
 {
-    factory.registerDataType("Decimal32", createExect<Decimal32>, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("Decimal64", createExect<Decimal64>, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("Decimal128", createExect<Decimal128>, DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("Decimal64", createExact<Decimal64>, DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("Decimal128", createExact<Decimal128>, DataTypeFactory::CaseInsensitive);

    factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive);
    factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive);

--- a/dbms/src/Functions/CMakeLists.txt
+++ b/dbms/src/Functions/CMakeLists.txt
@@ -11,6 +11,7 @@ add_library(clickhouse_functions ${clickhouse_functions_sources})
 target_link_libraries(clickhouse_functions
    PUBLIC
        clickhouse_dictionaries
+        clickhouse_dictionaries_embedded
        dbms
        ${CONSISTENT_HASHING_LIBRARY}
        consistent-hashing-sumbur
@@ -19,6 +20,9 @@ target_link_libraries(clickhouse_functions
        ${METROHASH_LIBRARIES}
        murmurhash
        ${BASE64_LIBRARY}
+
+    PRIVATE
+        ${Boost_FILESYSTEM_LIBRARY}
 )

 if (OPENSSL_CRYPTO_LIBRARY)

--- a/dbms/src/Functions/FunctionsVisitParam.h
+++ b/dbms/src/Functions/FunctionsVisitParam.h
@@ -100,7 +100,7 @@ struct ExtractRaw

        for (auto extract_begin = pos; pos != end; ++pos)
        {
-            if (*pos == current_expect_end)
+            if (current_expect_end && *pos == current_expect_end)
            {
                expects_end.pop_back();
                current_expect_end = expects_end.empty() ? 0 : expects_end.back();
@@ -192,7 +192,7 @@ struct ExtractParamImpl

            /// We check that the entry does not pass through the boundaries of strings.
            if (pos + needle.size() < begin + offsets[i])
-                res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i]);
+                res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i] - 1);  /// don't include terminating zero
            else
                res[i] = 0;


--- a/dbms/src/Functions/GatherUtils/CMakeLists.txt
+++ b/dbms/src/Functions/GatherUtils/CMakeLists.txt
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_functions_gatherutils .)
 add_library(clickhouse_functions_gatherutils ${clickhouse_functions_gatherutils_sources} ${clickhouse_functions_gatherutils_headers})
-target_link_libraries(clickhouse_functions_gatherutils PRIVATE clickhouse_common_io)
+target_link_libraries(clickhouse_functions_gatherutils PRIVATE dbms)

 if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
    # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.

--- a/dbms/src/Functions/URL/CMakeLists.txt
+++ b/dbms/src/Functions/URL/CMakeLists.txt
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_functions_url .)
 add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
-target_link_libraries(clickhouse_functions_url PRIVATE clickhouse_common_io)
-target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include 
+target_link_libraries(clickhouse_functions_url PRIVATE dbms)
+target_include_directories(clickhouse_functions_url PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) # ${CMAKE_CURRENT_BINARY_DIR}/include

 if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
    # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.

--- a/dbms/src/Functions/array/CMakeLists.txt
+++ b/dbms/src/Functions/array/CMakeLists.txt
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(clickhouse_functions_array .)
 add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers})
-target_link_libraries(clickhouse_functions_array PRIVATE clickhouse_common_io clickhouse_functions_gatherutils)
+target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils)

 if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
    # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.

--- a/dbms/src/Interpreters/ThreadStatusExt.cpp
+++ b/dbms/src/Interpreters/ThreadStatusExt.cpp
@@ -7,12 +7,26 @@
 #include <Interpreters/QueryThreadLog.h>
 #include <Interpreters/ProcessList.h>

+#if defined(__linux__)
+#include <sys/time.h>
+#include <sys/resource.h>
+
+#include <Common/hasLinuxCapability.h>
+#endif
+
+
 /// Implement some methods of ThreadStatus and CurrentThread here to avoid extra linking dependencies in clickhouse_common_io
 /// TODO It doesn't make sense.

 namespace DB
 {

+namespace ErrorCodes
+{
+    extern const int CANNOT_SET_THREAD_PRIORITY;
+}
+
+
 void ThreadStatus::attachQueryContext(Context & query_context_)
 {
    query_context = &query_context_;
@@ -95,6 +109,23 @@ void ThreadStatus::attachQuery(const ThreadGroupStatusPtr & thread_group_, bool
        thread_group->thread_numbers.emplace_back(thread_number);
    }

+#if defined(__linux__)
+    /// Set "nice" value if required.
+    if (query_context)
+    {
+        Int32 new_os_thread_priority = query_context->getSettingsRef().os_thread_priority;
+        if (new_os_thread_priority && hasLinuxCapability(CAP_SYS_NICE))
+        {
+            LOG_TRACE(log, "Setting nice to " << new_os_thread_priority);
+
+            if (0 != setpriority(PRIO_PROCESS, os_thread_id, new_os_thread_priority))
+                throwFromErrno("Cannot 'setpriority'", ErrorCodes::CANNOT_SET_THREAD_PRIORITY);
+
+            os_thread_priority = new_os_thread_priority;
+        }
+    }
+#endif
+
    initPerformanceCounters();

    thread_state = ThreadState::AttachedToQuery;
@@ -175,6 +206,18 @@ void ThreadStatus::detachQuery(bool exit_if_already_detached, bool thread_exits)
    thread_group.reset();

    thread_state = thread_exits ? ThreadState::Died : ThreadState::DetachedFromQuery;
+
+#if defined(__linux__)
+    if (os_thread_priority)
+    {
+        LOG_TRACE(log, "Resetting nice");
+
+        if (0 != setpriority(PRIO_PROCESS, os_thread_id, 0))
+            LOG_ERROR(log, "Cannot 'setpriority' back to zero: " << errnoToString(ErrorCodes::CANNOT_SET_THREAD_PRIORITY, errno));
+
+        os_thread_priority = 0;
+    }
+#endif
 }

 void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log)

--- a/dbms/src/Storages/CMakeLists.txt
+++ b/dbms/src/Storages/CMakeLists.txt
 add_subdirectory(System)
-add_subdirectory(Kafka)

 if(ENABLE_TESTS)
    add_subdirectory(tests)

--- a/dbms/src/Storages/Kafka/CMakeLists.txt
+++ b/dbms/src/Storages/Kafka/CMakeLists.txt
-if(USE_RDKAFKA)
-    include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
-    add_headers_and_sources(clickhouse_storage_kafka .)
-    add_library(clickhouse_storage_kafka ${clickhouse_storage_kafka_sources})
-    target_link_libraries(clickhouse_storage_kafka PRIVATE clickhouse_common_io ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
-    if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
-        target_include_directories(clickhouse_storage_kafka SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
-    endif()
-endif()
--- a/dbms/src/Storages/System/CMakeLists.txt
+++ b/dbms/src/Storages/System/CMakeLists.txt
@@ -12,7 +12,7 @@ include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 add_headers_and_sources(storages_system .)
 list (APPEND storages_system_sources ${CONFIG_BUILD})
 add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources})
-target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper)
+target_link_libraries(clickhouse_storages_system PRIVATE dbms common string_utils clickhouse_common_zookeeper clickhouse_parsers)

 add_custom_target(generate-contributors ./StorageSystemContributors.sh SOURCES StorageSystemContributors.sh WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
    # BYPRODUCTS StorageSystemContributors.generated.cpp

--- a/dbms/src/TableFunctions/CMakeLists.txt
+++ b/dbms/src/TableFunctions/CMakeLists.txt
@@ -5,4 +5,4 @@ list(REMOVE_ITEM clickhouse_table_functions_sources ITableFunction.cpp TableFunc
 list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h)

 add_library(clickhouse_table_functions ${clickhouse_table_functions_sources})
-target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY})
+target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms ${Poco_Foundation_LIBRARY})
--- a/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.reference
+++ b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.reference
+
--- a/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.sql
+++ b/dbms/tests/queries/0_stateless/00961_visit_param_buffer_underflow.sql
+SELECT visitParamExtractRaw('\"a\":', 'a');
--- a/dbms/tests/queries/0_stateless/00962_visit_param_various.reference
+++ b/dbms/tests/queries/0_stateless/00962_visit_param_various.reference
+123
+Hello
+Hello
+0
--- a/dbms/tests/queries/0_stateless/00962_visit_param_various.sql
+++ b/dbms/tests/queries/0_stateless/00962_visit_param_various.sql
+SELECT visitParamExtractUInt('"a":123', 'a');
+SELECT visitParamExtractString('"a":"Hello"', 'a');
+SELECT visitParamExtractRaw('"a":Hello}', 'a');
+
+SELECT sum(ignore(visitParamExtractRaw(concat('{"a":', reinterpretAsString(rand64())), 'a'))) FROM numbers(1000000);
--- a/dbms/tests/queries/0_stateless/00963_achimbab.reference
+++ b/dbms/tests/queries/0_stateless/00963_achimbab.reference
+{
+	"meta":
+	[
+		{
+			"name": "total",
+			"type": "Nullable(UInt8)"
+		},
+		{
+			"name": "arrayElement(k, 1)",
+			"type": "Nullable(UInt16)"
+		},
+		{
+			"name": "arrayElement(k, 2)",
+			"type": "Nullable(UInt16)"
+		}
+	],
+
+	"data":
+	[
+		{
+			"total": 1,
+			"arrayElement(k, 1)": null,
+			"arrayElement(k, 2)": 4
+		},
+		{
+			"total": 1,
+			"arrayElement(k, 1)": null,
+			"arrayElement(k, 2)": 2
+		},
+		{
+			"total": 1,
+			"arrayElement(k, 1)": null,
+			"arrayElement(k, 2)": 1
+		},
+		{
+			"total": 1,
+			"arrayElement(k, 1)": null,
+			"arrayElement(k, 2)": 3
+		},
+		{
+			"total": 1,
+			"arrayElement(k, 1)": null,
+			"arrayElement(k, 2)": 5
+		}
+	],
+
+	"totals":
+	{
+		"total": 1,
+		"arrayElement(k, 1)": null,
+		"arrayElement(k, 2)": null
+	},
+
+	"rows": 5,
+
+	"rows_before_limit_at_least": 5
+}
--- a/dbms/tests/queries/0_stateless/00963_achimbab.sql
+++ b/dbms/tests/queries/0_stateless/00963_achimbab.sql
+SET output_format_write_statistics = 0;
+
+select 
+    sum(cnt) > 0 as total,
+    k[1], k[2]
+    from
+    (
+        select
+            arrayMap( x ->  x % 3 ? toNullable(number%5 + x) : null, range(3)) as k,
+            number % 4 ? toNullable( rand() ) : Null  as cnt
+        from system.numbers_mt
+        where number < 1000000
+        limit 1000000
+    ) 
+group by k with totals 
+order by total desc 
+SETTINGS max_threads = 100, max_execution_time = 120 
+format JSON;
--- a/debian/clickhouse-server.postinst
+++ b/debian/clickhouse-server.postinst
@@ -121,9 +121,9 @@ Please fix this and reinstall this package." >&2
    TMPFILE=/tmp/test_setcap.sh

    command -v setcap >/dev/null \
-        && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \
-        && setcap "cap_net_admin,cap_ipc_lock+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \
-        || echo "Cannot set 'net_admin' or 'ipc_lock' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually."
+        && echo > $TMPFILE && chmod a+x $TMPFILE && $TMPFILE && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" $TMPFILE && $TMPFILE && rm $TMPFILE \
+        && setcap "cap_net_admin,cap_ipc_lock,cap_sys_nice+ep" "${CLICKHOUSE_BINDIR}/${CLICKHOUSE_GENERIC_PROGRAM}" \
+        || echo "Cannot set 'net_admin' or 'ipc_lock' or 'sys_nice' capability for clickhouse binary. This is optional. Taskstats accounting will be disabled. To enable taskstats accounting you may add the required capability later manually."

    # Clean old dynamic compilation results
    if [ -d "${CLICKHOUSE_DATADIR_FROM_CONFIG}/build" ]; then

--- a/debian/clickhouse-server.service
+++ b/debian/clickhouse-server.service
@@ -11,7 +11,7 @@ RuntimeDirectory=clickhouse-server
 ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid
 LimitCORE=infinity
 LimitNOFILE=500000
-CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK
+CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE

 [Install]
 WantedBy=multi-user.target
--- a/docker/packager/packager
+++ b/docker/packager/packager
@@ -146,7 +146,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di
        cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0')

    if split_binary:
-        cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')
+        cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -DGLIBC_COMPATIBILITY=ON')

    if version:
        result.append("VERSION_STRING='{}'".format(version))