Merge branch 'master' of github.com:yandex/ClickHouse

cmake/print_include_directories.cmake

@@ -13,6 +13,9 @@ list(APPEND dirs ${dirs1})
 get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
+get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES)
+list(APPEND dirs ${dirs1})
+
 if (USE_INTERNAL_BOOST_LIBRARY)
     get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
     list(APPEND dirs ${dirs1})

contrib/CMakeLists.txt

@@ -46,6 +46,7 @@ if (USE_INTERNAL_METROHASH_LIBRARY)
 endif ()
 
 add_subdirectory (murmurhash)
+add_subdirectory (croaring)
 
 if (USE_INTERNAL_BTRIE_LIBRARY)
     add_subdirectory (libbtrie)

contrib/arrow-cmake/CMakeLists.txt

@@ -198,7 +198,7 @@ list(APPEND PARQUET_SRCS
 add_library(${PARQUET_LIBRARY} ${LINK_MODE} ${PARQUET_SRCS})
 target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src)
 include(${ClickHouse_SOURCE_DIR}/contrib/thrift/build/cmake/ConfigureChecks.cmake) # makes config.h
-target_link_libraries(${PARQUET_LIBRARY} PRIVATE ${ARROW_LIBRARY} ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
+target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} ${Boost_REGEX_LIBRARY})
 target_include_directories(${PARQUET_LIBRARY} PRIVATE ${Boost_INCLUDE_DIRS})
 
 if(SANITIZE STREQUAL "undefined")

contrib/croaring/CMakeLists.txt

+add_library(roaring
+	roaring.c
+	roaring.h
+	roaring.hh)
+
+target_include_directories (roaring PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})

contrib/croaring/LICENSE

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 The CRoaring authors 
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

contrib/croaring/README.txt

+download from https://github.com/RoaringBitmap/CRoaring/archive/v0.2.57.tar.gz
+and use ./amalgamation.sh generate

dbms/CMakeLists.txt

@@ -137,12 +137,10 @@ endif ()
 add_subdirectory(src/Common/ZooKeeper)
 add_subdirectory(src/Common/Config)
 
-if (MAKE_STATIC_LIBRARIES)
+if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
     add_library(dbms ${dbms_headers} ${dbms_sources})
 else ()
     add_library(dbms SHARED ${dbms_headers} ${dbms_sources})
-    set_target_properties (dbms PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse)
-    install (TARGETS dbms LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
 endif ()
 
 if (USE_EMBEDDED_COMPILER)
@@ -201,6 +199,8 @@ target_link_libraries (clickhouse_common_io
     Threads::Threads
         PRIVATE
     ${CMAKE_DL_LIBS}
+        PUBLIC
+    roaring	
 )
 
 target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR})

dbms/programs/CMakeLists.txt

@@ -7,7 +7,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_
 option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL})
-option (ENABLE_CLICKHOUSE_PERFORMANCE "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL})
+option (ENABLE_CLICKHOUSE_PERFORMANCE_TEST "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL})
@@ -15,8 +15,64 @@ option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_
 option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL})
 option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL})
 
+if(NOT (MAKE_STATIC_LIBRARIES OR SPLIT_SHARED_LIBRARIES))
+    set(CLICKHOUSE_ONE_SHARED 1)
+endif()
+
 configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h)
 
+
+
+macro(clickhouse_target_link_split_lib target name)
+    if(NOT CLICKHOUSE_ONE_SHARED)
+        target_link_libraries(${target} PRIVATE clickhouse-${name}-lib)
+    else()
+        target_link_libraries(${target} PRIVATE clickhouse-lib)
+    endif()
+endmacro()
+
+macro(clickhouse_program_link_split_binary name)
+    clickhouse_target_link_split_lib(clickhouse-${name} ${name})
+endmacro()
+
+macro(clickhouse_program_add_library name)
+    string(TOUPPER ${name} name_uc)
+    string(REPLACE "-" "_" name_uc ${name_uc})
+
+    # Some dark magic
+    set(CLICKHOUSE_${name_uc}_SOURCES ${CLICKHOUSE_${name_uc}_SOURCES} PARENT_SCOPE)
+    set(CLICKHOUSE_${name_uc}_LINK ${CLICKHOUSE_${name_uc}_LINK} PARENT_SCOPE)
+    set(CLICKHOUSE_${name_uc}_INCLUDE ${CLICKHOUSE_${name_uc}_INCLUDE} PARENT_SCOPE)
+
+    if(NOT CLICKHOUSE_ONE_SHARED)
+        add_library(clickhouse-${name}-lib ${LINK_MODE} ${CLICKHOUSE_${name_uc}_SOURCES})
+
+        set(_link ${CLICKHOUSE_${name_uc}_LINK}) # can't use ${} in if()
+        if(_link)
+            target_link_libraries(clickhouse-${name}-lib ${CLICKHOUSE_${name_uc}_LINK})
+        endif()
+
+        set(_include ${CLICKHOUSE_${name_uc}_INCLUDE}) # can't use ${} in if()
+        if (_include)
+            target_include_directories(clickhouse-${name}-lib ${CLICKHOUSE_${name_uc}_INCLUDE})
+        endif()
+    endif()
+endmacro()
+
+macro(clickhouse_program_add_executable name)
+    if(CLICKHOUSE_SPLIT_BINARY)
+        add_executable(clickhouse-${name} clickhouse-${name}.cpp)
+        clickhouse_program_link_split_binary(${name})
+        install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+    endif()
+endmacro()
+
+macro(clickhouse_program_add name)
+    clickhouse_program_add_library(${name})
+    clickhouse_program_add_executable(${name})
+endmacro()
+
+
 add_subdirectory (server)
 add_subdirectory (client)
 add_subdirectory (local)
@@ -33,6 +89,13 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
     add_subdirectory (odbc-bridge)
 endif ()
 
+if (CLICKHOUSE_ONE_SHARED)
+    add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_PERFORMANCE_TEST_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_COMPILER_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
+    target_link_libraries(clickhouse-lib PUBLIC ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_PERFORMANCE_TEST_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_COMPILER_LINK}  ${CLICKHOUSE_ODBC_BRIDGE_LINK})
+    set_target_properties(clickhouse-lib PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse)
+    target_include_directories(clickhouse-lib ${CLICKHOUSE_SERVER_INCLUDE} ${CLICKHOUSE_CLIENT_INCLUDE} ${CLICKHOUSE_LOCAL_INCLUDE} ${CLICKHOUSE_BENCHMARK_INCLUDE} ${CLICKHOUSE_PERFORMANCE_TEST_INCLUDE} ${CLICKHOUSE_COPIER_INCLUDE} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE} ${CLICKHOUSE_COMPRESSOR_INCLUDE} ${CLICKHOUSE_FORMAT_INCLUDE} ${CLICKHOUSE_OBFUSCATOR_INCLUDE} ${CLICKHOUSE_COMPILER_INCLUDE} ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE})
+endif()
+
 if (CLICKHOUSE_SPLIT_BINARY)
     set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
             clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier)
@@ -71,7 +134,7 @@ else ()
     if (ENABLE_CLICKHOUSE_BENCHMARK)
         target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib)
     endif ()
-    if (ENABLE_CLICKHOUSE_PERFORMANCE)
+    if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
         target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib)
     endif ()
     if (ENABLE_CLICKHOUSE_COPIER)
@@ -114,7 +177,7 @@ else ()
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
     endif ()
-    if (ENABLE_CLICKHOUSE_PERFORMANCE)
+    if (ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
         add_custom_target (clickhouse-performance-test ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-performance-test DEPENDS clickhouse)
         install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-performance-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
         list(APPEND CLICKHOUSE_BUNDLE clickhouse-performance-test)

dbms/programs/benchmark/CMakeLists.txt

-add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp)
-target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse_aggregate_functions clickhouse-client-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
-target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
+set(CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
+set(CLICKHOUSE_BENCHMARK_LINK PRIVATE clickhouse_aggregate_functions clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_BENCHMARK_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
-    target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
-    install (TARGETS clickhouse-benchmark ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(benchmark)
+
+if(NOT CLICKHOUSE_ONE_SHARED)
+    target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse-client-lib)
+endif()

dbms/programs/client/CMakeLists.txt

-add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
-target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
-if (READLINE_INCLUDE_DIR)
-    target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
-endif ()
+set(CLICKHOUSE_CLIENT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp)
+set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_CLIENT_INCLUDE SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-client clickhouse-client.cpp)
-    target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
-    install (TARGETS clickhouse-client ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(client)
 
 install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)

dbms/programs/compressor/CMakeLists.txt

-add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
-target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+# Also in utils
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    # Also in utils
-    add_executable (clickhouse-compressor clickhouse-compressor.cpp)
-    target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
-    install (TARGETS clickhouse-compressor ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+set(CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
+set(CLICKHOUSE_COMPRESSOR_LINK PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+#set(CLICKHOUSE_COMPRESSOR_INCLUDE SYSTEM PRIVATE ...)
+
+clickhouse_program_add(compressor)

dbms/programs/config_tools.h.in

@@ -6,7 +6,7 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT
 #cmakedefine01 ENABLE_CLICKHOUSE_LOCAL
 #cmakedefine01 ENABLE_CLICKHOUSE_BENCHMARK
-#cmakedefine01 ENABLE_CLICKHOUSE_PERFORMANCE
+#cmakedefine01 ENABLE_CLICKHOUSE_PERFORMANCE_TEST
 #cmakedefine01 ENABLE_CLICKHOUSE_COPIER
 #cmakedefine01 ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG
 #cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR

dbms/programs/copier/CMakeLists.txt

-add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp)
-target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon)
+set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp)
+set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_functions clickhouse_aggregate_functions daemon)
+#set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ...)
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-copier clickhouse-copier.cpp)
-    target_link_libraries (clickhouse-copier clickhouse-copier-lib)
-    install (TARGETS clickhouse-copier ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(copier)

dbms/programs/extract-from-config/CMakeLists.txt

-add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp)
-target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
+set(CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+#set(CLICKHOUSE_EXTRACT_FROM_CONFIG_INCLUDE SYSTEM PRIVATE ...)
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
-    target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
-    install (TARGETS clickhouse-extract-from-config ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(extract-from-config)

dbms/programs/format/CMakeLists.txt

-add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp)
-target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-format clickhouse-format.cpp)
-    target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
-    install (TARGETS clickhouse-format ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+set(CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
+set(CLICKHOUSE_FORMAT_LINK PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY})
+#set(CLICKHOUSE_FORMAT_INCLUDE SYSTEM PRIVATE ...)
+
+clickhouse_program_add(format)

dbms/programs/local/CMakeLists.txt

-add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp)
-target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
+set(CLICKHOUSE_LOCAL_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY})
+#set(CLICKHOUSE_LOCAL_INCLUDE SYSTEM PRIVATE ...)
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-local clickhouse-local.cpp)
-    target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
-    install (TARGETS clickhouse-local ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+clickhouse_program_add(local)
+
+if(NOT CLICKHOUSE_ONE_SHARED)
+    target_link_libraries(clickhouse-local-lib PRIVATE clickhouse-server-lib)
 endif ()

dbms/programs/main.cpp

@@ -38,7 +38,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK)
 int mainEntryClickHouseBenchmark(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE)
+#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
 int mainEntryClickHousePerformanceTest(int argc, char ** argv);
 #endif
 #if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
@@ -84,7 +84,7 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER)
     {"server", mainEntryClickHouseServer},
 #endif
-#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE)
+#if ENABLE_CLICKHOUSE_PERFORMANCE_TEST || !defined(ENABLE_CLICKHOUSE_PERFORMANCE_TEST)
     {"performance-test", mainEntryClickHousePerformanceTest},
 #endif
 #if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)

dbms/programs/obfuscator/CMakeLists.txt

-add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp)
-target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
+set(CLICKHOUSE_OBFUSCATOR_LINK PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
+#set(CLICKHOUSE_OBFUSCATOR_INCLUDE SYSTEM PRIVATE ...)
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
-    set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
-    target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
-    install (TARGETS clickhouse-obfuscator ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(obfuscator)

dbms/programs/odbc-bridge/CMakeLists.txt

-add_headers_and_sources(clickhouse_odbc_bridge .)
-
-add_library (clickhouse-odbc-bridge-lib ${LINK_MODE} ${clickhouse_odbc_bridge_sources})
-
-target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE daemon dbms clickhouse_common_io)
-target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
+set(CLICKHOUSE_ODBC_BRIDGE_SOURCES
+    ${CMAKE_CURRENT_SOURCE_DIR}/ColumnInfoHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/getIdentifierQuote.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/HandlerFactory.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/IdentifierQuoteHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/MainHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ODBCBlockInputStream.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/odbc-bridge.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ODBCBridge.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/PingHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/validateODBCConnectionString.cpp
+)
+
+set(CLICKHOUSE_ODBC_BRIDGE_LINK PRIVATE daemon dbms clickhouse_common_io)
+set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
 if (USE_POCO_SQLODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY})
-    target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
+    set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQLODBC_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR})
 endif ()
 if (Poco_SQL_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_SQL_LIBRARY})
 endif ()
 
 if (USE_POCO_DATAODBC)
-    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY})
-    target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
+    set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_DataODBC_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_INCLUDE ${CLICKHOUSE_ODBC_BRIDGE_INCLUDE} SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR})
 endif()
 if (Poco_Data_FOUND)
-    target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY})
+    set(CLICKHOUSE_ODBC_BRIDGE_LINK ${CLICKHOUSE_ODBC_BRIDGE_LINK} PRIVATE ${Poco_Data_LIBRARY})
 endif ()
 
-
-if (ENABLE_TESTS)
-    add_subdirectory (tests)
-endif ()
+clickhouse_program_add_library(odbc-bridge)
 
 # clickhouse-odbc-bridge is always a separate binary.
 # Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers.
@@ -32,5 +38,11 @@ endif ()
 SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 
 add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
-target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
+
+clickhouse_program_link_split_binary(odbc-bridge)
+
 install (TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
+
+if (ENABLE_TESTS)
+    add_subdirectory (tests)
+endif ()

dbms/programs/odbc-bridge/tests/CMakeLists.txt

 add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
-target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io)
+clickhouse_target_link_split_lib(validate-odbc-connection-string odbc-bridge)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io)

dbms/programs/performance-test/CMakeLists.txt

-add_library (clickhouse-performance-test-lib ${LINK_MODE}
-  JSONString.cpp
-  StopConditionsSet.cpp
-  TestStopConditions.cpp
-  TestStats.cpp
-  ConfigPreprocessor.cpp
-  PerformanceTest.cpp
-  PerformanceTestInfo.cpp
-  executeQuery.cpp
-  applySubstitutions.cpp
-  ReportBuilder.cpp
-  PerformanceTestSuite.cpp
-)
-target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
-target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
+set(CLICKHOUSE_PERFORMANCE_TEST_SOURCES 
+  ${CMAKE_CURRENT_SOURCE_DIR}/JSONString.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/StopConditionsSet.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/TestStopConditions.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/TestStats.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/ConfigPreprocessor.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTest.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestInfo.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/executeQuery.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/applySubstitutions.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/ReportBuilder.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTestSuite.cpp
+ )
 
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
-    target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
-    install (TARGETS clickhouse-performance-test ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+set(CLICKHOUSE_PERFORMANCE_TEST_LINK PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
+set(CLICKHOUSE_PERFORMANCE_TEST_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
+
+clickhouse_program_add(performance-test)

dbms/programs/server/CMakeLists.txt

-add_library (clickhouse-server-lib ${LINK_MODE}
-    HTTPHandler.cpp
-    InterserverIOHTTPHandler.cpp
-    MetricsTransmitter.cpp
-    NotFoundHandler.cpp
-    PingRequestHandler.cpp
-    ReplicasStatusHandler.cpp
-    RootRequestHandler.cpp
-    Server.cpp
-    TCPHandler.cpp
-    )
+set(CLICKHOUSE_SERVER_SOURCES
+    ${CMAKE_CURRENT_SOURCE_DIR}/HTTPHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/InterserverIOHTTPHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/NotFoundHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/PingRequestHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/ReplicasStatusHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/RootRequestHandler.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/TCPHandler.cpp
+   )
 
-target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_dictionaries clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
+set(CLICKHOUSE_SERVER_LINK PRIVATE clickhouse_dictionaries clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY})
 if (USE_POCO_NETSSL)
-    target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
+    set(CLICKHOUSE_SERVER_LINK ${CLICKHOUSE_SERVER_LINK} PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY})
 endif ()
+set(CLICKHOUSE_SERVER_INCLUDE PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
 
-target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
-
-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-server clickhouse-server.cpp)
-    target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib)
-    install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-endif ()
+clickhouse_program_add(server)
 
 if (GLIBC_COMPATIBILITY)
     set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
@@ -31,7 +26,4 @@ if (GLIBC_COMPATIBILITY)
     #add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort --version-sort --reverse | perl -lnE 'warn($_), exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") # old
 endif ()
 
-install (
-    FILES config.xml users.xml
-    DESTINATION  ${CLICKHOUSE_ETC_DIR}/clickhouse-server
-    COMPONENT clickhouse)
+install(FILES config.xml users.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server COMPONENT clickhouse)

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.cpp

+#include <AggregateFunctions/AggregateFunctionFactory.h>
+#include <AggregateFunctions/AggregateFunctionGroupBitmap.h>
+#include <AggregateFunctions/Helpers.h>
+#include <AggregateFunctions/FactoryHelpers.h>
+
+
+namespace DB
+{
+
+namespace
+{
+
+template <template <typename> class Data>
+AggregateFunctionPtr createAggregateFunctionBitmap(const std::string & name, const DataTypes & argument_types, const Array & parameters)
+{
+    assertNoParameters(name, parameters);
+    assertUnary(name, argument_types);
+
+    if (!argument_types[0]->canBeUsedInBitOperations())
+        throw Exception("The type " + argument_types[0]->getName() + " of argument for aggregate function " + name
+            + " is illegal, because it cannot be used in Bitmap operations",
+            ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+    AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionBitmap, Data>(*argument_types[0], argument_types[0]));
+
+    if (!res)
+        throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
+
+    return res;
+}
+
+}
+
+void registerAggregateFunctionsBitmap(AggregateFunctionFactory & factory)
+{
+    factory.registerFunction("groupBitmap", createAggregateFunctionBitmap<AggregateFunctionGroupBitmapData>);
+
+}
+
+}

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmap.h

+#pragma once
+
+#include <Columns/ColumnVector.h>
+#include <boost/noncopyable.hpp>
+#include <AggregateFunctions/IAggregateFunction.h>
+#include <AggregateFunctions/AggregateFunctionGroupBitmapData.h>
+#include <DataTypes/DataTypesNumber.h>
+
+namespace DB
+{
+
+/// Counts bitmap operation on numbers.
+template <typename T, typename Data>
+class AggregateFunctionBitmap final : public IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>
+{
+public:
+    AggregateFunctionBitmap(const DataTypePtr & type)
+            : IAggregateFunctionDataHelper<Data, AggregateFunctionBitmap<T, Data>>({type}, {}) {}
+
+    String getName() const override { return Data::name(); }
+
+    DataTypePtr getReturnType() const override
+    {
+        return std::make_shared<DataTypeNumber<T>>();
+    }
+
+    void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
+    {
+        this->data(place).rbs.add(static_cast<const ColumnVector<T> &>(*columns[0]).getData()[row_num]);
+    }
+
+    void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
+    {
+        this->data(place).rbs.merge(this->data(rhs).rbs);
+    }
+
+    void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
+    {
+        this->data(place).rbs.write(buf);
+    }
+
+    void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
+    {
+        this->data(place).rbs.read(buf);
+    }
+
+    void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
+    {
+        static_cast<ColumnVector<T> &>(to).getData().push_back(this->data(place).rbs.size());
+    }
+
+    const char * getHeaderFilePath() const override { return __FILE__; }
+};
+
+
+}

dbms/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h

+#pragma once
+
+#include <roaring.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <boost/noncopyable.hpp>
+#include <roaring.hh>
+#include <Common/HashTable/SmallTable.h>
+
+namespace DB
+{
+/**
+  * For a small number of values - an array of fixed size "on the stack".
+  * For large, roaring_bitmap_t is allocated.
+  * For a description of the roaring_bitmap_t, see: https://github.com/RoaringBitmap/CRoaring
+  */
+template <typename T, UInt8 small_set_size>
+class RoaringBitmapWithSmallSet : private boost::noncopyable
+{
+private:
+    using Small = SmallSet<T, small_set_size>;
+    using ValueBuffer = std::vector<T>;
+    Small small;
+    roaring_bitmap_t * rb = nullptr;
+
+    void toLarge()
+    {
+        rb = roaring_bitmap_create();
+
+        for (const auto & x : small)
+            roaring_bitmap_add(rb, x.getValue());
+    }
+
+public:
+    bool isLarge() const { return rb != nullptr; }
+
+    bool isSmall() const { return rb == nullptr; }
+
+    ~RoaringBitmapWithSmallSet()
+    {
+        if (isLarge())
+            roaring_bitmap_free(rb);
+    }
+
+    void add(T value)
+    {
+        if (isSmall())
+        {
+            if (small.find(value) == small.end())
+            {
+                if (!small.full())
+                    small.insert(value);
+                else
+                {
+                    toLarge();
+                    roaring_bitmap_add(rb, value);
+                }
+            }
+        }
+        else
+            roaring_bitmap_add(rb, value);
+    }
+
+    UInt64 size() const { return isSmall() ? small.size() : roaring_bitmap_get_cardinality(rb); }
+
+    void merge(const RoaringBitmapWithSmallSet & r1)
+    {
+        if (r1.isLarge())
+        {
+            if (isSmall())
+                toLarge();
+
+            roaring_bitmap_or_inplace(rb, r1.rb);
+        }
+        else
+        {
+            for (const auto & x : r1.small)
+                add(x.getValue());
+        }
+    }
+
+    void read(DB::ReadBuffer & in)
+    {
+        bool is_large;
+        readBinary(is_large, in);
+
+        if (is_large)
+        {
+            toLarge();
+            UInt32 cardinality;
+            readBinary(cardinality, in);
+            db_roaring_bitmap_add_many(in, rb, cardinality);
+        }
+        else
+            small.read(in);
+    }
+
+    void write(DB::WriteBuffer & out) const
+    {
+        writeBinary(isLarge(), out);
+
+        if (isLarge())
+        {
+            UInt32 cardinality = roaring_bitmap_get_cardinality(rb);
+            writePODBinary(cardinality, out);
+            db_ra_to_uint32_array(out, &rb->high_low_container);
+        }
+        else
+            small.write(out);
+    }
+
+
+    roaring_bitmap_t * getRb() const { return rb; }
+
+    Small & getSmall() const { return small; }
+
+    /**
+     * Get a new roaring_bitmap_t from elements of small
+     */
+    roaring_bitmap_t * getNewRbFromSmall() const
+    {
+        roaring_bitmap_t * smallRb = roaring_bitmap_create();
+        for (const auto & x : small)
+            roaring_bitmap_add(smallRb, x.getValue());
+        return smallRb;
+    }
+
+    /**
+     * Computes the intersection between two bitmaps
+     */
+    void rb_and(const RoaringBitmapWithSmallSet & r1)
+    {
+        ValueBuffer buffer;
+        if (isSmall() && r1.isSmall())
+        {
+            // intersect
+            for (const auto & x : small)
+                if (r1.small.find(x.getValue()) != r1.small.end())
+                    buffer.push_back(x.getValue());
+
+            // Clear out the original values
+            small.clear();
+
+            for (const auto & value : buffer)
+                small.insert(value);
+
+            buffer.clear();
+        }
+        else if (isSmall() && r1.isLarge())
+        {
+            for (const auto & x : small)
+                if (roaring_bitmap_contains(r1.rb, x.getValue()))
+                    buffer.push_back(x.getValue());
+
+            // Clear out the original values
+            small.clear();
+
+            for (const auto & value : buffer)
+                small.insert(value);
+
+            buffer.clear();
+        }
+        else
+        {
+            roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+            roaring_bitmap_and_inplace(rb, rb1);
+            if (r1.isSmall())
+                roaring_bitmap_free(rb1);
+        }
+    }
+
+    /**
+     * Computes the union between two bitmaps.
+     */
+    void rb_or(const RoaringBitmapWithSmallSet & r1) { merge(r1); }
+
+    /**
+     * Computes the symmetric difference (xor) between two bitmaps.
+     */
+    void rb_xor(const RoaringBitmapWithSmallSet & r1)
+    {
+        if (isSmall())
+            toLarge();
+        roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+        roaring_bitmap_xor_inplace(rb, rb1);
+        if (r1.isSmall())
+            roaring_bitmap_free(rb1);
+    }
+
+    /**
+     * Computes the difference (andnot) between two bitmaps
+     */
+    void rb_andnot(const RoaringBitmapWithSmallSet & r1)
+    {
+        ValueBuffer buffer;
+        if (isSmall() && r1.isSmall())
+        {
+            // subtract
+            for (const auto & x : small)
+                if (r1.small.find(x.getValue()) == r1.small.end())
+                    buffer.push_back(x.getValue());
+
+            // Clear out the original values
+            small.clear();
+
+            for (const auto & value : buffer)
+                small.insert(value);
+
+            buffer.clear();
+        }
+        else if (isSmall() && r1.isLarge())
+        {
+            for (const auto & x : small)
+                if (!roaring_bitmap_contains(r1.rb, x.getValue()))
+                    buffer.push_back(x.getValue());
+
+            // Clear out the original values
+            small.clear();
+
+            for (const auto & value : buffer)
+                small.insert(value);
+
+            buffer.clear();
+        }
+        else
+        {
+            roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+            roaring_bitmap_andnot_inplace(rb, rb1);
+            if (r1.isSmall())
+                roaring_bitmap_free(rb1);
+        }
+    }
+
+    /**
+     * Computes the cardinality of the intersection between two bitmaps.
+     */
+    UInt64 rb_and_cardinality(const RoaringBitmapWithSmallSet & r1) const
+    {
+        UInt64 retSize = 0;
+        if (isSmall() && r1.isSmall())
+        {
+            for (const auto & x : small)
+                if (r1.small.find(x.getValue()) != r1.small.end())
+                    retSize++;
+        }
+        else if (isSmall() && r1.isLarge())
+        {
+            for (const auto & x : small)
+                if (roaring_bitmap_contains(r1.rb, x.getValue()))
+                    retSize++;
+        }
+        else
+        {
+            roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+            retSize = roaring_bitmap_and_cardinality(rb, rb1);
+            if (r1.isSmall())
+                roaring_bitmap_free(rb1);
+        }
+        return retSize;
+    }
+
+    /**
+     * Computes the cardinality of the union between two bitmaps.
+    */
+    UInt64 rb_or_cardinality(const RoaringBitmapWithSmallSet & r1) const
+    {
+        UInt64 c1 = size();
+        UInt64 c2 = r1.size();
+        UInt64 inter = rb_and_cardinality(r1);
+        return c1 + c2 - inter;
+    }
+
+    /**
+     * Computes the cardinality of the symmetric difference (andnot) between two bitmaps.
+    */
+    UInt64 rb_xor_cardinality(const RoaringBitmapWithSmallSet & r1) const
+    {
+        UInt64 c1 = size();
+        UInt64 c2 = r1.size();
+        UInt64 inter = rb_and_cardinality(r1);
+        return c1 + c2 - 2 * inter;
+    }
+
+    /**
+     * Computes the cardinality of the difference (andnot) between two bitmaps.
+     */
+    UInt64 rb_andnot_cardinality(const RoaringBitmapWithSmallSet & r1) const
+    {
+        UInt64 c1 = size();
+        UInt64 inter = rb_and_cardinality(r1);
+        return c1 - inter;
+    }
+
+    /**
+     * Return 1 if the two bitmaps contain the same elements.
+     */
+    UInt8 rb_equals(const RoaringBitmapWithSmallSet & r1)
+    {
+        if (isSmall())
+            toLarge();
+        roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+        UInt8 is_true = roaring_bitmap_equals(rb, rb1);
+        if (r1.isSmall())
+            roaring_bitmap_free(rb1);
+        return is_true;
+    }
+
+    /**
+     * Check whether two bitmaps intersect.
+     */
+    UInt8 rb_intersect(const RoaringBitmapWithSmallSet & r1)
+    {
+        if (isSmall())
+            toLarge();
+        roaring_bitmap_t * rb1 = r1.isSmall() ? r1.getNewRbFromSmall() : r1.getRb();
+        UInt8 is_true = roaring_bitmap_intersect(rb, rb1);
+        if (r1.isSmall())
+            roaring_bitmap_free(rb1);
+        return is_true;
+    }
+
+    /**
+     * Remove value
+     */
+    void rb_remove(UInt64 offsetid)
+    {
+        if (isSmall())
+            toLarge();
+        roaring_bitmap_remove(rb, offsetid);
+    }
+
+    /**
+     * compute (in place) the negation of the roaring bitmap within a specified
+     * interval: [range_start, range_end). The number of negated values is
+     * range_end - range_start.
+     * Areas outside the range are passed through unchanged.
+     */
+    void rb_flip(UInt64 offsetstart, UInt64 offsetend)
+    {
+        if (isSmall())
+            toLarge();
+        roaring_bitmap_flip_inplace(rb, offsetstart, offsetend);
+    }
+
+    /**
+     * returns the number of integers that are smaller or equal to offsetid.
+     */
+    UInt64 rb_rank(UInt64 offsetid)
+    {
+        if (isSmall())
+            toLarge();
+        return roaring_bitmap_rank(rb, offsetid);
+    }
+
+    /**
+     * Convert elements to integer array, return number of elements
+     */
+    template <typename Element>
+    UInt64 rb_to_array(PaddedPODArray<Element> & res_data) const
+    {
+        UInt64 count = 0;
+        if (isSmall())
+        {
+            for (const auto & x : small)
+            {
+                res_data.emplace_back(x.getValue());
+                count++;
+            }
+        }
+        else
+        {
+            roaring_uint32_iterator_t iterator;
+            roaring_init_iterator(rb, &iterator);
+            while (iterator.has_value)
+            {
+                res_data.emplace_back(iterator.current_value);
+                roaring_advance_uint32_iterator(&iterator);
+                count++;
+            }
+        }
+        return count;
+    }
+
+private:
+    /// To read and write the DB Buffer directly, migrate code from CRoaring
+    void db_roaring_bitmap_add_many(DB::ReadBuffer & dbBuf, roaring_bitmap_t * r, size_t n_args)
+    {
+        void * container = NULL; // hold value of last container touched
+        uint8_t typecode = 0; // typecode of last container touched
+        uint32_t prev = 0; // previous valued inserted
+        size_t i = 0; // index of value
+        int containerindex = 0;
+        if (n_args == 0)
+            return;
+        uint32_t val;
+        readBinary(val, dbBuf);
+        container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
+        prev = val;
+        i++;
+        for (; i < n_args; i++)
+        {
+            readBinary(val, dbBuf);
+            if (((prev ^ val) >> 16) == 0)
+            { // no need to seek the container, it is at hand
+                // because we already have the container at hand, we can do the
+                // insertion
+                // automatically, bypassing the roaring_bitmap_add call
+                uint8_t newtypecode = typecode;
+                void * container2 = container_add(container, val & 0xFFFF, typecode, &newtypecode);
+                // rare instance when we need to
+                if (container2 != container)
+                {
+                    // change the container type
+                    container_free(container, typecode);
+                    ra_set_container_at_index(&r->high_low_container, containerindex, container2, newtypecode);
+                    typecode = newtypecode;
+                    container = container2;
+                }
+            }
+            else
+            {
+                container = containerptr_roaring_bitmap_add(r, val, &typecode, &containerindex);
+            }
+            prev = val;
+        }
+    }
+
+    void db_ra_to_uint32_array(DB::WriteBuffer & dbBuf, roaring_array_t * ra) const
+    {
+        size_t ctr = 0;
+        for (Int32 i = 0; i < ra->size; ++i)
+        {
+            Int32 num_added = db_container_to_uint32_array(dbBuf, ra->containers[i], ra->typecodes[i], ((UInt32)ra->keys[i]) << 16);
+            ctr += num_added;
+        }
+    }
+
+    UInt32 db_container_to_uint32_array(DB::WriteBuffer & dbBuf, const void * container, UInt8 typecode, UInt32 base) const
+    {
+        container = container_unwrap_shared(container, &typecode);
+        switch (typecode)
+        {
+            case BITSET_CONTAINER_TYPE_CODE:
+                return db_bitset_container_to_uint32_array(dbBuf, (const bitset_container_t *)container, base);
+            case ARRAY_CONTAINER_TYPE_CODE:
+                return db_array_container_to_uint32_array(dbBuf, (const array_container_t *)container, base);
+            case RUN_CONTAINER_TYPE_CODE:
+                return db_run_container_to_uint32_array(dbBuf, (const run_container_t *)container, base);
+        }
+        return 0;
+    }
+
+    UInt32 db_bitset_container_to_uint32_array(DB::WriteBuffer & dbBuf, const bitset_container_t * cont, UInt32 base) const
+    {
+        return (UInt32)db_bitset_extract_setbits(dbBuf, cont->array, BITSET_CONTAINER_SIZE_IN_WORDS, base);
+    }
+
+    size_t db_bitset_extract_setbits(DB::WriteBuffer & dbBuf, UInt64 * bitset, size_t length, UInt32 base) const
+    {
+        UInt32 outpos = 0;
+        for (size_t i = 0; i < length; ++i)
+        {
+            UInt64 w = bitset[i];
+            while (w != 0)
+            {
+                UInt64 t = w & (~w + 1); // on x64, should compile to BLSI (careful: the Intel compiler seems to fail)
+                UInt32 r = __builtin_ctzll(w); // on x64, should compile to TZCNT
+                UInt32 val = r + base;
+                writePODBinary(val, dbBuf);
+                outpos++;
+                w ^= t;
+            }
+            base += 64;
+        }
+        return outpos;
+    }
+
+    int db_array_container_to_uint32_array(DB::WriteBuffer & dbBuf, const array_container_t * cont, UInt32 base) const
+    {
+        UInt32 outpos = 0;
+        for (Int32 i = 0; i < cont->cardinality; ++i)
+        {
+            const UInt32 val = base + cont->array[i];
+            writePODBinary(val, dbBuf);
+            outpos++;
+        }
+        return outpos;
+    }
+
+    int db_run_container_to_uint32_array(DB::WriteBuffer & dbBuf, const run_container_t * cont, UInt32 base) const
+    {
+        UInt32 outpos = 0;
+        for (Int32 i = 0; i < cont->n_runs; ++i)
+        {
+            UInt32 run_start = base + cont->runs[i].value;
+            UInt16 le = cont->runs[i].length;
+            for (Int32 j = 0; j <= le; ++j)
+            {
+                UInt32 val = run_start + j;
+                writePODBinary(val, dbBuf);
+                outpos++;
+            }
+        }
+        return outpos;
+    }
+};
+
+template <typename T>
+struct AggregateFunctionGroupBitmapData
+{
+    RoaringBitmapWithSmallSet<T, 32> rbs;
+    static const char * name() { return "groupBitmap"; }
+};
+
+
+}

dbms/src/AggregateFunctions/registerAggregateFunctions.cpp

@@ -26,6 +26,7 @@ void registerAggregateFunctionUniqCombined(AggregateFunctionFactory &);
 void registerAggregateFunctionUniqUpTo(AggregateFunctionFactory &);
 void registerAggregateFunctionTopK(AggregateFunctionFactory &);
 void registerAggregateFunctionsBitwise(AggregateFunctionFactory &);
+void registerAggregateFunctionsBitmap(AggregateFunctionFactory &);
 void registerAggregateFunctionsMaxIntersections(AggregateFunctionFactory &);
 void registerAggregateFunctionEntropy(AggregateFunctionFactory &);
 
@@ -63,6 +64,7 @@ void registerAggregateFunctions()
         registerAggregateFunctionUniqUpTo(factory);
         registerAggregateFunctionTopK(factory);
         registerAggregateFunctionsBitwise(factory);
+        registerAggregateFunctionsBitmap(factory);
         registerAggregateFunctionsMaxIntersections(factory);
         registerAggregateFunctionHistogram(factory);
         registerAggregateFunctionRetention(factory);

dbms/src/Columns/ColumnFixedString.cpp

@@ -4,6 +4,7 @@
 #include <Common/Arena.h>
 #include <Common/SipHash.h>
 #include <Common/memcpySmall.h>
+#include <Common/memcmpSmall.h>
 
 #include <DataStreams/ColumnGathererStream.h>
 
@@ -106,8 +107,7 @@ struct ColumnFixedString::less
     explicit less(const ColumnFixedString & parent_) : parent(parent_) {}
     bool operator()(size_t lhs, size_t rhs) const
     {
-        /// TODO: memcmp slows down.
-        int res = memcmp(&parent.chars[lhs * parent.n], &parent.chars[rhs * parent.n], parent.n);
+        int res = memcmpSmallAllowOverflow15(parent.chars.data() + lhs * parent.n, parent.chars.data() + rhs * parent.n, parent.n);
         return positive ? (res < 0) : (res > 0);
     }
 };

dbms/src/Columns/ColumnFixedString.h

 #pragma once
 
-#include <string.h> // memcmp
-
 #include <Common/PODArray.h>
+#include <Common/memcmpSmall.h>
 #include <Columns/IColumn.h>
 #include <Columns/ColumnVectorHelper.h>
 
@@ -98,7 +97,7 @@ public:
     int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
     {
         const ColumnFixedString & rhs = static_cast<const ColumnFixedString &>(rhs_);
-        return memcmp(&chars[p1 * n], &rhs.chars[p2 * n], n);
+        return memcmpSmallAllowOverflow15(chars.data() + p1 * n, rhs.chars.data() + p2 * n, n);
     }
 
     void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;

dbms/src/Columns/ColumnString.cpp

 #include <Core/Defines.h>
 #include <Common/Arena.h>
+#include <Common/memcmpSmall.h>
 #include <Columns/Collator.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsCommon.h>
@@ -239,15 +240,11 @@ struct ColumnString::less
     explicit less(const ColumnString & parent_) : parent(parent_) {}
     bool operator()(size_t lhs, size_t rhs) const
     {
-        size_t left_len = parent.sizeAt(lhs);
-        size_t right_len = parent.sizeAt(rhs);
+        int res = memcmpSmallAllowOverflow15(
+            parent.chars.data() + parent.offsetAt(lhs), parent.sizeAt(lhs) - 1,
+            parent.chars.data() + parent.offsetAt(rhs), parent.sizeAt(rhs) - 1);
 
-        int res = memcmp(&parent.chars[parent.offsetAt(lhs)], &parent.chars[parent.offsetAt(rhs)], std::min(left_len, right_len));
-
-        if (res != 0)
-            return positive ? (res < 0) : (res > 0);
-        else
-            return positive ? (left_len < right_len) : (left_len > right_len);
+        return positive ? (res < 0) : (res > 0);
     }
 };
 

dbms/src/Columns/ColumnString.h

@@ -6,6 +6,7 @@
 #include <Common/PODArray.h>
 #include <Common/SipHash.h>
 #include <Common/memcpySmall.h>
+#include <Common/memcmpSmall.h>
 
 
 class Collator;
@@ -210,16 +211,7 @@ public:
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
     {
         const ColumnString & rhs = static_cast<const ColumnString &>(rhs_);
-
-        const size_t size = sizeAt(n);
-        const size_t rhs_size = rhs.sizeAt(m);
-
-        int cmp = memcmp(&chars[offsetAt(n)], &rhs.chars[rhs.offsetAt(m)], std::min(size, rhs_size));
-
-        if (cmp != 0)
-            return cmp;
-        else
-            return size > rhs_size ? 1 : (size < rhs_size ? -1 : 0);
+        return memcmpSmallAllowOverflow15(chars.data() + offsetAt(n), sizeAt(n) - 1, rhs.chars.data() + rhs.offsetAt(m), rhs.sizeAt(m) - 1);
     }
 
     /// Variant of compareAt for string comparison with respect of collation.

dbms/src/Common/memcmpSmall.h

+#pragma once
+
+#include <cstdint>
+#include <algorithm>
+
+#ifdef __SSE2__
+#include <emmintrin.h>
+
+
+namespace detail
+{
+
+template <typename T>
+inline int cmp(T a, T b)
+{
+    if (a < b)
+        return -1;
+    if (a > b)
+        return 1;
+    return 0;
+}
+
+}
+
+/** All functions works under the following assumptions:
+  * - it's possible to read up to 15 excessive bytes after end of 'a' and 'b' region;
+  * - memory regions are relatively small and extra loop unrolling is not worth to do.
+  */
+
+/** Variant when memory regions may have different sizes.
+  */
+template <typename Char>
+inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
+{
+    size_t min_size = std::min(a_size, b_size);
+
+    for (size_t offset = 0; offset < min_size; offset += 16)
+    {
+        uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
+        mask = ~mask;
+
+        if (mask)
+        {
+            offset += __builtin_ctz(mask);
+
+            if (offset >= min_size)
+                break;
+
+            return detail::cmp(a[offset], b[offset]);
+        }
+    }
+
+    return detail::cmp(a_size, b_size);
+}
+
+
+/** Variant when memory regions have same size.
+  * TODO Check if the compiler can optimize previous function when the caller pass identical sizes.
+  */
+template <typename Char>
+inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size)
+{
+    for (size_t offset = 0; offset < size; offset += 16)
+    {
+        uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
+        mask = ~mask;
+
+        if (mask)
+        {
+            offset += __builtin_ctz(mask);
+
+            if (offset >= size)
+                return 0;
+
+            return detail::cmp(a[offset], b[offset]);
+        }
+    }
+
+    return 0;
+}
+
+
+/** Compare memory regions for equality.
+  */
+template <typename Char>
+inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
+{
+    if (a_size != b_size)
+        return false;
+
+    for (size_t offset = 0; offset < a_size; offset += 16)
+    {
+        uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
+        mask = ~mask;
+
+        if (mask)
+        {
+            offset += __builtin_ctz(mask);
+            return offset >= a_size;
+        }
+    }
+
+    return true;
+}
+
+
+/** Variant when the caller know in advance that the size is a multiple of 16.
+  */
+template <typename Char>
+inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
+{
+    for (size_t offset = 0; offset < size; offset += 16)
+    {
+        uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(a + offset)),
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(b + offset))));
+        mask = ~mask;
+
+        if (mask)
+        {
+            offset += __builtin_ctz(mask);
+            return detail::cmp(a[offset], b[offset]);
+        }
+    }
+
+    return 0;
+}
+
+
+/** Variant when the size is 16 exactly.
+  */
+template <typename Char>
+inline int memcmp16(const Char * a, const Char * b)
+{
+    uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(a)),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(b))));
+    mask = ~mask;
+
+    if (mask)
+    {
+        auto offset = __builtin_ctz(mask);
+        return detail::cmp(a[offset], b[offset]);
+    }
+
+    return 0;
+}
+
+
+/** Variant when the size is 16 exactly.
+  */
+inline bool memequal16(const void * a, const void * b)
+{
+    return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8(
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(a)),
+        _mm_loadu_si128(reinterpret_cast<const __m128i *>(b))));
+}
+
+
+/** Compare memory region to zero */
+inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
+{
+    const __m128i zero16 = _mm_setzero_si128();
+
+    for (size_t offset = 0; offset < size; offset += 16)
+    {
+        uint16_t mask = _mm_movemask_epi8(_mm_cmpeq_epi8(zero16,
+            _mm_loadu_si128(reinterpret_cast<const __m128i *>(reinterpret_cast<const char *>(data) + offset))));
+        mask = ~mask;
+
+        if (mask)
+        {
+            offset += __builtin_ctz(mask);
+            return offset >= size;
+        }
+    }
+
+    return true;
+}
+
+
+#else
+
+#include <cstring>
+
+template <typename Char>
+inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
+{
+    return memcmp(a, b, std::min(a_size, b_size));
+}
+
+template <typename Char>
+inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size)
+{
+    return memcmp(a, b, size);
+}
+
+template <typename Char>
+inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size)
+{
+    return a_size == b_size && 0 == memcmp(a, b, a_size);
+}
+
+template <typename Char>
+inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
+{
+    return memcmp(a, b, size);
+}
+
+template <typename Char>
+inline int memcmp16(const Char * a, const Char * b)
+{
+    return memcmp(a, b, 16);
+}
+
+inline bool memequal16(const void * a, const void * b)
+{
+    return 0 == memcmp(a, b, 16);
+}
+
+inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
+{
+    const char * pos = reinterpret_cast<const char *>(data);
+    const char * end = pos + size;
+
+    for (; pos < end; ++pos)
+        if (*pos)
+            return false;
+
+    return true;
+}
+
+#endif

dbms/src/Common/memcpySmall.h

 #pragma once
 
 #include <string.h>
-#include <Core/Defines.h>
 
 #ifdef __SSE2__
 #include <emmintrin.h>

dbms/src/Compression/CMakeLists.txt

@@ -3,6 +3,7 @@ add_headers_and_sources(clickhouse_compression .)
 add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources})
 target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY})
 target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
+target_include_directories(clickhouse_compression SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
 
 if (NOT USE_INTERNAL_LZ4_LIBRARY)
     target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})

dbms/src/Functions/EmptyImpl.h

 #pragma once
-#include <cstring>
+
+#include <Common/memcmpSmall.h>
 #include <Columns/ColumnString.h>
 #include <Functions/FunctionFactory.h>
 
@@ -38,11 +39,9 @@ struct EmptyImpl
 
     static void vector_fixed_to_vector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res)
     {
-        std::vector<char> empty_chars(n);
         size_t size = data.size() / n;
-
         for (size_t i = 0; i < size; ++i)
-            res[i] = negative ^ (0 == memcmp(&data[i * n], empty_chars.data(), n));
+            res[i] = negative ^ memoryIsZeroSmallAllowOverflow15(data.data() + i * n, n);
     }
 
     static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)

dbms/src/Functions/FunctionsBitmap.cpp

+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionsBitmap.h>
+
+
+namespace DB
+{
+
+void registerFunctionsBitmap(FunctionFactory & factory)
+{
+    factory.registerFunction<FunctionBitmapBuild>();
+    factory.registerFunction<FunctionBitmapToArray>();
+
+    factory.registerFunction<FunctionBitmapSelfCardinality>();
+    factory.registerFunction<FunctionBitmapAndCardinality>();
+    factory.registerFunction<FunctionBitmapOrCardinality>();
+    factory.registerFunction<FunctionBitmapXorCardinality>();
+    factory.registerFunction<FunctionBitmapAndnotCardinality>();
+
+    factory.registerFunction<FunctionBitmapAnd>();
+    factory.registerFunction<FunctionBitmapOr>();
+    factory.registerFunction<FunctionBitmapXor>();
+    factory.registerFunction<FunctionBitmapAndnot>();
+
+}
+}

dbms/src/Functions/arrayIndex.h

@@ -6,9 +6,11 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnString.h>
+#include <Columns/ColumnFixedString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnNullable.h>
 #include <Common/FieldVisitors.h>
+#include <Common/memcmpSmall.h>
 
 
 namespace DB
@@ -272,8 +274,7 @@ struct ArrayIndexNumNullImpl
     }
 };
 
-/// Implementation for arrays of strings when the 2nd function argument
-/// is a NULL value.
+/// Implementation for arrays of strings when the 2nd function argument is a NULL value.
 template <typename IndexConv>
 struct ArrayIndexStringNullImpl
 {
@@ -311,12 +312,11 @@ struct ArrayIndexStringImpl
 {
     static void vector_const(
         const ColumnString::Chars & data, const ColumnArray::Offsets & offsets, const ColumnString::Offsets & string_offsets,
-        const String & value,
+        const ColumnString::Chars & value, ColumnString::Offset value_size,
         PaddedPODArray<typename IndexConv::ResultType> & result,
         const PaddedPODArray<UInt8> * null_map_data)
     {
         const auto size = offsets.size();
-        const auto value_size = value.size();
         result.resize(size);
 
         ColumnArray::Offset current_offset = 0;
@@ -331,12 +331,12 @@ struct ArrayIndexStringImpl
                     ? 0
                     : string_offsets[current_offset + j - 1];
 
-                ColumnArray::Offset string_size = string_offsets[current_offset + j] - string_pos;
+                ColumnArray::Offset string_size = string_offsets[current_offset + j] - string_pos - 1;
 
                 if (null_map_data && (*null_map_data)[current_offset + j])
                 {
                 }
-                else if (string_size == value_size + 1 && 0 == memcmp(value.data(), &data[string_pos], value_size))
+                else if (memequalSmallAllowOverflow15(value.data(), value_size, &data[string_pos], string_size))
                 {
                     if (!IndexConv::apply(j, current))
                         break;
@@ -381,7 +381,7 @@ struct ArrayIndexStringImpl
                     if (null_map_item && (*null_map_item)[i])
                         hit = true;
                 }
-                else if (string_size == value_size && 0 == memcmp(&item_values[value_pos], &data[string_pos], value_size))
+                else if (memequalSmallAllowOverflow15(&item_values[value_pos], value_size, &data[string_pos], string_size))
                     hit = true;
 
                 if (hit)
@@ -708,16 +708,32 @@ private:
         const auto item_arg = block.getByPosition(arguments[1]).column.get();
 
         if (item_arg->onlyNull())
+        {
             ArrayIndexStringNullImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(),
                 col_nested->getOffsets(), col_res->getData(), null_map_data);
+        }
         else if (const auto item_arg_const = checkAndGetColumnConstStringOrFixedString(item_arg))
-            ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(),
-                col_nested->getOffsets(), item_arg_const->getValue<String>(), col_res->getData(),
-                null_map_data);
+        {
+            const ColumnString * item_const_string = checkAndGetColumn<ColumnString>(&item_arg_const->getDataColumn());
+            const ColumnFixedString * item_const_fixedstring = checkAndGetColumn<ColumnFixedString>(&item_arg_const->getDataColumn());
+
+            if (item_const_string)
+                ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(), col_nested->getOffsets(),
+                    item_const_string->getChars(), item_const_string->getDataAt(0).size,
+                    col_res->getData(), null_map_data);
+            else if (item_const_fixedstring)
+                ArrayIndexStringImpl<IndexConv>::vector_const(col_nested->getChars(), col_array->getOffsets(), col_nested->getOffsets(),
+                    item_const_fixedstring->getChars(), item_const_fixedstring->getN(),
+                    col_res->getData(), null_map_data);
+            else
+                throw Exception("Logical error: ColumnConst contains not String nor FixedString column", ErrorCodes::ILLEGAL_COLUMN);
+        }
         else if (const auto item_arg_vector = checkAndGetColumn<ColumnString>(item_arg))
+        {
             ArrayIndexStringImpl<IndexConv>::vector_vector(col_nested->getChars(), col_array->getOffsets(),
                 col_nested->getOffsets(), item_arg_vector->getChars(), item_arg_vector->getOffsets(),
                 col_res->getData(), null_map_data, null_map_item);
+        }
         else
             return false;
 

dbms/src/Functions/registerFunctions.cpp

@@ -13,6 +13,7 @@ namespace DB
 void registerFunctionsArithmetic(FunctionFactory &);
 void registerFunctionsArray(FunctionFactory &);
 void registerFunctionsTuple(FunctionFactory &);
+void registerFunctionsBitmap(FunctionFactory &);
 void registerFunctionsCoding(FunctionFactory &);
 void registerFunctionsComparison(FunctionFactory &);
 void registerFunctionsConditional(FunctionFactory &);
@@ -53,6 +54,7 @@ void registerFunctions()
     registerFunctionsArithmetic(factory);
     registerFunctionsArray(factory);
     registerFunctionsTuple(factory);
+    registerFunctionsBitmap(factory);
     registerFunctionsCoding(factory);
     registerFunctionsComparison(factory);
     registerFunctionsConditional(factory);

dbms/src/Interpreters/Settings.h

@@ -75,7 +75,7 @@ struct Settings
     M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
     \
     M(SettingBool, compile, false, "Whether query compilation is enabled.") \
-    M(SettingBool, compile_expressions, true, "Compile some scalar functions and operators to native code.") \
+    M(SettingBool, compile_expressions, false, "Compile some scalar functions and operators to native code.") \
     M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
     M(SettingUInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled") \
     M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \

dbms/tests/performance/string_sort/string_sort.xml

+<test>
+    <name>String sorting</name>
+
+    <preconditions>
+        <table_exists>hits_10m_single</table_exists>
+    </preconditions>
+
+    <type>loop</type>
+
+    <stop_conditions>
+        <all_of>
+            <iterations>5</iterations>
+            <min_time_not_changing_for_ms>10000</min_time_not_changing_for_ms>
+        </all_of>
+        <any_of>
+            <iterations>50</iterations>
+            <total_time_ms>60000</total_time_ms>
+        </any_of>
+    </stop_conditions>
+
+    <substitutions>
+       <substitution>
+           <name>str1</name>
+           <values>
+               <value>URL</value>
+               <value>Referer</value>
+               <value>Title</value>
+               <value>SearchPhrase</value>
+               <value>MobilePhoneModel</value>
+               <value>PageCharset</value>
+           </values>
+       </substitution>
+       <substitution>
+           <name>str2</name>
+           <values>
+               <value>URL</value>
+               <value>Referer</value>
+               <value>Title</value>
+               <value>SearchPhrase</value>
+               <value>MobilePhoneModel</value>
+               <value>PageCharset</value>
+           </values>
+       </substitution>
+    </substitutions>
+
+    <query><![CDATA[SELECT {str1} FROM hits_10m_single ORDER BY {str1} LIMIT 10]]></query>
+    <query><![CDATA[SELECT {str1} FROM hits_10m_single ORDER BY {str1} LIMIT 9000000, 10]]></query>
+
+    <query><![CDATA[SELECT {str1}, {str2} FROM hits_10m_single ORDER BY {str1}, {str2} LIMIT 10]]></query>
+    <query><![CDATA[SELECT {str1}, {str2} FROM hits_10m_single ORDER BY {str1}, {str2} LIMIT 9000000, 10]]></query>
+
+    <main_metric>
+        <min_time/>
+    </main_metric>
+</test>

dbms/tests/queries/0_stateless/00829_bitmap_function.reference

+[1,2,3,4,5]
+[3]
+[1,2,3,4,5]
+[1,2,4,5]
+[1,2]
+5
+1
+5
+4
+2
+70
+2019-01-01	50	[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50]
+2019-01-02	60	[11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70]
+60	50	70	40	20	30
+60	50	70	40	20	30
+2019-01-01	50
+2019-01-02	60

dbms/tests/queries/0_stateless/00829_bitmap_function.sql

+SELECT bitmapToArray(bitmapBuild([1, 2, 3, 4, 5]));
+SELECT bitmapToArray(bitmapAnd(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
+SELECT bitmapToArray(bitmapOr(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
+SELECT bitmapToArray(bitmapXor(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
+SELECT bitmapToArray(bitmapAndnot(bitmapBuild([1,2,3]),bitmapBuild([3,4,5])));
+SELECT bitmapCardinality(bitmapBuild([1, 2, 3, 4, 5]));
+SELECT bitmapAndCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
+SELECT bitmapOrCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
+SELECT bitmapXorCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
+SELECT bitmapAndnotCardinality(bitmapBuild([1,2,3]),bitmapBuild([3,4,5]));
+
+DROP TABLE IF EXISTS test.bitmap_test;
+CREATE TABLE test.bitmap_test(pickup_date Date, city_id UInt32, uid UInt32)ENGINE = Memory;
+INSERT INTO test.bitmap_test SELECT '2019-01-01', 1, number FROM numbers(1,50);
+INSERT INTO test.bitmap_test SELECT '2019-01-02', 1, number FROM numbers(11,60);
+
+
+SELECT groupBitmap( uid ) AS user_num FROM test.bitmap_test;
+
+SELECT pickup_date, groupBitmap( uid ) AS user_num, bitmapToArray(groupBitmapState( uid )) AS users FROM test.bitmap_test GROUP BY pickup_date;
+
+SELECT
+    bitmapCardinality(day_today) AS today_users,
+    bitmapCardinality(day_before) AS before_users,
+    bitmapOrCardinality(day_today, day_before) AS all_users,
+    bitmapAndCardinality(day_today, day_before) AS old_users,
+    bitmapAndnotCardinality(day_today, day_before) AS new_users,
+    bitmapXorCardinality(day_today, day_before) AS diff_users
+FROM
+(
+ SELECT city_id, groupBitmapState( uid ) AS day_today FROM test.bitmap_test WHERE pickup_date = '2019-01-02' GROUP BY city_id
+ )
+ALL LEFT JOIN
+(
+ SELECT city_id, groupBitmapState( uid ) AS day_before FROM test.bitmap_test WHERE pickup_date = '2019-01-01' GROUP BY city_id
+)
+USING city_id;
+
+SELECT
+    bitmapCardinality(day_today) AS today_users,
+    bitmapCardinality(day_before) AS before_users,
+    bitmapCardinality(bitmapOr(day_today, day_before))ll_users,
+    bitmapCardinality(bitmapAnd(day_today, day_before)) AS old_users,
+    bitmapCardinality(bitmapAndnot(day_today, day_before)) AS new_users,
+    bitmapCardinality(bitmapXor(day_today, day_before)) AS diff_users
+FROM
+(
+ SELECT city_id, groupBitmapState( uid ) AS day_today FROM test.bitmap_test WHERE pickup_date = '2019-01-02' GROUP BY city_id
+ )
+ALL LEFT JOIN
+(
+ SELECT city_id, groupBitmapState( uid ) AS day_before FROM test.bitmap_test WHERE pickup_date = '2019-01-01' GROUP BY city_id
+)
+USING city_id;
+
+
+DROP TABLE IF EXISTS test.bitmap_state_test;
+CREATE TABLE test.bitmap_state_test
+(
+	pickup_date Date,
+	city_id UInt32,
+    uv AggregateFunction( groupBitmap, UInt32 )	
+)
+ENGINE = AggregatingMergeTree( pickup_date, ( pickup_date, city_id ), 8192);
+
+INSERT INTO test.bitmap_state_test SELECT 
+    pickup_date, 
+    city_id,
+    groupBitmapState(uid) AS uv
+FROM test.bitmap_test
+GROUP BY pickup_date, city_id;
+	
+SELECT pickup_date, groupBitmapMerge(uv) AS users from test.bitmap_state_test group by pickup_date;
+
+DROP TABLE IF EXISTS test.bitmap_test;
+DROP TABLE IF EXISTS test.bitmap_state_test;
+

dbms/tests/queries/0_stateless/00912_string_comparison.sql

+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' AS a, prefix || 'y' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'y' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' || prefix AS a, prefix || 'x' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'x' || prefix AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'y' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'y' || prefix AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+WITH substring('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, number) AS prefix, prefix || 'x' || prefix AS a, prefix || 'x' AS b SELECT a = b, a < b, a > b, a <= b, a >= b FROM numbers(40);
+
+WITH arrayJoin(['aaa', 'bbb']) AS a, 'aaa\0bbb' AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
+WITH arrayJoin(['aaa', 'zzz']) AS a, 'aaa\0bbb' AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
+WITH arrayJoin(['aaa', 'bbb']) AS a, materialize('aaa\0bbb') AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
+WITH arrayJoin(['aaa', 'zzz']) AS a, materialize('aaa\0bbb') AS b SELECT a = b, a < b, a > b, a <= b, a >= b;
+
+SELECT empty(toFixedString('', 1 + randConstant() % 100));

docs/en/query_language/agg_functions/reference.md

@@ -179,6 +179,48 @@ binary     decimal
 01101000 = 104
 ```
 
+
+##groupBitmap
+
+Bitmap or Aggregate calculations from a unsigned integer column, return cardinality of type UInt64, if add suffix -State, then return [bitmap object](../functions/bitmap_functions.md).
+
+```
+groupBitmap(expr)
+```
+
+**Parameters**
+
+`expr` – An expression that results in `UInt*` type.
+
+**Return value**
+
+Value of the `UInt64` type.
+
+**Example**
+
+Test data:
+
+```
+userid
+1
+1
+2
+3
+```
+
+Query:
+
+```
+SELECT groupBitmap(userid) as num FROM t
+```
+
+Result:
+
+```
+num
+3
+```
+
 ## min(x) {#agg_function-min}
 
 Calculates the minimum.

docs/redirects.txt

@@ -86,6 +86,7 @@ functions/arithmetic_functions.md query_language/functions/arithmetic_functions.
 functions/array_functions.md query_language/functions/array_functions.md
 functions/array_join.md query_language/functions/array_join.md
 functions/bit_functions.md query_language/functions/bit_functions.md
+functions/bitmap_functions.md query_language/functions/bitmap_functions.md
 functions/comparison_functions.md query_language/functions/comparison_functions.md
 functions/conditional_functions.md query_language/functions/conditional_functions.md
 functions/date_time_functions.md query_language/functions/date_time_functions.md

docs/toc_en.yml

@@ -115,6 +115,7 @@ nav:
     - 'Working with Arrays': 'query_language/functions/array_functions.md'
     - 'Splitting and Merging Strings and Arrays': 'query_language/functions/splitting_merging_functions.md'
     - 'Bit': 'query_language/functions/bit_functions.md'
+    - 'Bitmap functions': 'query_language/functions/bitmap_functions.md'    
     - 'Hash': 'query_language/functions/hash_functions.md'
     - 'Generating Pseudo-Random Numbers': 'query_language/functions/random_functions.md'
     - 'Encoding': 'query_language/functions/encoding_functions.md'

utils/compressor/CMakeLists.txt

 find_package (Threads)
 
-add_executable (util-clickhouse-compressor main.cpp)
-target_link_libraries (util-clickhouse-compressor PRIVATE clickhouse-compressor-lib)
-set_target_properties(util-clickhouse-compressor PROPERTIES OUTPUT_NAME "clickhouse-compressor")
-
-#install (TARGETS util-clickhouse-compressor RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-compressor)
-
 add_executable (zstd_test zstd_test.cpp)
 target_link_libraries (zstd_test PRIVATE ${ZSTD_LIBRARY} common Threads::Threads)