Build fixes (#6016)

* Fix building without submodules

* Fix more gcc9 warnings

* was wrong!
../dbms/src/IO/WriteBufferAIO.cpp:277:54: error: result of comparison 'ssize_t' (aka 'long') > 9223372036854775807 is always false [-Werror,-Wtautological-type-limit-compare]
    if ((static_cast<ssize_t>(flush_buffer.offset()) > std::numeric_limits<off_t>::max()) ||
         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

* mimalloc off MI_OVERRIDE

* Fix freebsd build

* Fix shared build

* Fix build in gcc9

* Fix split build

* fix

* fix

* fix

* fix

* clean

* zstd

* Update CMakeLists.txt

* Update Defines.h

* Try fix -Wsign-compare

* Freebsd fix

* Add missing libs

* Fix double-conversion include for copy-headers

* Fix zlib link

* Fix includes for arcadia

* Fix includes for arcadia

* Fix includes for arcadia

* Freebsd fix

* Arcadia fixes

* Update QueryProfiler.cpp

* Freebsd fix

* clean

* 19.11: Fixes for arcadia porting (#6223)

* fix glibc-compatibility

* M_LIBRARY fix

* wip

* Fix

* Fix arm build

* unwind fix

* Update CMakeLists.txt

CMakeLists.txt

@@ -262,10 +262,10 @@ if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
     set (CMAKE_C_FLAGS                   "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
 endif ()
 
-if (NOT SANITIZE)
+if (NOT SANITIZE AND NOT SPLIT_SHARED_LIBRARIES)
     set (CMAKE_EXE_LINKER_FLAGS              "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined")
     set (CMAKE_SHARED_LINKER_FLAGS           "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
-endif()
+endif ()
 
 include (cmake/find_unwind.cmake)
 
@@ -311,8 +311,11 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L
     # There are two variants of C++ library: libc++ (from LLVM compiler infrastructure) and libstdc++ (from GCC).
 
     if (USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
-        # TODO: Allow to use non-static library as well.
-        set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
+        if (USE_STATIC_LIBRARIES)
+            set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_static${${CMAKE_POSTFIX_VARIABLE}}.a")
+        else ()
+            set (EXCEPTION_HANDLING_LIBRARY "${ClickHouse_BINARY_DIR}/contrib/libunwind-cmake/libunwind_shared${${CMAKE_POSTFIX_VARIABLE}}.so")
+        endif ()
     else ()
         set (EXCEPTION_HANDLING_LIBRARY "-lgcc_eh")
     endif ()
@@ -352,6 +355,10 @@ if (OS_LINUX AND NOT UNBUNDLED AND (GLIBC_COMPATIBILITY OR USE_INTERNAL_UNWIND_L
     message(STATUS "Default libraries: ${DEFAULT_LIBS}")
 endif ()
 
+if (NOT GLIBC_COMPATIBILITY)
+    set (M_LIBRARY m)
+endif ()
+
 if (DEFAULT_LIBS)
     # Add default libs to all targets as the last dependency.
     set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})

cmake/Modules/FindODBC.cmake

@@ -129,7 +129,7 @@ find_package_handle_standard_args(ODBC
 )
 
 if(ODBC_FOUND)
-  set(ODBC_LIBRARIES ${ODBC_LIBRARY} ${_odbc_required_libs_paths})
+  set(ODBC_LIBRARIES ${ODBC_LIBRARY} ${_odbc_required_libs_paths} ${LTDL_LIBRARY})
   set(ODBC_INCLUDE_DIRS ${ODBC_INCLUDE_DIR})
   set(ODBC_DEFINITIONS ${PC_ODBC_CFLAGS_OTHER})
 endif()

cmake/print_include_directories.cmake

@@ -16,7 +16,12 @@ list(APPEND dirs ${dirs1})
 get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
-if (USE_INTERNAL_BOOST_LIBRARY)
+if (TARGET double-conversion)
+    get_property (dirs1 TARGET double-conversion PROPERTY INCLUDE_DIRECTORIES)
+    list(APPEND dirs ${dirs1})
+endif ()
+
+if (TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY})
     get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES)
     list(APPEND dirs ${dirs1})
 endif ()

contrib/arrow-cmake/CMakeLists.txt

@@ -44,6 +44,7 @@ set( thriftcpp_threads_SOURCES
 add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES})
 set_target_properties(${THRIFT_LIBRARY} PROPERTIES CXX_STANDARD 14) # REMOVE after https://github.com/apache/thrift/pull/1641
 target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src PRIVATE ${Boost_INCLUDE_DIRS})
+target_link_libraries(${THRIFT_LIBRARY} PRIVATE Threads::Threads)
 
 
 

contrib/brotli-cmake/CMakeLists.txt

@@ -31,3 +31,7 @@ set(SRCS
 add_library(brotli ${SRCS})
 
 target_include_directories(brotli PUBLIC ${BROTLI_SOURCE_DIR}/include)
+
+if(M_LIBRARY)
+    target_link_libraries(brotli PRIVATE ${M_LIBRARY})
+endif()

contrib/double-conversion-cmake/CMakeLists.txt

@@ -10,5 +10,4 @@ ${LIBRARY_DIR}/double-conversion/fast-dtoa.cc
 ${LIBRARY_DIR}/double-conversion/fixed-dtoa.cc
 ${LIBRARY_DIR}/double-conversion/strtod.cc)
 
-target_include_directories(double-conversion SYSTEM PUBLIC "${LIBRARY_DIR}")
-
+target_include_directories(double-conversion SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}")

contrib/h3-cmake/CMakeLists.txt

@@ -25,3 +25,6 @@ add_library(h3 ${SRCS})
 target_include_directories(h3 SYSTEM PUBLIC ${H3_SOURCE_DIR}/include)
 target_include_directories(h3 SYSTEM PUBLIC ${H3_BINARY_DIR}/include)
 target_compile_definitions(h3 PRIVATE H3_HAVE_VLA)
+if(M_LIBRARY)
+    target_link_libraries(h3 PRIVATE ${M_LIBRARY})
+endif()

contrib/librdkafka-cmake/CMakeLists.txt

@@ -65,7 +65,7 @@ add_library(rdkafka ${SRCS})
 target_include_directories(rdkafka SYSTEM PUBLIC include)
 target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
 target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
-target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY})
+target_link_libraries(rdkafka PRIVATE ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${LZ4_LIBRARY} ${LIBGSASL_LIBRARY} Threads::Threads)
 if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY)
-    target_link_libraries(rdkafka PUBLIC ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
+    target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
 endif()

contrib/libunwind-cmake/CMakeLists.txt

@@ -26,6 +26,7 @@ set(LIBUNWIND_SOURCES
 
 add_library(unwind_static ${LIBUNWIND_SOURCES})
 
-target_include_directories(unwind_static PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
+target_include_directories(unwind_static SYSTEM BEFORE PUBLIC ${LIBUNWIND_SOURCE_DIR}/include)
 target_compile_definitions(unwind_static PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
 target_compile_options(unwind_static PRIVATE -fno-exceptions -funwind-tables -fno-sanitize=all -nostdinc++ -fno-rtti)
+target_link_libraries(unwind_static PRIVATE Threads::Threads ${CMAKE_DL_LIBS})

contrib/libxml2-cmake/CMakeLists.txt

@@ -52,7 +52,10 @@ set(SRCS
 )
 add_library(libxml2 ${SRCS})
 
-target_link_libraries(libxml2 ${ZLIB_LIBRARIES})
+target_link_libraries(libxml2 PRIVATE ${ZLIB_LIBRARIES} ${CMAKE_DL_LIBS})
+if(M_LIBRARY)
+    target_link_libraries(libxml2 PRIVATE ${M_LIBRARY})
+endif()
 
 target_include_directories(libxml2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include)
 target_include_directories(libxml2 PUBLIC ${LIBXML2_SOURCE_DIR}/include)

contrib/mariadb-connector-c-cmake/CMakeLists.txt

@@ -60,6 +60,11 @@ endif()
 
 add_library(mysqlclient ${SRCS})
 
+target_link_libraries(mysqlclient PRIVATE ${CMAKE_DL_LIBS} Threads::Threads)
+if(M_LIBRARY)
+    target_link_libraries(mysqlclient PRIVATE ${M_LIBRARY})
+endif()
+
 if(OPENSSL_LIBRARIES)
     target_link_libraries(mysqlclient PRIVATE ${OPENSSL_LIBRARIES})
     target_compile_definitions(mysqlclient PRIVATE -D HAVE_OPENSSL -D HAVE_TLS)

contrib/unixodbc-cmake/CMakeLists.txt

@@ -32,6 +32,7 @@ target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl/libltdl)
 target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc)
 
 target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2)
+target_link_libraries(ltdl PRIVATE ${CMAKE_DL_LIBS})
 
 
 set(SRCS

dbms/CMakeLists.txt

@@ -160,9 +160,6 @@ if (OS_FREEBSD)
 endif ()
 
 if (USE_UNWIND)
-    target_compile_definitions (clickhouse_common_io PRIVATE USE_UNWIND=1)
-    target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${UNWIND_INCLUDE_DIR})
-
     if (NOT USE_INTERNAL_UNWIND_LIBRARY_FOR_EXCEPTION_HANDLING)
         target_link_libraries (clickhouse_common_io PRIVATE ${UNWIND_LIBRARY})
     endif ()
@@ -253,10 +250,6 @@ target_link_libraries(clickhouse_common_io
     roaring
 )
 
-if(ZSTD_LIBRARY)
-    target_link_libraries(clickhouse_common_io PUBLIC ${ZSTD_LIBRARY})
-endif()
-
 if (USE_RDKAFKA)
     target_link_libraries(dbms PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY})
     if(NOT USE_INTERNAL_RDKAFKA_LIBRARY)
@@ -312,11 +305,14 @@ target_include_directories(dbms SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
 if (NOT USE_INTERNAL_LZ4_LIBRARY)
     target_include_directories(dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
 endif ()
+
+if (ZSTD_LIBRARY)
+    target_link_libraries(dbms PRIVATE ${ZSTD_LIBRARY})
+endif()
 if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
     target_include_directories(dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
 endif ()
 
-
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
     target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()

dbms/src/Common/Arena.h

@@ -5,7 +5,9 @@
 #include <vector>
 #include <boost/noncopyable.hpp>
 #include <common/likely.h>
-#include <sanitizer/asan_interface.h>
+#if __has_include(<sanitizer/asan_interface.h>)
+#   include <sanitizer/asan_interface.h>
+#endif
 #include <Core/Defines.h>
 #include <Common/memcpySmall.h>
 #include <Common/ProfileEvents.h>

dbms/src/Common/ArenaWithFreeLists.h

 #pragma once
 
-#include <sanitizer/asan_interface.h>
+#if __has_include(<sanitizer/asan_interface.h>)
+#   include <sanitizer/asan_interface.h>
+#endif
+#include <Core/Defines.h>
 #include <Common/Arena.h>
 #include <Common/BitHelpers.h>
 

dbms/src/Common/MiAllocator.cpp

-#include <Common/config.h>
+#include "MiAllocator.h"
 
 #if USE_MIMALLOC
-
-#include "MiAllocator.h"
 #include <mimalloc.h>
 
 #include <Common/Exception.h>

dbms/src/Common/MiAllocator.h

@@ -2,10 +2,7 @@
 
 #include <Common/config.h>
 
-#if !USE_MIMALLOC
-#error "do not include this file until USE_MIMALLOC is set to 1"
-#endif
-
+#if USE_MIMALLOC
 #include <cstddef>
 
 namespace DB
@@ -26,3 +23,5 @@ struct MiAllocator
 };
 
 }
+
+#endif

dbms/src/Common/QueryProfiler.cpp

@@ -114,7 +114,7 @@ namespace
         out.next();
     }
 
-    const UInt32 TIMER_PRECISION = 1e9;
+    [[maybe_unused]] const UInt32 TIMER_PRECISION = 1e9;
 }
 
 namespace ErrorCodes
@@ -158,7 +158,12 @@ QueryProfilerBase<ProfilerImpl>::QueryProfilerBase(const Int32 thread_id, const
         struct sigevent sev;
         sev.sigev_notify = SIGEV_THREAD_ID;
         sev.sigev_signo = pause_signal;
+
+#if defined(__FreeBSD__)
+        sev._sigev_un._threadid = thread_id;
+#else
         sev._sigev_un._tid = thread_id;
+#endif
         if (timer_create(clock_type, &sev, &timer_id))
             throwFromErrno("Failed to create thread timer", ErrorCodes::CANNOT_CREATE_TIMER);
 

dbms/src/Common/StackTrace.cpp

 #include <common/SimpleCache.h>
 #include <common/demangle.h>
-
+#include <Common/config.h>
 #include <Common/StackTrace.h>
 #include <Common/SymbolIndex.h>
 #include <Common/Dwarf.h>
 #include <Common/Elf.h>
-
 #include <sstream>
 #include <filesystem>
 #include <unordered_map>
@@ -31,6 +30,8 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext
                 error << " Access: write.";
             else
                 error << " Access: read.";
+#else
+            UNUSED(context);
 #endif
 
             switch (info.si_code)

dbms/src/Common/TraceCollector.cpp

@@ -46,6 +46,7 @@ TraceCollector::TraceCollector(std::shared_ptr<TraceLog> & trace_log)
     if (-1 == fcntl(trace_pipe.fds_rw[1], F_SETFL, flags | O_NONBLOCK))
         throwFromErrno("Cannot set non-blocking mode of pipe", ErrorCodes::CANNOT_FCNTL);
 
+#if !defined(__FreeBSD__)
     /** Increase pipe size to avoid slowdown during fine-grained trace collection.
       */
     constexpr int max_pipe_capacity_to_set = 1048576;
@@ -57,6 +58,7 @@ TraceCollector::TraceCollector(std::shared_ptr<TraceLog> & trace_log)
             throwFromErrno("Cannot increase pipe capacity to " + toString(pipe_size * 2), ErrorCodes::CANNOT_FCNTL);
 
     LOG_TRACE(log, "Pipe capacity is " << formatReadableSizeWithBinarySuffix(std::min(pipe_size, max_pipe_capacity_to_set)));
+#endif
 
     thread = ThreadFromGlobalPool(&TraceCollector::run, this);
 }

dbms/src/Common/config.h.in

@@ -9,5 +9,5 @@
 #cmakedefine01 USE_CPUINFO
 #cmakedefine01 USE_BROTLI
 #cmakedefine01 USE_MIMALLOC
-
+#cmakedefine01 USE_UNWIND
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY

dbms/src/Compression/CompressedReadBufferBase.cpp

 #include "CompressedReadBufferBase.h"
 
 #include <vector>
-
 #include <string.h>
 #include <city.h>
-#include <zstd.h>
-
 #include <Common/PODArray.h>
 #include <Common/ProfileEvents.h>
 #include <Common/Exception.h>

dbms/src/Compression/CompressedWriteBuffer.cpp

 #include <memory>
 #include <city.h>
-#include <lz4.h>
-#include <lz4hc.h>
-#include <zstd.h>
 #include <string.h>
 
 #include <common/unaligned.h>

dbms/src/Compression/ICompressionCodec.cpp

@@ -7,7 +7,6 @@
 #include <IO/ReadBufferFromFileBase.h>
 #include <Common/typeid_cast.h>
 #include <Compression/CompressionFactory.h>
-#include <zstd.h>
 
 namespace ProfileEvents
 {

dbms/src/Core/Defines.h

@@ -140,6 +140,11 @@
 /// It could be any magic number.
 #define DBMS_DISTRIBUTED_SENDS_MAGIC_NUMBER 0xCAFECABE
 
+#if !__has_include(<sanitizer/asan_interface.h>)
+#   define ASAN_UNPOISON_MEMORY_REGION(a, b)
+#   define ASAN_POISON_MEMORY_REGION(a, b)
+#endif
+
 /// A macro for suppressing warnings about unused variables or function results.
 /// Useful for structured bindings which have no standard way to declare this.
 #define UNUSED(...) (void)(__VA_ARGS__)

dbms/src/Formats/CapnProtoRowInputStream.cpp

-#include "config_formats.h"
-#if USE_CAPNP
-
 #include "CapnProtoRowInputStream.h"
 
+#if USE_CAPNP
 #include <IO/ReadBuffer.h>
 #include <Interpreters/Context.h>
 #include <Formats/FormatFactory.h>

dbms/src/Formats/ParquetBlockInputStream.cpp

-#include "config_formats.h"
-#if USE_PARQUET
-#    include "ParquetBlockInputStream.h"
+#include "ParquetBlockInputStream.h"
 
+#if USE_PARQUET
 #    include <algorithm>
 #    include <iterator>
 #    include <vector>

dbms/src/Formats/ParquetBlockOutputStream.cpp

-#include "config_formats.h"
-#if USE_PARQUET
-#    include "ParquetBlockOutputStream.h"
+#include "ParquetBlockOutputStream.h"
 
+#if USE_PARQUET
 // TODO: clean includes
 #    include <Columns/ColumnDecimal.h>
 #    include <Columns/ColumnFixedString.h>

dbms/src/Formats/ProtobufColumnMatcher.cpp

-#include "config_formats.h"
-#if USE_PROTOBUF
-
 #include "ProtobufColumnMatcher.h"
-
+#if USE_PROTOBUF
 #include <Common/Exception.h>
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/descriptor.pb.h>

dbms/src/Formats/ProtobufReader.h

@@ -9,7 +9,7 @@
 #if USE_PROTOBUF
 
 #include <boost/noncopyable.hpp>
-#include <Formats/ProtobufColumnMatcher.h>
+#include "ProtobufColumnMatcher.h"
 #include <IO/ReadBuffer.h>
 #include <memory>
 

dbms/src/Formats/ProtobufRowInputStream.cpp

-#include "config_formats.h"
-#if USE_PROTOBUF
-
 #include "ProtobufRowInputStream.h"
 
+#if USE_PROTOBUF
 #include <Core/Block.h>
 #include <Formats/BlockInputStreamFromRowInputStream.h>
 #include <Formats/FormatFactory.h>

dbms/src/Formats/ProtobufWriter.h

@@ -7,7 +7,7 @@
 #include "config_formats.h"
 #if USE_PROTOBUF
 
-#include <Formats/ProtobufColumnMatcher.h>
+#include "ProtobufColumnMatcher.h"
 #include <IO/WriteBufferFromString.h>
 #include <boost/noncopyable.hpp>
 #include <Common/PODArray.h>

dbms/src/Functions/CMakeLists.txt

@@ -22,6 +22,7 @@ target_link_libraries(clickhouse_functions
         ${BASE64_LIBRARY}
 
     PRIVATE
+        ${ZLIB_LIBRARIES}
         ${Boost_FILESYSTEM_LIBRARY}
 )
 

dbms/src/IO/ReadBufferAIO.cpp

@@ -254,7 +254,7 @@ void ReadBufferAIO::prepare()
     /// Region of the disk from which we want to read data.
     const off_t region_begin = first_unread_pos_in_file;
 
-    if ((requested_byte_count > std::numeric_limits<off_t>::max()) ||
+    if ((requested_byte_count > static_cast<size_t>(std::numeric_limits<off_t>::max())) ||
         (first_unread_pos_in_file > (std::numeric_limits<off_t>::max() - static_cast<off_t>(requested_byte_count))))
         throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR);
 

dbms/src/IO/ReadBufferFromHDFS.cpp

-#include <Common/config.h>
+#include "ReadBufferFromHDFS.h"
 
 #if USE_HDFS
-
-#include <IO/ReadBufferFromHDFS.h>
 #include <IO/HDFSCommon.h>
 #include <Poco/URI.h>
 #include <hdfs/hdfs.h>

dbms/src/IO/ReadBufferFromHDFS.h

 #pragma once
 
+#include <Common/config.h>
+
 #if USE_HDFS
 #include <IO/ReadBuffer.h>
 #include <IO/BufferWithOwnMemory.h>

dbms/src/IO/WriteBufferAIO.cpp

@@ -274,7 +274,7 @@ void WriteBufferAIO::prepare()
     /// Region of the disk in which we want to write data.
     const off_t region_begin = pos_in_file;
 
-    if ((flush_buffer.offset() > std::numeric_limits<off_t>::max()) ||
+    if ((flush_buffer.offset() > static_cast<size_t>(std::numeric_limits<off_t>::max())) ||
         (pos_in_file > (std::numeric_limits<off_t>::max() - static_cast<off_t>(flush_buffer.offset()))))
         throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR);