Use submodule contrib/capnproto, remove include_directories (#1746)

* Add submodule contrib/capnproto

* Use contrib/capnproto, remove include_directories

* Fix include

* Clean

* Fix werror

* Update doc

* Fix

* Final fix

* Fxi

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "contrib/googletest"]
 	path = contrib/googletest
 	url = https://github.com/google/googletest.git
+[submodule "contrib/capnproto"]
+	path = contrib/capnproto
+	url = https://github.com/capnproto/capnproto.git

cmake/find_capnp.cmake

 option (ENABLE_CAPNP "Enable Cap'n Proto" ON)
 
 if (ENABLE_CAPNP)
+
+    # cmake 3.5.1 bug:
+    # capnproto uses this cmake feature:
+    # target_compile_features(kj PUBLIC cxx_constexpr)
+    # old cmake adds -std=gnu++11 to end of all compile commands (even if -std=gnu++17 already present in compile string)
+    # cmake 3.9.1 (ubuntu artful) have no this bug (c++17 support added to cmake 3.8.2)
+    if (CMAKE_VERSION VERSION_LESS "3.8.0")
+       set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT 0)
+       set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
+    else ()
+       set (USE_INTERNAL_CAPNP_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
+    endif ()
+
+    option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${USE_INTERNAL_CAPNP_LIBRARY_DEFAULT})
+
+    if (USE_INTERNAL_CAPNP_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/CMakeLists.txt")
+       message (WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
+       set (USE_INTERNAL_CAPNP_LIBRARY 0)
+       set (MISSING_INTERNAL_CAPNP_LIBRARY 1)
+    endif ()
+
+    if (NOT USE_INTERNAL_CAPNP_LIBRARY)
         set (CAPNP_PATHS "/usr/local/opt/capnp/lib")
         set (CAPNP_INCLUDE_PATHS "/usr/local/opt/capnp/include")
         find_library (CAPNP capnp PATHS ${CAPNP_PATHS})
         find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS})
         find_library (KJ kj PATHS ${CAPNP_PATHS})
-	set (CAPNP_LIBS ${CAPNP} ${CAPNPC} ${KJ})
-
+        set (CAPNP_LIBRARY ${CAPNP} ${CAPNPC} ${KJ})
         find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS})
-	if (CAPNP_INCLUDE_DIR AND CAPNP_LIBS)
-	    include_directories (${CAPNP_INCLUDE_DIR})
+    endif ()
+
+    if (CAPNP_INCLUDE_DIR AND CAPNP_LIBRARY)
         set(USE_CAPNP 1)
+    elseif (NOT MISSING_INTERNAL_CAPNP_LIBRARY)
+        set (USE_INTERNAL_CAPNP_LIBRARY 1)
+        set (CAPNP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src")
+        set (CAPNP_LIBRARY capnpc)
+        set (USE_CAPNP 1)
     endif ()
+
 endif ()
 
 if (USE_CAPNP)
-    message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBS}")
+    message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_INCLUDE_DIR} : ${CAPNP_LIBRARY}")
 else ()
     message (STATUS "Build without capnp (support for Cap'n Proto format will be disabled)")
 endif ()

cmake/find_icu4c.cmake

@@ -10,7 +10,6 @@ if (ENABLE_ICU)
 
     find_path (ICU_INCLUDE_DIR NAMES unicode/unistr.h PATHS ${ICU_INCLUDE_PATHS})
     if (ICU_INCLUDE_DIR AND ICU_LIBS)
-        include_directories (${ICU_INCLUDE_DIR})
         set(USE_ICU 1)
     endif ()
 endif ()

cmake/find_openssl.cmake

@@ -20,8 +20,4 @@ if (NOT OPENSSL_FOUND)
     endif ()
 endif ()
 
-if (OPENSSL_FOUND)
-    include_directories (${OPENSSL_INCLUDE_DIR})
-endif ()
-
 message (STATUS "Using openssl=${OPENSSL_FOUND}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")

cmake/find_poco.cmake

@@ -10,7 +10,6 @@ if (NOT USE_INTERNAL_POCO_LIBRARY)
 endif ()
 
 if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY)
-    #include_directories (${Poco_INCLUDE_DIRS})
 else ()
 
     set (USE_INTERNAL_POCO_LIBRARY 1)
@@ -73,8 +72,6 @@ else ()
     set (Poco_Data_LIBRARY PocoData)
     set (Poco_XML_LIBRARY PocoXML)
 
-    #include_directories (BEFORE ${Poco_INCLUDE_DIRS})
-
 endif ()
 
 message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${Poco_MongoDB_FOUND}, DataODBC=${Poco_DataODBC_FOUND}, NetSSL=${Poco_NetSSL_FOUND}")

cmake/find_rdkafka.cmake

@@ -16,7 +16,6 @@ if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
 endif ()
 
 if (RDKAFKA_LIBRARY AND RDKAFKA_INCLUDE_DIR)
-    include_directories (${RDKAFKA_INCLUDE_DIR})
     set (USE_RDKAFKA 1)
 elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY)
     set (USE_INTERNAL_RDKAFKA_LIBRARY 1)

cmake/find_re2.cmake

@@ -6,7 +6,6 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
 endif ()
 
 if (RE2_LIBRARY AND RE2_INCLUDE_DIR)
-    include_directories (${RE2_INCLUDE_DIR})
     set (RE2_ST_LIBRARY ${RE2_LIBRARY})
 else ()
     set (USE_INTERNAL_RE2_LIBRARY 1)

cmake/find_readline_edit.cmake

@@ -35,8 +35,6 @@ elseif (EDIT_LIB AND TERMCAP_LIB)
 endif ()
 
 if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR)
-    include_directories (${READLINE_INCLUDE_DIR})
-
     include (CheckCXXSourceRuns)
 
     set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS})

cmake/find_sparsehash.cmake

@@ -5,7 +5,6 @@ if (NOT USE_INTERNAL_SPARCEHASH_LIBRARY)
 endif ()
 
 if (SPARCEHASH_INCLUDE_DIR)
-    include_directories (${SPARCEHASH_INCLUDE_DIR})
 else ()
     set (USE_INTERNAL_SPARCEHASH_LIBRARY 1)
     set (SPARCEHASH_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libsparsehash")

cmake/find_zookeeper.cmake

@@ -17,7 +17,6 @@ if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
 endif ()
 
 if (ZOOKEEPER_LIBRARY AND ZOOKEEPER_INCLUDE_DIR)
-    include_directories (${ZOOKEEPER_INCLUDE_DIR})
 else ()
     set (USE_INTERNAL_ZOOKEEPER_LIBRARY 1)
     set(WANT_CPPUNIT 0 CACHE BOOL "")

contrib/CMakeLists.txt

@@ -78,13 +78,23 @@ if (NOT ARCH_ARM)
 endif ()
 
 if (USE_INTERNAL_RDKAFKA_LIBRARY)
-    set(RDKAFKA_BUILD_EXAMPLES OFF CACHE BOOL "")
-    set(RDKAFKA_BUILD_TESTS OFF CACHE BOOL "")
-    set(RDKAFKA_BUILD_STATIC ON CACHE BOOL "")
+    set(RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "")
+    set(RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "")
+    set(RDKAFKA_BUILD_STATIC ON CACHE INTERNAL "")
     mark_as_advanced(ZLIB_INCLUDE_DIR)
     add_subdirectory (librdkafka)
 endif ()
 
+if (USE_INTERNAL_CAPNP_LIBRARY)
+    set(BUILD_TESTING 0 CACHE INTERNAL "")
+    set (_save ${CMAKE_CXX_EXTENSIONS})
+    set (CMAKE_CXX_EXTENSIONS)
+    add_subdirectory (capnproto/c++)
+    set (CMAKE_CXX_EXTENSIONS ${_save})
+    target_include_directories(${CAPNP_LIBRARY} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/capnproto/c++/src>)
+endif ()
+
+
 if (USE_INTERNAL_POCO_LIBRARY)
     set (_save ${ENABLE_TESTS})
     set (ENABLE_TESTS 0)

capnproto @ c949a18d

+Subproject commit c949a18da5f041a36cc218c5c4b79c7705999b4f

dbms/CMakeLists.txt

@@ -167,6 +167,14 @@ target_link_libraries (dbms
     ${BTRIE_LIBRARIES}
 )
 
+if (NOT USE_INTERNAL_RE2_LIBRARY)
+    target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
+endif ()
+
+if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
+    target_include_directories (clickhouse_common_io BEFORE PUBLIC ${ZOOKEEPER_INCLUDE_DIR})
+endif ()
+
 if (Poco_DataODBC_FOUND)
     target_link_libraries (dbms ${Poco_DataODBC_LIBRARY})
 endif()
@@ -183,10 +191,21 @@ target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
 
 if (USE_ICU)
     target_link_libraries (dbms ${ICU_LIBS})
+    target_include_directories (dbms PRIVATE ${ICU_INCLUDE_DIR})
 endif ()
 
 if (USE_CAPNP)
-    target_link_libraries (dbms ${CAPNP_LIBS})
+    target_link_libraries (dbms ${CAPNP_LIBRARY})
+    if (NOT USE_INTERNAL_CAPNP_LIBRARY)
+        target_include_directories (dbms BEFORE PRIVATE ${CAPNP_INCLUDE_DIR})
+    endif ()
+endif ()
+
+if (USE_RDKAFKA)
+    target_link_libraries (dbms ${RDKAFKA_LIBRARY})
+    if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
+        target_include_directories (dbms BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
+    endif ()
 endif ()
 
 target_link_libraries (dbms
@@ -209,10 +228,6 @@ target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
 target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
 target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
 
-if (USE_RDKAFKA)
-    target_link_libraries (dbms ${RDKAFKA_LIBRARY})
-endif ()
-
 # also for copy_headers.sh:
 target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
 

dbms/src/Common/config_build.cpp.in

@@ -22,5 +22,17 @@ const char * auto_config_build[]
     "BUILD_COMPILE_DEFINITIONS", "@BUILD_COMPILE_DEFINITIONS@",
     "BUILD_INCLUDE_DIRECTORIES", "@BUILD_INCLUDE_DIRECTORIES@",
     "STATIC", "@USE_STATIC_LIBRARIES@",
+    "USE_CAPNP", "@USE_CAPNP@",
+    "USE_ICU", "@USE_ICU@",
+    "USE_MYSQL", "@USE_MYSQL@",
+    "USE_RE2_ST", "@USE_RE2_ST@",
+    "USE_VECTORCLASS", "@USE_VECTORCLASS@",
+    "USE_RDKAFKA", "@USE_RDKAFKA@",
+    "USE_CAPNP", "@USE_CAPNP@",
+    "USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@",
+    "USE_Poco_DataODBC", "@Poco_DataODBC_FOUND",
+    "USE_Poco_MongoDB", "@Poco_MongoDB_FOUND",
+    "USE_Poco_NetSSL", "@Poco_NetSSL_FOUND",
+
     nullptr, nullptr
 };

dbms/src/DataStreams/CapnProtoRowInputStream.cpp

@@ -77,6 +77,7 @@ Field convertNodeToField(capnp::DynamicValue::Reader value)
         case capnp::DynamicValue::ANY_POINTER:
             throw Exception("ANY_POINTER type not supported");
     }
+    return Field();
 }
 
 capnp::StructSchema::Field getFieldOrThrow(capnp::StructSchema node, const std::string & field)

dbms/src/Server/CMakeLists.txt

@@ -27,6 +27,7 @@ target_link_libraries (clickhouse-extract-from-config-lib clickhouse_common_io $
 
 add_library (clickhouse-client-lib Client.cpp)
 target_link_libraries (clickhouse-client-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_include_directories (clickhouse-client-lib PRIVATE ${READLINE_INCLUDE_DIR})
 install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
 
 add_library (clickhouse-benchmark-lib ${SPLIT_SHARED} Benchmark.cpp)

libs/libmysqlxx/CMakeLists.txt

@@ -49,6 +49,7 @@ if (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
 endif ()
 
 target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBRARIES})
+target_include_directories (mysqlxx PRIVATE ${OPENSSL_INCLUDE_DIR})
 
 if (ENABLE_TESTS)
     add_subdirectory (src/tests)

utils/build/build_debian.sh

@@ -8,7 +8,7 @@
 # install compiler and libs
 sudo apt install -y git bash cmake gcc-7 g++-7 libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev libltdl-dev libssl-dev
 # for -DUNBUNDLED=1 mode:
-#sudo apt install -y libboost-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev libzstd-dev
+#sudo apt install -y libboost-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libzookeeper-mt-dev libsparsehash-dev librdkafka-dev libcapnp-dev libpoco-dev libsparsehash-dev libgoogle-perftools-dev libunwind-dev libzstd-dev
 
 # install testing only stuff if you want:
 sudo apt install -y python python-lxml python-termcolor curl perl