Merge pull request #342 from proller/cmake

Reorganize includes, less depends in .h files

CMakeLists.txt

@@ -154,48 +154,25 @@ else ()
 	set (CLICKHOUSE_ETC_DIR ${CMAKE_INSTALL_PREFIX}/etc)
 endif ()
 
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdouble-conversion/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libsparsehash/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libre2/)
-include_directories (BEFORE ${ClickHouse_BINARY_DIR}/contrib/libre2/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/)
-include_directories (BEFORE ${ClickHouse_BINARY_DIR}/libs/libcommon/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libpocoext/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libmysqlxx/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/libs/libzkutil/include/)
-
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/dbms/include)
-
 include (cmake/find_openssl.cmake)
 include (cmake/find_icu4c.cmake)
 include (cmake/find_boost.cmake)
 include (cmake/find_poco.cmake)
 include (cmake/find_libtool.cmake)
-include (cmake/find_mysqlclient.cmake)
 include (cmake/find_rt.cmake)
 if (ENABLE_LIBTCMALLOC)
     include (cmake/find_gperftools.cmake)
 endif ()
 
-# Directory for Yandex specific files
-set (CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)
-
 add_subdirectory (contrib)
 add_subdirectory (libs)
 add_subdirectory (utils)
 add_subdirectory (dbms)
 
+# Directory for Yandex specific files
+set (CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)
 if (EXISTS ${CLICKHOUSE_PRIVATE_DIR})
-	add_subdirectory (private)
+	add_subdirectory (${CLICKHOUSE_PRIVATE_DIR})
 endif ()
 
 message (STATUS "C_FLAGS    = ${CMAKE_C_FLAGS}")

cmake/dbms_include.cmake

+
+include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
+
+# TODO:
+# move code with incldes from .h to .cpp and clean this list:
+include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libpocoext/include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libzkutil/include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libmysqlxx/include)
+include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libzookeeper/include)
+include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libcityhash/include)
+include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libdouble-conversion)

contrib/liblz4/CMakeLists.txt

+include_directories (BEFORE include)
+
 add_library (lz4
     src/lz4.c
     src/lz4hc.c

contrib/libre2/CMakeLists.txt

@@ -38,6 +38,8 @@ set (re2_sources
 # In order to avoid redundant locks in some cases, we use not thread-safe version of the library (re2_st).
 add_definitions (-DNDEBUG)
 
+include_directories (BEFORE .)
+
 add_library (re2 ${re2_sources})
 add_library (re2_st ${re2_sources})
 

dbms/CMakeLists.txt

-include_directories (BEFORE include)
-#include_directories (/usr/include/mysql)
+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
+
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libsparsehash)
+include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libre2/)
+include_directories (BEFORE ${ClickHouse_BINARY_DIR}/contrib/libre2/)
+include_directories (${ClickHouse_SOURCE_DIR}/libs/libdaemon/include/)
 
 add_subdirectory (src)
 

dbms/include/DB/Core/StringRef.h

 #pragma once
 
-#include <string.h>
-#include <city.h>
-
 #include <string>
 #include <vector>
-
 #include <functional>
 #include <ostream>
 
@@ -13,6 +9,8 @@
 	#include <emmintrin.h>
 #endif
 
+#include <city.h>
+
 #include <DB/Core/Types.h>
 #include <DB/Common/unaligned.h>
 
@@ -215,11 +213,11 @@ inline size_t hashLessThan8(const char * data, size_t size)
 
 	if (size > 0)
 	{
-		uint8 a = data[0];
-		uint8 b = data[size >> 1];
-		uint8 c = data[size - 1];
-		uint32 y = static_cast<uint32>(a) + (static_cast<uint32>(b) << 8);
-		uint32 z = size + (static_cast<uint32>(c) << 2);
+		uint8_t a = data[0];
+		uint8_t b = data[size >> 1];
+		uint8_t c = data[size - 1];
+		uint32_t y = static_cast<uint32_t>(a) + (static_cast<uint32_t>(b) << 8);
+		uint32_t z = size + (static_cast<uint32_t>(c) << 2);
 		return shiftMix(y * k2 ^ z * k3) * k2;
 	}
 

dbms/include/DB/IO/HashingReadBuffer.h

 #pragma once
 
-#include <city.h>
 #include <DB/IO/ReadBuffer.h>
 #include <DB/IO/HashingWriteBuffer.h>
 

dbms/include/DB/IO/HashingWriteBuffer.h

 #pragma once
 
-#include <city.h>
 #include <DB/IO/WriteBuffer.h>
 #include <DB/IO/BufferWithOwnMemory.h>
 #include <DB/IO/ReadHelpers.h>
@@ -35,45 +34,7 @@ public:
 
 	/// вычисление хэша зависит от разбиения по блокам
 	/// поэтому нужно вычислить хэш от n полных кусочков и одного неполного
-	void calculateHash(DB::BufferBase::Position data, size_t len)
-	{
-		if (len)
-		{
-			/// если данных меньше, чем block_size то сложим их в свой буффер и посчитаем от них hash позже
-			if (block_pos + len < block_size)
-			{
-				memcpy(&BufferWithOwnMemory<Buffer>::memory[block_pos], data, len);
-				block_pos += len;
-			}
-			else
-			{
-				/// если в буффер уже что-то записано, то допишем его
-				if (block_pos)
-				{
-					size_t n = block_size - block_pos;
-					memcpy(&BufferWithOwnMemory<Buffer>::memory[block_pos], data, n);
-					append(&BufferWithOwnMemory<Buffer>::memory[0]);
-					len -= n;
-					data += n;
-					block_pos = 0;
-				}
-
-				while (len >= block_size)
-				{
-					append(data);
-					len -= block_size;
-					data += block_size;
-				}
-
-				/// запишем остаток в свой буфер
-				if (len)
-				{
-					memcpy(&BufferWithOwnMemory<Buffer>::memory[0], data, len);
-					block_pos = len;
-				}
-			}
-		}
-	}
+	void calculateHash(DB::BufferBase::Position data, size_t len);
 
 protected:
 	size_t block_pos;

dbms/include/DB/Interpreters/AggregationCommon.h

 #pragma once
 
-#include <city.h>
 #include <openssl/md5.h>
 
 #include <DB/Common/SipHash.h>

dbms/src/IO/HashingWriteBuffer.cpp

 #include <DB/IO/HashingWriteBuffer.h>
 #include <iomanip>
+#include <city.h>
+
+namespace DB
+{
+
+/// вычисление хэша зависит от разбиения по блокам
+/// поэтому нужно вычислить хэш от n полных кусочков и одного неполного
+template <class Buffer>
+void IHashingBuffer<Buffer>::calculateHash(DB::BufferBase::Position data, size_t len)
+{
+	if (len)
+	{
+		/// если данных меньше, чем block_size то сложим их в свой буффер и посчитаем от них hash позже
+		if (block_pos + len < block_size)
+		{
+			memcpy(&BufferWithOwnMemory<Buffer>::memory[block_pos], data, len);
+			block_pos += len;
+		}
+		else
+		{
+			/// если в буффер уже что-то записано, то допишем его
+			if (block_pos)
+			{
+				size_t n = block_size - block_pos;
+				memcpy(&BufferWithOwnMemory<Buffer>::memory[block_pos], data, n);
+				append(&BufferWithOwnMemory<Buffer>::memory[0]);
+				len -= n;
+				data += n;
+				block_pos = 0;
+			}
+
+			while (len >= block_size)
+			{
+				append(data);
+				len -= block_size;
+				data += block_size;
+			}
+
+			/// запишем остаток в свой буфер
+			if (len)
+			{
+				memcpy(&BufferWithOwnMemory<Buffer>::memory[0], data, len);
+				block_pos = len;
+			}
+		}
+	}
+}
+
+template class IHashingBuffer<DB::ReadBuffer>;
+template class IHashingBuffer<DB::WriteBuffer>;
+
+}
 
 /// UInt64 это 39 символов в 10 системе счисления
 static const size_t UINT64_DECIMAL_SIZE = 39;

dbms/src/Server/CMakeLists.txt

@@ -28,15 +28,17 @@ install_symlink_to_clickhouse(clickhouse-local)
 install_symlink_to_clickhouse(clickhouse-benchmark)
 
 INSTALL(
-    FILES config.xml users.xml
-    DESTINATION  ${CLICKHOUSE_ETC_DIR}/clickhouse-server
-    COMPONENT clickhouse)
+	FILES config.xml users.xml
+	DESTINATION  ${CLICKHOUSE_ETC_DIR}/clickhouse-server
+	COMPONENT clickhouse)
 
-INSTALL(
-    FILES ${CLICKHOUSE_PRIVATE_DIR}/Server/metrika/config.xml ${CLICKHOUSE_PRIVATE_DIR}/Server/metrika/users.xml
-    DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/metrika
-    COMPONENT clickhouse
-    OPTIONAL)
+if (EXISTS ${CLICKHOUSE_PRIVATE_DIR})
+	INSTALL(
+		FILES ${CLICKHOUSE_PRIVATE_DIR}/Server/metrika/config.xml ${CLICKHOUSE_PRIVATE_DIR}/Server/metrika/users.xml
+		DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server/metrika
+		COMPONENT clickhouse
+	OPTIONAL)
+endif ()
 
 if (NOT CMAKE_SYSTEM MATCHES "FreeBSD")
 	INSTALL(

libs/libcommon/CMakeLists.txt

@@ -9,7 +9,10 @@ set_source_files_properties(
 	src/ClickHouseRevision.cpp PROPERTIES OBJECT_DEPENDS ${REVISIONFILE}
 )
 
+include_directories (include)
+# for generated revision.h:
 include_directories (${CMAKE_CURRENT_BINARY_DIR}/src/)
+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
 
 if (APPLE)
 	if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" AND NOT "${CMAKE_SYSTEM_VERSION}" VERSION_LESS "16.1.0")

libs/libdaemon/CMakeLists.txt

+include_directories (include)
+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
+
 add_library (daemon
 	src/BaseDaemon.cpp
 	src/GraphiteWriter.cpp

libs/libmysqlxx/CMakeLists.txt

+
+include (cmake/find_mysqlclient.cmake)
+
+include_directories (include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
+
 add_library (mysqlxx
 	src/Connection.cpp
 	src/Exception.cpp

libs/libmysqlxx/include/mysqlxx/Types.h

@@ -3,10 +3,6 @@
 #include <string>
 #include <Poco/Types.h>
 
-#include <common/LocalDate.h>
-#include <common/LocalDateTime.h>
-
-
 struct st_mysql;
 using MYSQL = st_mysql;
 

libs/libmysqlxx/include/mysqlxx/Value.h

@@ -13,6 +13,8 @@
 #include <common/DateLUT.h>
 
 #include <mysqlxx/Types.h>
+#include <common/LocalDateTime.h>
+
 
 /// Обрезать длинный запрос до указанной длины для текста исключения.
 #define MYSQLXX_QUERY_PREVIEW_LENGTH 1000

libs/libpocoext/CMakeLists.txt

+include_directories (include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
+
 add_library (pocoext
 	src/LevelFilterChannel.cpp
 	src/ThreadNumber.cpp

libs/libzkutil/CMakeLists.txt

+include_directories (include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
+
+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
+
 add_library(zkutil
 	src/ZooKeeper.cpp
 	src/Lock.cpp

utils/CMakeLists.txt

+include(${CMAKE_SOURCE_DIR}/cmake/dbms_include.cmake)
+
 add_subdirectory (compressor)
 add_subdirectory (iotest)
 add_subdirectory (corrector_utf8)

utils/compressor/CMakeLists.txt

+
+include_directories (BEFORE ${CMAKE_SOURCE_DIR}/contrib/libzstd/include)
+
 add_executable (compressor main.cpp)
 target_link_libraries (compressor dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
 

utils/corrector_utf8/CMakeLists.txt

+include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
+
 add_executable(corrector_utf8 corrector_utf8.cpp)
 
 # Link the executable to the library.

utils/iotest/CMakeLists.txt

+include_directories (${CMAKE_SOURCE_DIR}/dbms/include)
+include_directories (${CMAKE_SOURCE_DIR}/libs/libcommon/include)
+
 add_executable (iotest iotest.cpp ${SRCS})
 target_link_libraries (iotest dbms)