Speed up CMake build process

Former-commit-id: 2edb2ae26d93ed9b7a8d25b89d30152e60257250

CHANGELOG.md

@@ -6,8 +6,9 @@ Please mark all change in change log and use the ticket from JIRA.
 
 ## Bug
 ## Improvement
-- \#64 - Improvement dump function in scheduler
+- \#64 - Improvement dump function in
 - \#82 - Move easyloggingpp into "external" directory
+- \#92 - Speed up CMake build process
 
 ## Feature
 ## Task

core/cmake/DefineOptions.cmake

@@ -55,21 +55,10 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
 define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
         "Show output from ExternalProjects rather than just logging to files" ON)
 
-define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
-Note that this requires linking Boost statically" OFF)
-
-define_option(MILVUS_BOOST_HEADER_ONLY "Use only BOOST headers" OFF)
-
-define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON)
-
 define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
 
-define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON)
-
 define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
 
-define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON)
-
 define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
 
 define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
@@ -78,16 +67,6 @@ define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
 
 define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
 
-define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
-
-if(CMAKE_VERSION VERSION_LESS 3.7)
-    set(MILVUS_WITH_ZSTD_DEFAULT OFF)
-else()
-    # ExternalProject_Add(SOURCE_SUBDIR) is available since CMake 3.7.
-    set(MILVUS_WITH_ZSTD_DEFAULT ON)
-endif()
-define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
-
 if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
     define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
     define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
@@ -95,6 +74,8 @@ endif()
 
 define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
 
+define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
+
 #----------------------------------------------------------------------
 if(MSVC)
     set_option_category("MSVC")

core/src/CMakeLists.txt

@@ -120,14 +120,10 @@ set(third_party_libs
         ${client_grpc_lib}
         yaml-cpp
         ${prometheus_lib}
-        ${boost_lib}
-        bzip2
-        lz4
-        snappy
-        zlib
-        zstd
         ${cuda_lib}
         mysqlpp
+        zlib
+        ${boost_lib}
         )
 
 if (MILVUS_ENABLE_PROFILING STREQUAL "ON")

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -299,12 +299,29 @@ macro(build_arrow)
             ${EP_COMMON_CMAKE_ARGS}
             -DARROW_BUILD_STATIC=ON
             -DARROW_BUILD_SHARED=OFF
-            -DARROW_PARQUET=OFF
             -DARROW_USE_GLOG=OFF
             -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
-            "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs"
+            -DARROW_CUDA=OFF
+            -DARROW_FLIGHT=OFF
+            -DARROW_GANDIVA=OFF
+            -DARROW_GANDIVA_JAVA=OFF
+            -DARROW_HDFS=OFF
+            -DARROW_HIVESERVER2=OFF
+            -DARROW_ORC=OFF
+            -DARROW_PARQUET=OFF
+            -DARROW_PLASMA=OFF
+            -DARROW_PLASMA_JAVA_CLIENT=OFF
+            -DARROW_PYTHON=OFF
+            -DARROW_WITH_BZ2=OFF
+            -DARROW_WITH_ZLIB=OFF
+            -DARROW_WITH_LZ4=OFF
+            -DARROW_WITH_SNAPPY=OFF
+            -DARROW_WITH_ZSTD=OFF
+            -DARROW_WITH_BROTLI=OFF
             -DCMAKE_BUILD_TYPE=Release
-            -DARROW_DEPENDENCY_SOURCE=BUNDLED) #Build all arrow dependencies from source instead of calling find_package first
+            -DARROW_DEPENDENCY_SOURCE=BUNDLED #Build all arrow dependencies from source instead of calling find_package first
+            -DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found
+            )
 
     
     if(USE_JFROG_CACHE STREQUAL "ON")

core/src/sdk/CMakeLists.txt

@@ -30,9 +30,6 @@ add_library(milvus_sdk STATIC
 
 target_link_libraries(milvus_sdk
         ${client_grpc_lib}
-        bzip2
-        lz4
-        snappy
         zlib
         )
 

core/thirdparty/versions.txt

-BOOST_VERSION=1.70.0
-BZIP2_VERSION=1.0.6
 EASYLOGGINGPP_VERSION=v9.96.7
 GTEST_VERSION=1.8.1
-LZ4_VERSION=v1.9.1
 MYSQLPP_VERSION=3.2.4
 PROMETHEUS_VERSION=v0.7.0
-SNAPPY_VERSION=1.1.7
 SQLITE_VERSION=3280000
 SQLITE_ORM_VERSION=master
 YAMLCPP_VERSION=0.6.2
-ZLIB_VERSION=v1.2.11
-ZSTD_VERSION=v1.4.0
 LIBUNWIND_VERSION=1.3.1
 GPERFTOOLS_VERSION=2.7
 GRPC_VERSION=master
+ZLIB_VERSION=v1.2.11
 
 # vim: set filetype=sh:

core/ubuntu_build_deps.sh

 #!/bin/bash
 
-sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev libboost-filesystem-dev libboost-serialization-dev
+sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \
+libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev
 
 sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so

core/unittest/CMakeLists.txt

@@ -102,7 +102,6 @@ set(unittest_libs
         sqlite
         libboost_system.a
         libboost_filesystem.a
-        lz4
         mysqlpp
         yaml-cpp
         gtest

core/unittest/server/CMakeLists.txt

@@ -59,9 +59,6 @@ set(client_grpc_lib
 target_link_libraries(test_server
         knowhere
         stdc++
-        snappy
-        bz2
-        zstd
         ${client_grpc_lib}
         ${unittest_libs}
         )