Optimize grpc download & build in CMakeLists (#3207)

* change thirdparty make args
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* add compile option to googletest
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* add ccache to gtest
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* rearrange grpc download &build
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* fix grpc download repeat problem
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>

* fix cannot find openblas.so.0 error
Signed-off-by: Nyangxuan <xuan.yang@zilliz.com>
Co-authored-by: Nyangxuan <xuan.yang@zilliz.com>

core/CMakeLists.txt

@@ -49,18 +49,22 @@ set( MILVUS_BINARY_DIR      ${PROJECT_BINARY_DIR} )
 set( MILVUS_ENGINE_SRC      ${PROJECT_SOURCE_DIR}/src )
 set( MILVUS_THIRDPARTY_SRC  ${PROJECT_SOURCE_DIR}/thirdparty )
 
+# This will set RPATH to all excutable TARGET
+# self-installed dynamic libraries will be correctly linked by excutable
+set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/lib" )
+set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
 # **************************** Dependencies ****************************
 include( CTest )
 include( BuildUtils )
+include( DefineOptions )
 import_mysql_inc()
+using_ccache_if_defined( MILVUS_USE_CCACHE )
 
 include( ExternalProject )
-include( DefineOptions )
 include( ThirdPartyPackages )
 
 add_subdirectory( thirdparty )
 
-using_ccache_if_defined( MILVUS_USE_CCACHE )
 
 # **************************** Compiler arguments ****************************
 

core/cmake/ThirdPartyPackages.cmake

@@ -17,7 +17,6 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
         yaml-cpp
         libunwind
         gperftools
-        GRPC
         ZLIB
         Opentracing
         fiu
@@ -46,8 +45,6 @@ macro(build_dependency DEPENDENCY_NAME)
         build_libunwind()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools")
         build_gperftools()
-    elseif ("${DEPENDENCY_NAME}" STREQUAL "GRPC")
-        build_grpc()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "ZLIB")
         build_zlib()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "Opentracing")
@@ -269,16 +266,6 @@ else ()
             "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz")
 endif ()
 
-if (DEFINED ENV{MILVUS_GRPC_URL})
-    set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}")
-else ()
-    set(GRPC_SOURCE_URL
-            "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip"
-            #"https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip"
-            #"https://gitee.com/quicksilver/grpc-milvus/repository/archive/${GRPC_VERSION}.zip"
-            )
-endif ()
-
 if (DEFINED ENV{MILVUS_ZLIB_URL})
     set(ZLIB_SOURCE_URL "$ENV{MILVUS_ZLIB_URL}")
 else ()
@@ -642,92 +629,6 @@ if (MILVUS_WITH_GPERFTOOLS)
     link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib)
 endif ()
 
-# ----------------------------------------------------------------------
-# GRPC
-
-macro(build_grpc)
-    message(STATUS "Building GRPC-${GRPC_VERSION} from source")
-    set(GRPC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/install")
-    set(GRPC_INCLUDE_DIR "${GRPC_PREFIX}/include")
-    set(GRPC_STATIC_LIB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GRPC++_STATIC_LIB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpc++${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GRPCPP_CHANNELZ_STATIC_LIB "${GRPC_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}grpcpp_channelz${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GRPC_PROTOBUF_LIB_DIR "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/libs/opt/protobuf")
-    set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
-    set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
-    ExternalProject_Add(grpc_ep
-            URL
-            ${GRPC_SOURCE_URL}
-            ${EP_LOG_OPTIONS}
-            URL_MD5
-            "478215c151a144c2d8625b49ff1b70aa"
-            CONFIGURE_COMMAND
-            ""
-            BUILD_IN_SOURCE
-            1
-            BUILD_COMMAND
-            ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-            INSTALL_COMMAND
-            ${MAKE} install prefix=${GRPC_PREFIX}
-            BUILD_BYPRODUCTS
-            ${GRPC_STATIC_LIB}
-            ${GRPC++_STATIC_LIB}
-            ${GRPCPP_CHANNELZ_STATIC_LIB}
-            ${GRPC_PROTOBUF_STATIC_LIB}
-            ${GRPC_PROTOC_STATIC_LIB})
-
-    ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-    file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")
-
-    add_library(grpc STATIC IMPORTED)
-    set_target_properties(grpc
-            PROPERTIES IMPORTED_LOCATION "${GRPC_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
-            INTERFACE_LINK_LIBRARIES "zlib")
-
-    add_library(grpc++ STATIC IMPORTED)
-    set_target_properties(grpc++
-            PROPERTIES IMPORTED_LOCATION "${GRPC++_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
-            INTERFACE_LINK_LIBRARIES "zlib")
-
-    add_library(grpcpp_channelz STATIC IMPORTED)
-    set_target_properties(grpcpp_channelz
-            PROPERTIES IMPORTED_LOCATION "${GRPCPP_CHANNELZ_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
-            INTERFACE_LINK_LIBRARIES "zlib")
-
-    add_library(grpc_protobuf STATIC IMPORTED)
-    set_target_properties(grpc_protobuf
-            PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOBUF_STATIC_LIB}"
-            INTERFACE_LINK_LIBRARIES "zlib")
-
-    add_library(grpc_protoc STATIC IMPORTED)
-    set_target_properties(grpc_protoc
-            PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOC_STATIC_LIB}"
-            INTERFACE_LINK_LIBRARIES "zlib")
-
-    add_dependencies(grpc grpc_ep)
-    add_dependencies(grpc++ grpc_ep)
-    add_dependencies(grpcpp_channelz grpc_ep)
-    add_dependencies(grpc_protobuf grpc_ep)
-    add_dependencies(grpc_protoc grpc_ep)
-endmacro()
-
-if (MILVUS_WITH_GRPC)
-    resolve_dependency(GRPC)
-
-    get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
-    include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
-    link_directories(SYSTEM ${GRPC_PREFIX}/lib)
-
-    set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
-    include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
-    link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
-endif ()
-
 # ----------------------------------------------------------------------
 # zlib
 
@@ -764,10 +665,10 @@ macro(build_zlib)
 endmacro()
 
 if (MILVUS_WITH_ZLIB)
-    resolve_dependency(ZLIB)
-
-    get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
-    include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+    # resolve_dependency(ZLIB)
+    #
+    # get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+    # include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
 endif ()
 
 # ----------------------------------------------------------------------

core/src/db/CMakeLists.txt

@@ -68,11 +68,9 @@ set( ENGINE_LIBS    pthread
                     libgfortran.a
                     )
 
-set( GRPC_LIB   grpcpp_channelz
+set( GRPC_LIB   libprotobuf
+                grpc++_reflection
                 grpc++
-                grpc
-                grpc_protobuf
-                grpc_protoc
                 )
 
 set( BOOST_LIB  libboost_system.a

core/src/grpc/cpp_gen.sh

 #!/bin/bash
 
-../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" status.proto
+../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" status.proto
 
-../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-status status.proto
+../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-status status.proto
 
-../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" milvus.proto
+../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="../../cmake-build-debug/thirdparty/grpc/grpc-build/grpc_cpp_plugin" milvus.proto
 
-../../cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto
\ No newline at end of file
+../../cmake-build-debug/thirdparty/grpc/grpc-build/third_party/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -170,7 +170,7 @@ if ("${MAKE}" STREQUAL "")
     endif ()
 endif ()
 
-set(MAKE_BUILD_ARGS "-j8")
+set(MAKE_BUILD_ARGS "-j6")
 
 
 # ----------------------------------------------------------------------

core/thirdparty/CMakeLists.txt

@@ -11,7 +11,7 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 #-------------------------------------------------------------------------------
 
-
+string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
 set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}")
 
@@ -24,10 +24,18 @@ endif ()
 # CC/CXX environment variables are captured on the first invocation of the
 # builder (e.g make or ninja) instead of when CMake is invoked into to build
 # directory. This leads to issues if the variables are exported in a subshell
-# and the invocation of make/ninja is in distinct subshell without the same
-# environment (CC/CXX).
-set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
+# and the invocation of make/ninja is in distinct subshell without the same # environment (CC/CXX).
+if ( CCACHE_FOUND )
+    set( EP_C_COMPILER "${CCACHE_FOUND} ${CMAKE_C_COMPILER}" )
+    set( EP_CXX_COMPILER "${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}" )
+else()
+    set( EP_C_COMPILER ${CMAKE_C_COMPILER} )
+    set( EP_CXX_COMPILER ${CMAKE_CXX_COMPILER} )
+endif()
+
+set(EP_COMMON_TOOLCHAIN 
+    -DCMAKE_C_COMPILER=${EP_C_COMPILER}
+    -DCMAKE_CXX_COMPILER=${EP_CXX_COMPILER})
 
 if (CMAKE_AR)
     set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR})
@@ -55,7 +63,14 @@ else ()
     set(EP_LOG_OPTIONS)
 endif ()
 
-set( MAKE_BUILD_ARGS "-j8" )
+set( MAKE_BUILD_ARGS "-j6" )
+
+set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
+# ----------------------------------------------------------------------
+# Find pthreads
+
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
 
 # ****************************** Thirdparty googletest ***************************************
 if (MILVUS_BUILD_TESTS)
@@ -68,8 +83,14 @@ if (MILVUS_BUILD_TESTS)
     add_subdirectory( gtest )
 endif()
 
-# ****************************** Thirdpart Common Config **************************************
-# INCLUDE DIRS need to be set at parent coupe.
-set( THIRDPARTY_INCLUDE_DIRS
-        ${GTEST_INCLUDE_DIR} 
-        PARENT_SCOPE )
+# ****************************** Thirdparty grpc ***************************************
+if (MILVUS_WITH_GRPC)
+    if (DEFINED ENV{MILVUS_GRPC_URL})
+        set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}")
+    else ()
+        set(GRPC_SOURCE_URL
+                "https://github.com/milvus-io/grpc-milvus/archive/${GRPC_VERSION}.zip"
+                )
+    endif ()
+    add_subdirectory( grpc )
+endif ()

core/thirdparty/grpc/CMakeLists.txt

+ # Another way is to use CMake's FetchContent module to clone gRPC at
+# configure time. This makes gRPC's source code available to your project,
+message(STATUS "Building GRPC-${GRPC_VERSION} from source")
+# ******************************************************************
+# FetchContent
+# ******************************************************************
+include(FetchContent)
+FetchContent_Declare(
+    grpc
+    URL     ${GRPC_SOURCE_URL}
+    URL_MD5 "478215c151a144c2d8625b49ff1b70aa"
+    SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-src
+    BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc-build
+    )
+set(FETCHCONTENT_QUIET OFF)
+
+# Compile options for grpc only
+add_compile_options(    ${EP_COMMON_TOOLCHAIN}
+                        "-D CMAKE_BUILD_TYPE=Release"
+                        "-D CMAKE_C_FLAGS=${EP_C_FLAGS}"
+                        "-D CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
+                        "-D CMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                        "-D CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+                        )
+
+# using openSSL rather than boringssl
+set(gRPC_SSL_PROVIDER "package" CACHE STRING "OPENSSL" FORCE)
+# zlib build by grpc
+set( gRPC_ZLIB_PROVIDER "module" )
+FetchContent_GetProperties( grpc )
+if ( NOT grpc_POPULATED )
+
+    FetchContent_Populate( grpc
+        )
+
+    # Adding the following targets:
+    # libprotobuf
+    # grpc++_reflection
+    # protoc (excutable)
+    # grpc++
+    add_subdirectory( ${grpc_SOURCE_DIR}
+                      ${grpc_BINARY_DIR}
+                      EXCLUDE_FROM_ALL
+                      )
+endif()
+
+# get grpc COMPILE_OPTIONS )
+get_property(var DIRECTORY "${grpc_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
+message(STATUS "grpc compile options: ${var}")
+
+
+# **************************************************************************
+# TODO grpc file dependee
+# generate cc files
+# **************************************************************************
+set( PROTOC_EXCUTABLE           $<TARGET_FILE:protoc> )
+set( GRPC_CPP_PLUGIN_EXCUTABLE  $<TARGET_FILE:grpc_cpp_plugin> )
+set( PROTO_PATH          "${MILVUS_SOURCE_DIR}/src/grpc" )
+
+# Proto file
+get_filename_component( milvus_proto         "${PROTO_PATH}/milvus.proto"    ABSOLUTE )
+get_filename_component( milvus_proto_path    "${PROTO_PATH}" PATH )
+
+get_filename_component( status_proto         "${PROTO_PATH}/status.proto"    ABSOLUTE )
+get_filename_component( status_proto_path    "${PROTO_PATH}"   PATH )
+
+
+# Generated sources
+set(milvus_proto_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.cc")
+set(milvus_proto_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.pb.h")
+set(milvus_grpc_srcs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.cc")
+set(milvus_grpc_hdrs "${PROTO_PATH}/gen-milvus-test/helloworld.grpc.pb.h")
+
+set(status_proto_srcs "${PROTO_PATH}/gen-status-test/helloworld.pb.cc")
+set(status_proto_hdrs "${PROTO_PATH}/gen-status-test/helloworld.pb.h")
+set(status_grpc_srcs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.cc")
+set(status_grpc_hdrs "${PROTO_PATH}/gen-status-test/helloworld.grpc.pb.h")
+
+add_custom_command(
+      OUTPUT "${milvus_proto_srcs}" "${milvus_proto_hdrs}" "${milvus_grpc_srcs}" "${milvus_grpc_hdrs}"
+      COMMAND ${PROTOC_EXCUTABLE}
+      ARGS  --grpc_out  "${PROTO_PATH}/gen-milvus-test"
+            --cpp_out   "${PROTO_PATH}/gen-milvus-test"
+            -I          "${milvus_proto_path}"
+            --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}"
+            "${milvus_proto}"
+      DEPENDS "${milvus_proto}" )
+
+add_custom_command(
+      OUTPUT "${status_proto_srcs}" "${status_proto_hdrs}" "${status_grpc_srcs}" "${status_grpc_hdrs}"
+      COMMAND ${PROTOC_EXCUTABLE}
+      ARGS  --grpc_out  "${PROTO_PATH}/gen-status-test"
+            --cpp_out   "${PROTO_PATH}/gen-status-test"
+            -I          "${status_proto_path}"
+            --plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN_EXCUTABLE}"
+            "${status_proto}"
+      DEPENDS "${status_proto}" )
+
+# Include generated *.pb.h files
+include_directories("${MILVUS_SOURCE_DIR}/gen-milvus-test")
+include_directories("${MILVUS_SOURCE_DIR}/gen-status-test")
+
+

core/thirdparty/gtest/CMakeLists.txt

@@ -67,9 +67,20 @@ execute_process(COMMAND             "${CMAKE_COMMAND}" --build .
                 WORKING_DIRECTORY   ${gtest_DOWNLOAD_DIR}
 )
 
+add_compile_options(    ${EP_COMMON_TOOLCHAIN}
+                        "-D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+                        "-D CMAKE_C_FLAGS=${EP_C_FLAGS}"
+                        "-D CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}"
+                        "-D CMAKE_CXX_FLAGS=${EP_CXX_FLAGS}"
+                        "-D CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}"
+)
 # Add googletest directly to our build. This adds the following targets:
 # gtest, gtest_main, gmock and gmock_main
 add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/googletest-src"
                  "${CMAKE_CURRENT_BINARY_DIR}/googletest-build"
                  EXCLUDE_FROM_ALL
 )
+
+# get_property(var DIRECTORY PROPERTY COMPILE_OPTIONS )
+get_property(var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
+message(STATUS "gtest compile options: ${var}")