diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 9577837bfd98af680d408788a500eb964c783daa..d4132e9ff1dc6534fbff06f361f3228757c81062 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 082433650c2ed70bc4a27210d08ea87c7e01566e..0b2f90fd638674d8be44ef58899f6cc45abfc5a0 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7f59561f82d61dd076164db49cd87940d5ca6d5b..5458cf7632b642857786673c1af0374d747446cb 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a11d40d7c96b3cbcfeeb7866eff94e24c496acb3..d6aa72860d061a465fb4aa10c329d6f5e4a3df64 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -28,6 +28,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-206 - Support SQ8 index type - MS-208 - Add buildinde interface for C++ SDK - MS-212 - Support Inner product metric type +- MS-241 - Build Faiss with MKL if using Intel CPU; else build with OpenBlas +- MS-242 - clean up cmake and change MAKE_BUILD_ARGS to be user defined variable ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/README.md b/cpp/README.md index 1b2f507db253250d52c2d892a7038a752c5615ec..b46515a1cf5d2a29d5f2e466aa3296c696687f2d 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,13 +1,13 @@ ### Compilation #### Step 1: install necessery tools - Install MySQL - centos7 : - yum install gfortran qt4 flex bison mysql-devel + yum install gfortran qt4 flex bison mysql-devel mysql ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client + + cd scripts && sudo ./requirements.sh If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 147663d0db7a444d20fbf24278ca0a4a69718a34..6a903f5806ec465f485c9b43b173974f1362261c 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -57,8 +57,6 @@ define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF) -define_option(MILVUS_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) - define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ Note that this requires linking Boost statically" ON) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9e19b037f8838a61e2ce5259b0541c7374d7de28..d08edceaf0d82a7bffa4102e65ac9638c8583823 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -5,7 +5,6 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, @@ -96,12 +95,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) - #message(STATUS "${DEPENDENCY_NAME} package not found") + #disable find_package for now build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -117,11 +112,9 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") -if(NOT MSVC) - # Set -fPIC on all external projects - set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") - set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") -endif() +# Set -fPIC on all external projects +set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") +set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") # CC/CXX environment variables are captured on the first invocation of the # builder (e.g make or ninja) instead of when CMake is invoked into to build @@ -159,20 +152,13 @@ endif() # Ensure that a default make is set if("${MAKE}" STREQUAL "") - if(NOT MSVC) - find_program(MAKE make) - endif() + find_program(MAKE make) endif() -set(MAKE_BUILD_ARGS "-j2") - -## Using make -j in sub-make is fragile -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() +if (NOT DEFINED MAKE_BUILD_ARGS) + set(MAKE_BUILD_ARGS "-j8") +endif() +message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}") # ---------------------------------------------------------------------- # Find pthreads @@ -285,7 +271,6 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") else () set(PROMETHEUS_SOURCE_URL - #"https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz" https://github.com/jupp0r/prometheus-cpp.git) endif() @@ -347,6 +332,7 @@ if(DEFINED ENV{MILVUS_AWS_URL}) else() set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif() + # ---------------------------------------------------------------------- # ARROW @@ -354,19 +340,13 @@ macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(ARROW_CUDA_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") - set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=ON @@ -375,8 +355,6 @@ macro(build_arrow) "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs" -DCMAKE_BUILD_TYPE=Release) -# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - externalproject_add(arrow_ep GIT_REPOSITORY ${ARROW_SOURCE_URL} @@ -384,14 +362,8 @@ macro(build_arrow) ${ARROW_VERSION} GIT_SHALLOW TRUE -# SOURCE_DIR -# ${ARROW_PREFIX} -# BINARY_DIR -# ${ARROW_PREFIX} SOURCE_SUBDIR cpp -# COMMAND -# "export \"ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}\"" ${EP_LOG_OPTIONS} CMAKE_ARGS ${ARROW_CMAKE_ARGS} @@ -400,21 +372,16 @@ macro(build_arrow) ${MAKE_BUILD_ARGS} INSTALL_COMMAND ${MAKE} install -# BUILD_IN_SOURCE -# 1 BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" -# "${ARROW_CUDA_STATIC_LIB}" ) -# ExternalProject_Add_StepDependencies(arrow_ep build thrift_ep) - file(MAKE_DIRECTORY "${ARROW_PREFIX}/include") add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") -# INTERFACE_LINK_LIBRARIES thrift) + add_dependencies(arrow arrow_ep) set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") @@ -438,9 +405,6 @@ endif() # Add Boost dependencies (code adapted from Apache Kudu (incubating)) set(Boost_USE_MULTITHREADED ON) -if(MSVC AND MILVUS_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() set(Boost_ADDITIONAL_VERSIONS "1.70.0" "1.70" @@ -530,59 +494,8 @@ if(MILVUS_BOOST_VENDORED) add_dependencies(boost_filesystem_static boost_ep) add_dependencies(boost_serialization_static boost_ep) -else() - if(MSVC) - # disable autolinking in boost - add_definitions(-DBOOST_ALL_NO_LIB) - endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - - if(MILVUS_BOOST_USE_SHARED) - # Find shared Boost libraries. - set(Boost_USE_STATIC_LIBS OFF) - set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS ON) - - if(MSVC) - # force all boost libraries to dynamic link - add_definitions(-DBOOST_ALL_DYN_LINK) - endif() - - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) - unset(BUILD_SHARED_LIBS_KEEP) - else() - # Find static boost headers and libs - # TODO Differentiate here between release and debug builds - set(Boost_USE_STATIC_LIBS ON) - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - endif() endif() -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - include_directories(SYSTEM ${Boost_INCLUDE_DIR}) link_directories(SYSTEM ${BOOST_LIB_DIR}) @@ -726,13 +639,6 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(MILVUS_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # LAPACK @@ -770,16 +676,23 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(MILVUS_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # FAISS +set(BUILD_FAISS_WITH_MKL false) + +if(EXISTS "/proc/cpuinfo") + FILE(READ /proc/cpuinfo PROC_CPUINFO) + + SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") + STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") + STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") + + if(${VENDOR_ID} STREQUAL "GenuineIntel") + set(BUILD_FAISS_WITH_MKL true) + endif() +endif() + macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") @@ -787,33 +700,35 @@ macro(build_faiss) set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") -# add_custom_target(faiss_dependencies) -# add_dependencies(faiss_dependencies openblas_ep) -# add_dependencies(faiss_dependencies openblas) -# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) -# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}" - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) -# if(OPENBLAS_STATIC_LIB) -# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) -# else() -# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) -# endif() -# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) + set(FAISS_CFLAGS ${EP_C_FLAGS}) + set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + message(STATUS "Build Faiss with MKL") + if(NOT DEFINED MKL_LIB_PATH) + set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") + message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") + endif() + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "CPPFLAGS=-DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" + "LDFLAGS=-L${MKL_LIB_PATH}" + "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") + + else() + message(STATUS "Build Faiss with OpenBlas/LAPACK") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") + endif() if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" -# "with_cuda_arch=\"-gencode=arch=compute_35,code=compute_35 \\ -# -gencode=arch=compute_52,code=compute_52 \\ -# -gencode=arch=compute_60,code=compute_60 \\ -# -gencode=arch=compute_61,code=compute_61\"" "--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\"" "--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\"" "--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\"" @@ -830,58 +745,66 @@ macro(build_faiss) CONFIGURE_COMMAND "./configure" ${FAISS_CONFIGURE_ARGS} -# BINARY_DIR -# ${FAISS_PREFIX} -# INSTALL_DIR -# ${FAISS_PREFIX} -# BUILD_COMMAND -# ${MAKE} ${MAKE_BUILD_ARGS} BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} all COMMAND cd gpu && ${MAKE} ${MAKE_BUILD_ARGS} BUILD_IN_SOURCE 1 -# INSTALL_DIR -# ${FAISS_PREFIX} INSTALL_COMMAND ${MAKE} install COMMAND ln -s faiss_ep ../faiss BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) -# DEPENDS -# ${faiss_dependencies}) - - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif() file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") - add_library(faiss STATIC IMPORTED) - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) - + add_library(faiss SHARED IMPORTED) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a + ${MKL_LIB_PATH}/libmkl_gnu_thread.a + ${MKL_LIB_PATH}/libmkl_core.a) + + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" ) + else() + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + endif() + add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + add_dependencies(faiss openblas_ep) + add_dependencies(faiss lapack_ep) + endif() endmacro() if(MILVUS_WITH_FAISS) - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + endif() resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) @@ -926,8 +849,6 @@ macro(build_gtest) set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - - ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -967,13 +888,11 @@ macro(build_gtest) endmacro() if (MILVUS_BUILD_TESTS) - #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) if(NOT GTEST_VENDORED) endif() - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) @@ -1011,32 +930,8 @@ macro(build_lz4) set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") - else() - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") - endif() - endif() - set(LZ4_STATIC_LIB - "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") - set(LZ4_BUILD_COMMAND - BUILD_COMMAND - msbuild.exe - /m - /p:Configuration=${CMAKE_BUILD_TYPE} - /p:Platform=x64 - /p:PlatformToolset=v140 - ${LZ4_RUNTIME_LIBRARY_LINKAGE} - /t:Build - ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) - else() - set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") - #set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh - # "AR=${CMAKE_AR}") - set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) - endif() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) # We need to copy the header in lib to directory outside of the build externalproject_add(lz4_ep @@ -1071,7 +966,6 @@ endmacro() if(MILVUS_WITH_LZ4) resolve_dependency(Lz4) - # TODO: Don't use global includes but rather target_include_directories get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/) include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) @@ -1097,16 +991,8 @@ macro(build_mysqlpp) externalproject_add(mysqlpp_ep URL ${MYSQLPP_SOURCE_URL} -# GIT_REPOSITORY -# ${MYSQLPP_SOURCE_URL} -# GIT_TAG -# ${MYSQLPP_VERSION} -# GIT_SHALLOW -# TRUE ${EP_LOG_OPTIONS} CONFIGURE_COMMAND -# "./bootstrap" -# COMMAND "./configure" ${MYSQLPP_CONFIGURE_ARGS} BUILD_COMMAND @@ -1167,10 +1053,6 @@ macro(build_prometheus) ${PROMETHEUS_VERSION} GIT_SHALLOW TRUE -# GIT_CONFIG -# recurse-submodules=true -# URL -# ${PROMETHEUS_SOURCE_URL} ${EP_LOG_OPTIONS} CMAKE_ARGS ${PROMETHEUS_CMAKE_ARGS} @@ -1214,21 +1096,15 @@ if(MILVUS_WITH_PROMETHEUS) resolve_dependency(Prometheus) - # TODO: Don't use global includes but rather target_include_directories - #get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES) - - #get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) - #get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) - #link_directories(${PROMETHEUS_PREFIX}/civetweb_ep-prefix/src/civetweb_ep) endif() # ---------------------------------------------------------------------- @@ -1276,8 +1152,6 @@ if(MILVUS_WITH_ROCKSDB) resolve_dependency(RocksDB) - # TODO: Don't use global includes but rather target_include_directories -# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) endif() @@ -1326,34 +1200,9 @@ macro(build_snappy) endmacro() if(MILVUS_WITH_SNAPPY) -# if(Snappy_SOURCE STREQUAL "AUTO") -# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy -# # errornously places them in ${CMAKE_ROOT}/Modules/ -# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong -# # location. -# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 -# # https://src.fedoraproject.org/rpms/snappy/pull-request/1 -# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt) -# endif() -# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) -# build_snappy() -# endif() -# elseif(Snappy_SOURCE STREQUAL "BUNDLED") -# build_snappy() -# elseif(Snappy_SOURCE STREQUAL "SYSTEM") -# # SnappyConfig.cmake is not installed on Ubuntu/Debian -# # TODO: Make a bug report upstream -# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt REQUIRED) -# endif() -# endif() resolve_dependency(Snappy) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -1425,75 +1274,11 @@ macro(build_sqlite_orm) endif () - #set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep") - #set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm") - -# set(SQLITE_ORM_STATIC_LIB -# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}") -# -# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14") -# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14") -# -# set(SQLITE_ORM_CMAKE_ARGS -# ${EP_COMMON_CMAKE_ARGS} -# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}" -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# -DSqliteOrm_BuildTests=off -# -DBUILD_TESTING=off) -# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}") -# -# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}") - -# externalproject_add(sqlite_orm_ep -# URL -# ${SQLITE_ORM_SOURCE_URL} -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix -# CONFIGURE_COMMAND -# "" -# BUILD_COMMAND -# "" -# INSTALL_COMMAND -# "" - #${EP_LOG_OPTIONS} - #${EP_LOG_OPTIONS} -# CMAKE_ARGS -# ${SQLITE_ORM_CMAKE_ARGS} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# BUILD_IN_SOURCE -# 1 -# BUILD_BYPRODUCTS -# "${SQLITE_ORM_STATIC_LIB}" -# ) -# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep) - - #set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h) -# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}") -# add_library(sqlite_orm STATIC IMPORTED) -## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}") -# set_target_properties( -# sqlite_orm -# PROPERTIES -# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}") -# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR}) -# target_link_libraries(sqlite_orm INTERFACE sqlite) -# -# add_dependencies(sqlite_orm sqlite_orm_ep) endmacro() if(MILVUS_WITH_SQLITE_ORM) resolve_dependency(SQLite_ORM) -# ExternalProject_Get_Property(sqlite_orm_ep source_dir) -# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep) include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") - #message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}") endif() # ---------------------------------------------------------------------- @@ -1533,18 +1318,7 @@ macro(build_thrift) endif() set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") - else() - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") - endif() - endif() - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - endif() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1555,60 +1329,6 @@ macro(build_thrift) endif() set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) - if(MSVC) - set(WINFLEXBISON_VERSION 2.4.9) - set(WINFLEXBISON_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") - externalproject_add( - winflexbison_ep - URL - https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip - URL_HASH - MD5=a2e979ea9928fbf8567e995e9c0df765 - SOURCE_DIR - ${WINFLEXBISON_PREFIX} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - "" - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) - - set(THRIFT_CMAKE_ARGS - "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" - "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" - "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" - "-DWITH_SHARED_LIB=OFF" - "-DWITH_PLUGIN=OFF" - ${THRIFT_CMAKE_ARGS}) - elseif(APPLE) - # Some other process always resets BISON_EXECUTABLE to the system default, - # thus we use our own variable here. - if(NOT DEFINED THRIFT_BISON_EXECUTABLE) - find_package(BISON 2.5.1) - - # In the case where we cannot find a system-wide installation, look for - # homebrew and ask for its bison installation. - if(NOT BISON_FOUND) - find_program(BREW_BIN brew) - if(BREW_BIN) - execute_process(COMMAND ${BREW_BIN} --prefix bison - OUTPUT_VARIABLE BISON_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) - set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") - find_package(BISON 2.5.1) - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - else() - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - endif() - set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" - ${THRIFT_CMAKE_ARGS}) - endif() - externalproject_add(thrift_ep URL ${THRIFT_SOURCE_URL} @@ -1637,8 +1357,7 @@ endmacro() if(MILVUS_WITH_THRIFT) resolve_dependency(Thrift) - # TODO: Don't use global includes but rather target_include_directories -# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/) + link_directories(SYSTEM ${THRIFT_PREFIX}/lib/) link_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep-build/lib) include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1684,8 +1403,7 @@ endmacro() if(MILVUS_WITH_YAMLCPP) resolve_dependency(yaml-cpp) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) @@ -1697,15 +1415,7 @@ endif() macro(build_zlib) message(STATUS "Building ZLIB-${ZLIB_VERSION} from source") set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep") - if(MSVC) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) - else() - set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) - endif() - else() - set(ZLIB_STATIC_LIB_NAME libz.a) - endif() + set(ZLIB_STATIC_LIB_NAME libz.a) set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) @@ -1734,8 +1444,7 @@ endmacro() if(MILVUS_WITH_ZLIB) resolve_dependency(ZLIB) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) endif() @@ -1757,22 +1466,15 @@ macro(build_zstd) -DZSTD_BUILD_STATIC=on -DZSTD_MULTITHREAD_SUPPORT=off) - if(MSVC) - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib") - if(MILVUS_USE_STATIC_CRT) - set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") - endif() - else() - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error - set(ZSTD_CMAKE_ARGS - ${ZSTD_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - endif() + + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") + + set(ZSTD_CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) if(CMAKE_VERSION VERSION_LESS 3.7) message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") @@ -1806,8 +1508,7 @@ endmacro() if(MILVUS_WITH_ZSTD) resolve_dependency(ZSTD) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ZSTD_PREFIX}/lib) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) @@ -1823,7 +1524,7 @@ macro(build_aws) ${EP_COMMON_TOOLCHAIN} "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_ONLY=s3 -DBUILD_SHARED_LIBS=off -DENABLE_TESTING=off @@ -1834,8 +1535,7 @@ macro(build_aws) "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWS_CPP_SDK_S3_STATIC_LIB "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error + set(AWS_CMAKE_ARGS ${AWS_CMAKE_ARGS} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -1843,10 +1543,6 @@ macro(build_aws) -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - if(CMAKE_VERSION VERSION_LESS 3.7) - message(FATAL_ERROR "Building AWS using ExternalProject requires at least CMake 3.7") - endif() - externalproject_add(aws_ep ${EP_LOG_OPTIONS} CMAKE_ARGS @@ -1861,8 +1557,6 @@ macro(build_aws) BUILD_BYPRODUCTS "${AWS_CPP_SDK_S3_STATIC_LIB}" "${AWS_CPP_SDK_CORE_STATIC_LIB}") - - file(MAKE_DIRECTORY "${AWS_PREFIX}/include") add_library(aws-cpp-sdk-s3 STATIC IMPORTED) @@ -1885,8 +1579,7 @@ endmacro() if(MILVUS_WITH_AWS) resolve_dependency(AWS) - - # TODO: Don't use global includes but rather target_include_directories + link_directories(SYSTEM ${AWS_PREFIX}/lib) get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/scripts/requirements.sh b/cpp/scripts/requirements.sh new file mode 100755 index 0000000000000000000000000000000000000000..5f8b74ad2c35970e396a719f77b8aa2e87ca4218 --- /dev/null +++ b/cpp/scripts/requirements.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB +apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB + +sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' +apt -y update && apt-get -y install intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 + +#sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' +#source /etc/profile diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 00e2e1e87d52bee259ff39b550b49dc1bd89bf90..5bea7818c95e6828a3fe629db566f82727922390 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -48,7 +48,6 @@ set(engine_files ${db_files} ${db_scheduler_files} ${wrapper_files} -# metrics/Metrics.cpp ${metrics_files} ) @@ -71,8 +70,6 @@ set(third_party_libs yaml-cpp libgpufaiss.a faiss - lapack - openblas prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core @@ -87,9 +84,19 @@ set(third_party_libs mysqlpp ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) + if (MEGASEARCH_WITH_ARROW STREQUAL "ON") set(third_party_libs ${third_party_libs} arrow) -endif() +endif() +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(third_party_libs ${third_party_libs} + ${MKL_LIBS} + ${MKL_LIBS}) +else() + set(third_party_libs ${third_party_libs} + lapack + openblas) +endif() if (GPU_VERSION STREQUAL "ON") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -187,6 +194,6 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION lib) #need to copy libmysqlpp.so + DESTINATION lib) add_subdirectory(sdk) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 311760948daa31f617a197852084b66bbe25290f..530e9feeec8dc2338569f724137306a419627e04 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -3,6 +3,7 @@ BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 FAISS_VERSION=7b07685 +MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 62e32f6d1db5dee8ae7bf3b2690b24af7515bc41..6a6f94a6321635137a4f0b02d98680aa75a913c6 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -28,7 +28,6 @@ set(unittest_libs easyloggingpp pthread metrics - openblas gfortran prometheus-cpp-pull prometheus-cpp-push diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index c0f3052848b7b966060c27f23309500a05aaba03..736219952cdf969104daa8bc77f6d78dbaf250d5 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -23,10 +23,7 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src - #${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} @@ -49,6 +46,11 @@ set(db_libs mysqlpp ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(db_test ${db_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION bin) + diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index ff2535e3fd676144353756295fb5ab62e835cc43..c4392505449dfe17678e7d95b4b95f6adbaf85fd 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -35,6 +35,10 @@ set(wrapper_libs zstd lz4 ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) set(topk_test_src diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 513b37b8c2543e1aa3908e4c0f194d4d4e712d3b..418544d0cafd6a7431020d91b58b56bc5a575992 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -10,8 +10,6 @@ include_directories(../../src) - - aux_source_directory(../../src/db db_srcs) aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) @@ -33,21 +31,10 @@ include_directories(../../third_party/build/include) link_directories(../../third_party/build/lib) include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") -#include_directories(../db/utils.h) include_directories(../../src/metrics) include_directories(/usr/include/mysql) -#set(metrics_src_files -# ../../src/metrics/Metrics.cpp -# ../../src/metrics/Metrics.h -# ../../src/metrics/PrometheusMetrics.cpp -# ../../src/metrics/MetricBase.h -# ../../src/server/ServerConfig.cpp -# ../../src/utils/CommonUtil.cpp -# ../../src/utils/TimeRecorder.cpp -# ) - set(count_test_src ${config_files} ${cache_srcs} @@ -77,5 +64,8 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) +endif() install(TARGETS metrics_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index e49aed500ce1bd5e36d6b6dd817cc7c13d75c2b6..40f5b1012884a820938c9a32c8dce92c5c8a4ed9 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -48,6 +48,10 @@ set(require_libs pthread ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(server_test ${require_libs} ${cuda_library}