diff --git a/core/build.sh b/core/build.sh index e844528ad39824feb8ad15f9357866dd96028aac..5abdaf175a09810fdb42c1f9efb5b3ca336e1b98 100755 --- a/core/build.sh +++ b/core/build.sh @@ -117,7 +117,7 @@ CMAKE_CMD="cmake \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ --DBUILD_FAISS_WITH_MKL=${WITH_MKL} \ +-DFAISS_WITH_MKL=${WITH_MKL} \ ../" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 79b5e0f1da8ef03a0757e6cc3f57660044396327..4d55fbee1872f9dd8aefac1d0f8f4ead27100888 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -24,7 +24,7 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status) include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus) add_subdirectory(index) -if (BUILD_FAISS_WITH_MKL) +if (FAISS_WITH_MKL) add_compile_definitions("WITH_MKL") endif () diff --git a/core/src/index/build.sh b/core/src/index/build.sh index a77a16cf0bb063f78faa7654fdb4fe1cfa719525..357ac5693ac4aa84d645b5f63b7d277707c60348 100644 --- a/core/src/index/build.sh +++ b/core/src/index/build.sh @@ -5,7 +5,7 @@ BUILD_UNITTEST="OFF" INSTALL_PREFIX=$(pwd)/cmake_build MAKE_CLEAN="OFF" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" +FAISS_WITH_MKL="OFF" USE_JFROG_CACHE="OFF" while getopts "p:d:t:uhrcgmj" arg @@ -31,7 +31,7 @@ do PROFILING="ON" ;; m) - BUILD_FAISS_WITH_MKL="ON" + FAISS_WITH_MKL="ON" ;; j) USE_JFROG_CACHE="ON" @@ -74,7 +74,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ - -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ + -DFAISS_WITH_MKL=${FAISS_WITH_MKL} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ ../" echo ${CMAKE_CMD} diff --git a/core/src/index/cmake/DefineOptionsCore.cmake b/core/src/index/cmake/DefineOptionsCore.cmake index 2f050cdf4ee362769792d879a51df6b52f6ef124..787a9c484ffbf6066dd98b04141a9c8b7915da11 100644 --- a/core/src/index/cmake/DefineOptionsCore.cmake +++ b/core/src/index/cmake/DefineOptionsCore.cmake @@ -79,7 +79,7 @@ define_option(KNOWHERE_WITH_FAISS "Build with FAISS library" ON) define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON) -define_option(BUILD_FAISS_WITH_MKL "Build FAISS with MKL" OFF) +define_option(FAISS_WITH_MKL "Build FAISS with MKL" OFF) #---------------------------------------------------------------------- set_option_category("Test and benchmark") diff --git a/core/src/index/cmake/FindFAISS.cmake b/core/src/index/cmake/FindFAISS.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e0e6337bcedae889da2ac66bb55fe76e661a815d --- /dev/null +++ b/core/src/index/cmake/FindFAISS.cmake @@ -0,0 +1,44 @@ +set(FAISS_STATIC_LIB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}) + +# First, find via if specified FAISS_ROOT +if (FAISS_ROOT) + find_library(FAISS_STATIC_LIB + NAMES ${FAISS_STATIC_LIB_NAME} + PATHS ${FAISS_ROOT} + PATH_SUFFIXES "lib" + NO_DEFAULT_PATH + ) + find_path(FAISS_INCLUDE_DIR + NAMES "faiss/Index.h" + PATHS ${FAISS_ROOT} + NO_DEFAULT_PATH + PATH_SUFFIXES "include" + ) +endif () + +find_package_handle_standard_args(FAISS REQUIRED_VARS FAISS_STATIC_LIB FAISS_INCLUDE_DIR) + +if (FAISS_FOUND) + if (NOT TARGET faiss) + add_library(faiss STATIC IMPORTED) + + set_target_properties( + faiss + PROPERTIES + IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + ) + + if (FAISS_WITH_MKL) + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}") + else () + set_target_properties( + faiss + PROPERTIES + INTERFACE_LINK_LIBRARIES "openblas;lapack") + endif () + endif () +endif () diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 60ac4b9eac0306d944c45ded75636828bba6713f..8563e37e87d98d93cd560a925b0bbf2a74a9b8aa 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -21,6 +21,7 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES GTest LAPACK OpenBLAS + MKL ) message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies") @@ -43,6 +44,8 @@ macro(build_dependency DEPENDENCY_NAME) build_openblas() elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS") build_faiss() + elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL") + build_mkl() else () message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () @@ -51,9 +54,9 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") find_package(${DEPENDENCY_NAME} MODULE) - if(NOT ${${DEPENDENCY_NAME}_FOUND}) - build_dependency(${DEPENDENCY_NAME}) - endif() + if (NOT ${${DEPENDENCY_NAME}_FOUND}) + build_dependency(${DEPENDENCY_NAME}) + endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -238,11 +241,11 @@ if (CUSTOMIZATION) # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 # set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0 - endif() -else() + endif () +else () set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz") set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50") -endif() +endif () message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") if (DEFINED ENV{KNOWHERE_ARROW_URL}) @@ -674,42 +677,22 @@ if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) endif () # ---------------------------------------------------------------------- -# FAISS - -macro(build_faiss) - message(STATUS "Building FAISS-${FAISS_VERSION} from source") +# MKL - if (NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) - endif () +macro(build_mkl) - if (EXISTS "/proc/cpuinfo") - FILE(READ /proc/cpuinfo PROC_CPUINFO) + if (FAISS_WITH_MKL) + if (EXISTS "/proc/cpuinfo") + FILE(READ /proc/cpuinfo PROC_CPUINFO) - SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") - STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") - STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") + SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") + STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") + STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") - if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel") - set(BUILD_FAISS_WITH_MKL OFF) + if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel") + set(FAISS_WITH_MKL OFF) + endif () endif () - endif () - - set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") - set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") - set(FAISS_STATIC_LIB - "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(FAISS_CONFIGURE_ARGS - "--prefix=${FAISS_PREFIX}" - "CFLAGS=${EP_C_FLAGS}" - "CXXFLAGS=${EP_CXX_FLAGS}" - --without-python) - - set(FAISS_CFLAGS ${EP_C_FLAGS}) - set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) - - if (BUILD_FAISS_WITH_MKL) find_path(MKL_LIB_PATH NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a" @@ -717,19 +700,38 @@ macro(build_faiss) if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND") message(FATAL_ERROR "Could not find MKL libraries") endif () - message(STATUS "Build Faiss with MKL. MKL lib path = ${MKL_LIB_PATH}") + message(STATUS "MKL lib path = ${MKL_LIB_PATH}") set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a ) + endif () +endmacro() + +# ---------------------------------------------------------------------- +# FAISS + +macro(build_faiss) + message(STATUS "Building FAISS-${FAISS_VERSION} from source") + + set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") + set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include") + set(FAISS_STATIC_LIB + "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(FAISS_CONFIGURE_ARGS + "--prefix=${FAISS_PREFIX}" + "CFLAGS=${EP_C_FLAGS}" + "CXXFLAGS=${EP_CXX_FLAGS}" + --without-python) + + if (FAISS_WITH_MKL) set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" "LDFLAGS=-L${MKL_LIB_PATH}" ) - else () message(STATUS "Build Faiss with OpenBlas/LAPACK") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} @@ -770,7 +772,7 @@ macro(build_faiss) BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) - if (NOT BUILD_FAISS_WITH_MKL) + if (NOT FAISS_WITH_MKL) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) endif () @@ -800,7 +802,7 @@ macro(build_faiss) BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) - if (NOT BUILD_FAISS_WITH_MKL) + if (NOT FAISS_WITH_MKL) ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) endif () @@ -815,7 +817,7 @@ macro(build_faiss) IMPORTED_LOCATION "${FAISS_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" ) - if (BUILD_FAISS_WITH_MKL) + if (FAISS_WITH_MKL) set_target_properties( faiss PROPERTIES @@ -834,7 +836,9 @@ endmacro() if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep) - if (NOT BUILD_FAISS_WITH_MKL) + if (FAISS_WITH_MKL) + resolve_dependency(MKL) + else () resolve_dependency(OpenBLAS) get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index 373965acf01ffa6ef7059b74bb0cad3a162e7000..5f8e4d6970af5bf161af7258e40ac955e7b1ea2a 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -54,7 +54,7 @@ set(depend_libs gfortran pthread ) -if (BUILD_FAISS_WITH_MKL) +if (FAISS_WITH_MKL) set(depend_libs ${depend_libs} "-Wl,--start-group \ ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index 13277aa40eeef12460b58770e550b3a43b27bfae..71636738aaf2cf6cc16309d54b19cfe6a328a147 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -8,7 +8,7 @@ set(depend_libs faiss arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a" ) -if (BUILD_FAISS_WITH_MKL) +if (FAISS_WITH_MKL) set(depend_libs ${depend_libs} "-Wl,--start-group \ ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ diff --git a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt index 6cad5abda5433039e0954396eb2ea2e0158ae9bd..e2fd7a6c85dc3776a220c13b2d483077e6dddee5 100644 --- a/core/src/index/unittest/faiss_benchmark/CMakeLists.txt +++ b/core/src/index/unittest/faiss_benchmark/CMakeLists.txt @@ -15,7 +15,7 @@ if (KNOWHERE_GPU_VERSION) faiss hdf5 arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a ) - if (BUILD_FAISS_WITH_MKL) + if (FAISS_WITH_MKL) set(depend_libs ${depend_libs} "-Wl,--start-group \ ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ diff --git a/core/src/index/unittest/faiss_ori/CMakeLists.txt b/core/src/index/unittest/faiss_ori/CMakeLists.txt index 829a27fd8705596b337b833a5da1d85c724ce7bf..907076ca6e33379cc37f656b5a89cf0d61c213d4 100644 --- a/core/src/index/unittest/faiss_ori/CMakeLists.txt +++ b/core/src/index/unittest/faiss_ori/CMakeLists.txt @@ -10,7 +10,7 @@ if (KNOWHERE_GPU_VERSION) faiss arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a ) - if (BUILD_FAISS_WITH_MKL) + if (FAISS_WITH_MKL) set(depend_libs ${depend_libs} "-Wl,--start-group \ ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \ diff --git a/core/src/index/unittest/test_nsg/CMakeLists.txt b/core/src/index/unittest/test_nsg/CMakeLists.txt index 269c0d21397e684b223a3e6ac779b0c18f4961b6..82dbb5110cb3788e543ab8612d13263a88a27ab4 100644 --- a/core/src/index/unittest/test_nsg/CMakeLists.txt +++ b/core/src/index/unittest/test_nsg/CMakeLists.txt @@ -5,7 +5,7 @@ add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO) find_package(OpenMP REQUIRED) -if (OPENMP_FOUND) +if (OpenMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") else ()