MS-249 Check machine hardware during initialize

Former-commit-id: ef7a11e20179fa803588f2d8add4232bf4bb4a3c

cpp/CHANGELOG.md

@@ -36,6 +36,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-256 - Add more cache config
 - MS-260 - Refine log
 - MS-249 - Check machine hardware during initialize
+- MS-261 - Update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
 
 ## New Feature
 - MS-180 - Add new mem manager

cpp/build.sh

@@ -8,8 +8,9 @@ MAKE_CLEAN="OFF"
 BUILD_COVERAGE="OFF"
 DB_PATH="/opt/milvus"
 PROFILING="OFF"
+BUILD_FAISS_WITH_MKL="OFF"
 
-while getopts "p:d:t:uhlrcg" arg
+while getopts "p:d:t:uhlrcgm" arg
 do
         case $arg in
              t)
@@ -40,6 +41,9 @@ do
              g)
                 PROFILING="ON"
                 ;;
+             m)
+                BUILD_FAISS_WITH_MKL="ON"
+                ;;
              h) # help
                 echo "
 
@@ -52,9 +56,10 @@ parameter:
 -r: remove previous build directory(default: OFF)
 -c: code coverage(default: OFF)
 -g: profiling(default: OFF)
+-m: build faiss with MKL(default: OFF)
 
 usage:
-./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
+./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m]
                 "
                 exit 0
                 ;;
@@ -83,6 +88,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
     -DBUILD_COVERAGE=${BUILD_COVERAGE} \
     -DMILVUS_DB_PATH=${DB_PATH} \
     -DMILVUS_ENABLE_PROFILING=${PROFILING} \
+    -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
     $@ ../"
     echo ${CMAKE_CMD}
 

cpp/cmake/ThirdPartyPackages.cmake

@@ -699,7 +699,9 @@ endmacro()
 # ----------------------------------------------------------------------
 # FAISS
 
-set(BUILD_FAISS_WITH_MKL false)
+if(NOT DEFINED BUILD_FAISS_WITH_MKL)
+    set(BUILD_FAISS_WITH_MKL OFF)
+endif()
 
 if(EXISTS "/proc/cpuinfo")
     FILE(READ /proc/cpuinfo PROC_CPUINFO)
@@ -708,8 +710,8 @@ if(EXISTS "/proc/cpuinfo")
     STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
     STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
 
-    if(${VENDOR_ID} STREQUAL "GenuineIntel")
-        set(BUILD_FAISS_WITH_MKL true)
+    if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
+        set(BUILD_FAISS_WITH_MKL OFF)
     endif()
 endif()
 
@@ -729,7 +731,7 @@ macro(build_faiss)
     set(FAISS_CFLAGS ${EP_C_FLAGS})
     set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
 
-    if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
         message(STATUS "Build Faiss with MKL")
         if(NOT DEFINED MKL_LIB_PATH)
             set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64")
@@ -750,10 +752,10 @@ macro(build_faiss)
     if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
         set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
                 "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
-                "--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\""
-                "--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\""
-                "--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\""
-                "--with-cuda-arch=\"-gencode=arch=compute_61,code=compute_61\""
+                "--with-cuda-arch=\"-gencode=arch=compute_35,code=sm_35\""
+                "--with-cuda-arch=\"-gencode=arch=compute_52,code=sm_52\""
+                "--with-cuda-arch=\"-gencode=arch=compute_60,code=sm_60\""
+                "--with-cuda-arch=\"-gencode=arch=compute_61,code=sm_61\""
                 )
     else()
         set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
@@ -767,26 +769,22 @@ macro(build_faiss)
             "./configure"
             ${FAISS_CONFIGURE_ARGS}
             BUILD_COMMAND
-            ${MAKE} ${MAKE_BUILD_ARGS} all
-            COMMAND
-            cd gpu && ${MAKE} ${MAKE_BUILD_ARGS}
+            ${MAKE} ${MAKE_BUILD_ARGS}
             BUILD_IN_SOURCE
             1
             INSTALL_COMMAND
             ${MAKE} install
-            COMMAND
-            ln -s faiss_ep ../faiss
             BUILD_BYPRODUCTS
             ${FAISS_STATIC_LIB})
         
-    if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
+    if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
         ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
     endif()
 
     file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
     add_library(faiss SHARED IMPORTED)
 
-    if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+    if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
         set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a
                      ${MKL_LIB_PATH}/libmkl_gnu_thread.a
                      ${MKL_LIB_PATH}/libmkl_core.a)
@@ -806,7 +804,7 @@ macro(build_faiss)
             
     add_dependencies(faiss faiss_ep)
 
-    if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
+    if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
         add_dependencies(faiss openblas_ep)
         add_dependencies(faiss lapack_ep)
     endif()
@@ -815,7 +813,7 @@ endmacro()
 
 if(MILVUS_WITH_FAISS)
 
-    if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
+    if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
         resolve_dependency(OpenBLAS)
         get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
         include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
@@ -830,10 +828,7 @@ if(MILVUS_WITH_FAISS)
     resolve_dependency(FAISS)
     get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
     include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
-    include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/")
-    link_directories(SYSTEM ${FAISS_PREFIX}/)
     link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
-    link_directories(SYSTEM ${FAISS_PREFIX}/gpu/)
 endif()
 
 # ----------------------------------------------------------------------

cpp/src/CMakeLists.txt

@@ -72,7 +72,6 @@ set(third_party_libs
         sqlite
         thrift
         yaml-cpp
-        libgpufaiss.a
         faiss
         prometheus-cpp-push
         prometheus-cpp-pull
@@ -94,7 +93,7 @@ set(third_party_libs
 if (MEGASEARCH_WITH_ARROW STREQUAL "ON")
     set(third_party_libs ${third_party_libs} arrow)
 endif()
-if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     set(third_party_libs ${third_party_libs}
             ${MKL_LIBS}
             ${MKL_LIBS})

cpp/thirdparty/versions.txt

@@ -2,7 +2,7 @@ ARROW_VERSION=zilliz
 BOOST_VERSION=1.70.0
 BZIP2_VERSION=1.0.6
 EASYLOGGINGPP_VERSION=v9.96.7
-FAISS_VERSION=7b07685
+FAISS_VERSION=v1.5.3
 MKL_VERSION=2019.4.243
 GTEST_VERSION=1.8.1
 JSONCONS_VERSION=0.126.0

cpp/unittest/db/CMakeLists.txt

@@ -39,7 +39,6 @@ set(db_test_src
 cuda_add_executable(db_test ${db_test_src})
 
 set(db_libs
-        libgpufaiss.a
         faiss
         cudart
         cublas
@@ -50,7 +49,7 @@ set(db_libs
         mysqlpp
         )
 
-if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
 endif()
 

cpp/unittest/faiss_wrapper/CMakeLists.txt

@@ -28,7 +28,6 @@ set(wrapper_libs
         stdc++
         boost_system_static
         boost_filesystem_static
-        libgpufaiss.a
         faiss
         cudart
         cublas
@@ -39,7 +38,7 @@ set(wrapper_libs
         zstd
         lz4
         )
-if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS})
 endif()
 

cpp/unittest/metrics/CMakeLists.txt

@@ -53,7 +53,6 @@ set(count_test_src
 add_executable(metrics_test ${count_test_src} ${require_files} )
 
 target_link_libraries(metrics_test
-        libgpufaiss.a
         faiss
         cudart
         cublas
@@ -68,7 +67,7 @@ target_link_libraries(metrics_test
         mysqlpp
         ${unittest_libs}
         )
-if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
 endif()
 

cpp/unittest/server/CMakeLists.txt

@@ -34,7 +34,6 @@ cuda_add_executable(server_test
 
 set(require_libs
         stdc++
-        libgpufaiss.a
         faiss
         cudart
         cublas
@@ -49,7 +48,7 @@ set(require_libs
         pthread
         )
 
-if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
+if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
     set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
 endif()