merge change from another pr

ci/scripts/build.sh

@@ -15,6 +15,7 @@ CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
 BUILD_TYPE="Debug"
 BUILD_UNITTEST="OFF"
 INSTALL_PREFIX="/opt/milvus"
+FAISS_ROOT=""
 BUILD_COVERAGE="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
@@ -22,7 +23,7 @@ GPU_VERSION="OFF"
 WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "o:t:b:gulcjmh" arg
+while getopts "o:t:b:f:gulcjmh" arg
 do
         case $arg in
              o)
@@ -34,6 +35,9 @@ do
              b)
                 CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
                 ;;
+             f)
+                FAISS_ROOT=$OPTARG # FAISS ROOT PATH
+                ;;
              g)
                 GPU_VERSION="ON";
                 ;;
@@ -60,6 +64,7 @@ parameter:
 -o: install prefix(default: /opt/milvus)
 -t: build type(default: Debug)
 -b: core code build directory
+-f: faiss root path
 -g: gpu version
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
@@ -69,7 +74,7 @@ parameter:
 -h: help
 
 usage:
-./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} [-u] [-l] [-c] [-j] [-m] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -94,8 +99,10 @@ CMAKE_CMD="cmake \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
--DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \
+-DFAISS_SOURCE=AUTO \
 ${MILVUS_CORE_DIR}"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

core/build.sh

@@ -14,16 +14,10 @@ CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 GPU_VERSION="OFF" #defaults to CPU version
 WITH_MKL="OFF"
+FAISS_ROOT=""
+FAISS_SOURCE="BUNDLED"
 
-CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
-wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
-if [ $? -eq 0 ]; then
-  CUSTOMIZATION="ON"
-else
-  CUSTOMIZATION="OFF"
-fi
-
-while getopts "p:d:t:ulrcgjhxzm" arg
+while getopts "p:d:t:f:ulrcgjhxzm" arg
 do
         case $arg in
              p)
@@ -35,6 +29,10 @@ do
              t)
                 BUILD_TYPE=$OPTARG # BUILD_TYPE
                 ;;
+             f)
+                FAISS_ROOT=$OPTARG
+                FAISS_SOURCE="AUTO"
+                ;;
              u)
                 echo "Build and run unittest cases" ;
                 BUILD_UNITTEST="ON";
@@ -73,6 +71,7 @@ parameter:
 -p: install prefix(default: $(pwd)/milvus)
 -d: db data path(default: /tmp/milvus)
 -t: build type(default: Debug)
+-f: faiss root path(default: empty)
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -r: remove previous build directory(default: OFF)
@@ -84,7 +83,7 @@ parameter:
 -h: help
 
 usage:
-./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
+./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -109,13 +108,14 @@ CMAKE_CMD="cmake \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DFAISS_ROOT=${FAISS_ROOT} \
+-DFAISS_SOURCE=${FAISS_SOURCE} \
 -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
 -DMILVUS_DB_PATH=${DB_PATH} \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
--DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
 -DMILVUS_GPU_VERSION=${GPU_VERSION} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 ../"

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -225,28 +225,12 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     set(${_LIB_NAME} "${_LIB_VERSION}")
 endforeach ()
 
-if (CUSTOMIZATION)
-    execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code)
-    message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}")
-    if (NOT return_code EQUAL 0)
-        MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}")
-    else ()
-        set(FAISS_SOURCE_URL ${FAISS_URL})
-        # set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
-        # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
-        # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
-        # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
-        # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
-        # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
-        # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
-        # set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
-        set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0
-    endif ()
+if (DEFINED ENV{FAISS_SOURCE_URL})
+    set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
 else ()
-    set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz")
+    set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz")
     set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50")
 endif ()
-message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}")
 
 if (DEFINED ENV{KNOWHERE_ARROW_URL})
     set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")

core/src/index/thirdparty/versions.txt

@@ -3,5 +3,5 @@ BOOST_VERSION=1.70.0
 GTEST_VERSION=1.8.1
 LAPACK_VERSION=v3.8.0
 OPENBLAS_VERSION=v0.3.6
-FAISS_VERSION=branch-0.3.0
+FAISS_VERSION=1.6.0
 MKL_VERSION=2019.5.281