diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 444537fe6b86609a1eec2ed95110d8743c36f5ff..27962ccb543f9c8e246753da20d1f7f6cc64c83d 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -15,6 +15,7 @@ CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" +FAISS_ROOT="" BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -22,7 +23,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:gulcjmh" arg +while getopts "o:t:b:f:gulcjmh" arg do case $arg in o) @@ -34,6 +35,9 @@ do b) CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR ;; + f) + FAISS_ROOT=$OPTARG # FAISS ROOT PATH + ;; g) GPU_VERSION="ON"; ;; @@ -60,6 +64,7 @@ parameter: -o: install prefix(default: /opt/milvus) -t: build type(default: Debug) -b: core code build directory +-f: faiss root path -g: gpu version -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) @@ -69,7 +74,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -94,8 +99,10 @@ CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ --DBUILD_FAISS_WITH_MKL=${WITH_MKL} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_WITH_MKL=${WITH_MKL} \ -DArrow_SOURCE=AUTO \ +-DFAISS_SOURCE=AUTO \ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/build.sh b/core/build.sh index 5abdaf175a09810fdb42c1f9efb5b3ca336e1b98..9b690a02614811a0458db02ca0cddcd884f8edc4 100755 --- a/core/build.sh +++ b/core/build.sh @@ -14,16 +14,10 @@ CUSTOMIZATION="OFF" # default use ori faiss CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version WITH_MKL="OFF" +FAISS_ROOT="" +FAISS_SOURCE="BUNDLED" -CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}" -wget -q --method HEAD ${CUSTOMIZED_FAISS_URL} -if [ $? -eq 0 ]; then - CUSTOMIZATION="ON" -else - CUSTOMIZATION="OFF" -fi - -while getopts "p:d:t:ulrcgjhxzm" arg +while getopts "p:d:t:f:ulrcgjhxzm" arg do case $arg in p) @@ -35,6 +29,10 @@ do t) BUILD_TYPE=$OPTARG # BUILD_TYPE ;; + f) + FAISS_ROOT=$OPTARG + FAISS_SOURCE="AUTO" + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -73,6 +71,7 @@ parameter: -p: install prefix(default: $(pwd)/milvus) -d: db data path(default: /tmp/milvus) -t: build type(default: Debug) +-f: faiss root path(default: empty) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -r: remove previous build directory(default: OFF) @@ -84,7 +83,7 @@ parameter: -h: help usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] +./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h] " exit 0 ;; @@ -109,13 +108,14 @@ CMAKE_CMD="cmake \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ +-DFAISS_ROOT=${FAISS_ROOT} \ +-DFAISS_SOURCE=${FAISS_SOURCE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DCUSTOMIZATION=${CUSTOMIZATION} \ --DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DFAISS_WITH_MKL=${WITH_MKL} \ ../" diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 8563e37e87d98d93cd560a925b0bbf2a74a9b8aa..c046bc3a56d491537e106e4d8a78212bc717832a 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -225,28 +225,12 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach () -if (CUSTOMIZATION) - execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}") - else () - set(FAISS_SOURCE_URL ${FAISS_URL}) - # set(FAISS_MD5 "a589663865a8558205533c8ac414278c") - # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0 - # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0 - # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 - # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 - # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 - # set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 - set(FAISS_MD5 "2293cdb209c3718e3b19f3edae8b32b3") # commit-id a13c1205dc52977a9ad3b33a14efa958604a8bff branch-0.3.0 - endif () +if (DEFINED ENV{FAISS_SOURCE_URL}) + set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}") else () - set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/1.6.0.tar.gz") + set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz") set(FAISS_MD5 "b02c1a53234f5acc9bea1b0c55524f50") endif () -message(STATUS "FAISS URL = ${FAISS_SOURCE_URL}") if (DEFINED ENV{KNOWHERE_ARROW_URL}) set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}") diff --git a/core/src/index/thirdparty/versions.txt b/core/src/index/thirdparty/versions.txt index 380c9dedad0d50f66a2a4a07580f4fcb84b5ae5a..efcef26fa94f1ea509d5fc85049744638c0e8abb 100644 --- a/core/src/index/thirdparty/versions.txt +++ b/core/src/index/thirdparty/versions.txt @@ -3,5 +3,5 @@ BOOST_VERSION=1.70.0 GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 OPENBLAS_VERSION=v0.3.6 -FAISS_VERSION=branch-0.3.0 +FAISS_VERSION=1.6.0 MKL_VERSION=2019.5.281