Merge branch 'update_build_config' into 'branch-0.5.0'

MS-635 update

See merge request megasearch/milvus!733

Former-commit-id: bdf282c9006a7516b20fa70d1abca22d5b7dd1e3

ci/jenkinsfile/milvus_build.groovy

@@ -10,7 +10,12 @@ container('milvus-build-env') {
                         sh "git config --global user.name \"test\""
                         withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
                             sh "./build.sh -l"
-                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c"
+                            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' \
+                            && export JFROG_USER_NAME='${USERNAME}' \
+                            && export JFROG_PASSWORD='${PASSWORD}' \
+                            && export FAISS_URL='http://192.168.1.105:6060/jinhai/faiss/-/archive/branch-0.2.1/faiss-branch-0.2.1.tar.gz' \
+                            && ./build.sh -t ${params.BUILD_TYPE} -j -u -c"
+
                             sh "./coverage.sh -u root -p Fantast1c -t 192.168.1.194"
                         }
                     }

core/CMakeLists.txt

@@ -132,6 +132,10 @@ include(ThirdPartyPackages)
 
 config_summary()
 
+if (CUSTOMIZATION)
+    add_definitions(-DCUSTOMIZATION)
+endif (CUSTOMIZATION)
+
 add_subdirectory(src)
 
 if (BUILD_UNIT_TEST STREQUAL "ON")

core/build.sh

@@ -9,10 +9,16 @@ DB_PATH="/opt/milvus"
 PROFILING="OFF"
 USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
-CUSTOMIZATION="ON"
+CUSTOMIZATION="OFF" # default use ori faiss
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-wget -q --method HEAD
+CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}"
+wget -q --method HEAD ${CUSTOMIZED_FAISS_URL}
+if [ $? -eq 0 ]; then
+  CUSTOMIZATION="ON"
+else
+  CUSTOMIZATION="OFF"
+fi
 
 while getopts "p:d:t:ulrcgjhx" arg
 do
@@ -49,7 +55,7 @@ do
                 USE_JFROG_CACHE="ON"
                 ;;
              x)
-                CUSTOMIZATION="OFF"
+                CUSTOMIZATION="OFF" # force use ori faiss
                 ;;
              h) # help
                 echo "
@@ -94,6 +100,7 @@ CMAKE_CMD="cmake \
 -DMILVUS_ENABLE_PROFILING=${PROFILING} \
 -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
+-DFAISS_URL=${CUSTOMIZED_FAISS_URL} \
 ../"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD}

core/src/CMakeLists.txt

@@ -27,10 +27,6 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
 #this statement must put here, since the CORE_INCLUDE_DIRS is defined in code/CMakeList.txt
 add_subdirectory(index)
 
-if (CUSTOMIZATION)
-    add_definitions(-DCUSTOMIZATION)
-endif (CUSTOMIZATION)
-
 set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
 foreach (dir ${CORE_INCLUDE_DIRS})
     include_directories(${dir})

core/src/index/CMakeLists.txt

@@ -86,10 +86,6 @@ include(DefineOptionsCore)
 include(BuildUtilsCore)
 include(ThirdPartyPackagesCore)
 
-if (CUSTOMIZATION)
-    add_definitions(-DCUSTOMIZATION)
-endif (CUSTOMIZATION)
-
 add_subdirectory(knowhere)
 
 if (BUILD_COVERAGE STREQUAL "ON")

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -233,18 +233,16 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
 endforeach()
 
 if(CUSTOMIZATION)
-    set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
-    # set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
-    # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
-    # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
-    # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
-    set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
-
-    execute_process(COMMAND wget -q --method HEAD ${FAISS_SOURCE_URL} RESULT_VARIABLE return_code)
-    message(STATUS "Check the remote cache file ${FAISS_SOURCE_URL}. return code = ${return_code}")
+    execute_process(COMMAND wget -q --method HEAD ${FAISS_URL} RESULT_VARIABLE return_code)
+    message(STATUS "Check the remote cache file ${FAISS_URL}. return code = ${return_code}")
     if (NOT return_code EQUAL 0)
-        set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")
-        set(CUSTOMIZATION FALSE PARENT_SCOPE)
+        MESSAGE(FATAL_ERROR "Can't access to ${FAISS_URL}")
+    else()
+        # set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
+        # set(FAISS_MD5 "57da9c4f599cc8fa4260488b1c96e1cc") # commit-id 6dbdf75987c34a2c853bd172ea0d384feea8358c branch-0.2.0
+        # set(FAISS_MD5 "21deb1c708490ca40ecb899122c01403") # commit-id 643e48f479637fd947e7b93fa4ca72b38ecc9a39 branch-0.2.0
+        # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
+        set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
     endif()
 else()
     set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz")

core/unittest/CMakeLists.txt

@@ -16,10 +16,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #-------------------------------------------------------------------------------
-if (CUSTOMIZATION)
-    add_definitions(-DCUSTOMIZATION)
-endif (CUSTOMIZATION)
-
 include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
 
 foreach(dir ${CORE_INCLUDE_DIRS})