Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0

Former-commit-id: 0c93c5c46c54a2e523142950a6f3bdc270235312

ci/jenkinsfile/milvus_build.groovy

@@ -7,7 +7,7 @@ container('milvus-build-env') {
                     dir ("cpp") {
                         sh "git config --global user.email \"test@zilliz.com\""
                         sh "git config --global user.name \"test\""
-                        sh "./build.sh -t ${params.BUILD_TYPE} -u -c"
+                        sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c"
                     }
                 } catch (exc) {
                     updateGitlabCommitStatus name: 'Build Engine', state: 'failed'

cpp/cmake/ThirdPartyPackages.cmake

@@ -321,6 +321,7 @@ else ()
     set(GTEST_SOURCE_URL
             "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
 endif()
+set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596")
 
 if (DEFINED ENV{MILVUS_JSONCONS_URL})
     set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}")
@@ -470,7 +471,7 @@ macro(build_arrow)
             -DCMAKE_BUILD_TYPE=Release)
 
     if(USE_JFROG_CACHE STREQUAL "ON")
-        execute_process(COMMAND sh -c "git ls-remote --heads ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
+        execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
         if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
             string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
             set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
@@ -882,7 +883,7 @@ macro(build_openblas)
     set(OPENBLAS_STATIC_LIB
             "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
-        if(USE_JFROG_CACHE STREQUAL "ON")
+    if(USE_JFROG_CACHE STREQUAL "ON")
         set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz")
         set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}")
         set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}")
@@ -910,33 +911,30 @@ macro(build_openblas)
                     ${OPENBLAS_STATIC_LIB})
 
             ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
-
-            file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
         else()
             ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
         endif()
     else()
         externalproject_add(openblas_ep
-                    URL
-                    ${OPENBLAS_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    ""
-                    BUILD_IN_SOURCE
-                    1
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    INSTALL_COMMAND
-                    ${MAKE}
-                    PREFIX=${OPENBLAS_PREFIX}
-                    install
-                    BUILD_BYPRODUCTS
-                    ${OPENBLAS_STATIC_LIB})
-
-        file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
+                URL
+                ${OPENBLAS_SOURCE_URL}
+                ${EP_LOG_OPTIONS}
+                CONFIGURE_COMMAND
+                ""
+                BUILD_IN_SOURCE
+                1
+                BUILD_COMMAND
+                ${MAKE}
+                ${MAKE_BUILD_ARGS}
+                INSTALL_COMMAND
+                ${MAKE}
+                PREFIX=${OPENBLAS_PREFIX}
+                install
+                BUILD_BYPRODUCTS
+                ${OPENBLAS_STATIC_LIB})
     endif()
 
+    file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
     add_library(openblas STATIC IMPORTED)
     set_target_properties(
             openblas
@@ -1212,19 +1210,49 @@ macro(build_gtest)
     set(GMOCK_STATIC_LIB
             "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
     )
-    ExternalProject_Add(googletest_ep
-                        URL
-                        ${GTEST_SOURCE_URL}
-                        BUILD_COMMAND
-                        ${MAKE}
-                        ${MAKE_BUILD_ARGS}
-                        BUILD_BYPRODUCTS
-                        ${GTEST_STATIC_LIB}
-                        ${GTEST_MAIN_STATIC_LIB}
-                        ${GMOCK_STATIC_LIB}
-                        CMAKE_ARGS
-                        ${GTEST_CMAKE_ARGS}
-                        ${EP_LOG_OPTIONS})
+
+    if(USE_JFROG_CACHE STREQUAL "ON")
+        set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
+        set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
+        set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
+
+        file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
+        list(GET status 0 status_code)
+        message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
+        if (NOT status_code EQUAL 0)
+            ExternalProject_Add(googletest_ep
+                    URL
+                    ${GTEST_SOURCE_URL}
+                    BUILD_COMMAND
+                    ${MAKE}
+                    ${MAKE_BUILD_ARGS}
+                    BUILD_BYPRODUCTS
+                    ${GTEST_STATIC_LIB}
+                    ${GTEST_MAIN_STATIC_LIB}
+                    ${GMOCK_STATIC_LIB}
+                    CMAKE_ARGS
+                    ${GTEST_CMAKE_ARGS}
+                    ${EP_LOG_OPTIONS})
+
+            ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
+        else()
+            ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
+        endif()
+    else()
+        ExternalProject_Add(googletest_ep
+                URL
+                ${GTEST_SOURCE_URL}
+                BUILD_COMMAND
+                ${MAKE}
+                ${MAKE_BUILD_ARGS}
+                BUILD_BYPRODUCTS
+                ${GTEST_STATIC_LIB}
+                ${GTEST_MAIN_STATIC_LIB}
+                ${GMOCK_STATIC_LIB}
+                CMAKE_ARGS
+                ${GTEST_CMAKE_ARGS}
+                ${EP_LOG_OPTIONS})
+    endif()
 
     # The include directory must exist before it is referenced by a target.
     file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
@@ -1474,29 +1502,74 @@ macro(build_prometheus)
             "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}"
             -DCMAKE_BUILD_TYPE=Release)
 
-    externalproject_add(prometheus_ep
-            GIT_REPOSITORY
-            ${PROMETHEUS_SOURCE_URL}
-            GIT_TAG
-            ${PROMETHEUS_VERSION}
-            GIT_SHALLOW
-            TRUE
-            ${EP_LOG_OPTIONS}
-            CMAKE_ARGS
-            ${PROMETHEUS_CMAKE_ARGS}
-            BUILD_COMMAND
-            ${MAKE}
-            ${MAKE_BUILD_ARGS}
-            BUILD_IN_SOURCE
-            1
-            INSTALL_COMMAND
-            ${MAKE}
-            "DESTDIR=${PROMETHEUS_PREFIX}"
-            install
-            BUILD_BYPRODUCTS
-            "${PROMETHEUS_CORE_STATIC_LIB}"
-            "${PROMETHEUS_PUSH_STATIC_LIB}"
-            "${PROMETHEUS_PULL_STATIC_LIB}")
+    if(USE_JFROG_CACHE STREQUAL "ON")
+        execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID)
+        if(${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
+            string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}")
+            set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz")
+            set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
+            set(PROMETHEUS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
+
+            file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
+            list(GET status 0 status_code)
+            message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
+            if (NOT status_code EQUAL 0)
+                externalproject_add(prometheus_ep
+                        GIT_REPOSITORY
+                        ${PROMETHEUS_SOURCE_URL}
+                        GIT_TAG
+                        ${PROMETHEUS_VERSION}
+                        GIT_SHALLOW
+                        TRUE
+                        ${EP_LOG_OPTIONS}
+                        CMAKE_ARGS
+                        ${PROMETHEUS_CMAKE_ARGS}
+                        BUILD_COMMAND
+                        ${MAKE}
+                        ${MAKE_BUILD_ARGS}
+                        BUILD_IN_SOURCE
+                        1
+                        INSTALL_COMMAND
+                        ${MAKE}
+                        "DESTDIR=${PROMETHEUS_PREFIX}"
+                        install
+                        BUILD_BYPRODUCTS
+                        "${PROMETHEUS_CORE_STATIC_LIB}"
+                        "${PROMETHEUS_PUSH_STATIC_LIB}"
+                        "${PROMETHEUS_PULL_STATIC_LIB}")
+
+                ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL})
+            else()
+                ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
+            endif()
+        else()
+            message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!")
+        endif()
+    else()
+        externalproject_add(prometheus_ep
+                GIT_REPOSITORY
+                ${PROMETHEUS_SOURCE_URL}
+                GIT_TAG
+                ${PROMETHEUS_VERSION}
+                GIT_SHALLOW
+                TRUE
+                ${EP_LOG_OPTIONS}
+                CMAKE_ARGS
+                ${PROMETHEUS_CMAKE_ARGS}
+                BUILD_COMMAND
+                ${MAKE}
+                ${MAKE_BUILD_ARGS}
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE}
+                "DESTDIR=${PROMETHEUS_PREFIX}"
+                install
+                BUILD_BYPRODUCTS
+                "${PROMETHEUS_CORE_STATIC_LIB}"
+                "${PROMETHEUS_PUSH_STATIC_LIB}"
+                "${PROMETHEUS_PULL_STATIC_LIB}")
+    endif()
 
     file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include")
     add_library(prometheus-cpp-push STATIC IMPORTED)

cpp/src/db/ExecutionEngine.h

@@ -19,7 +19,8 @@ enum class EngineType {
     FAISS_IDMAP = 1,
     FAISS_IVFFLAT,
     FAISS_IVFSQ8,
-    MAX_VALUE = FAISS_IVFSQ8,
+    NSG_MIX,
+    MAX_VALUE = NSG_MIX,
 };
 
 class ExecutionEngine {

cpp/src/db/ExecutionEngineImpl.cpp

@@ -69,6 +69,10 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_MIX);
             break;
         }
+        case EngineType::NSG_MIX: {
+            index = GetVecIndexFactory(IndexType::NSG_MIX);
+            break;
+        }
         default: {
             ENGINE_LOG_ERROR << "Invalid engine type";
             return nullptr;

cpp/src/sdk/include/MilvusApi.h

@@ -19,6 +19,7 @@ enum class IndexType {
     cpu_idmap,
     gpu_ivfflat,
     gpu_ivfsq8,
+    mix_nsg,
 };
 
 /**

cpp/src/server/thrift_impl/RequestTask.cpp

@@ -36,6 +36,7 @@ namespace {
                 {1, engine::EngineType::FAISS_IDMAP},
                 {2, engine::EngineType::FAISS_IVFFLAT},
                 {3, engine::EngineType::FAISS_IVFSQ8},
+                {4, engine::EngineType::NSG_MIX},
         };
 
         if(map_type.find(type) == map_type.end()) {
@@ -51,6 +52,7 @@ namespace {
                 {engine::EngineType::FAISS_IDMAP, 1},
                 {engine::EngineType::FAISS_IVFFLAT, 2},
                 {engine::EngineType::FAISS_IVFSQ8, 3},
+                {engine::EngineType::NSG_MIX, 4},
         };
 
         if(map_type.find(type) == map_type.end()) {

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -7,6 +7,7 @@
 #include "knowhere/index/vector_index/idmap.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/index/vector_index/cpu_kdt_rng.h"
+#include "knowhere/index/vector_index/nsg_index.h"
 #include "knowhere/common/exception.h"
 
 #include "vec_index.h"
@@ -103,10 +104,10 @@ VecIndexPtr GetVecIndexFactory(const IndexType &type) {
             index = std::make_shared<zilliz::knowhere::GPUIVFSQ>(0);
             return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFSQ8_MIX);
         }
-            //case IndexType::NSG: { // TODO(linxj): bug.
-            //    index = std::make_shared<zilliz::knowhere::NSG>();
-            //    break;
-            //}
+        case IndexType::NSG_MIX: { // TODO(linxj): bug.
+            index = std::make_shared<zilliz::knowhere::NSG>(0);
+            break;
+        }
         default: {
             return nullptr;
         }
@@ -199,6 +200,18 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
             if (!cfg.contains("nbits")) { cfg["nbits"] = int(8); }
             break;
         }
+        case IndexType::NSG_MIX: {
+            auto scale_factor = round(cfg["dim"].as<int>() / 128.0);
+            scale_factor = scale_factor >= 4 ? 4 : scale_factor;
+            cfg["nlist"] = int(size / 1000000.0 * 8192);
+            if (!cfg.contains("nprobe")) { cfg["nprobe"] = 6 + 10 * scale_factor; }
+            if (!cfg.contains("knng")) { cfg["knng"] = 100 + 100 * scale_factor; }
+            if (!cfg.contains("search_length")) { cfg["search_length"] = 40 + 5 * scale_factor; }
+            if (!cfg.contains("out_degree")) { cfg["out_degree"] = 50 + 5 * scale_factor; }
+            if (!cfg.contains("candidate_pool_size")) { cfg["candidate_pool_size"] = 200 + 100 * scale_factor; }
+            WRAPPER_LOG_DEBUG << pretty_print(cfg);
+            break;
+        }
     }
 }
 

cpp/src/wrapper/knowhere/vec_index.h

@@ -32,7 +32,7 @@ enum class IndexType {
     FAISS_IVFPQ_GPU,
     SPTAG_KDT_RNT_CPU,
     FAISS_IVFSQ8_MIX,
-    //NSG,
+    NSG_MIX,
 };
 
 class VecIndex {

knowhere @ 6a4f242e

-Subproject commit 02550a43b5146bd7976b8b2b3fc37ca885d1e880
+Subproject commit 6a4f242ec40c539f1e6d9a7305282f7ff03c19d2

cpp/unittest/index_wrapper/knowhere_test.cpp

@@ -108,6 +108,12 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
                                             64, 100000, 10, 10,
                                             Config::object{{"dim", 64}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}},
                                             Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 5}}
+                            ),
+                            std::make_tuple(IndexType::NSG_MIX, "Default",
+                                            128, 250000, 10, 10,
+                                            Config::object{{"dim", 128}, {"nlist", 8192}, {"nprobe", 16}, {"metric_type", "L2"},
+                                                           {"knng", 200}, {"search_length", 40}, {"out_degree", 60}, {"candidate_pool_size", 200}},
+                                            Config::object{{"k", 10}, {"search_length", 20}}
                             )
                             //std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default",
                             //                64, 10000, 10, 10,