Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0

Former-commit-id: 0c93c5c46c54a2e523142950a6f3bdc270235312

Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0
Former-commit-id: 0c93c5c46c54a2e523142950a6f3bdc270235312
45bb9d67 · kun yu · 409175f9 · 720c170a · 45bb9d67 · 45bb9d67
10 changed file
--- a/ci/jenkinsfile/milvus_build.groovy
+++ b/ci/jenkinsfile/milvus_build.groovy
@@ -7,7 +7,7 @@ container('milvus-build-env') {
                    dir ("cpp") {
                        sh "git config --global user.email \"test@zilliz.com\""
                        sh "git config --global user.name \"test\""
-                        sh "./build.sh -t ${params.BUILD_TYPE} -u -c"
+                        sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c"
                    }
                } catch (exc) {
                    updateGitlabCommitStatus name: 'Build Engine', state: 'failed'

--- a/cpp/cmake/ThirdPartyPackages.cmake
+++ b/cpp/cmake/ThirdPartyPackages.cmake
@@ -321,6 +321,7 @@ else ()
    set(GTEST_SOURCE_URL
            "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
 endif()
+set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596")
 if (DEFINED ENV{MILVUS_JSONCONS_URL})
    set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}")
@@ -470,7 +471,7 @@ macro(build_arrow)
            -DCMAKE_BUILD_TYPE=Release)
    if(USE_JFROG_CACHE STREQUAL "ON")
-        execute_process(COMMAND sh -c "git ls-remote --heads ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
+        execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
        if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
            string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
            set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
@@ -882,7 +883,7 @@ macro(build_openblas)
    set(OPENBLAS_STATIC_LIB
            "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
-        if(USE_JFROG_CACHE STREQUAL "ON")
+    if(USE_JFROG_CACHE STREQUAL "ON")
        set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz")
        set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}")
        set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}")
@@ -910,33 +911,30 @@ macro(build_openblas)
                    ${OPENBLAS_STATIC_LIB})
            ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
-            file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
        else()
            ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
        endif()
    else()
        externalproject_add(openblas_ep
-                    URL
+                URL
-                    ${OPENBLAS_SOURCE_URL}
+                ${OPENBLAS_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
+                ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
+                CONFIGURE_COMMAND
-                    ""
+                ""
-                    BUILD_IN_SOURCE
+                BUILD_IN_SOURCE
-                    1
+                1
-                    BUILD_COMMAND
+                BUILD_COMMAND
-                    ${MAKE}
+                ${MAKE}
-                    ${MAKE_BUILD_ARGS}
+                ${MAKE_BUILD_ARGS}
-                    INSTALL_COMMAND
+                INSTALL_COMMAND
-                    ${MAKE}
+                ${MAKE}
-                    PREFIX=${OPENBLAS_PREFIX}
+                PREFIX=${OPENBLAS_PREFIX}
-                    install
+                install
-                    BUILD_BYPRODUCTS
+                BUILD_BYPRODUCTS
-                    ${OPENBLAS_STATIC_LIB})
+                ${OPENBLAS_STATIC_LIB})
-        file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
    endif()
+    file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
    add_library(openblas STATIC IMPORTED)
    set_target_properties(
            openblas
@@ -1212,19 +1210,49 @@ macro(build_gtest)
    set(GMOCK_STATIC_LIB
            "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
    )
-    ExternalProject_Add(googletest_ep
-                        URL
+    if(USE_JFROG_CACHE STREQUAL "ON")
-                        ${GTEST_SOURCE_URL}
+        set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
-                        BUILD_COMMAND
+        set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
-                        ${MAKE}
+        set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
-                        ${MAKE_BUILD_ARGS}
-                        BUILD_BYPRODUCTS
+        file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
-                        ${GTEST_STATIC_LIB}
+        list(GET status 0 status_code)
-                        ${GTEST_MAIN_STATIC_LIB}
+        message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-                        ${GMOCK_STATIC_LIB}
+        if (NOT status_code EQUAL 0)
-                        CMAKE_ARGS
+            ExternalProject_Add(googletest_ep
-                        ${GTEST_CMAKE_ARGS}
+                    URL
-                        ${EP_LOG_OPTIONS})
+                    ${GTEST_SOURCE_URL}
+                    BUILD_COMMAND
+                    ${MAKE}
+                    ${MAKE_BUILD_ARGS}
+                    BUILD_BYPRODUCTS
+                    ${GTEST_STATIC_LIB}
+                    ${GTEST_MAIN_STATIC_LIB}
+                    ${GMOCK_STATIC_LIB}
+                    CMAKE_ARGS
+                    ${GTEST_CMAKE_ARGS}
+                    ${EP_LOG_OPTIONS})
+            ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
+        else()
+            ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
+        endif()
+    else()
+        ExternalProject_Add(googletest_ep
+                URL
+                ${GTEST_SOURCE_URL}
+                BUILD_COMMAND
+                ${MAKE}
+                ${MAKE_BUILD_ARGS}
+                BUILD_BYPRODUCTS
+                ${GTEST_STATIC_LIB}
+                ${GTEST_MAIN_STATIC_LIB}
+                ${GMOCK_STATIC_LIB}
+                CMAKE_ARGS
+                ${GTEST_CMAKE_ARGS}
+                ${EP_LOG_OPTIONS})
+    endif()
    # The include directory must exist before it is referenced by a target.
    file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
@@ -1474,29 +1502,74 @@ macro(build_prometheus)
            "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}"
            -DCMAKE_BUILD_TYPE=Release)
-    externalproject_add(prometheus_ep
+    if(USE_JFROG_CACHE STREQUAL "ON")
-            GIT_REPOSITORY
+        execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID)
-            ${PROMETHEUS_SOURCE_URL}
+        if(${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
-            GIT_TAG
+            string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}")
-            ${PROMETHEUS_VERSION}
+            set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz")
-            GIT_SHALLOW
+            set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
-            TRUE
+            set(PROMETHEUS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
-            ${EP_LOG_OPTIONS}
-            CMAKE_ARGS
+            file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
-            ${PROMETHEUS_CMAKE_ARGS}
+            list(GET status 0 status_code)
-            BUILD_COMMAND
+            message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            ${MAKE}
+            if (NOT status_code EQUAL 0)
-            ${MAKE_BUILD_ARGS}
+                externalproject_add(prometheus_ep
-            BUILD_IN_SOURCE
+                        GIT_REPOSITORY
-            1
+                        ${PROMETHEUS_SOURCE_URL}
-            INSTALL_COMMAND
+                        GIT_TAG
-            ${MAKE}
+                        ${PROMETHEUS_VERSION}
-            "DESTDIR=${PROMETHEUS_PREFIX}"
+                        GIT_SHALLOW
-            install
+                        TRUE
-            BUILD_BYPRODUCTS
+                        ${EP_LOG_OPTIONS}
-            "${PROMETHEUS_CORE_STATIC_LIB}"
+                        CMAKE_ARGS
-            "${PROMETHEUS_PUSH_STATIC_LIB}"
+                        ${PROMETHEUS_CMAKE_ARGS}
-            "${PROMETHEUS_PULL_STATIC_LIB}")
+                        BUILD_COMMAND
+                        ${MAKE}
+                        ${MAKE_BUILD_ARGS}
+                        BUILD_IN_SOURCE
+                        1
+                        INSTALL_COMMAND
+                        ${MAKE}
+                        "DESTDIR=${PROMETHEUS_PREFIX}"
+                        install
+                        BUILD_BYPRODUCTS
+                        "${PROMETHEUS_CORE_STATIC_LIB}"
+                        "${PROMETHEUS_PUSH_STATIC_LIB}"
+                        "${PROMETHEUS_PULL_STATIC_LIB}")
+                ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL})
+            else()
+                ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
+            endif()
+        else()
+            message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!")
+        endif()
+    else()
+        externalproject_add(prometheus_ep
+                GIT_REPOSITORY
+                ${PROMETHEUS_SOURCE_URL}
+                GIT_TAG
+                ${PROMETHEUS_VERSION}
+                GIT_SHALLOW
+                TRUE
+                ${EP_LOG_OPTIONS}
+                CMAKE_ARGS
+                ${PROMETHEUS_CMAKE_ARGS}
+                BUILD_COMMAND
+                ${MAKE}
+                ${MAKE_BUILD_ARGS}
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE}
+                "DESTDIR=${PROMETHEUS_PREFIX}"
+                install
+                BUILD_BYPRODUCTS
+                "${PROMETHEUS_CORE_STATIC_LIB}"
+                "${PROMETHEUS_PUSH_STATIC_LIB}"
+                "${PROMETHEUS_PULL_STATIC_LIB}")
+    endif()
    file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include")
    add_library(prometheus-cpp-push STATIC IMPORTED)

--- a/cpp/src/db/ExecutionEngine.h
+++ b/cpp/src/db/ExecutionEngine.h
@@ -19,7 +19,8 @@ enum class EngineType {
    FAISS_IDMAP = 1,
    FAISS_IVFFLAT,
    FAISS_IVFSQ8,
-    MAX_VALUE = FAISS_IVFSQ8,
+    NSG_MIX,
+    MAX_VALUE = NSG_MIX,
 };
 class ExecutionEngine {

--- a/cpp/src/db/ExecutionEngineImpl.cpp
+++ b/cpp/src/db/ExecutionEngineImpl.cpp
@@ -69,6 +69,10 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
            index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_MIX);
            break;
        }
+        case EngineType::NSG_MIX: {
+            index = GetVecIndexFactory(IndexType::NSG_MIX);
+            break;
+        }
        default: {
            ENGINE_LOG_ERROR << "Invalid engine type";
            return nullptr;

--- a/cpp/src/sdk/include/MilvusApi.h
+++ b/cpp/src/sdk/include/MilvusApi.h
@@ -19,6 +19,7 @@ enum class IndexType {
    cpu_idmap,
    gpu_ivfflat,
    gpu_ivfsq8,
+    mix_nsg,
 };
 /**

--- a/cpp/src/server/thrift_impl/RequestTask.cpp
+++ b/cpp/src/server/thrift_impl/RequestTask.cpp
@@ -36,6 +36,7 @@ namespace {
                {1, engine::EngineType::FAISS_IDMAP},
                {2, engine::EngineType::FAISS_IVFFLAT},
                {3, engine::EngineType::FAISS_IVFSQ8},
+                {4, engine::EngineType::NSG_MIX},
        };
        if(map_type.find(type) == map_type.end()) {
@@ -51,6 +52,7 @@ namespace {
                {engine::EngineType::FAISS_IDMAP, 1},
                {engine::EngineType::FAISS_IVFFLAT, 2},
                {engine::EngineType::FAISS_IVFSQ8, 3},
+                {engine::EngineType::NSG_MIX, 4},
        };
        if(map_type.find(type) == map_type.end()) {

--- a/cpp/src/wrapper/knowhere/vec_index.cpp
+++ b/cpp/src/wrapper/knowhere/vec_index.cpp
@@ -7,6 +7,7 @@
 #include "knowhere/index/vector_index/idmap.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/index/vector_index/cpu_kdt_rng.h"
+#include "knowhere/index/vector_index/nsg_index.h"
 #include "knowhere/common/exception.h"
 #include "vec_index.h"
@@ -103,10 +104,10 @@ VecIndexPtr GetVecIndexFactory(const IndexType &type) {
            index = std::make_shared<zilliz::knowhere::GPUIVFSQ>(0);
            return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFSQ8_MIX);
        }
-            //case IndexType::NSG: { // TODO(linxj): bug.
+        case IndexType::NSG_MIX: { // TODO(linxj): bug.
-            //    index = std::make_shared<zilliz::knowhere::NSG>();
+            index = std::make_shared<zilliz::knowhere::NSG>(0);
-            //    break;
+            break;
-            //}
+        }
        default: {
            return nullptr;
        }
@@ -199,6 +200,18 @@ void AutoGenParams(const IndexType &type, const long &size, zilliz::knowhere::Co
            if (!cfg.contains("nbits")) { cfg["nbits"] = int(8); }
            break;
        }
+        case IndexType::NSG_MIX: {
+            auto scale_factor = round(cfg["dim"].as<int>() / 128.0);
+            scale_factor = scale_factor >= 4 ? 4 : scale_factor;
+            cfg["nlist"] = int(size / 1000000.0 * 8192);
+            if (!cfg.contains("nprobe")) { cfg["nprobe"] = 6 + 10 * scale_factor; }
+            if (!cfg.contains("knng")) { cfg["knng"] = 100 + 100 * scale_factor; }
+            if (!cfg.contains("search_length")) { cfg["search_length"] = 40 + 5 * scale_factor; }
+            if (!cfg.contains("out_degree")) { cfg["out_degree"] = 50 + 5 * scale_factor; }
+            if (!cfg.contains("candidate_pool_size")) { cfg["candidate_pool_size"] = 200 + 100 * scale_factor; }
+            WRAPPER_LOG_DEBUG << pretty_print(cfg);
+            break;
+        }
    }
 }

--- a/cpp/src/wrapper/knowhere/vec_index.h
+++ b/cpp/src/wrapper/knowhere/vec_index.h
@@ -32,7 +32,7 @@ enum class IndexType {
    FAISS_IVFPQ_GPU,
    SPTAG_KDT_RNT_CPU,
    FAISS_IVFSQ8_MIX,
-    //NSG,
+    NSG_MIX,
 };
 class VecIndex {

--- a/knowhere @ 6a4f242e
+++ b/knowhere @ 6a4f242e
-Subproject commit 02550a43b5146bd7976b8b2b3fc37ca885d1e880
+Subproject commit 6a4f242ec40c539f1e6d9a7305282f7ff03c19d2
--- a/cpp/unittest/index_wrapper/knowhere_test.cpp
+++ b/cpp/unittest/index_wrapper/knowhere_test.cpp
@@ -108,6 +108,12 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
                                            64, 100000, 10, 10,
                                            Config::object{{"dim", 64}, {"nlist", 1000}, {"nbits", 8}, {"metric_type", "L2"}},
                                            Config::object{{"dim", 64}, {"k", 10}, {"nprobe", 5}}
+                            ),
+                            std::make_tuple(IndexType::NSG_MIX, "Default",
+                                            128, 250000, 10, 10,
+                                            Config::object{{"dim", 128}, {"nlist", 8192}, {"nprobe", 16}, {"metric_type", "L2"},
+                                                           {"knng", 200}, {"search_length", 40}, {"out_degree", 60}, {"candidate_pool_size", 200}},
+                                            Config::object{{"k", 10}, {"search_length", 20}}
                            )
                            //std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default",
                            //                64, 10000, 10, 10,