MS-596 enable knowhere and fix compile error

Former-commit-id: 75c435ef22ceec6f07e92e42f99b369657e3911a

cpp/src/core/CMakeLists.txt

@@ -46,9 +46,11 @@ if(NOT CMAKE_BUILD_TYPE)
 endif(NOT CMAKE_BUILD_TYPE)
 
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -fopenmp")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
 else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -fopenmp")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
 endif()
 MESSAGE(STATUS "CMAKE_CXX_FLAGS" ${CMAKE_CXX_FLAGS})
 

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp

@@ -24,6 +24,7 @@
 
 #include "NSG.h"
 #include "knowhere/common/Exception.h"
+#include "knowhere/common/Log.h"
 #include "knowhere/common/Timer.h"
 #include "NSGHelper.h"
 
@@ -83,8 +84,9 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con
     for (int i = 0; i < ntotal; ++i) {
         total_degree += nsg[i].size();
     }
-    std::cout << "graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024;
-    std::cout << "average degree: " << total_degree / ntotal;
+
+    KNOWHERE_LOG_DEBUG << "Graph physical size: " << total_degree * sizeof(node_t) / 1024 / 1024 << "m";
+    KNOWHERE_LOG_DEBUG << "Average degree: " << total_degree / ntotal;
     /////
 
     is_trained = true;

cpp/src/core/test/CMakeLists.txt

@@ -29,7 +29,7 @@ set(util_srcs
         ${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/ArrowAdapter.cpp
         ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Exception.cpp
         ${CORE_SOURCE_DIR}/knowhere/knowhere/common/Timer.cpp
-        utils.cpp
+        ${CORE_SOURCE_DIR}/test/utils.cpp
         )
 
 #<IVF-TEST>
@@ -52,18 +52,10 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
 
 #<IDMAP-TEST>
 set(idmap_srcs
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
         ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVF.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp
-        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp
         )
 if(NOT TARGET test_idmap)
-    add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs})
+    add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${ivf_srcs} ${util_srcs})
 endif()
 target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
 
@@ -86,5 +78,5 @@ install(TARGETS test_idmap DESTINATION unittest)
 install(TARGETS test_kdt DESTINATION unittest)
 
 #add_subdirectory(faiss_ori)
-#add_subdirectory(test_nsg)
+add_subdirectory(test_nsg)
 

cpp/src/core/test/test_nsg/CMakeLists.txt

 ##############################
-include_directories(/usr/local/include/gperftools)
-link_directories(/usr/local/lib)
+#include_directories(/usr/local/include/gperftools)
+#link_directories(/usr/local/lib)
 
 add_definitions(-std=c++11 -O3 -lboost -march=native -Wall -DINFO)
 
@@ -13,29 +13,15 @@ else ()
 endif ()
 message(${OpenMP_CXX_FLAGS})
 
-include_directories(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg)
-
-aux_source_directory(${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg nsg_src)
+include_directories(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg)
+aux_source_directory(${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/nsg nsg_src)
 
 set(interface_src
-        ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/nsg_index.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
-        ${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
-        ../utils.cpp
+        ${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNSG.cpp
         )
 
 if(NOT TARGET test_nsg)
-    add_executable(test_nsg
-            test_nsg.cpp
-            ${interface_src}
-            ${nsg_src}
-            ${util_srcs}
-            )
+    add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${ivf_srcs})
 endif()
 
 target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})

cpp/src/core/test/test_nsg/test_nsg.cpp

@@ -19,10 +19,11 @@
 #include <gtest/gtest.h>
 #include <memory>
 
-#include "knowhere/common/exception.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/nsg_index.h"
-#include "knowhere/index/vector_index/nsg/nsg_io.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/FaissBaseIndex.h"
+#include "knowhere/index/vector_index/IndexNSG.h"
+#include "knowhere/index/vector_index/nsg/NSGIO.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 
 #include "../utils.h"
 
@@ -32,16 +33,31 @@ using ::testing::TestWithParam;
 using ::testing::Values;
 using ::testing::Combine;
 
-constexpr int64_t DEVICE_ID = 0;
+constexpr int64_t DEVICE_ID = 1;
 
-class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Config, Config>> {
+ class NSGInterfaceTest : public DataGen, public ::testing::Test {
  protected:
     void SetUp() override {
         //Init_with_default();
         FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_ID, 1024*1024*200, 1024*1024*600, 2);
-        Generate(256, 10000, 1);
+        Generate(256, 1000000, 1);
         index_ = std::make_shared<NSG>();
-        std::tie(train_cfg, search_cfg) = GetParam();
+
+        auto tmp_conf = std::make_shared<NSGCfg>();
+        tmp_conf->gpu_id = DEVICE_ID;
+        tmp_conf->knng = 100;
+        tmp_conf->nprobe = 32;
+        tmp_conf->nlist = 16384;
+        tmp_conf->search_length = 60;
+        tmp_conf->out_degree = 70;
+        tmp_conf->candidate_pool_size = 500;
+        tmp_conf->metric_type = METRICTYPE::L2;
+        train_conf = tmp_conf;
+
+        auto tmp2_conf = std::make_shared<NSGCfg>();
+        tmp2_conf->k = k;
+        tmp2_conf->search_length = 30;
+        search_conf = tmp2_conf;
     }
 
     void TearDown() override {
@@ -50,18 +66,10 @@ class NSGInterfaceTest : public DataGen, public TestWithParam<::std::tuple<Confi
 
  protected:
     std::shared_ptr<NSG> index_;
-    Config train_cfg;
-    Config search_cfg;
+    Config train_conf;
+    Config search_conf;
 };
 
-INSTANTIATE_TEST_CASE_P(NSGparameters, NSGInterfaceTest,
-                        Values(std::make_tuple(
-                            // search length > out_degree
-                            Config::object{{"nlist", 128}, {"nprobe", 50}, {"knng", 100}, {"metric_type", "L2"},
-                                           {"search_length", 60}, {"out_degree", 70}, {"candidate_pool_size", 500}},
-                            Config::object{{"k", 20}, {"search_length", 30}}))
-);
-
 void AssertAnns(const DatasetPtr &result,
                 const int &nq,
                 const int &k) {
@@ -71,17 +79,17 @@ void AssertAnns(const DatasetPtr &result,
     }
 }
 
-TEST_P(NSGInterfaceTest, basic_test) {
+TEST_F(NSGInterfaceTest, basic_test) {
     assert(!xb.empty());
 
-    auto model = index_->Train(base_dataset, train_cfg);
-    auto result = index_->Search(query_dataset, search_cfg);
+    auto model = index_->Train(base_dataset, train_conf);
+    auto result = index_->Search(query_dataset, search_conf);
     AssertAnns(result, nq, k);
 
     auto binaryset = index_->Serialize();
     auto new_index = std::make_shared<NSG>();
     new_index->Load(binaryset);
-    auto new_result = new_index->Search(query_dataset, Config::object{{"k", k}});
+    auto new_result = new_index->Search(query_dataset, search_conf);
     AssertAnns(result, nq, k);
 
     ASSERT_EQ(index_->Count(), nb);

cpp/src/wrapper/ConfAdapter.cpp

@@ -145,6 +145,10 @@ NSGConfAdapter::Match(const TempMetaConf &metaconf) {
     conf->out_degree = 50 + 5 * scale_factor;
     conf->candidate_pool_size = 200 + 100 * scale_factor;
     MatchBase(conf);
+
+//    WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist
+//    << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d
+//    << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng;
     return conf;
 }
 

cpp/src/wrapper/ConfAdapter.h

@@ -47,6 +47,9 @@ class ConfAdapter {
     virtual knowhere::Config
     MatchSearch(const TempMetaConf &metaconf, const IndexType &type);
 
+//    virtual void
+//    Dump(){}
+
  protected:
     static void MatchBase(knowhere::Config conf);
 };

cpp/src/wrapper/VecIndex.cpp

@@ -137,7 +137,7 @@ GetVecIndexFactory(const IndexType &type, const Config &cfg) {
             index = std::make_shared<zilliz::knowhere::GPUIVFSQ>(gpu_device);
             break;
         }
-        case IndexType::NSG_MIX: { // TODO(linxj): bug.
+        case IndexType::NSG_MIX: {
             index = std::make_shared<zilliz::knowhere::NSG>(gpu_device);
             break;
         }

cpp/unittest/wrapper/wrapper_test.cpp

@@ -173,7 +173,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
                             std::make_tuple(IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10),
                             std::make_tuple(IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10),
                             std::make_tuple(IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10),
-//                            std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
+                            std::make_tuple(IndexType::NSG_MIX, "Default", 128, 250000, 10, 10),
 //                            std::make_tuple(IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 250000, 10, 10),
                             std::make_tuple(IndexType::FAISS_IDMAP, "Default", 64, 100000, 10, 10)
                         )