fix by reduce knng size

core/src/index/CMakeLists.txt

@@ -88,14 +88,14 @@ endif ()
 include(ThirdPartyPackagesCore)
 
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt")
     if (KNOWHERE_GPU_VERSION)
         set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
     endif ()
 else ()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt")
     if (KNOWHERE_GPU_VERSION)
-        set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
+        set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g")
     endif ()
 endif ()
 

core/src/index/knowhere/knowhere/common/Config.h

@@ -18,6 +18,8 @@
 #pragma once
 
 #include <memory>
+#include <sstream>
+#include "Log.h"
 
 namespace knowhere {
 
@@ -50,6 +52,18 @@ struct Cfg {
     CheckValid() {
         return true;
     }
+
+    void
+    Dump() {
+        KNOWHERE_LOG_DEBUG << DumpImpl().str();
+    }
+
+    virtual std::stringstream
+    DumpImpl() {
+        std::stringstream ss;
+        ss << "dim: " << d << ", metric: " << int(metric_type) << ", gpuid: " << gpu_id << ", k: " << k;
+        return ss;
+    }
 };
 using Config = std::shared_ptr<Cfg>;
 

core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp

@@ -34,4 +34,26 @@ GetMetricType(METRICTYPE& type) {
     KNOWHERE_THROW_MSG("Metric type is invalid");
 }
 
+std::stringstream
+IVFCfg::DumpImpl() {
+    auto ss = Cfg::DumpImpl();
+    ss << ", nlist: " << nlist << ", nprobe: " << nprobe;
+    return ss;
+}
+
+std::stringstream
+IVFSQCfg::DumpImpl() {
+    auto ss = IVFCfg::DumpImpl();
+    ss << ", nbits: " << nbits;
+    return ss;
+}
+
+std::stringstream
+NSGCfg::DumpImpl() {
+    auto ss = IVFCfg::DumpImpl();
+    ss << ", knng: " << knng << ", search_length: " << search_length << ", out_degree: " << out_degree
+       << ", candidate: " << candidate_pool_size;
+    return ss;
+}
+
 }  // namespace knowhere

core/src/index/knowhere/knowhere/index/vector_index/helpers/IndexParameter.h

@@ -53,6 +53,9 @@ struct IVFCfg : public Cfg {
 
     IVFCfg() = default;
 
+    std::stringstream
+    DumpImpl() override;
+
     bool
     CheckValid() override {
         return true;
@@ -69,6 +72,9 @@ struct IVFSQCfg : public IVFCfg {
         : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) {
     }
 
+    std::stringstream
+    DumpImpl() override;
+
     IVFSQCfg() = default;
 
     bool
@@ -119,6 +125,9 @@ struct NSGCfg : public IVFCfg {
 
     NSGCfg() = default;
 
+    std::stringstream
+    DumpImpl() override;
+
     bool
     CheckValid() override {
         return true;

core/src/wrapper/ConfAdapter.cpp

@@ -47,7 +47,8 @@ ConfAdapter::Match(const TempMetaConf& metaconf) {
     auto conf = std::make_shared<knowhere::Cfg>();
     conf->d = metaconf.dim;
     conf->metric_type = metaconf.metric_type;
-    conf->gpu_id = conf->gpu_id;
+    conf->gpu_id = metaconf.gpu_id;
+    conf->k = metaconf.k;
     MatchBase(conf);
     return conf;
 }
@@ -65,7 +66,7 @@ IVFConfAdapter::Match(const TempMetaConf& metaconf) {
     conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
     conf->d = metaconf.dim;
     conf->metric_type = metaconf.metric_type;
-    conf->gpu_id = conf->gpu_id;
+    conf->gpu_id = metaconf.gpu_id;
     MatchBase(conf);
     return conf;
 }
@@ -114,7 +115,7 @@ IVFSQConfAdapter::Match(const TempMetaConf& metaconf) {
     conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
     conf->d = metaconf.dim;
     conf->metric_type = metaconf.metric_type;
-    conf->gpu_id = conf->gpu_id;
+    conf->gpu_id = metaconf.gpu_id;
     conf->nbits = 8;
     MatchBase(conf);
     return conf;
@@ -126,7 +127,7 @@ IVFPQConfAdapter::Match(const TempMetaConf& metaconf) {
     conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
     conf->d = metaconf.dim;
     conf->metric_type = metaconf.metric_type;
-    conf->gpu_id = conf->gpu_id;
+    conf->gpu_id = metaconf.gpu_id;
     conf->nbits = 8;
 
     if (!(conf->d % 4))
@@ -175,21 +176,17 @@ NSGConfAdapter::Match(const TempMetaConf& metaconf) {
     conf->nlist = MatchNlist(metaconf.size, metaconf.nlist);
     conf->d = metaconf.dim;
     conf->metric_type = metaconf.metric_type;
-    conf->gpu_id = conf->gpu_id;
+    conf->gpu_id = metaconf.gpu_id;
+    conf->k = metaconf.k;
 
-    double factor = metaconf.size / TYPICAL_COUNT;
     auto scale_factor = round(metaconf.dim / 128.0);
     scale_factor = scale_factor >= 4 ? 4 : scale_factor;
-    conf->nprobe = conf->nlist > 10000 ? conf->nlist * 0.02 : conf->nlist * 0.1;
-    conf->knng = (100 + 100 * scale_factor) * factor;
-    conf->search_length = (40 + 5 * scale_factor) * factor;
-    conf->out_degree = (50 + 5 * scale_factor) * factor;
-    conf->candidate_pool_size = (200 + 100 * scale_factor) * factor;
+    conf->nprobe = int64_t(conf->nlist * 0.01);
+    conf->knng = 40 + 10 * scale_factor; // the size of knng
+    conf->search_length = 40 + 5 * scale_factor;
+    conf->out_degree = 50 + 5 * scale_factor;
+    conf->candidate_pool_size = 200 + 100 * scale_factor;
     MatchBase(conf);
-
-    //    WRAPPER_LOG_DEBUG << "nlist: " << conf->nlist
-    //    << ", gpu_id: " << conf->gpu_id << ", d: " << conf->d
-    //    << ", nprobe: " << conf->nprobe << ", knng: " << conf->knng;
     return conf;
 }
 

core/src/wrapper/ConfAdapter.h

@@ -46,9 +46,6 @@ class ConfAdapter {
     virtual knowhere::Config
     MatchSearch(const TempMetaConf& metaconf, const IndexType& type);
 
-    //    virtual void
-    //    Dump(){}
-
  protected:
     static void
     MatchBase(knowhere::Config conf);

core/unittest/wrapper/test_wrapper.cpp

@@ -56,10 +56,6 @@ class KnowhereWrapperTest
         index_ = GetVecIndexFactory(index_type);
         conf = ParamGenerator::GetInstance().GenBuild(index_type, tempconf);
         searchconf = ParamGenerator::GetInstance().GenSearchConf(index_type, tempconf);
-
-//        conf->k = k;
-//        conf->d = dim;
-//        conf->gpu_id = DEVICEID;
     }
 
     void TearDown() override {
@@ -97,6 +93,7 @@ INSTANTIATE_TEST_CASE_P(WrapperParam, KnowhereWrapperTest,
 
 TEST_P(KnowhereWrapperTest, BASE_TEST) {
     EXPECT_EQ(index_->GetType(), index_type);
+    //    conf->Dump();
 
     auto elems = nq * k;
     std::vector<int64_t> res_ids(elems);
@@ -191,3 +188,119 @@ TEST(whatever, test_config) {
     auto pq_conf = std::make_shared<milvus::engine::IVFPQConfAdapter>();
     pq_conf->Match(conf);
 }
+
+// #include "knowhere/index/vector_index/IndexIDMAP.h"
+// #include "src/wrapper/VecImpl.h"
+// #include "src/index/unittest/utils.h"
+// The two case below prove NSG is concern with data distribution
+// Further work: 1. Use right basedata and pass it by milvus
+//                  a. batch size is 100000 [Pass]
+//                  b. transfer all at once [Pass]
+//               2. Use SIFT1M in test and check time cost []
+// TEST_P(KnowhereWrapperTest, nsgwithidmap) {
+//     auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
+//     auto ori_xb = xb;
+//     auto ori_ids = ids;
+//     std::vector<float> temp_xb;
+//     std::vector<int64_t> temp_ids;
+//     nb = 50000;
+//     for (int i = 0; i < 20; ++i) {
+//         GenData(dim, nb, nq, xb, xq, ids, k, gt_ids, gt_dis);
+//         assert(xb.size() == nb*dim);
+// //#define IDMAP
+// #ifdef IDMAP
+//         temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
+//         temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
+//         if (i == 0) {
+//             idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
+//         } else {
+//             idmap->Add(nb, temp_xb.data(), temp_ids.data());
+//         }
+//         temp_xb.clear();
+//         temp_ids.clear();
+// #else
+//         temp_xb.insert(temp_xb.end(), xb.data(), xb.data() + nb*dim);
+//         temp_ids.insert(temp_ids.end(), ori_ids.data()+nb*i, ori_ids.data() + nb*(i+1));
+// #endif
+//     }
+
+// #ifdef IDMAP
+//     auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
+//     auto x = idmap_idx->Count();
+//     index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
+// #else
+//     assert(temp_xb.size() == 1000000*128);
+//     index_->BuildAll(1000000, temp_xb.data(), ori_ids.data(), conf);
+// #endif
+// }
+
+// TEST_P(KnowhereWrapperTest, nsgwithsidmap) {
+//     auto idmap = GetVecIndexFactory(milvus::engine::IndexType::FAISS_IDMAP);
+//     auto ori_xb = xb;
+//     std::vector<float> temp_xb;
+//     std::vector<int64_t> temp_ids;
+//     nb = 50000;
+//     for (int i = 0; i < 20; ++i) {
+// #define IDMAP
+// #ifdef IDMAP
+//         temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
+//         temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
+//         if (i == 0) {
+//             idmap->BuildAll(nb, temp_xb.data(), temp_ids.data(), conf);
+//         } else {
+//             idmap->Add(nb, temp_xb.data(), temp_ids.data());
+//         }
+//         temp_xb.clear();
+//         temp_ids.clear();
+// #else
+//         temp_xb.insert(temp_xb.end(), ori_xb.data()+nb*dim*i, ori_xb.data() + nb*dim*(i+1));
+//         temp_ids.insert(temp_ids.end(), ids.data()+nb*i, ids.data() + nb*(i+1));
+// #endif
+//     }
+
+// #ifdef IDMAP
+//     auto idmap_idx = std::dynamic_pointer_cast<milvus::engine::BFIndex>(idmap);
+//     auto x = idmap_idx->Count();
+//     index_->BuildAll(idmap_idx->Count(), idmap_idx->GetRawVectors(), idmap_idx->GetRawIds(), conf);
+// #else
+//     index_->BuildAll(1000000, temp_xb.data(), temp_ids.data(), conf);
+// #endif
+
+//     // The code use to store raw base data
+//     FileIOWriter writer("/tmp/newraw");
+//     ori_xb.shrink_to_fit();
+//     std::cout << "size" << ori_xb.size();
+//     writer(static_cast<void*>(ori_xb.data()), ori_xb.size()* sizeof(float));
+//     std::cout << "Finish!" << std::endl;
+// }
+
+// void load_data(char* filename, float*& data, unsigned& num,
+//                unsigned& dim) {  // load data with sift10K pattern
+//     std::ifstream in(filename, std::ios::binary);
+//     if (!in.is_open()) {
+//         std::cout << "open file error" << std::endl;
+//         exit(-1);
+//     }
+//     in.read((char*)&dim, 4);
+//     in.seekg(0, std::ios::end);
+//     std::ios::pos_type ss = in.tellg();
+//     size_t fsize = (size_t)ss;
+//     num = (unsigned)(fsize / (dim + 1) / 4);
+//     data = new float[(size_t)num * (size_t)dim];
+
+//     in.seekg(0, std::ios::beg);
+//     for (size_t i = 0; i < num; i++) {
+//         in.seekg(4, std::ios::cur);
+//         in.read((char*)(data + i * dim), dim * 4);
+//     }
+//     in.close();
+// }
+
+// TEST_P(KnowhereWrapperTest, Sift1M) {
+//     float* data = nullptr;
+//     unsigned points_num, dim;
+//     load_data("/mnt/112d53a6-5592-4360-a33b-7fd789456fce/workspace/Data/sift/sift_base.fvecs", data, points_num,
+//     dim); std::cout << points_num << " " << dim << std::endl;
+
+//     index_->BuildAll(points_num, data, ids.data(), conf);
+// }
\ No newline at end of file

core/unittest/wrapper/utils.cpp

@@ -117,6 +117,11 @@ void
 DataGenBase::GenData(const int& dim, const int& nb, const int& nq, std::vector<float>& xb, std::vector<float>& xq,
                      std::vector<int64_t>& ids, const int& k, std::vector<int64_t>& gt_ids,
                      std::vector<float>& gt_dis) {
+    xb.clear();
+    xq.clear();
+    ids.clear();
+    gt_ids.clear();
+    gt_dis.clear();
     xb.resize(nb * dim);
     xq.resize(nq * dim);
     ids.resize(nb);

core/unittest/wrapper/utils.h

@@ -25,7 +25,6 @@
 #include <cstdio>
 #include <fstream>
 
-
 #include "wrapper/VecIndex.h"
 #include "wrapper/utils.h"
 #include "knowhere/index/vector_index/helpers/IndexParameter.h"
@@ -90,17 +89,20 @@ class ParamGenerator {
         return instance;
     }
 
-    knowhere::Config GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
+    knowhere::Config 
+    GenSearchConf(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
         auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type);
         return adapter->MatchSearch(conf, type);
     }
 
-    knowhere::Config GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
+    knowhere::Config 
+    GenBuild(const milvus::engine::IndexType& type, const milvus::engine::TempMetaConf& conf) {
         auto adapter = milvus::engine::AdapterMgr::GetInstance().GetAdapter(type);
         return adapter->Match(conf);
     }
 
-    knowhere::Config Gen(const milvus::engine::IndexType& type) {
+    knowhere::Config
+    Gen(const milvus::engine::IndexType& type) {
         switch (type) {
             case milvus::engine::IndexType::FAISS_IDMAP: {
                 auto tempconf = std::make_shared<knowhere::Cfg>();