Fix warning master (#2894)

* fix clean unused variable warning in knowhere
Signed-off-by: Ncqy <yaya645@126.com>

* clean unused variable warning in knowhere
Signed-off-by: Ncqy <yaya645@126.com>

* clean unused variable warning in knowhere
Signed-off-by: Ncqy <yaya645@126.com>

* clean unused variable warning in knowhere
Signed-off-by: Ncqy <yaya645@126.com>

* fix konwhere warning
Signed-off-by: Ncqy <yaya645@126.com>

CHANGELOG.md

@@ -42,7 +42,7 @@ Please mark all changes in change log and use the issue from GitHub
 -   \#2561 Clean util dependencies with other modules
 -   \#2612 Move all APIs in utils into namespace milvus 
 -   \#2675 Print out system memory size when report invalid cpu cache size 
--   \#2828 Let Faiss not to compile half float by default
+-   \#2828 Let Faiss not compile half float by default
 -   \#2841 Replace IndexType/EngineType/MetricType
 -   \#2858 Unify index name in db
 -   \#2884 Using BlockingQueue in JobMgr

core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp

@@ -86,7 +86,7 @@ IndexAnnoy::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) {
         return;
     }
 
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR(dataset_ptr)
 
     metric_type_ = config[Metric::TYPE];
     if (metric_type_ == Metric::L2) {
@@ -110,7 +110,7 @@ IndexAnnoy::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     auto k = config[meta::TOPK].get<int64_t>();
     auto search_k = config[IndexParams::search_k].get<int64_t>();
     auto all_num = rows * k;

core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp

@@ -44,7 +44,7 @@ BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) {
     if (!index_) {
         KNOWHERE_THROW_MSG("index not initialize");
     }
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     int64_t k = config[meta::TOPK].get<int64_t>();
     auto elems = rows * k;
@@ -135,7 +135,7 @@ BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
 
     index_->add_with_ids(rows, (uint8_t*)p_data, p_ids);
 }
@@ -177,7 +177,7 @@ BinaryIDMAP::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config)
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     std::vector<int64_t> new_ids(rows);
     for (int i = 0; i < rows; ++i) {

core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp

@@ -48,7 +48,7 @@ BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     try {
         int64_t k = config[meta::TOPK].get<int64_t>();
@@ -147,7 +147,7 @@ BinaryIVF::Dim() {
 
 void
 BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR(dataset_ptr)
 
     int64_t nlist = config[IndexParams::nlist];
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());

core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp

@@ -78,7 +78,8 @@ IndexHNSW::Load(const BinarySet& index_binary) {
 void
 IndexHNSW::Train(const DatasetPtr& dataset_ptr, const Config& config) {
     try {
-        GETTENSOR(dataset_ptr)
+        int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);
+        int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS);
 
         hnswlib::SpaceInterface<float>* space;
         if (config[Metric::TYPE] == Metric::L2) {
@@ -102,7 +103,7 @@ IndexHNSW::Add(const DatasetPtr& dataset_ptr, const Config& config) {
 
     std::lock_guard<std::mutex> lk(mutex_);
 
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
 
     //     if (normalize) {
     //         std::vector<float> ep_norm_vector(Dim());
@@ -135,7 +136,7 @@ IndexHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config) {
     if (!index_) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     size_t k = config[meta::TOPK].get<int64_t>();
     size_t id_size = sizeof(int64_t) * k;

core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp

@@ -68,7 +68,7 @@ IDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
     index_->add_with_ids(rows, (float*)p_data, p_ids);
 }
 
@@ -96,7 +96,7 @@ IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) {
     if (!index_) {
         KNOWHERE_THROW_MSG("index not initialize");
     }
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     int64_t k = config[meta::TOPK].get<int64_t>();
     auto elems = rows * k;

core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp

@@ -64,7 +64,7 @@ IVF::Load(const BinarySet& binary_set) {
 
 void
 IVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
     faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
@@ -80,7 +80,7 @@ IVF::Add(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
     index_->add_with_ids(rows, (float*)p_data, p_ids);
 }
 
@@ -91,7 +91,7 @@ IVF::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
     index_->add(rows, (float*)p_data);
 }
 
@@ -101,7 +101,7 @@ IVF::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     try {
         fiu_do_on("IVF.Search.throw_std_exception", throw std::exception());

core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp

@@ -33,7 +33,7 @@ namespace knowhere {
 
 void
 IVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
     faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);

core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp

@@ -35,7 +35,7 @@ namespace knowhere {
 
 void
 IVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     // std::stringstream index_type;
     // index_type << "IVF" << config[IndexParams::nlist] << ","

core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp

@@ -78,7 +78,7 @@ NSG::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     try {
         auto elems = rows * config[meta::TOPK].get<int64_t>();
@@ -139,7 +139,7 @@ NSG::Train(const DatasetPtr& dataset_ptr, const Config& config) {
     b_params.out_degree = config[IndexParams::out_degree];
     b_params.search_length = config[IndexParams::search_length];
 
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR(dataset_ptr)
 
     impl::NsgIndex::Metric_Type metric;
     auto metric_str = config[Metric::TYPE].get<std::string>();

core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp

@@ -32,7 +32,7 @@ ConvertToMetadataSet(const DatasetPtr& dataset_ptr) {
 
 std::shared_ptr<SPTAG::VectorSet>
 ConvertToVectorSet(const DatasetPtr& dataset_ptr) {
-    GETTENSOR(dataset_ptr);
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     size_t num_bytes = rows * dim * sizeof(float);
     SPTAG::ByteArray byte_array((uint8_t*)p_data, num_bytes, false);
 
@@ -42,7 +42,7 @@ ConvertToVectorSet(const DatasetPtr& dataset_ptr) {
 
 std::vector<SPTAG::QueryResult>
 ConvertToQueryResult(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr);
+    GET_TENSOR_DATA_DIM(dataset_ptr);
 
     int64_t k = config[meta::TOPK].get<int64_t>();
     std::vector<SPTAG::QueryResult> query_results(rows, SPTAG::QueryResult(nullptr, k, true));

core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h

@@ -18,15 +18,21 @@
 namespace milvus {
 namespace knowhere {
 
-#define GETTENSOR(dataset_ptr)                            \
-    int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);   \
+#define GET_TENSOR_DATA(dataset_ptr)                      \
     int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS); \
     const void* p_data = dataset_ptr->Get<const void*>(meta::TENSOR);
 
-#define GETTENSORWITHIDS(dataset_ptr)                                 \
-    int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);               \
-    int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS);             \
-    const void* p_data = dataset_ptr->Get<const void*>(meta::TENSOR); \
+#define GET_TENSOR_DATA_DIM(dataset_ptr) \
+    GET_TENSOR_DATA(dataset_ptr)         \
+    int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);
+
+#define GET_TENSOR_DATA_ID(dataset_ptr) \
+    GET_TENSOR_DATA(dataset_ptr)        \
+    const int64_t* p_ids = dataset_ptr->Get<const int64_t*>(meta::IDS);
+
+#define GET_TENSOR(dataset_ptr)                         \
+    GET_TENSOR_DATA(dataset_ptr)                        \
+    int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM); \
     const int64_t* p_ids = dataset_ptr->Get<const int64_t*>(meta::IDS);
 
 extern DatasetPtr

core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp

@@ -30,7 +30,7 @@ namespace knowhere {
 
 void
 GPUIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);

core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp

@@ -27,7 +27,7 @@ namespace knowhere {
 
 void
 GPUIVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);

core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp

@@ -28,7 +28,7 @@ namespace knowhere {
 
 void
 GPUIVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     // std::stringstream index_type;

core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp

@@ -28,7 +28,7 @@ namespace knowhere {
 
 void
 GPUIVFSQNR::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     // std::stringstream index_type;

core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp

@@ -30,7 +30,7 @@ namespace knowhere {
 
 void
 IVFSQHybrid::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     std::stringstream index_type;

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp

@@ -79,7 +79,8 @@ IndexHNSW_NM::Load(const BinarySet& index_binary) {
 void
 IndexHNSW_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
     try {
-        GETTENSOR(dataset_ptr)
+        int64_t dim = dataset_ptr->Get<int64_t>(meta::DIM);
+        int64_t rows = dataset_ptr->Get<int64_t>(meta::ROWS);
 
         hnswlib_nm::SpaceInterface<float>* space;
         if (config[Metric::TYPE] == Metric::L2) {
@@ -106,7 +107,7 @@ IndexHNSW_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) {
 
     std::lock_guard<std::mutex> lk(mutex_);
 
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
 
     auto base = index_->getCurrentElementCount();
     auto pp_data = const_cast<void*>(p_data);
@@ -123,7 +124,7 @@ IndexHNSW_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) {
     if (!index_) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     size_t k = config[meta::TOPK].get<int64_t>();
     size_t id_size = sizeof(int64_t) * k;

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp

@@ -73,7 +73,7 @@ IndexHNSW_SQ8NR::Load(const BinarySet& index_binary) {
 void
 IndexHNSW_SQ8NR::Train(const DatasetPtr& dataset_ptr, const Config& config) {
     try {
-        GETTENSOR(dataset_ptr)
+        GET_TENSOR_DATA_DIM(dataset_ptr)
 
         hnswlib_nm::SpaceInterface<float>* space;
         if (config[Metric::TYPE] == Metric::L2) {
@@ -103,7 +103,7 @@ IndexHNSW_SQ8NR::Add(const DatasetPtr& dataset_ptr, const Config& config) {
 
     std::lock_guard<std::mutex> lk(mutex_);
 
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
 
     auto base = index_->getCurrentElementCount();
     auto pp_data = const_cast<void*>(p_data);
@@ -120,7 +120,7 @@ IndexHNSW_SQ8NR::Query(const DatasetPtr& dataset_ptr, const Config& config) {
     if (!index_) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     size_t k = config[meta::TOPK].get<int64_t>();
     size_t id_size = sizeof(int64_t) * k;

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp

@@ -116,7 +116,7 @@ IVFSQNR_NM::Load(const BinarySet& binary_set) {
 
 void
 IVFSQNR_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
     faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
@@ -134,7 +134,7 @@ IVFSQNR_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
     index_->add_with_ids_without_codes(rows, (float*)p_data, p_ids);
 
     ArrangeCodes(dataset_ptr, config);
@@ -147,7 +147,7 @@ IVFSQNR_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
     index_->add_without_codes(rows, (float*)p_data);
 
     ArrangeCodes(dataset_ptr, config);
@@ -175,7 +175,7 @@ IVFSQNR_NM::CopyCpuToGpu(const int64_t device_id, const Config& config) {
 
 void
 IVFSQNR_NM::ArrangeCodes(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     // Construct arranged sq8 data from original data
     const float* original_data = (const float*)p_data;

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp

@@ -102,7 +102,7 @@ IVF_NM::Load(const BinarySet& binary_set) {
 
 void
 IVF_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());
     faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
@@ -118,7 +118,7 @@ IVF_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSORWITHIDS(dataset_ptr)
+    GET_TENSOR_DATA_ID(dataset_ptr)
     index_->add_with_ids_without_codes(rows, (float*)p_data, p_ids);
 }
 
@@ -129,7 +129,7 @@ IVF_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) {
     }
 
     std::lock_guard<std::mutex> lk(mutex_);
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
     index_->add_without_codes(rows, (float*)p_data);
 }
 
@@ -139,7 +139,7 @@ IVF_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA(dataset_ptr)
 
     try {
         fiu_do_on("IVF_NM.Search.throw_std_exception", throw std::exception());

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp

@@ -79,7 +79,7 @@ NSG_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) {
         KNOWHERE_THROW_MSG("index not initialize or trained");
     }
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
 
     try {
         auto topK = config[meta::TOPK].get<int64_t>();
@@ -143,7 +143,7 @@ NSG_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
 
     auto p_ids = dataset_ptr->Get<const int64_t*>(meta::IDS);
 
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     impl::NsgIndex::Metric_Type metric_type_nsg;
     if (config[Metric::TYPE].get<std::string>() == "IP") {
         metric_type_nsg = impl::NsgIndex::Metric_Type::Metric_Type_IP;

core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVFSQNR_NM.cpp

@@ -33,7 +33,7 @@ namespace knowhere {
 
 void
 GPUIVFSQNR_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get<std::string>());

core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp

@@ -31,7 +31,7 @@ namespace knowhere {
 
 void
 GPUIVF_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) {
-    GETTENSOR(dataset_ptr)
+    GET_TENSOR_DATA_DIM(dataset_ptr)
     gpu_id_ = config[knowhere::meta::DEVICEID];
 
     auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);