From 01437cef39a04733e584574cb4a66c0a9475aad7 Mon Sep 17 00:00:00 2001 From: cqy123456 <39671710+cqy123456@users.noreply.github.com> Date: Sat, 18 Jul 2020 12:47:29 +0800 Subject: [PATCH] Fix warning master (#2894) * fix clean unused variable warning in knowhere Signed-off-by: cqy * clean unused variable warning in knowhere Signed-off-by: cqy * clean unused variable warning in knowhere Signed-off-by: cqy * clean unused variable warning in knowhere Signed-off-by: cqy * fix konwhere warning Signed-off-by: cqy --- CHANGELOG.md | 2 +- .../knowhere/index/vector_index/IndexAnnoy.cpp | 4 ++-- .../index/vector_index/IndexBinaryIDMAP.cpp | 6 +++--- .../index/vector_index/IndexBinaryIVF.cpp | 4 ++-- .../knowhere/index/vector_index/IndexHNSW.cpp | 7 ++++--- .../knowhere/index/vector_index/IndexIDMAP.cpp | 4 ++-- .../knowhere/index/vector_index/IndexIVF.cpp | 8 ++++---- .../knowhere/index/vector_index/IndexIVFPQ.cpp | 2 +- .../knowhere/index/vector_index/IndexIVFSQ.cpp | 2 +- .../knowhere/index/vector_index/IndexNSG.cpp | 4 ++-- .../vector_index/adapter/SptagAdapter.cpp | 4 ++-- .../index/vector_index/adapter/VectorAdapter.h | 18 ++++++++++++------ .../index/vector_index/gpu/IndexGPUIVF.cpp | 2 +- .../index/vector_index/gpu/IndexGPUIVFPQ.cpp | 2 +- .../index/vector_index/gpu/IndexGPUIVFSQ.cpp | 2 +- .../index/vector_index/gpu/IndexGPUIVFSQNR.cpp | 2 +- .../vector_index/gpu/IndexIVFSQHybrid.cpp | 2 +- .../index/vector_offset_index/IndexHNSW_NM.cpp | 7 ++++--- .../vector_offset_index/IndexHNSW_SQ8NR.cpp | 6 +++--- .../vector_offset_index/IndexIVFSQNR_NM.cpp | 8 ++++---- .../index/vector_offset_index/IndexIVF_NM.cpp | 8 ++++---- .../index/vector_offset_index/IndexNSG_NM.cpp | 4 ++-- .../gpu/IndexGPUIVFSQNR_NM.cpp | 2 +- .../vector_offset_index/gpu/IndexGPUIVF_NM.cpp | 2 +- 24 files changed, 60 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb018af..122d06f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,7 +42,7 @@ Please mark all changes in change log and use the issue from GitHub - \#2561 Clean util dependencies with other modules - \#2612 Move all APIs in utils into namespace milvus - \#2675 Print out system memory size when report invalid cpu cache size -- \#2828 Let Faiss not to compile half float by default +- \#2828 Let Faiss not compile half float by default - \#2841 Replace IndexType/EngineType/MetricType - \#2858 Unify index name in db - \#2884 Using BlockingQueue in JobMgr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp index 466acf45..f0694e50 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp @@ -86,7 +86,7 @@ IndexAnnoy::BuildAll(const DatasetPtr& dataset_ptr, const Config& config) { return; } - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR(dataset_ptr) metric_type_ = config[Metric::TYPE]; if (metric_type_ == Metric::L2) { @@ -110,7 +110,7 @@ IndexAnnoy::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) auto k = config[meta::TOPK].get(); auto search_k = config[IndexParams::search_k].get(); auto all_num = rows * k; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp index 056a955a..5e3531a5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp @@ -44,7 +44,7 @@ BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) int64_t k = config[meta::TOPK].get(); auto elems = rows * k; @@ -135,7 +135,7 @@ BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) index_->add_with_ids(rows, (uint8_t*)p_data, p_ids); } @@ -177,7 +177,7 @@ BinaryIDMAP::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) } std::lock_guard lk(mutex_); - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) std::vector new_ids(rows); for (int i = 0; i < rows; ++i) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp index 8036a428..41b2d1db 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp @@ -48,7 +48,7 @@ BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) try { int64_t k = config[meta::TOPK].get(); @@ -147,7 +147,7 @@ BinaryIVF::Dim() { void BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR(dataset_ptr) int64_t nlist = config[IndexParams::nlist]; faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp index 758ef74a..45ed68e0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp @@ -78,7 +78,8 @@ IndexHNSW::Load(const BinarySet& index_binary) { void IndexHNSW::Train(const DatasetPtr& dataset_ptr, const Config& config) { try { - GETTENSOR(dataset_ptr) + int64_t dim = dataset_ptr->Get(meta::DIM); + int64_t rows = dataset_ptr->Get(meta::ROWS); hnswlib::SpaceInterface* space; if (config[Metric::TYPE] == Metric::L2) { @@ -102,7 +103,7 @@ IndexHNSW::Add(const DatasetPtr& dataset_ptr, const Config& config) { std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) // if (normalize) { // std::vector ep_norm_vector(Dim()); @@ -135,7 +136,7 @@ IndexHNSW::Query(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) size_t k = config[meta::TOPK].get(); size_t id_size = sizeof(int64_t) * k; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 612de455..01af8621 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -68,7 +68,7 @@ IDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) index_->add_with_ids(rows, (float*)p_data, p_ids); } @@ -96,7 +96,7 @@ IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) int64_t k = config[meta::TOPK].get(); auto elems = rows * k; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index fdd8265c..12bc63a4 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -64,7 +64,7 @@ IVF::Load(const BinarySet& binary_set) { void IVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type); @@ -80,7 +80,7 @@ IVF::Add(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) index_->add_with_ids(rows, (float*)p_data, p_ids); } @@ -91,7 +91,7 @@ IVF::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) index_->add(rows, (float*)p_data); } @@ -101,7 +101,7 @@ IVF::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) try { fiu_do_on("IVF.Search.throw_std_exception", throw std::exception()); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp index 662fefba..513f825b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp @@ -33,7 +33,7 @@ namespace knowhere { void IVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp index f1da52a9..8d4102f4 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp @@ -35,7 +35,7 @@ namespace knowhere { void IVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) // std::stringstream index_type; // index_type << "IVF" << config[IndexParams::nlist] << "," diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp index 782e08c3..651ba047 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp @@ -78,7 +78,7 @@ NSG::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) try { auto elems = rows * config[meta::TOPK].get(); @@ -139,7 +139,7 @@ NSG::Train(const DatasetPtr& dataset_ptr, const Config& config) { b_params.out_degree = config[IndexParams::out_degree]; b_params.search_length = config[IndexParams::search_length]; - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR(dataset_ptr) impl::NsgIndex::Metric_Type metric; auto metric_str = config[Metric::TYPE].get(); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp b/core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp index 030aa866..cf07bd23 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp @@ -32,7 +32,7 @@ ConvertToMetadataSet(const DatasetPtr& dataset_ptr) { std::shared_ptr ConvertToVectorSet(const DatasetPtr& dataset_ptr) { - GETTENSOR(dataset_ptr); + GET_TENSOR_DATA_DIM(dataset_ptr) size_t num_bytes = rows * dim * sizeof(float); SPTAG::ByteArray byte_array((uint8_t*)p_data, num_bytes, false); @@ -42,7 +42,7 @@ ConvertToVectorSet(const DatasetPtr& dataset_ptr) { std::vector ConvertToQueryResult(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr); + GET_TENSOR_DATA_DIM(dataset_ptr); int64_t k = config[meta::TOPK].get(); std::vector query_results(rows, SPTAG::QueryResult(nullptr, k, true)); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h b/core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h index 44086796..9fe4e5b9 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.h @@ -18,15 +18,21 @@ namespace milvus { namespace knowhere { -#define GETTENSOR(dataset_ptr) \ - int64_t dim = dataset_ptr->Get(meta::DIM); \ +#define GET_TENSOR_DATA(dataset_ptr) \ int64_t rows = dataset_ptr->Get(meta::ROWS); \ const void* p_data = dataset_ptr->Get(meta::TENSOR); -#define GETTENSORWITHIDS(dataset_ptr) \ - int64_t dim = dataset_ptr->Get(meta::DIM); \ - int64_t rows = dataset_ptr->Get(meta::ROWS); \ - const void* p_data = dataset_ptr->Get(meta::TENSOR); \ +#define GET_TENSOR_DATA_DIM(dataset_ptr) \ + GET_TENSOR_DATA(dataset_ptr) \ + int64_t dim = dataset_ptr->Get(meta::DIM); + +#define GET_TENSOR_DATA_ID(dataset_ptr) \ + GET_TENSOR_DATA(dataset_ptr) \ + const int64_t* p_ids = dataset_ptr->Get(meta::IDS); + +#define GET_TENSOR(dataset_ptr) \ + GET_TENSOR_DATA(dataset_ptr) \ + int64_t dim = dataset_ptr->Get(meta::DIM); \ const int64_t* p_ids = dataset_ptr->Get(meta::IDS); extern DatasetPtr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp index 3552a190..22e606e0 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp @@ -30,7 +30,7 @@ namespace knowhere { void GPUIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp index d9465761..3c742d79 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp @@ -27,7 +27,7 @@ namespace knowhere { void GPUIVFPQ::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp index bcb5c024..66df444a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp @@ -28,7 +28,7 @@ namespace knowhere { void GPUIVFSQ::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; // std::stringstream index_type; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp index 8b5f64ae..4e23bda3 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQNR.cpp @@ -28,7 +28,7 @@ namespace knowhere { void GPUIVFSQNR::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; // std::stringstream index_type; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp index dde0fcd2..577adb5a 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp @@ -30,7 +30,7 @@ namespace knowhere { void IVFSQHybrid::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; std::stringstream index_type; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp index dbf7bf93..54119a9f 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_NM.cpp @@ -79,7 +79,8 @@ IndexHNSW_NM::Load(const BinarySet& index_binary) { void IndexHNSW_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { try { - GETTENSOR(dataset_ptr) + int64_t dim = dataset_ptr->Get(meta::DIM); + int64_t rows = dataset_ptr->Get(meta::ROWS); hnswlib_nm::SpaceInterface* space; if (config[Metric::TYPE] == Metric::L2) { @@ -106,7 +107,7 @@ IndexHNSW_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) { std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) auto base = index_->getCurrentElementCount(); auto pp_data = const_cast(p_data); @@ -123,7 +124,7 @@ IndexHNSW_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) size_t k = config[meta::TOPK].get(); size_t id_size = sizeof(int64_t) * k; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp index 5b34701e..2cecfeb1 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.cpp @@ -73,7 +73,7 @@ IndexHNSW_SQ8NR::Load(const BinarySet& index_binary) { void IndexHNSW_SQ8NR::Train(const DatasetPtr& dataset_ptr, const Config& config) { try { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) hnswlib_nm::SpaceInterface* space; if (config[Metric::TYPE] == Metric::L2) { @@ -103,7 +103,7 @@ IndexHNSW_SQ8NR::Add(const DatasetPtr& dataset_ptr, const Config& config) { std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) auto base = index_->getCurrentElementCount(); auto pp_data = const_cast(p_data); @@ -120,7 +120,7 @@ IndexHNSW_SQ8NR::Query(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) size_t k = config[meta::TOPK].get(); size_t id_size = sizeof(int64_t) * k; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp index 8fe13697..cfa86102 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVFSQNR_NM.cpp @@ -116,7 +116,7 @@ IVFSQNR_NM::Load(const BinarySet& binary_set) { void IVFSQNR_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type); @@ -134,7 +134,7 @@ IVFSQNR_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) index_->add_with_ids_without_codes(rows, (float*)p_data, p_ids); ArrangeCodes(dataset_ptr, config); @@ -147,7 +147,7 @@ IVFSQNR_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) index_->add_without_codes(rows, (float*)p_data); ArrangeCodes(dataset_ptr, config); @@ -175,7 +175,7 @@ IVFSQNR_NM::CopyCpuToGpu(const int64_t device_id, const Config& config) { void IVFSQNR_NM::ArrangeCodes(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) // Construct arranged sq8 data from original data const float* original_data = (const float*)p_data; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp index cc97ef93..30d79fac 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp @@ -102,7 +102,7 @@ IVF_NM::Load(const BinarySet& binary_set) { void IVF_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); faiss::Index* coarse_quantizer = new faiss::IndexFlat(dim, metric_type); @@ -118,7 +118,7 @@ IVF_NM::Add(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSORWITHIDS(dataset_ptr) + GET_TENSOR_DATA_ID(dataset_ptr) index_->add_with_ids_without_codes(rows, (float*)p_data, p_ids); } @@ -129,7 +129,7 @@ IVF_NM::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) { } std::lock_guard lk(mutex_); - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) index_->add_without_codes(rows, (float*)p_data); } @@ -139,7 +139,7 @@ IVF_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA(dataset_ptr) try { fiu_do_on("IVF_NM.Search.throw_std_exception", throw std::exception()); diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp index 496048d2..ffc321d5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp @@ -79,7 +79,7 @@ NSG_NM::Query(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) try { auto topK = config[meta::TOPK].get(); @@ -143,7 +143,7 @@ NSG_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { auto p_ids = dataset_ptr->Get(meta::IDS); - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) impl::NsgIndex::Metric_Type metric_type_nsg; if (config[Metric::TYPE].get() == "IP") { metric_type_nsg = impl::NsgIndex::Metric_Type::Metric_Type_IP; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVFSQNR_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVFSQNR_NM.cpp index 347c36fe..23a3b304 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVFSQNR_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVFSQNR_NM.cpp @@ -33,7 +33,7 @@ namespace knowhere { void GPUIVFSQNR_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; faiss::MetricType metric_type = GetMetricType(config[Metric::TYPE].get()); diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp b/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp index 0edca4a4..0f697bd7 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp @@ -31,7 +31,7 @@ namespace knowhere { void GPUIVF_NM::Train(const DatasetPtr& dataset_ptr, const Config& config) { - GETTENSOR(dataset_ptr) + GET_TENSOR_DATA_DIM(dataset_ptr) gpu_id_ = config[knowhere::meta::DEVICEID]; auto gpu_res = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_); -- GitLab