From 8b627cacfdc98b669bea6ad6695c4d75d4d6a6c6 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Thu, 28 May 2020 10:22:14 +0800 Subject: [PATCH] improve knowhere coverage (#2444) * increase nb for NSG code coverage Signed-off-by: yudong.cai * test more APIs in test_annoy Signed-off-by: yudong.cai * disable get_vector_by_id and search_by_id Signed-off-by: yudong.cai * improve code coverage Signed-off-by: yudong.cai * improve code coverage Signed-off-by: yudong.cai * update unittest Signed-off-by: yudong.cai * install test_instructionset Signed-off-by: yudong.cai * update changelog Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngine.h | 2 + core/src/db/engine/ExecutionEngineImpl.cpp | 2 + core/src/db/engine/ExecutionEngineImpl.h | 2 + .../src/index/knowhere/knowhere/index/Index.h | 5 -- .../index/vector_index/IndexAnnoy.cpp | 10 ---- .../knowhere/index/vector_index/IndexAnnoy.h | 3 -- .../index/vector_index/IndexBinaryIDMAP.cpp | 4 ++ .../index/vector_index/IndexBinaryIDMAP.h | 4 ++ .../index/vector_index/IndexBinaryIVF.cpp | 6 ++- .../index/vector_index/IndexBinaryIVF.h | 4 ++ .../index/vector_index/IndexIDMAP.cpp | 4 ++ .../knowhere/index/vector_index/IndexIDMAP.h | 4 ++ .../knowhere/index/vector_index/IndexIVF.cpp | 2 + .../knowhere/index/vector_index/IndexIVF.h | 4 ++ .../knowhere/index/vector_index/VecIndex.h | 4 ++ .../index/vector_index/gpu/IndexGPUIVF.cpp | 8 +-- core/src/index/thirdparty/faiss/Index.cpp | 2 + core/src/index/thirdparty/faiss/Index.h | 2 + .../index/thirdparty/faiss/IndexBinary.cpp | 2 + core/src/index/thirdparty/faiss/IndexBinary.h | 2 + .../index/thirdparty/faiss/IndexBinaryIVF.cpp | 2 + .../index/thirdparty/faiss/IndexBinaryIVF.h | 2 + core/src/index/thirdparty/faiss/IndexIVF.cpp | 2 + core/src/index/thirdparty/faiss/IndexIVF.h | 2 + .../index/thirdparty/faiss/MetaIndexes.cpp | 2 + core/src/index/thirdparty/faiss/MetaIndexes.h | 2 + core/src/index/unittest/CMakeLists.txt | 1 + core/src/index/unittest/test_annoy.cpp | 27 ++++++---- core/src/index/unittest/test_binaryidmap.cpp | 22 +++++--- core/src/index/unittest/test_binaryivf.cpp | 54 +++++++++++-------- .../index/unittest/test_customized_index.cpp | 6 +-- core/src/index/unittest/test_hnsw.cpp | 10 ++++ core/src/index/unittest/test_idmap.cpp | 10 ++-- .../index/unittest/test_instructionset.cpp | 7 ++- core/src/index/unittest/test_ivf.cpp | 9 ++-- core/src/index/unittest/test_nsg.cpp | 34 +++++++----- core/src/index/unittest/test_sptag.cpp | 8 ++- core/src/index/unittest/utils.cpp | 12 +++-- core/src/index/unittest/utils.h | 6 +-- 40 files changed, 202 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c214877..1bea91ca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Please mark all change in change log and use the issue from GitHub - \#2370 Clean compile warning - \#2381 Upgrade FAISS to 1.6.3 - \#2410 Logging build index progress +- \#2441 Improve Knowhere code coverage ## Task diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index 31c63fbb3..b1b9c761a 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -109,11 +109,13 @@ class ExecutionEngine { // virtual Status // Merge(const std::string& location) = 0; +#if 0 virtual Status GetVectorByID(const int64_t id, float* vector, bool hybrid) = 0; virtual Status GetVectorByID(const int64_t id, uint8_t* vector, bool hybrid) = 0; +#endif virtual Status ExecBinaryQuery(query::GeneralQueryPtr general_query, faiss::ConcurrentBitsetPtr& bitset, diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index dcc6a4ec3..96513b128 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -1248,6 +1248,7 @@ ExecutionEngineImpl::Search(int64_t n, const uint8_t* data, int64_t k, const mil return Status::OK(); } +#if 0 Status ExecutionEngineImpl::GetVectorByID(const int64_t id, float* vector, bool hybrid) { if (index_ == nullptr) { @@ -1299,6 +1300,7 @@ ExecutionEngineImpl::GetVectorByID(const int64_t id, uint8_t* vector, bool hybri return Status::OK(); } +#endif Status ExecutionEngineImpl::Cache() { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index b62411518..7e482119b 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -63,11 +63,13 @@ class ExecutionEngineImpl : public ExecutionEngine { Status CopyToCpu() override; +#if 0 Status GetVectorByID(const int64_t id, float* vector, bool hybrid) override; Status GetVectorByID(const int64_t id, uint8_t* vector, bool hybrid) override; +#endif Status ExecBinaryQuery(query::GeneralQueryPtr general_query, faiss::ConcurrentBitsetPtr& bitset, diff --git a/core/src/index/knowhere/knowhere/index/Index.h b/core/src/index/knowhere/knowhere/index/Index.h index 87ba3198b..c5a5047ba 100644 --- a/core/src/index/knowhere/knowhere/index/Index.h +++ b/core/src/index/knowhere/knowhere/index/Index.h @@ -37,11 +37,6 @@ class ToIndexData : public milvus::cache::DataObj { explicit ToIndexData(int64_t size) : size_(size) { } - int64_t - Size() override { - return size_; - } - private: int64_t size_ = 0; }; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp index 5d0eedb4b..990f8ebfa 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp @@ -151,7 +151,6 @@ IndexAnnoy::Count() { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } - return index_->get_n_items(); } @@ -160,17 +159,8 @@ IndexAnnoy::Dim() { if (!index_) { KNOWHERE_THROW_MSG("index not initialize"); } - return index_->get_dim(); } -int64_t -IndexAnnoy::IndexSize() { - if (index_size_ != -1) { - return index_size_; - } - - return index_size_ = Dim() * Count() * sizeof(float); -} } // namespace knowhere } // namespace milvus diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h index cbef69c5a..fa78743e2 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexAnnoy.h @@ -62,9 +62,6 @@ class IndexAnnoy : public VecIndex { int64_t Dim() override; - int64_t - IndexSize() override; - private: MetricType metric_type_; std::shared_ptr> index_ = nullptr; diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp index 7f70a5ce8..a38fba01b 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp @@ -72,6 +72,7 @@ BinaryIDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) { return ret_ds; } +#if 0 DatasetPtr BinaryIDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { @@ -109,6 +110,7 @@ BinaryIDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { return ret_ds; } +#endif void BinaryIDMAP::Add(const DatasetPtr& dataset_ptr, const Config& config) { @@ -169,6 +171,7 @@ BinaryIDMAP::AddWithoutIds(const DatasetPtr& dataset_ptr, const Config& config) index_->add_with_ids(rows, (uint8_t*)p_data, new_ids.data()); } +#if 0 DatasetPtr BinaryIDMAP::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { @@ -189,6 +192,7 @@ BinaryIDMAP::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) ret_ds->Set(meta::TENSOR, p_x); return ret_ds; } +#endif void BinaryIDMAP::QueryImpl(int64_t n, const uint8_t* data, int64_t k, float* distances, int64_t* labels, diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h index 4fc5fe760..bf3b57808 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.h @@ -50,8 +50,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex { DatasetPtr Query(const DatasetPtr&, const Config&) override; +#if 0 DatasetPtr QueryById(const DatasetPtr& dataset_ptr, const Config& config) override; +#endif int64_t Count() override { @@ -68,8 +70,10 @@ class BinaryIDMAP : public VecIndex, public FaissBaseBinaryIndex { return Count() * Dim() / 8; } +#if 0 DatasetPtr GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) override; +#endif virtual const uint8_t* GetRawVectors(); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp index f951f3b33..616f4c8d2 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.cpp @@ -83,6 +83,7 @@ BinaryIVF::Query(const DatasetPtr& dataset_ptr, const Config& config) { } } +#if 0 DatasetPtr BinaryIVF::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_ || !index_->is_trained) { @@ -126,6 +127,7 @@ BinaryIVF::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG(e.what()); } } +#endif void BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { @@ -140,13 +142,14 @@ BinaryIVF::Train(const DatasetPtr& dataset_ptr, const Config& config) { index_ = index; } +#if 0 DatasetPtr BinaryIVF::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_ || !index_->is_trained) { KNOWHERE_THROW_MSG("index not initialize or trained"); } - // GETBINARYTENSOR(dataset_ptr) + // GETBINARYTENSOR(dataset_ptr) // auto rows = dataset_ptr->Get(meta::ROWS); auto p_data = dataset_ptr->Get(meta::IDS); auto elems = dataset_ptr->Get(meta::DIM); @@ -166,6 +169,7 @@ BinaryIVF::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG(e.what()); } } +#endif std::shared_ptr BinaryIVF::GenParams(const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h index 7e201db84..e46a64e8d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexBinaryIVF.h @@ -62,8 +62,10 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex { DatasetPtr Query(const DatasetPtr& dataset_ptr, const Config& config) override; +#if 0 DatasetPtr QueryById(const DatasetPtr& dataset_ptr, const Config& config) override; +#endif int64_t Count() override { @@ -75,8 +77,10 @@ class BinaryIVF : public VecIndex, public FaissBaseBinaryIndex { return index_->d; } +#if 0 DatasetPtr GetVectorById(const DatasetPtr& dataset_ptr, const Config& config); +#endif protected: virtual std::shared_ptr diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp index 786976b54..8f1babc84 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp @@ -113,6 +113,7 @@ IDMAP::Query(const DatasetPtr& dataset_ptr, const Config& config) { return ret_ds; } +#if 0 DatasetPtr IDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { @@ -139,6 +140,7 @@ IDMAP::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { ret_ds->Set(meta::DISTANCE, p_dist); return ret_ds; } +#endif VecIndexPtr IDMAP::CopyCpuToGpu(const int64_t device_id, const Config& config) { @@ -179,6 +181,7 @@ IDMAP::GetRawIds() { } } +#if 0 DatasetPtr IDMAP::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_) { @@ -198,6 +201,7 @@ IDMAP::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { ret_ds->Set(meta::TENSOR, p_x); return ret_ds; } +#endif void IDMAP::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, const Config& config) { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h index e80f9bf92..f15c665d4 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.h @@ -48,8 +48,10 @@ class IDMAP : public VecIndex, public FaissBaseIndex { DatasetPtr Query(const DatasetPtr&, const Config&) override; +#if 0 DatasetPtr QueryById(const DatasetPtr& dataset, const Config& config) override; +#endif int64_t Count() override { @@ -66,8 +68,10 @@ class IDMAP : public VecIndex, public FaissBaseIndex { return Count() * Dim() * sizeof(FloatType); } +#if 0 DatasetPtr GetVectorById(const DatasetPtr& dataset, const Config& config) override; +#endif VecIndexPtr CopyCpuToGpu(const int64_t, const Config&); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp index 0d208dc08..94133acc2 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp @@ -142,6 +142,7 @@ IVF::Query(const DatasetPtr& dataset_ptr, const Config& config) { } } +#if 0 DatasetPtr IVF::QueryById(const DatasetPtr& dataset_ptr, const Config& config) { if (!index_ || !index_->is_trained) { @@ -214,6 +215,7 @@ IVF::GetVectorById(const DatasetPtr& dataset_ptr, const Config& config) { KNOWHERE_THROW_MSG(e.what()); } } +#endif void IVF::Seal() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h index cb688eddb..612abc6bd 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.h @@ -53,8 +53,10 @@ class IVF : public VecIndex, public FaissBaseIndex { DatasetPtr Query(const DatasetPtr&, const Config&) override; +#if 0 DatasetPtr QueryById(const DatasetPtr& dataset, const Config& config) override; +#endif int64_t Count() override { @@ -66,8 +68,10 @@ class IVF : public VecIndex, public FaissBaseIndex { return index_->d; } +#if 0 DatasetPtr GetVectorById(const DatasetPtr& dataset, const Config& config) override; +#endif virtual void Seal(); diff --git a/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h b/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h index 7eb121832..9f49166f5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/VecIndex.h @@ -45,10 +45,12 @@ class VecIndex : public Index { virtual DatasetPtr Query(const DatasetPtr& dataset, const Config& config) = 0; +#if 0 virtual DatasetPtr QueryById(const DatasetPtr& dataset, const Config& config) { return nullptr; } +#endif // virtual DatasetPtr // QueryByRange(const DatasetPtr&, const Config&) = 0; @@ -72,10 +74,12 @@ class VecIndex : public Index { return index_mode_; } +#if 0 virtual DatasetPtr GetVectorById(const DatasetPtr& dataset, const Config& config) { return nullptr; } +#endif faiss::ConcurrentBitsetPtr GetBlacklist() { diff --git a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp index 171749d32..5735f5f84 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp @@ -147,10 +147,10 @@ GPUIVF::QueryImpl(int64_t n, const float* data, int64_t k, float* distances, int ResScope rs(res_, gpu_id_); // if query size > 2048 we search by blocks to avoid malloc issue - size_t block_size = 2048; - size_t dim = device_index->d; - for (size_t i = 0; i < n; i += block_size) { - size_t search_size = (n - i > block_size) ? block_size : (n - i); + const int64_t block_size = 2048; + int64_t dim = device_index->d; + for (int64_t i = 0; i < n; i += block_size) { + int64_t search_size = (n - i > block_size) ? block_size : (n - i); device_index->search(search_size, (float*)data + i * dim, k, distances + i * k, labels + i * k, bitset_); } } else { diff --git a/core/src/index/thirdparty/faiss/Index.cpp b/core/src/index/thirdparty/faiss/Index.cpp index 81e8baa5f..d5748f719 100644 --- a/core/src/index/thirdparty/faiss/Index.cpp +++ b/core/src/index/thirdparty/faiss/Index.cpp @@ -52,6 +52,7 @@ void Index::add_with_ids( FAISS_THROW_MSG ("add_with_ids not implemented for this type of index"); } +#if 0 void Index::get_vector_by_id (idx_t n, const idx_t *xid, float *x, ConcurrentBitsetPtr bitset) { FAISS_THROW_MSG ("get_vector_by_id not implemented for this type of index"); } @@ -60,6 +61,7 @@ void Index::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, ConcurrentBitsetPtr bitset) { FAISS_THROW_MSG ("search_by_id not implemented for this type of index"); } +#endif size_t Index::remove_ids(const IDSelector& /*sel*/) { FAISS_THROW_MSG ("remove_ids not implemented for this type of index"); diff --git a/core/src/index/thirdparty/faiss/Index.h b/core/src/index/thirdparty/faiss/Index.h index 829d2a600..9a0967962 100644 --- a/core/src/index/thirdparty/faiss/Index.h +++ b/core/src/index/thirdparty/faiss/Index.h @@ -117,6 +117,7 @@ struct Index { float *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) const = 0; +#if 0 /** query n raw vectors from the index by ids. * * return n raw vectors. @@ -140,6 +141,7 @@ struct Index { */ virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr); +#endif /** query n vectors of dimension d to the index. * diff --git a/core/src/index/thirdparty/faiss/IndexBinary.cpp b/core/src/index/thirdparty/faiss/IndexBinary.cpp index 965068f08..fc41fe481 100644 --- a/core/src/index/thirdparty/faiss/IndexBinary.cpp +++ b/core/src/index/thirdparty/faiss/IndexBinary.cpp @@ -36,6 +36,7 @@ void IndexBinary::add_with_ids(idx_t, const uint8_t *, const idx_t *) { FAISS_THROW_MSG("add_with_ids not implemented for this type of index"); } +#if 0 void IndexBinary::get_vector_by_id (idx_t n, const idx_t *xid, uint8_t *x, ConcurrentBitsetPtr bitset) { FAISS_THROW_MSG("get_vector_by_id not implemented for this type of index"); } @@ -44,6 +45,7 @@ void IndexBinary::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *dis ConcurrentBitsetPtr bitset) { FAISS_THROW_MSG("search_by_id not implemented for this type of index"); } +#endif size_t IndexBinary::remove_ids(const IDSelector&) { FAISS_THROW_MSG("remove_ids not implemented for this type of index"); diff --git a/core/src/index/thirdparty/faiss/IndexBinary.h b/core/src/index/thirdparty/faiss/IndexBinary.h index 86a45668f..4141a7d63 100644 --- a/core/src/index/thirdparty/faiss/IndexBinary.h +++ b/core/src/index/thirdparty/faiss/IndexBinary.h @@ -99,6 +99,7 @@ struct IndexBinary { int32_t *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) const = 0; +#if 0 /** Query n raw vectors from the index by ids. * * return n raw vectors. @@ -122,6 +123,7 @@ struct IndexBinary { */ virtual void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr); +#endif /** Query n vectors of dimension d to the index. * diff --git a/core/src/index/thirdparty/faiss/IndexBinaryIVF.cpp b/core/src/index/thirdparty/faiss/IndexBinaryIVF.cpp index 79fa1256b..57ab21dd8 100644 --- a/core/src/index/thirdparty/faiss/IndexBinaryIVF.cpp +++ b/core/src/index/thirdparty/faiss/IndexBinaryIVF.cpp @@ -157,6 +157,7 @@ void IndexBinaryIVF::search(idx_t n, const uint8_t *x, idx_t k, indexIVF_stats.search_time += getmillisecs() - t0; } +#if 0 void IndexBinaryIVF::get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, ConcurrentBitsetPtr bitset) { make_direct_map(true); @@ -181,6 +182,7 @@ void IndexBinaryIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t * search(n, x, k, distances, labels, bitset); delete []x; } +#endif void IndexBinaryIVF::reconstruct(idx_t key, uint8_t *recons) const { idx_t lo = direct_map.get (key); diff --git a/core/src/index/thirdparty/faiss/IndexBinaryIVF.h b/core/src/index/thirdparty/faiss/IndexBinaryIVF.h index 3a45b82ff..c3cd7e744 100644 --- a/core/src/index/thirdparty/faiss/IndexBinaryIVF.h +++ b/core/src/index/thirdparty/faiss/IndexBinaryIVF.h @@ -115,11 +115,13 @@ struct IndexBinaryIVF : IndexBinary { void search(idx_t n, const uint8_t *x, idx_t k, int32_t *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) const override; +#if 0 /** get raw vectors by ids */ void get_vector_by_id(idx_t n, const idx_t *xid, uint8_t *x, ConcurrentBitsetPtr bitset = nullptr) override; void search_by_id (idx_t n, const idx_t *xid, idx_t k, int32_t *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) override; +#endif void range_search(idx_t n, const uint8_t *x, int radius, RangeSearchResult *result, diff --git a/core/src/index/thirdparty/faiss/IndexIVF.cpp b/core/src/index/thirdparty/faiss/IndexIVF.cpp index f7687cfe6..fa6f050ca 100644 --- a/core/src/index/thirdparty/faiss/IndexIVF.cpp +++ b/core/src/index/thirdparty/faiss/IndexIVF.cpp @@ -316,6 +316,7 @@ void IndexIVF::search (idx_t n, const float *x, idx_t k, indexIVF_stats.search_time += getmillisecs() - t0; } +#if 0 void IndexIVF::get_vector_by_id (idx_t n, const idx_t *xid, float *x, ConcurrentBitsetPtr bitset) { make_direct_map(true); @@ -340,6 +341,7 @@ void IndexIVF::search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distance search(n, x, k, distances, labels, bitset); delete []x; } +#endif void IndexIVF::search_preassigned (idx_t n, const float *x, idx_t k, const idx_t *keys, diff --git a/core/src/index/thirdparty/faiss/IndexIVF.h b/core/src/index/thirdparty/faiss/IndexIVF.h index 2409f9aab..744f27f33 100644 --- a/core/src/index/thirdparty/faiss/IndexIVF.h +++ b/core/src/index/thirdparty/faiss/IndexIVF.h @@ -192,11 +192,13 @@ struct IndexIVF: Index, Level1Quantizer { float *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) const override; +#if 0 /** get raw vectors by ids */ void get_vector_by_id (idx_t n, const idx_t *xid, float *x, ConcurrentBitsetPtr bitset = nullptr) override; void search_by_id (idx_t n, const idx_t *xid, idx_t k, float *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) override; +#endif void range_search (idx_t n, const float* x, float radius, RangeSearchResult* result, diff --git a/core/src/index/thirdparty/faiss/MetaIndexes.cpp b/core/src/index/thirdparty/faiss/MetaIndexes.cpp index 0851733f3..0094e17ba 100644 --- a/core/src/index/thirdparty/faiss/MetaIndexes.cpp +++ b/core/src/index/thirdparty/faiss/MetaIndexes.cpp @@ -93,6 +93,7 @@ void IndexIDMapTemplate::search } } +#if 0 template void IndexIDMapTemplate::get_vector_by_id(idx_t n, const idx_t *xid, component_t *x, ConcurrentBitsetPtr bitset) @@ -117,6 +118,7 @@ void IndexIDMapTemplate::search_by_id (idx_t n, const idx_t *xid, idx_t index->search(n, x, k, distances, labels, bitset); delete []x; } +#endif template void IndexIDMapTemplate::range_search diff --git a/core/src/index/thirdparty/faiss/MetaIndexes.h b/core/src/index/thirdparty/faiss/MetaIndexes.h index adba16e16..cfac6a557 100644 --- a/core/src/index/thirdparty/faiss/MetaIndexes.h +++ b/core/src/index/thirdparty/faiss/MetaIndexes.h @@ -42,10 +42,12 @@ struct IndexIDMapTemplate : IndexT { distance_t* distances, idx_t* labels, ConcurrentBitsetPtr bitset = nullptr) const override; +#if 0 void get_vector_by_id(idx_t n, const idx_t *xid, component_t *x, ConcurrentBitsetPtr bitset = nullptr) override; void search_by_id (idx_t n, const idx_t *xid, idx_t k, distance_t *distances, idx_t *labels, ConcurrentBitsetPtr bitset = nullptr) override; +#endif void train(idx_t n, const component_t* x) override; diff --git a/core/src/index/unittest/CMakeLists.txt b/core/src/index/unittest/CMakeLists.txt index 5d88731d7..f29ca1af0 100644 --- a/core/src/index/unittest/CMakeLists.txt +++ b/core/src/index/unittest/CMakeLists.txt @@ -80,6 +80,7 @@ if (NOT TARGET test_instructionset) add_executable(test_instructionset test_instructionset.cpp) endif () target_link_libraries(test_instructionset ${depend_libs} ${unittest_libs}) +install(TARGETS test_instructionset DESTINATION unittest) ################################################################################ # diff --git a/core/src/index/unittest/test_annoy.cpp b/core/src/index/unittest/test_annoy.cpp index 9faf0b0e9..7787c4408 100644 --- a/core/src/index/unittest/test_annoy.cpp +++ b/core/src/index/unittest/test_annoy.cpp @@ -28,7 +28,6 @@ class AnnoyTest : public DataGen, public TestWithParam { void SetUp() override { IndexType = GetParam(); - // std::cout << "IndexType from GetParam() is: " << IndexType << std::endl; Generate(128, 10000, 10); index_ = std::make_shared(); conf = milvus::knowhere::Config{ @@ -38,8 +37,6 @@ class AnnoyTest : public DataGen, public TestWithParam { {milvus::knowhere::IndexParams::search_k, 100}, {milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2}, }; - - // Init_with_default(); } protected: @@ -53,10 +50,20 @@ INSTANTIATE_TEST_CASE_P(AnnoyParameters, AnnoyTest, Values("Annoy")); TEST_P(AnnoyTest, annoy_basic) { assert(!xb.empty()); - // index_->Train(base_dataset, conf); + // null faiss index + { + ASSERT_ANY_THROW(index_->Train(base_dataset, conf)); + ASSERT_ANY_THROW(index_->Query(query_dataset, conf)); + ASSERT_ANY_THROW(index_->Serialize(conf)); + ASSERT_ANY_THROW(index_->Add(base_dataset, conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, conf)); + ASSERT_ANY_THROW(index_->Count()); + ASSERT_ANY_THROW(index_->Dim()); + } + index_->BuildAll(base_dataset, conf); // Train + Add - EXPECT_EQ(index_->Count(), nb); - EXPECT_EQ(index_->Dim(), dim); + ASSERT_EQ(index_->Count(), nb); + ASSERT_EQ(index_->Dim(), dim); auto result = index_->Query(query_dataset, conf); AssertAnns(result, nq, k); @@ -89,8 +96,8 @@ TEST_P(AnnoyTest, annoy_delete) { assert(!xb.empty()); index_->BuildAll(base_dataset, conf); // Train + Add - EXPECT_EQ(index_->Count(), nb); - EXPECT_EQ(index_->Dim(), dim); + ASSERT_EQ(index_->Count(), nb); + ASSERT_EQ(index_->Dim(), dim); faiss::ConcurrentBitsetPtr bitset = std::make_shared(nb); for (auto i = 0; i < nq; ++i) { @@ -191,8 +198,8 @@ TEST_P(AnnoyTest, annoy_serialize) { binaryset.Append("annoy_dim", dim_data, bin_dim->size); index_->Load(binaryset); - EXPECT_EQ(index_->Count(), nb); - EXPECT_EQ(index_->Dim(), dim); + ASSERT_EQ(index_->Count(), nb); + ASSERT_EQ(index_->Dim(), dim); auto result = index_->Query(query_dataset, conf); AssertAnns(result, nq, conf[milvus::knowhere::meta::TOPK]); } diff --git a/core/src/index/unittest/test_binaryidmap.cpp b/core/src/index/unittest/test_binaryidmap.cpp index d770455c5..cfbdc85a8 100644 --- a/core/src/index/unittest/test_binaryidmap.cpp +++ b/core/src/index/unittest/test_binaryidmap.cpp @@ -49,6 +49,14 @@ TEST_P(BinaryIDMAPTest, binaryidmap_basic) { {milvus::knowhere::Metric::TYPE, MetricType}, }; + // null faiss index + { + ASSERT_ANY_THROW(index_->Serialize()); + ASSERT_ANY_THROW(index_->Query(query_dataset, conf)); + ASSERT_ANY_THROW(index_->Add(nullptr, conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(nullptr, conf)); + } + index_->Train(base_dataset, conf); index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); @@ -62,7 +70,7 @@ TEST_P(BinaryIDMAPTest, binaryidmap_basic) { auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto result2 = index_->Query(query_dataset, conf); + auto result2 = new_index->Query(query_dataset, conf); AssertAnns(result2, nq, k); // PrintResult(re_result, nq, k); @@ -72,11 +80,11 @@ TEST_P(BinaryIDMAPTest, binaryidmap_basic) { } index_->SetBlacklist(concurrent_bitset_ptr); - auto result3 = index_->Query(query_dataset, conf); - AssertAnns(result3, nq, k, CheckMode::CHECK_NOT_EQUAL); + auto result_bs_1 = index_->Query(query_dataset, conf); + AssertAnns(result_bs_1, nq, k, CheckMode::CHECK_NOT_EQUAL); - // auto result4 = index_->SearchById(id_dataset, conf); - // AssertAneq(result4, nq, k); + // auto result4 = index_->SearchById(id_dataset, conf); + // AssertAneq(result4, nq, k); } TEST_P(BinaryIDMAPTest, binaryidmap_serialize) { @@ -98,7 +106,7 @@ TEST_P(BinaryIDMAPTest, binaryidmap_serialize) { { // serialize index index_->Train(base_dataset, conf); - index_->Add(base_dataset, milvus::knowhere::Config()); + index_->AddWithoutIds(base_dataset, milvus::knowhere::Config()); auto re_result = index_->Query(query_dataset, conf); AssertAnns(re_result, nq, k); // PrintResult(re_result, nq, k); @@ -120,6 +128,6 @@ TEST_P(BinaryIDMAPTest, binaryidmap_serialize) { EXPECT_EQ(index_->Dim(), dim); auto result = index_->Query(query_dataset, conf); AssertAnns(result, nq, k); - // PrintResult(result, nq, k); + // PrintResult(result, nq, k); } } diff --git a/core/src/index/unittest/test_binaryivf.cpp b/core/src/index/unittest/test_binaryivf.cpp index 4559674f6..76e099db8 100644 --- a/core/src/index/unittest/test_binaryivf.cpp +++ b/core/src/index/unittest/test_binaryivf.cpp @@ -60,7 +60,15 @@ INSTANTIATE_TEST_CASE_P(METRICParameters, BinaryIVFTest, TEST_P(BinaryIVFTest, binaryivf_basic) { assert(!xb_bin.empty()); - index_->Train(base_dataset, conf); + // null faiss index + { + ASSERT_ANY_THROW(index_->Serialize()); + ASSERT_ANY_THROW(index_->Query(query_dataset, conf)); + ASSERT_ANY_THROW(index_->Add(nullptr, conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(nullptr, conf)); + } + + index_->BuildAll(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dim(), dim); @@ -77,11 +85,13 @@ TEST_P(BinaryIVFTest, binaryivf_basic) { auto result2 = index_->Query(query_dataset, conf); AssertAnns(result2, nq, k, CheckMode::CHECK_NOT_EQUAL); +#if 0 auto result3 = index_->QueryById(id_dataset, conf); AssertAnns(result3, nq, k, CheckMode::CHECK_NOT_EQUAL); - // auto result4 = index_->GetVectorById(xid_dataset, conf); - // AssertBinVeceq(result4, base_dataset, xid_dataset, nq, dim/8); + auto result4 = index_->GetVectorById(xid_dataset, conf); + AssertBinVeceq(result4, base_dataset, xid_dataset, nq, dim/8); +#endif } TEST_P(BinaryIVFTest, binaryivf_serialize) { @@ -93,32 +103,32 @@ TEST_P(BinaryIVFTest, binaryivf_serialize) { reader(ret, bin->size); }; - // { - // // serialize index-model - // auto model = index_->Train(base_dataset, conf); - // auto binaryset = model->Serialize(); - // auto bin = binaryset.GetByName("BinaryIVF"); + // { + // // serialize index-model + // auto model = index_->Train(base_dataset, conf); + // auto binaryset = model->Serialize(); + // auto bin = binaryset.GetByName("BinaryIVF"); // - // std::string filename = "/tmp/binaryivf_test_model_serialize.bin"; - // auto load_data = new uint8_t[bin->size]; - // serialize(filename, bin, load_data); + // std::string filename = "/tmp/binaryivf_test_model_serialize.bin"; + // auto load_data = new uint8_t[bin->size]; + // serialize(filename, bin, load_data); // - // binaryset.clear(); - // auto data = std::make_shared(); - // data.reset(load_data); - // binaryset.Append("BinaryIVF", data, bin->size); + // binaryset.clear(); + // auto data = std::make_shared(); + // data.reset(load_data); + // binaryset.Append("BinaryIVF", data, bin->size); // - // model->Load(binaryset); + // model->Load(binaryset); // - // index_->set_index_model(model); - // index_->Add(base_dataset, conf); - // auto result = index_->Query(query_dataset, conf); - // AssertAnns(result, nq, conf[milvus::knowhere::meta::TOPK]); - // } + // index_->set_index_model(model); + // index_->Add(base_dataset, conf); + // auto result = index_->Query(query_dataset, conf); + // AssertAnns(result, nq, conf[milvus::knowhere::meta::TOPK]); + // } { // serialize index - index_->Train(base_dataset, conf); + index_->BuildAll(base_dataset, conf); // index_->set_index_model(model); // index_->Add(base_dataset, conf); auto binaryset = index_->Serialize(); diff --git a/core/src/index/unittest/test_customized_index.cpp b/core/src/index/unittest/test_customized_index.cpp index fa610915c..03c55d450 100644 --- a/core/src/index/unittest/test_customized_index.cpp +++ b/core/src/index/unittest/test_customized_index.cpp @@ -82,20 +82,20 @@ TEST_F(SingleIndexTest, IVFSQHybrid) { ASSERT_ANY_THROW(cpu_idx->CopyCpuToGpuWithQuantizer(-1, conf)); auto pair = cpu_idx->CopyCpuToGpuWithQuantizer(DEVICEID, conf); auto gpu_idx = pair.first; - auto quantization = pair.second; auto result = gpu_idx->Query(query_dataset, conf); AssertAnns(result, nq, conf[milvus::knowhere::meta::TOPK]); - // PrintResult(result, nq, k); + // PrintResult(result, nq, k); milvus::json quantizer_conf{{milvus::knowhere::meta::DEVICEID, DEVICEID}, {"mode", 2}}; for (int i = 0; i < 2; ++i) { auto hybrid_idx = std::make_shared(DEVICEID); hybrid_idx->Load(binaryset); + auto quantization = hybrid_idx->LoadQuantizer(quantizer_conf); auto new_idx = hybrid_idx->LoadData(quantization, quantizer_conf); auto result = new_idx->Query(query_dataset, conf); AssertAnns(result, nq, conf[milvus::knowhere::meta::TOPK]); - // PrintResult(result, nq, k); + // PrintResult(result, nq, k); } } diff --git a/core/src/index/unittest/test_hnsw.cpp b/core/src/index/unittest/test_hnsw.cpp index 71c18cee8..48f458fca 100644 --- a/core/src/index/unittest/test_hnsw.cpp +++ b/core/src/index/unittest/test_hnsw.cpp @@ -47,6 +47,16 @@ INSTANTIATE_TEST_CASE_P(HNSWParameters, HNSWTest, Values("HNSW")); TEST_P(HNSWTest, HNSW_basic) { assert(!xb.empty()); + // null faiss index + { + ASSERT_ANY_THROW(index_->Serialize()); + ASSERT_ANY_THROW(index_->Query(query_dataset, conf)); + ASSERT_ANY_THROW(index_->Add(nullptr, conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(nullptr, conf)); + ASSERT_ANY_THROW(index_->Count()); + ASSERT_ANY_THROW(index_->Dim()); + } + index_->Train(base_dataset, conf); index_->Add(base_dataset, conf); EXPECT_EQ(index_->Count(), nb); diff --git a/core/src/index/unittest/test_idmap.cpp b/core/src/index/unittest/test_idmap.cpp index 3ac90e58d..2abbd8e27 100644 --- a/core/src/index/unittest/test_idmap.cpp +++ b/core/src/index/unittest/test_idmap.cpp @@ -98,15 +98,17 @@ TEST_P(IDMAPTest, idmap_basic) { auto binaryset = index_->Serialize(); auto new_index = std::make_shared(); new_index->Load(binaryset); - auto result2 = index_->Query(query_dataset, conf); + auto result2 = new_index->Query(query_dataset, conf); AssertAnns(result2, nq, k); // PrintResult(re_result, nq, k); - auto result3 = index_->QueryById(id_dataset, conf); +#if 0 + auto result3 = new_index->QueryById(id_dataset, conf); AssertAnns(result3, nq, k); - auto result4 = index_->GetVectorById(xid_dataset, conf); + auto result4 = new_index->GetVectorById(xid_dataset, conf); AssertVec(result4, base_dataset, xid_dataset, 1, dim); +#endif faiss::ConcurrentBitsetPtr concurrent_bitset_ptr = std::make_shared(nb); for (int64_t i = 0; i < nq; ++i) { @@ -117,11 +119,13 @@ TEST_P(IDMAPTest, idmap_basic) { auto result_bs_1 = index_->Query(query_dataset, conf); AssertAnns(result_bs_1, nq, k, CheckMode::CHECK_NOT_EQUAL); +#if 0 auto result_bs_2 = index_->QueryById(id_dataset, conf); AssertAnns(result_bs_2, nq, k, CheckMode::CHECK_NOT_EQUAL); auto result_bs_3 = index_->GetVectorById(xid_dataset, conf); AssertVec(result_bs_3, base_dataset, xid_dataset, 1, dim, CheckMode::CHECK_NOT_EQUAL); +#endif } TEST_P(IDMAPTest, idmap_serialize) { diff --git a/core/src/index/unittest/test_instructionset.cpp b/core/src/index/unittest/test_instructionset.cpp index cead50d27..f1eb5746a 100644 --- a/core/src/index/unittest/test_instructionset.cpp +++ b/core/src/index/unittest/test_instructionset.cpp @@ -20,7 +20,8 @@ #include #include -TEST(InstructionSetTest, INSTRUCTION_SET_TEST) { +void +ShowInstructionSet() { auto& outstream = std::cout; auto support_message = [&outstream](const std::string& isa_feature, bool is_supported) { @@ -88,3 +89,7 @@ TEST(InstructionSetTest, INSTRUCTION_SET_TEST) { support_message("XOP", instruction_set_inst.XOP()); support_message("XSAVE", instruction_set_inst.XSAVE()); } + +TEST(InstructionSetTest, INSTRUCTION_SET_TEST) { + ASSERT_NO_FATAL_FAILURE(ShowInstructionSet()); +} diff --git a/core/src/index/unittest/test_ivf.cpp b/core/src/index/unittest/test_ivf.cpp index f6d9767e4..1827d0b4d 100644 --- a/core/src/index/unittest/test_ivf.cpp +++ b/core/src/index/unittest/test_ivf.cpp @@ -102,7 +102,7 @@ TEST_P(IVFTest, ivf_basic_cpu) { ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, conf_)); index_->Train(base_dataset, conf_); - index_->Add(base_dataset, conf_); + index_->AddWithoutIds(base_dataset, conf_); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dim(), dim); @@ -111,6 +111,7 @@ TEST_P(IVFTest, ivf_basic_cpu) { // PrintResult(result, nq, k); if (index_type_ != milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) { +#if 0 auto result2 = index_->QueryById(id_dataset, conf_); AssertAnns(result2, nq, k); @@ -122,6 +123,7 @@ TEST_P(IVFTest, ivf_basic_cpu) { /* for SQ8, sometimes the mean diff can bigger than 20% */ // AssertVec(result3, base_dataset, xid_dataset, 1, dim, CheckMode::CHECK_APPROXIMATE_EQUAL); } +#endif faiss::ConcurrentBitsetPtr concurrent_bitset_ptr = std::make_shared(nb); for (int64_t i = 0; i < nq; ++i) { @@ -133,12 +135,14 @@ TEST_P(IVFTest, ivf_basic_cpu) { AssertAnns(result_bs_1, nq, k, CheckMode::CHECK_NOT_EQUAL); // PrintResult(result, nq, k); +#if 0 auto result_bs_2 = index_->QueryById(id_dataset, conf_); AssertAnns(result_bs_2, nq, k, CheckMode::CHECK_NOT_EQUAL); // PrintResult(result, nq, k); auto result_bs_3 = index_->GetVectorById(xid_dataset, conf_); AssertVec(result_bs_3, base_dataset, xid_dataset, 1, dim, CheckMode::CHECK_NOT_EQUAL); +#endif } #ifdef MILVUS_GPU_VERSION @@ -157,8 +161,7 @@ TEST_P(IVFTest, ivf_basic_gpu) { ASSERT_ANY_THROW(index_->Add(base_dataset, conf_)); ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, conf_)); - index_->Train(base_dataset, conf_); - index_->Add(base_dataset, conf_); + index_->BuildAll(base_dataset, conf_); EXPECT_EQ(index_->Count(), nb); EXPECT_EQ(index_->Dim(), dim); diff --git a/core/src/index/unittest/test_nsg.cpp b/core/src/index/unittest/test_nsg.cpp index 0d9ca2278..c4dc46a86 100644 --- a/core/src/index/unittest/test_nsg.cpp +++ b/core/src/index/unittest/test_nsg.cpp @@ -33,7 +33,7 @@ using ::testing::Combine; using ::testing::TestWithParam; using ::testing::Values; -constexpr int64_t DEVICEID = 0; +constexpr int64_t DEVICE_GPU0 = 0; class NSGInterfaceTest : public DataGen, public ::testing::Test { protected: @@ -41,10 +41,10 @@ class NSGInterfaceTest : public DataGen, public ::testing::Test { SetUp() override { #ifdef MILVUS_GPU_VERSION int64_t MB = 1024 * 1024; - milvus::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICEID, MB * 200, MB * 600, 1); + milvus::knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(DEVICE_GPU0, MB * 200, MB * 600, 1); #endif int nsg_dim = 256; - Generate(nsg_dim, nb, nq); + Generate(nsg_dim, 20000, nq); index_ = std::make_shared(); train_conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, 256}, @@ -80,12 +80,14 @@ TEST_F(NSGInterfaceTest, basic_test) { fiu_init(0); // untrained index { - ASSERT_ANY_THROW(index_->Query(query_dataset, search_conf)); ASSERT_ANY_THROW(index_->Serialize()); + ASSERT_ANY_THROW(index_->Query(query_dataset, search_conf)); + ASSERT_ANY_THROW(index_->Add(base_dataset, search_conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(base_dataset, search_conf)); } - train_conf[milvus::knowhere::meta::DEVICEID] = DEVICEID; - index_->Train(base_dataset, train_conf); + train_conf[milvus::knowhere::meta::DEVICEID] = -1; + index_->BuildAll(base_dataset, train_conf); auto result = index_->Query(query_dataset, search_conf); AssertAnns(result, nq, k); @@ -96,16 +98,24 @@ TEST_F(NSGInterfaceTest, basic_test) { fiu_disable("NSG.Serialize.throw_exception"); } - auto new_index = std::make_shared(); - new_index->Load(binaryset); + /* test NSG GPU train */ + auto new_index_1 = std::make_shared(DEVICE_GPU0); + train_conf[milvus::knowhere::meta::DEVICEID] = DEVICE_GPU0; + new_index_1->BuildAll(base_dataset, train_conf); + auto new_result_1 = new_index_1->Query(query_dataset, search_conf); + AssertAnns(new_result_1, nq, k); + + /* test NSG index load */ + auto new_index_2 = std::make_shared(); + new_index_2->Load(binaryset); { fiu_enable("NSG.Load.throw_exception", 1, nullptr, 0); - ASSERT_ANY_THROW(new_index->Load(binaryset)); + ASSERT_ANY_THROW(new_index_2->Load(binaryset)); fiu_disable("NSG.Load.throw_exception"); } - auto new_result = new_index->Query(query_dataset, search_conf); - AssertAnns(result, nq, k); + auto new_result_2 = new_index_2->Query(query_dataset, search_conf); + AssertAnns(new_result_2, nq, k); ASSERT_EQ(index_->Count(), nb); ASSERT_EQ(index_->Dim(), dim); @@ -129,7 +139,7 @@ TEST_F(NSGInterfaceTest, compare_test) { TEST_F(NSGInterfaceTest, delete_test) { assert(!xb.empty()); - train_conf[milvus::knowhere::meta::DEVICEID] = DEVICEID; + train_conf[milvus::knowhere::meta::DEVICEID] = DEVICE_GPU0; index_->Train(base_dataset, train_conf); auto result = index_->Query(query_dataset, search_conf); diff --git a/core/src/index/unittest/test_sptag.cpp b/core/src/index/unittest/test_sptag.cpp index 42d257e3c..65349e5e8 100644 --- a/core/src/index/unittest/test_sptag.cpp +++ b/core/src/index/unittest/test_sptag.cpp @@ -60,7 +60,13 @@ INSTANTIATE_TEST_CASE_P(SPTAGParameters, SPTAGTest, Values("KDT", "BKT")); TEST_P(SPTAGTest, sptag_basic) { assert(!xb.empty()); - index_->Train(base_dataset, conf); + // null faiss index + { + ASSERT_ANY_THROW(index_->Add(nullptr, conf)); + ASSERT_ANY_THROW(index_->AddWithoutIds(nullptr, conf)); + } + + index_->BuildAll(base_dataset, conf); // index_->Add(base_dataset, conf); auto result = index_->Query(query_dataset, conf); AssertAnns(result, nq, k); diff --git a/core/src/index/unittest/utils.cpp b/core/src/index/unittest/utils.cpp index 15c2f4b70..df9c2a082 100644 --- a/core/src/index/unittest/utils.cpp +++ b/core/src/index/unittest/utils.cpp @@ -161,6 +161,7 @@ AssertAnns(const milvus::knowhere::DatasetPtr& result, const int nq, const int k } } +#if 0 void AssertVec(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::DatasetPtr& base_dataset, const milvus::knowhere::DatasetPtr& id_dataset, const int n, const int dim, const CheckMode check_mode) { @@ -194,18 +195,19 @@ AssertVec(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::Da } void -AssertBinVeceq(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::DatasetPtr& base_dataset, - const milvus::knowhere::DatasetPtr& id_dataset, const int n, const int dim) { - auto base = base_dataset->Get(milvus::knowhere::meta::TENSOR); +AssertBinVec(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::DatasetPtr& base_dataset, + const milvus::knowhere::DatasetPtr& id_dataset, const int n, const int dim, const CheckMode check_mode) { + auto base = (uint8_t*)base_dataset->Get(milvus::knowhere::meta::TENSOR); auto ids = id_dataset->Get(milvus::knowhere::meta::IDS); - auto x = result->Get(milvus::knowhere::meta::TENSOR); + auto x = result->Get(milvus::knowhere::meta::TENSOR); for (auto i = 0; i < 1; i++) { auto id = ids[i]; for (auto j = 0; j < dim; j++) { - EXPECT_EQ(*(base + id * dim + j), *(x + i * dim + j)); + ASSERT_EQ(*(base + id * dim + j), *(x + i * dim + j)); } } } +#endif void PrintResult(const milvus::knowhere::DatasetPtr& result, const int& nq, const int& k) { diff --git a/core/src/index/unittest/utils.h b/core/src/index/unittest/utils.h index 1f6fb6356..2d1076255 100644 --- a/core/src/index/unittest/utils.h +++ b/core/src/index/unittest/utils.h @@ -76,9 +76,9 @@ AssertVec(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::Da const CheckMode check_mode = CheckMode::CHECK_EQUAL); void -AssertBinVeceq(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::DatasetPtr& base_dataset, - const milvus::knowhere::DatasetPtr& id_dataset, const int n, const int dim, - const CheckMode check_mode = CheckMode::CHECK_EQUAL); +AssertBinVec(const milvus::knowhere::DatasetPtr& result, const milvus::knowhere::DatasetPtr& base_dataset, + const milvus::knowhere::DatasetPtr& id_dataset, const int n, const int dim, + const CheckMode check_mode = CheckMode::CHECK_EQUAL); void PrintResult(const milvus::knowhere::DatasetPtr& result, const int& nq, const int& k); -- GitLab