From 8574b8b93f474d4e74433be8b7485b9eb89c7b4e Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR <48198922+Yukikaze-CZR@users.noreply.github.com> Date: Wed, 4 Dec 2019 19:37:13 +0800 Subject: [PATCH] add code coverage rate and fix #670 (#693) --- .../index/vector_index/IndexIVFSQHybrid.cpp | 56 ------------------- .../index/vector_index/IndexIVFSQHybrid.h | 2 +- .../index/vector_index/helpers/Cloner.cpp | 2 + core/src/index/unittest/Helper.h | 2 + core/src/wrapper/gpu/GPUVecImpl.cpp | 2 + core/unittest/db/test_db.cpp | 21 ++++++- 6 files changed, 27 insertions(+), 58 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index f2a963ee..4fecfb9d 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -295,61 +295,5 @@ FaissIVFQuantizer::~FaissIVFQuantizer() { // else do nothing } -#else - -QuantizerPtr -IVFSQHybrid::LoadQuantizer(const Config& conf) { - return knowhere::QuantizerPtr(); -} - -void -IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { -} - -void -IVFSQHybrid::UnsetQuantizer() { -} - -VectorIndexPtr -IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { - return nullptr; -} - -std::pair -IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) { - KNOWHERE_THROW_MSG("Not yet implemented"); -} - -IndexModelPtr -IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { - return GPUIVFSQ::Train(dataset, config); -} - -VectorIndexPtr -IVFSQHybrid::CopyGpuToCpu(const Config& config) { - return GPUIVFSQ::CopyGpuToCpu(config); -} - -VectorIndexPtr -IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { - return IVF::CopyCpuToGpu(device_id, config); -} - -void -IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, - const Config& cfg) { - GPUIVF::search_impl(n, data, k, distances, labels, cfg); -} - -void -IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { - GPUIVF::LoadImpl(index_binary); -} - -void -IVFSQHybrid::set_index_model(IndexModelPtr model) { - GPUIVF::set_index_model(model); -} - #endif } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index d2a3be6c..5846d3e5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -36,7 +36,6 @@ struct FaissIVFQuantizer : public Quantizer { ~FaissIVFQuantizer() override; }; using FaissIVFQuantizerPtr = std::shared_ptr; -#endif class IVFSQHybrid : public GPUIVFSQ { public: @@ -93,5 +92,6 @@ class IVFSQHybrid : public GPUIVFSQ { int64_t gpu_mode = 0; // 0,1,2 int64_t quantizer_gpu_id_ = -1; }; +#endif } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 363e629e..4c5a7ec9 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -38,9 +38,11 @@ CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) { VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config) { +#ifdef CUSTOMIZATION if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyCpuToGpu(device_id, config); } +#endif if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyGpuToGpu(device_id, config); diff --git a/core/src/index/unittest/Helper.h b/core/src/index/unittest/Helper.h index 5d7df41b..9095cd6f 100644 --- a/core/src/index/unittest/Helper.h +++ b/core/src/index/unittest/Helper.h @@ -53,8 +53,10 @@ IndexFactory(const std::string& type) { return std::make_shared(DEVICEID); } else if (type == "GPUIVFSQ") { return std::make_shared(DEVICEID); +#ifdef CUSTOMIZATION } else if (type == "IVFSQHybrid") { return std::make_shared(DEVICEID); +#endif #endif } } diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index e40f5038..8fca6949 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -73,6 +73,7 @@ IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { return Status::OK(); } +#ifdef CUSTOMIZATION knowhere::QuantizerPtr IVFHybridIndex::LoadQuantizer(const Config& conf) { // TODO(linxj): Hardcode here @@ -158,6 +159,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c } return std::make_pair(nullptr, nullptr); } +#endif } // namespace engine } // namespace milvus diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index dbed92b3..8db7d3f9 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -351,6 +351,19 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + #ifdef CUSTOMIZATION // test FAISS_IVFSQ8H optimizer index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; @@ -375,7 +388,13 @@ TEST_F(DBTest, SEARCH_TEST) { { // search by specify index file milvus::engine::meta::DatesT dates; - std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; + std::vector file_ids; + // sometimes this case run fast to merge file and build index, old file will be deleted immediately, + // so the QueryByFileID cannot get files to search + // input 100 files ids to avoid random failure of this case + for (int i = 0; i < 100; i++) { + file_ids.push_back(std::to_string(i)); + } result_ids.clear(); result_dists.clear(); stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_dists); -- GitLab