From 1d9ccd9bb7027a9395e27e6b2eaa9abfaca7384c Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Wed, 15 Jul 2020 17:42:44 +0800 Subject: [PATCH] add MILVUS_SUPPORT_SPTAG for some code (#2866) * add MILVUS_SUPPORT_SPTAG for some code Signed-off-by: yudong.cai * use knowhere::IndexEnum in db Signed-off-by: yudong.cai * keep EngineType order Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/src/db/Utils.cpp | 2 + core/src/db/meta/MetaTypes.h | 56 ++++++++++--------- .../delivery/request/DeleteByIDRequest.cpp | 2 + core/unittest/db/test_engine.cpp | 2 + 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da4fc075..5840c801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Please mark all changes in change log and use the issue from GitHub - \#2612 Move all APIs in utils into namespace milvus - \#2675 Print out system memory size when report invalid cpu cache size - \#2841 Replace IndexType/EngineType/MetricType +- \#2858 Unify index name in db ## Task diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index d22c882b..38fffb74 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -254,8 +254,10 @@ GetIndexName(int32_t index_type) { {(int32_t)engine::EngineType::FAISS_IVFSQ8NR, "IVFSQ8NR"}, {(int32_t)engine::EngineType::FAISS_IVFSQ8H, "IVFSQ8H"}, {(int32_t)engine::EngineType::FAISS_PQ, "PQ"}, +#ifdef MILVUS_SUPPORT_SPTAG {(int32_t)engine::EngineType::SPTAG_KDT, "KDT"}, {(int32_t)engine::EngineType::SPTAG_BKT, "BKT"}, +#endif {(int32_t)engine::EngineType::FAISS_BIN_IDMAP, "IDMAP"}, {(int32_t)engine::EngineType::FAISS_BIN_IVFFLAT, "IVFFLAT"}, {(int32_t)engine::EngineType::HNSW_SQ8NR, "HNSW_SQ8NR"}, diff --git a/core/src/db/meta/MetaTypes.h b/core/src/db/meta/MetaTypes.h index c7685b01..268e88ce 100644 --- a/core/src/db/meta/MetaTypes.h +++ b/core/src/db/meta/MetaTypes.h @@ -17,6 +17,7 @@ #include #include "db/Constants.h" +#include "knowhere/index/IndexType.h" #include "src/version.h" namespace milvus { @@ -26,34 +27,39 @@ namespace engine { enum class EngineType { INVALID = 0, FAISS_IDMAP = 1, - FAISS_IVFFLAT, - FAISS_IVFSQ8, - NSG_MIX, - FAISS_IVFSQ8H, - FAISS_PQ, - SPTAG_KDT, - SPTAG_BKT, - FAISS_BIN_IDMAP, - FAISS_BIN_IVFFLAT, - HNSW, - ANNOY, - FAISS_IVFSQ8NR, - HNSW_SQ8NR, + FAISS_IVFFLAT = 2, + FAISS_IVFSQ8 = 3, + NSG_MIX = 4, + FAISS_IVFSQ8H = 5, + FAISS_PQ = 6, +#ifdef MILVUS_SUPPORT_SPTAG + SPTAG_KDT = 7, + SPTAG_BKT = 8, +#endif + FAISS_BIN_IDMAP = 9, + FAISS_BIN_IVFFLAT = 10, + HNSW = 11, + ANNOY = 12, + FAISS_IVFSQ8NR = 13, + HNSW_SQ8NR = 14, MAX_VALUE = HNSW_SQ8NR, }; -static std::map s_map_engine_type = {{"FLAT", EngineType::FAISS_IDMAP}, - {"IVFFLAT", EngineType::FAISS_IVFFLAT}, - {"IVFSQ8", EngineType::FAISS_IVFSQ8}, - {"RNSG", EngineType::NSG_MIX}, - {"IVFSQ8H", EngineType::FAISS_IVFSQ8H}, - {"IVFPQ", EngineType::FAISS_PQ}, - {"SPTAGKDT", EngineType::SPTAG_KDT}, - {"SPTAGBKT", EngineType::SPTAG_BKT}, - {"HNSW", EngineType::HNSW}, - {"ANNOY", EngineType::ANNOY}, - {"IVFSQ8NR", EngineType::FAISS_IVFSQ8NR}, - {"HNSW_SQ8NR", EngineType::HNSW_SQ8NR}}; +static std::map s_map_engine_type = { + {knowhere::IndexEnum::INDEX_FAISS_IDMAP, EngineType::FAISS_IDMAP}, + {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, EngineType::FAISS_IVFFLAT}, + {knowhere::IndexEnum::INDEX_FAISS_IVFPQ, EngineType::FAISS_PQ}, + {knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, EngineType::FAISS_IVFSQ8}, + {knowhere::IndexEnum::INDEX_FAISS_IVFSQ8NR, EngineType::FAISS_IVFSQ8NR}, + {knowhere::IndexEnum::INDEX_FAISS_IVFSQ8H, EngineType::FAISS_IVFSQ8H}, + {knowhere::IndexEnum::INDEX_NSG, EngineType::NSG_MIX}, +#ifdef MILVUS_SUPPORT_SPTAG + {knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, EngineType::SPTAG_KDT}, + {knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, EngineType::SPTAG_BKT}, +#endif + {knowhere::IndexEnum::INDEX_HNSW, EngineType::HNSW}, + {knowhere::IndexEnum::INDEX_HNSW_SQ8NR, EngineType::HNSW_SQ8NR}, + {knowhere::IndexEnum::INDEX_ANNOY, EngineType::ANNOY}}; enum class MetricType { L2 = 1, // Euclidean Distance diff --git a/core/src/server/delivery/request/DeleteByIDRequest.cpp b/core/src/server/delivery/request/DeleteByIDRequest.cpp index 9487f545..117186aa 100644 --- a/core/src/server/delivery/request/DeleteByIDRequest.cpp +++ b/core/src/server/delivery/request/DeleteByIDRequest.cpp @@ -68,6 +68,7 @@ DeleteByIDRequest::OnExecute() { } // Check collection's index type supports delete +#ifdef MILVUS_SUPPORT_SPTAG if (collection_schema.engine_type_ == (int32_t)engine::EngineType::SPTAG_BKT || collection_schema.engine_type_ == (int32_t)engine::EngineType::SPTAG_KDT) { std::string err_msg = @@ -75,6 +76,7 @@ DeleteByIDRequest::OnExecute() { LOG_SERVER_ERROR_ << err_msg; return Status(SERVER_UNSUPPORTED_ERROR, err_msg); } +#endif rc.RecordSection("check validation"); diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 250dc8dd..47092d30 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -116,6 +116,7 @@ TEST_F(EngineTest, FACTORY_TEST) { ASSERT_TRUE(engine_ptr != nullptr); } +#ifdef MILVUS_SUPPORT_SPTAG { auto engine_ptr = milvus::engine::EngineFactory::Build( 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, @@ -150,6 +151,7 @@ TEST_F(EngineTest, FACTORY_TEST) { milvus::engine::MetricType::L2, index_params)); fiu_disable("ExecutionEngineImpl.throw_exception"); } +#endif } TEST_F(EngineTest, ENGINE_IMPL_TEST) { -- GitLab