From dc31742538b942ac933b406d29247e68499438bf Mon Sep 17 00:00:00 2001 From: op-hunter Date: Tue, 14 Jul 2020 09:45:08 +0800 Subject: [PATCH] Hnsw sq8 support (#2835) * modify for SQ8 Signed-off-by: shengjun.li * add hnsw-sq8-support Signed-off-by: cmli * add unittest 4 hnsw_sq8nr and fix several bugs Signed-off-by: cmli Co-authored-by: shengjun.li Co-authored-by: cmli --- CHANGELOG.md | 1 + .../default/DefaultVectorCompressFormat.cpp | 1 + core/src/db/engine/ExecutionEngine.h | 17 ++++++++++++----- core/src/db/engine/ExecutionEngineImpl.cpp | 4 ++++ .../index/vector_offset_index/IndexHNSW_SQ8NR.h | 2 +- core/src/server/ValidationUtil.cpp | 2 ++ core/src/server/web_impl/Constants.h | 1 + sdk/examples/utils/Utils.cpp | 5 ++++- sdk/include/MilvusApi.h | 1 + 9 files changed, 27 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ba72de..3011f54c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Please mark all changes in change log and use the issue from GitHub - \#2689 Construct Knowhere Index Without Data - \#2798 hnsw-sq8 support - \#2802 Add new index: IVFSQ8NR +- \#2834 add C++ sdk support 4 hnsw_sq8nr ## Improvement - \#2543 Remove secondary_path related code diff --git a/core/src/codecs/default/DefaultVectorCompressFormat.cpp b/core/src/codecs/default/DefaultVectorCompressFormat.cpp index 6b43363e..74d8d703 100644 --- a/core/src/codecs/default/DefaultVectorCompressFormat.cpp +++ b/core/src/codecs/default/DefaultVectorCompressFormat.cpp @@ -46,6 +46,7 @@ DefaultVectorCompressFormat::read(const storage::FSHandlerPtr& fs_ptr, const std return; } + compress = std::make_shared(); compress->data = std::shared_ptr(new uint8_t[length]); compress->size = length; diff --git a/core/src/db/engine/ExecutionEngine.h b/core/src/db/engine/ExecutionEngine.h index 0672be01..a59d05e5 100644 --- a/core/src/db/engine/ExecutionEngine.h +++ b/core/src/db/engine/ExecutionEngine.h @@ -52,11 +52,18 @@ enum class EngineType { MAX_VALUE = HNSW_SQ8NR, }; -static std::map s_map_engine_type = { - {"FLAT", EngineType::FAISS_IDMAP}, {"IVFFLAT", EngineType::FAISS_IVFFLAT}, {"IVFSQ8", EngineType::FAISS_IVFSQ8}, - {"RNSG", EngineType::NSG_MIX}, {"IVFSQ8H", EngineType::FAISS_IVFSQ8H}, {"IVFPQ", EngineType::FAISS_PQ}, - {"SPTAGKDT", EngineType::SPTAG_KDT}, {"SPTAGBKT", EngineType::SPTAG_BKT}, {"HNSW", EngineType::HNSW}, - {"ANNOY", EngineType::ANNOY}, {"IVFSQ8NR", EngineType::FAISS_IVFSQ8NR}}; +static std::map s_map_engine_type = {{"FLAT", EngineType::FAISS_IDMAP}, + {"IVFFLAT", EngineType::FAISS_IVFFLAT}, + {"IVFSQ8", EngineType::FAISS_IVFSQ8}, + {"RNSG", EngineType::NSG_MIX}, + {"IVFSQ8H", EngineType::FAISS_IVFSQ8H}, + {"IVFPQ", EngineType::FAISS_PQ}, + {"SPTAGKDT", EngineType::SPTAG_KDT}, + {"SPTAGBKT", EngineType::SPTAG_BKT}, + {"HNSW", EngineType::HNSW}, + {"ANNOY", EngineType::ANNOY}, + {"IVFSQ8NR", EngineType::FAISS_IVFSQ8NR}, + {"HNSW_SQ8NR", EngineType::HNSW_SQ8NR}}; enum class MetricType { L2 = 1, // Euclidean Distance diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 2493d392..a1e69a3e 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -245,6 +245,10 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_HNSW, mode); break; } + case EngineType::HNSW_SQ8NR: { + index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_HNSW_SQ8NR, mode); + break; + } case EngineType::ANNOY: { index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_ANNOY, mode); break; diff --git a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.h b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.h index 42390e4e..9c3012e9 100644 --- a/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.h +++ b/core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.h @@ -25,7 +25,7 @@ namespace knowhere { class IndexHNSW_SQ8NR : public VecIndex { public: IndexHNSW_SQ8NR() { - index_type_ = IndexEnum::INDEX_HNSW; + index_type_ = IndexEnum::INDEX_HNSW_SQ8NR; } BinarySet diff --git a/core/src/server/ValidationUtil.cpp b/core/src/server/ValidationUtil.cpp index 76388ef6..c05ff9b1 100644 --- a/core/src/server/ValidationUtil.cpp +++ b/core/src/server/ValidationUtil.cpp @@ -248,6 +248,7 @@ ValidateIndexParams(const milvus::json& index_params, const engine::meta::Collec } break; } + case (int32_t)engine::EngineType::HNSW_SQ8NR: case (int32_t)engine::EngineType::HNSW: { auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64); if (!status.ok()) { @@ -297,6 +298,7 @@ ValidateSearchParams(const milvus::json& search_params, const engine::meta::Coll } break; } + case (int32_t)engine::EngineType::HNSW_SQ8NR: case (int32_t)engine::EngineType::HNSW: { auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 4096); if (!status.ok()) { diff --git a/core/src/server/web_impl/Constants.h b/core/src/server/web_impl/Constants.h index e8016cad..3ad8c415 100644 --- a/core/src/server/web_impl/Constants.h +++ b/core/src/server/web_impl/Constants.h @@ -28,6 +28,7 @@ extern const char* NAME_ENGINE_TYPE_IVFSQ8H; extern const char* NAME_ENGINE_TYPE_RNSG; extern const char* NAME_ENGINE_TYPE_IVFPQ; extern const char* NAME_ENGINE_TYPE_HNSW; +extern const char* NAME_ENGINE_TYPE_HNSW_SQ8NR; extern const char* NAME_ENGINE_TYPE_ANNOY; extern const char* NAME_METRIC_TYPE_L2; diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index 831b2ac6..ace75e9e 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -113,6 +113,8 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { return "SPTAGBKT"; case milvus::IndexType::HNSW: return "HNSW"; + case milvus::IndexType::HNSW_SQ8NR: + return "HNSW_SQ8NR"; case milvus::IndexType::ANNOY: return "ANNOY"; case milvus::IndexType::IVFSQ8NR: @@ -367,4 +369,5 @@ Utils::PrintTopKHybridQueryResult(milvus::TopKHybridQueryResult& topk_query_resu } } -} // namespace milvus_sdk \ No newline at end of file +} // namespace milvus_sdk + diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 2f250d2a..49e24870 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -40,6 +40,7 @@ enum class IndexType { HNSW = 11, ANNOY = 12, IVFSQ8NR = 13, + HNSW_SQ8NR = 14, }; enum class MetricType { -- GitLab