Hnsw sq8 support (#2835)

* modify for SQ8
Signed-off-by: Nshengjun.li <shengjun.li@zilliz.com>

* add hnsw-sq8-support
Signed-off-by: Ncmli <chengming.li@zilliz.com>

* add unittest 4 hnsw_sq8nr and fix several bugs
Signed-off-by: Ncmli <chengming.li@zilliz.com>
Co-authored-by: Nshengjun.li <shengjun.li@zilliz.com>
Co-authored-by: Ncmli <chengming.li@zilliz.com>

CHANGELOG.md

@@ -33,6 +33,7 @@ Please mark all changes in change log and use the issue from GitHub
 -   \#2689 Construct Knowhere Index Without Data
 -   \#2798 hnsw-sq8 support
 -   \#2802 Add new index: IVFSQ8NR
+-   \#2834 add C++ sdk support 4 hnsw_sq8nr
 
 ## Improvement
 -   \#2543 Remove secondary_path related code

core/src/codecs/default/DefaultVectorCompressFormat.cpp

@@ -46,6 +46,7 @@ DefaultVectorCompressFormat::read(const storage::FSHandlerPtr& fs_ptr, const std
         return;
     }
 
+    compress = std::make_shared<knowhere::Binary>();
     compress->data = std::shared_ptr<uint8_t[]>(new uint8_t[length]);
     compress->size = length;
 

core/src/db/engine/ExecutionEngine.h

@@ -52,11 +52,18 @@ enum class EngineType {
     MAX_VALUE = HNSW_SQ8NR,
 };
 
-static std::map<std::string, EngineType> s_map_engine_type = {
-    {"FLAT", EngineType::FAISS_IDMAP},   {"IVFFLAT", EngineType::FAISS_IVFFLAT},  {"IVFSQ8", EngineType::FAISS_IVFSQ8},
-    {"RNSG", EngineType::NSG_MIX},       {"IVFSQ8H", EngineType::FAISS_IVFSQ8H},  {"IVFPQ", EngineType::FAISS_PQ},
-    {"SPTAGKDT", EngineType::SPTAG_KDT}, {"SPTAGBKT", EngineType::SPTAG_BKT},     {"HNSW", EngineType::HNSW},
-    {"ANNOY", EngineType::ANNOY},        {"IVFSQ8NR", EngineType::FAISS_IVFSQ8NR}};
+static std::map<std::string, EngineType> s_map_engine_type = {{"FLAT", EngineType::FAISS_IDMAP},
+                                                              {"IVFFLAT", EngineType::FAISS_IVFFLAT},
+                                                              {"IVFSQ8", EngineType::FAISS_IVFSQ8},
+                                                              {"RNSG", EngineType::NSG_MIX},
+                                                              {"IVFSQ8H", EngineType::FAISS_IVFSQ8H},
+                                                              {"IVFPQ", EngineType::FAISS_PQ},
+                                                              {"SPTAGKDT", EngineType::SPTAG_KDT},
+                                                              {"SPTAGBKT", EngineType::SPTAG_BKT},
+                                                              {"HNSW", EngineType::HNSW},
+                                                              {"ANNOY", EngineType::ANNOY},
+                                                              {"IVFSQ8NR", EngineType::FAISS_IVFSQ8NR},
+                                                              {"HNSW_SQ8NR", EngineType::HNSW_SQ8NR}};
 
 enum class MetricType {
     L2 = 1,              // Euclidean Distance

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -245,6 +245,10 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_HNSW, mode);
             break;
         }
+        case EngineType::HNSW_SQ8NR: {
+            index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_HNSW_SQ8NR, mode);
+            break;
+        }
         case EngineType::ANNOY: {
             index = vec_index_factory.CreateVecIndex(knowhere::IndexEnum::INDEX_ANNOY, mode);
             break;

core/src/index/knowhere/knowhere/index/vector_offset_index/IndexHNSW_SQ8NR.h

@@ -25,7 +25,7 @@ namespace knowhere {
 class IndexHNSW_SQ8NR : public VecIndex {
  public:
     IndexHNSW_SQ8NR() {
-        index_type_ = IndexEnum::INDEX_HNSW;
+        index_type_ = IndexEnum::INDEX_HNSW_SQ8NR;
     }
 
     BinarySet

core/src/server/ValidationUtil.cpp

@@ -248,6 +248,7 @@ ValidateIndexParams(const milvus::json& index_params, const engine::meta::Collec
             }
             break;
         }
+        case (int32_t)engine::EngineType::HNSW_SQ8NR:
         case (int32_t)engine::EngineType::HNSW: {
             auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 4, 64);
             if (!status.ok()) {
@@ -297,6 +298,7 @@ ValidateSearchParams(const milvus::json& search_params, const engine::meta::Coll
             }
             break;
         }
+        case (int32_t)engine::EngineType::HNSW_SQ8NR:
         case (int32_t)engine::EngineType::HNSW: {
             auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 4096);
             if (!status.ok()) {

core/src/server/web_impl/Constants.h

@@ -28,6 +28,7 @@ extern const char* NAME_ENGINE_TYPE_IVFSQ8H;
 extern const char* NAME_ENGINE_TYPE_RNSG;
 extern const char* NAME_ENGINE_TYPE_IVFPQ;
 extern const char* NAME_ENGINE_TYPE_HNSW;
+extern const char* NAME_ENGINE_TYPE_HNSW_SQ8NR;
 extern const char* NAME_ENGINE_TYPE_ANNOY;
 
 extern const char* NAME_METRIC_TYPE_L2;

sdk/examples/utils/Utils.cpp

@@ -113,6 +113,8 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) {
             return "SPTAGBKT";
         case milvus::IndexType::HNSW:
             return "HNSW";
+        case milvus::IndexType::HNSW_SQ8NR:
+            return "HNSW_SQ8NR";
         case milvus::IndexType::ANNOY:
             return "ANNOY";
         case milvus::IndexType::IVFSQ8NR:
@@ -367,4 +369,5 @@ Utils::PrintTopKHybridQueryResult(milvus::TopKHybridQueryResult& topk_query_resu
     }
 }
 
-}  // namespace milvus_sdk
\ No newline at end of file
+}  // namespace milvus_sdk
+

sdk/include/MilvusApi.h

@@ -40,6 +40,7 @@ enum class IndexType {
     HNSW = 11,
     ANNOY = 12,
     IVFSQ8NR = 13,
+    HNSW_SQ8NR = 14,
 };
 
 enum class MetricType {