From 25b8c3738382ca8365804aa341f1519a4f7f081b Mon Sep 17 00:00:00 2001
From: dragondriver <jiquan.long@zilliz.com>
Date: Tue, 13 Jul 2021 16:05:36 +0800
Subject: [PATCH] Fix 6421, split raw vectors to serveral smaller one (#6449)

Signed-off-by: dragondriver <jiquan.long@zilliz.com>
---
 internal/core/src/indexbuilder/IndexWrapper.cpp | 13 +++++++++++++
 internal/core/src/indexbuilder/IndexWrapper.h   |  3 +++
 2 files changed, 16 insertions(+)

diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp
index 12b6e90ce..5c022fd96 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.cpp
+++ b/internal/core/src/indexbuilder/IndexWrapper.cpp
@@ -21,6 +21,7 @@
 #include "indexbuilder/utils.h"
 #include "index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.h"
 #include "index/knowhere/knowhere/common/Timer.h"
+#include "index/knowhere/knowhere/common/Utils.h"
 
 namespace milvus {
 namespace indexbuilder {
@@ -239,6 +240,10 @@ IndexWrapper::Serialize() {
         std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
         memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
         binarySet.Append(RAW_DATA, raw_data, raw_data_.size());
+        auto slice_size = get_index_file_slice_size();
+        // https://github.com/milvus-io/milvus/issues/6421
+        // Disassemble will only divide the raw vectors, other keys was already divided
+        knowhere::Disassemble(slice_size * 1024 * 1024, binarySet);
     }
 
     namespace indexcgo = milvus::proto::indexcgo;
@@ -317,6 +322,14 @@ IndexWrapper::get_index_mode() {
     return mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
 }
 
+int64_t
+IndexWrapper::get_index_file_slice_size() {
+    if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
+        return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>();
+    }
+    return 4;  // by default
+}
+
 std::unique_ptr<IndexWrapper::QueryResult>
 IndexWrapper::Query(const knowhere::DatasetPtr& dataset) {
     return std::move(QueryImpl(dataset, config_));
diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h
index ad5227b10..e462400b1 100644
--- a/internal/core/src/indexbuilder/IndexWrapper.h
+++ b/internal/core/src/indexbuilder/IndexWrapper.h
@@ -65,6 +65,9 @@ class IndexWrapper {
     knowhere::IndexMode
     get_index_mode();
 
+    int64_t
+    get_index_file_slice_size();
+
     template <typename T>
     std::optional<T>
     get_config_by_name(std::string name);
-- 
GitLab