Fix 6421, split raw vectors to serveral smaller one (#6449)

Signed-off-by: Ndragondriver <jiquan.long@zilliz.com>

internal/core/src/indexbuilder/IndexWrapper.cpp

@@ -21,6 +21,7 @@
 #include "indexbuilder/utils.h"
 #include "index/knowhere/knowhere/index/vector_index/ConfAdapterMgr.h"
 #include "index/knowhere/knowhere/common/Timer.h"
+#include "index/knowhere/knowhere/common/Utils.h"
 
 namespace milvus {
 namespace indexbuilder {
@@ -239,6 +240,10 @@ IndexWrapper::Serialize() {
         std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
         memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
         binarySet.Append(RAW_DATA, raw_data, raw_data_.size());
+        auto slice_size = get_index_file_slice_size();
+        // https://github.com/milvus-io/milvus/issues/6421
+        // Disassemble will only divide the raw vectors, other keys was already divided
+        knowhere::Disassemble(slice_size * 1024 * 1024, binarySet);
     }
 
     namespace indexcgo = milvus::proto::indexcgo;
@@ -317,6 +322,14 @@ IndexWrapper::get_index_mode() {
     return mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
 }
 
+int64_t
+IndexWrapper::get_index_file_slice_size() {
+    if (config_.contains(knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE)) {
+        return config_[knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE].get<int64_t>();
+    }
+    return 4;  // by default
+}
+
 std::unique_ptr<IndexWrapper::QueryResult>
 IndexWrapper::Query(const knowhere::DatasetPtr& dataset) {
     return std::move(QueryImpl(dataset, config_));

internal/core/src/indexbuilder/IndexWrapper.h

@@ -65,6 +65,9 @@ class IndexWrapper {
     knowhere::IndexMode
     get_index_mode();
 
+    int64_t
+    get_index_file_slice_size();
+
     template <typename T>
     std::optional<T>
     get_config_by_name(std::string name);