Abandon using protobuf to pass binaryset parameter (#15626)

Signed-off-by: Nzhenshan.cao <zhenshan.cao@zilliz.com>

internal/core/CMakeLists.txt

@@ -285,6 +285,10 @@ install(FILES ${CMAKE_BINARY_DIR}/src/indexbuilder/libmilvus_indexbuilder${CMAKE
         DESTINATION lib
 )
 
+install(FILES ${CMAKE_BINARY_DIR}/src/common/libmilvus_common${CMAKE_SHARED_LIBRARY_SUFFIX}
+        DESTINATION lib
+)
+
 # Install common
 install(
         DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/common/

internal/core/src/common/CMakeLists.txt

@@ -13,10 +13,11 @@ set(COMMON_SRC
         Schema.cpp
         Types.cpp
         SystemProperty.cpp
+        vector_index_c.cpp
         )
 
-add_library(milvus_common
+add_library(milvus_common SHARED
         ${COMMON_SRC}
         )
 
-target_link_libraries(milvus_common knowhere milvus_proto yaml-cpp )
+target_link_libraries(milvus_common milvus_utils milvus_config log knowhere milvus_proto yaml-cpp )

internal/core/src/common/vector_index_c.cpp

+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "knowhere/common/BinarySet.h"
+#include "common/vector_index_c.h"
+
+CStatus
+NewBinarySet(CBinarySet* c_binary_set) {
+    try {
+        auto binary_set = std::make_unique<milvus::knowhere::BinarySet>();
+        *c_binary_set = binary_set.release();
+        auto status = CStatus();
+        status.error_code = Success;
+        status.error_msg = "";
+        return status;
+    } catch (std::exception& e) {
+        auto status = CStatus();
+        status.error_code = UnexpectedError;
+        status.error_msg = strdup(e.what());
+        return status;
+    }
+}
+
+void
+DeleteBinarySet(CBinarySet c_binary_set) {
+    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+    delete binary_set;
+}
+
+CStatus
+AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) {
+    auto status = CStatus();
+    try {
+        auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+        std::string index_key(c_index_key);
+        uint8_t* index = (uint8_t*)index_binary;
+        std::shared_ptr<uint8_t[]> data(index, [](void*) {});
+        binary_set->Append(index_key, data, index_size);
+
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::exception& e) {
+        status.error_code = UnexpectedError;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
+}
+
+int
+GetBinarySetSize(CBinarySet c_binary_set) {
+    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+    return binary_set->binary_map_.size();
+}
+
+void
+GetBinarySetKeys(CBinarySet c_binary_set, void* datas) {
+    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+    auto& map_ = binary_set->binary_map_;
+    const char** datas_ = (const char**)datas;
+    std::size_t i = 0;
+    for (auto it = map_.begin(); it != map_.end(); ++it, i++) {
+        datas_[i] = it->first.c_str();
+    }
+}
+
+int
+GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) {
+    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+    int64_t ret_ = 0;
+    try {
+        std::string key_(key);
+        auto binary = binary_set->GetByName(key_);
+        ret_ = binary->size;
+    } catch (std::exception& e) {
+    }
+    return ret_;
+}
+
+CStatus
+CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) {
+    auto status = CStatus();
+    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+    try {
+        auto binary = binary_set->GetByName(key);
+        status.error_code = Success;
+        status.error_msg = "";
+        memcpy((uint8_t*)data, binary->data.get(), binary->size);
+    } catch (std::exception& e) {
+        status.error_code = UnexpectedError;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
+}

internal/core/src/common/vector_index_c.h

+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "common/type_c.h"
+
+typedef void* CBinarySet;
+
+CStatus
+NewBinarySet(CBinarySet* c_binary_set);
+
+void
+DeleteBinarySet(CBinarySet c_binary_set);
+
+CStatus
+AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key);
+
+int
+GetBinarySetSize(CBinarySet c_binary_set);
+
+void
+GetBinarySetKeys(CBinarySet c_binary_set, void* datas);
+
+int
+GetBinarySetValueSize(CBinarySet c_set, const char* key);
+
+// Note: the memory of data has been allocated outside
+CStatus
+CopyBinarySetValue(void* data, const char* key, CBinarySet c_set);
+
+#ifdef __cplusplus
+}
+#endif

internal/core/src/indexbuilder/CMakeLists.txt

@@ -28,12 +28,8 @@ endif ()
 # link order matters
 target_link_libraries(milvus_indexbuilder
         knowhere
-        milvus_config
         milvus_common
-        milvus_utils
-        milvus_proto
         ${TBB}
-        log
         ${PLATFORM_LIBS}
         pthread
         )

internal/core/src/indexbuilder/IndexWrapper.cpp

@@ -40,7 +40,8 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria
     AssertInfo(index_ != nullptr, "[IndexWrapper]Index is null after create index");
 }
 
-template <typename ParamsT>  // ugly here, ParamsT will just be MapParams later
+template <typename ParamsT>
+// ugly here, ParamsT will just be MapParams later
 void
 IndexWrapper::parse_impl(const std::string& serialized_params_str, knowhere::Config& conf) {
     bool deserialized_success;
@@ -164,7 +165,7 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
         }
     }
     auto conf_adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
-    std::cout << "config_ when build index: " << config_ << std::endl;
+    std::cout << "Konwhere BuildWithoutIds config_ is " << config_ << std::endl;
     AssertInfo(conf_adapter->CheckTrain(config_, index_mode), "something wrong in index parameters!");
 
     if (is_in_need_id_list(index_type)) {
@@ -228,6 +229,23 @@ IndexWrapper::StoreRawData(const knowhere::DatasetPtr& dataset) {
     }
 }
 
+std::unique_ptr<milvus::knowhere::BinarySet>
+IndexWrapper::SerializeBinarySet() {
+    auto ret = std::make_unique<milvus::knowhere::BinarySet>(index_->Serialize(config_));
+    auto index_type = get_index_type();
+
+    if (is_in_nm_list(index_type)) {
+        std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
+        memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
+        ret->Append(RAW_DATA, raw_data, raw_data_.size());
+        auto slice_size = get_index_file_slice_size();
+        // https://github.com/milvus-io/milvus/issues/6421
+        // Disassemble will only divide the raw vectors, other keys were already divided
+        knowhere::Disassemble(slice_size * 1024 * 1024, *ret);
+    }
+    return std::move(ret);
+}
+
 /*
  * brief Return serialized binary set
  * TODO: use a more efficient method to manage memory, consider std::vector later
@@ -236,6 +254,7 @@ std::unique_ptr<IndexWrapper::Binary>
 IndexWrapper::Serialize() {
     auto binarySet = index_->Serialize(config_);
     auto index_type = get_index_type();
+
     if (is_in_nm_list(index_type)) {
         std::shared_ptr<uint8_t[]> raw_data(new uint8_t[raw_data_.size()], std::default_delete<uint8_t[]>());
         memcpy(raw_data.get(), raw_data_.data(), raw_data_.size());
@@ -266,6 +285,21 @@ IndexWrapper::Serialize() {
     return binary;
 }
 
+void
+IndexWrapper::LoadFromBinarySet(milvus::knowhere::BinarySet& binary_set) {
+    auto& map_ = binary_set.binary_map_;
+    for (auto it = map_.begin(); it != map_.end(); ++it) {
+        if (it->first == RAW_DATA) {
+            raw_data_.clear();
+            auto data_size = it->second->size;
+            raw_data_.resize(data_size);
+            memcpy(raw_data_.data(), it->second->data.get(), data_size);
+            break;
+        }
+    }
+    index_->Load(binary_set);
+}
+
 void
 IndexWrapper::Load(const char* serialized_sliced_blob_buffer, int32_t size) {
     namespace indexcgo = milvus::proto::indexcgo;

internal/core/src/indexbuilder/IndexWrapper.h

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "knowhere/index/vector_index/VecIndex.h"
+#include "knowhere/common/BinarySet.h"
 
 namespace milvus::indexbuilder {
 
@@ -37,6 +38,12 @@ class IndexWrapper {
     std::unique_ptr<Binary>
     Serialize();
 
+    std::unique_ptr<milvus::knowhere::BinarySet>
+    SerializeBinarySet();
+
+    void
+    LoadFromBinarySet(milvus::knowhere::BinarySet&);
+
     void
     Load(const char* serialized_sliced_blob_buffer, int32_t size);
 

internal/core/src/indexbuilder/index_c.cpp

@@ -10,9 +10,13 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
 #include <string>
+
 #ifndef __APPLE__
+
 #include <malloc.h>
+
 #endif
+
 #include "knowhere/index/vector_index/adapter/VectorAdapter.h"
 #include "indexbuilder/IndexWrapper.h"
 #include "indexbuilder/index_c.h"
@@ -90,6 +94,22 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve
     return status;
 }
 
+CStatus
+SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set) {
+    auto status = CStatus();
+    try {
+        auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
+        auto binary = cIndex->SerializeBinarySet();
+        *c_binary_set = binary.release();
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::exception& e) {
+        status.error_code = UnexpectedError;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
+}
+
 CStatus
 SerializeToSlicedBuffer(CIndex index, CBinary* c_binary) {
     auto status = CStatus();
@@ -140,6 +160,22 @@ LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, in
     return status;
 }
 
+CStatus
+LoadFromBinarySet(CIndex index, CBinarySet c_binary_set) {
+    auto status = CStatus();
+    try {
+        auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
+        auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
+        cIndex->LoadFromBinarySet(*binary_set);
+        status.error_code = Success;
+        status.error_msg = "";
+    } catch (std::exception& e) {
+        status.error_code = UnexpectedError;
+        status.error_msg = strdup(e.what());
+    }
+    return status;
+}
+
 CStatus
 QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res) {
     auto status = CStatus();

internal/core/src/indexbuilder/index_c.h

@@ -18,6 +18,7 @@ extern "C" {
 #include <stdint.h>
 #include "segcore/collection_c.h"
 #include "common/type_c.h"
+#include "common/vector_index_c.h"
 
 typedef void* CIndex;
 typedef void* CIndexQueryResult;
@@ -41,6 +42,9 @@ BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* ve
 CStatus
 SerializeToSlicedBuffer(CIndex index, CBinary* c_binary);
 
+CStatus
+SerializeToBinarySet(CIndex index, CBinarySet* c_binary_set);
+
 int64_t
 GetCBinarySize(CBinary c_binary);
 
@@ -54,6 +58,9 @@ DeleteCBinary(CBinary c_binary);
 CStatus
 LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size);
 
+CStatus
+LoadFromBinarySet(CIndex index, CBinarySet c_binary_set);
+
 CStatus
 QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors, CIndexQueryResult* res);
 

internal/core/src/segcore/CMakeLists.txt

@@ -40,16 +40,12 @@ endif ()
 
 target_link_libraries(milvus_segcore
         ${PLATFORM_LIBS}
-        log
         pthread
         ${TBB}
         ${OpenMP_CXX_FLAGS}
-        knowhere
         milvus_common
-        milvus_config
-        milvus_proto
+        knowhere
         milvus_query
-        milvus_utils
 #        gperftools
         )
 

internal/core/src/segcore/load_index_c.cpp

@@ -106,47 +106,3 @@ AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
         return status;
     }
 }
-
-CStatus
-NewBinarySet(CBinarySet* c_binary_set) {
-    try {
-        auto binary_set = std::make_unique<milvus::knowhere::BinarySet>();
-        *c_binary_set = binary_set.release();
-        auto status = CStatus();
-        status.error_code = Success;
-        status.error_msg = "";
-        return status;
-    } catch (std::exception& e) {
-        auto status = CStatus();
-        status.error_code = UnexpectedError;
-        status.error_msg = strdup(e.what());
-        return status;
-    }
-}
-
-void
-DeleteBinarySet(CBinarySet c_binary_set) {
-    auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
-    delete binary_set;
-}
-
-CStatus
-AppendBinaryIndex(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) {
-    try {
-        auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
-        std::string index_key(c_index_key);
-        uint8_t* index = (uint8_t*)index_binary;
-        std::shared_ptr<uint8_t[]> data(index, [](void*) {});
-        binary_set->Append(index_key, data, index_size);
-
-        auto status = CStatus();
-        status.error_code = Success;
-        status.error_msg = "";
-        return status;
-    } catch (std::exception& e) {
-        auto status = CStatus();
-        status.error_code = UnexpectedError;
-        status.error_msg = strdup(e.what());
-        return status;
-    }
-}

internal/core/src/segcore/load_index_c.h

@@ -17,11 +17,11 @@ extern "C" {
 #include <stdint.h>
 #include <stdlib.h>
 
+#include "common/vector_index_c.h"
 #include "common/type_c.h"
 #include "segcore/collection_c.h"
 
 typedef void* CLoadIndexInfo;
-typedef void* CBinarySet;
 
 CStatus
 NewLoadIndexInfo(CLoadIndexInfo* c_load_index_info);
@@ -38,15 +38,6 @@ AppendFieldInfo(CLoadIndexInfo c_load_index_info, int64_t field_id);
 CStatus
 AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set);
 
-CStatus
-NewBinarySet(CBinarySet* c_binary_set);
-
-void
-DeleteBinarySet(CBinarySet c_binary_set);
-
-CStatus
-AppendBinaryIndex(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key);
-
 #ifdef __cplusplus
 }
 #endif

internal/core/unittest/test_index_wrapper.cpp

@@ -315,16 +315,16 @@ TEST(IVFFLATNMWrapper, Codec) {
     auto dataset = GenDataset(flat_nb, metric_type, false);
     auto xb_data = dataset.get_col<float>(0);
     auto xb_dataset = milvus::knowhere::GenDataset(flat_nb, DIM, xb_data.data());
-    auto index =
+    auto index_wrapper =
         std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
-    ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
+    ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset));
 
-    auto binary = index->Serialize();
-    auto copy_index =
+    auto binary = index_wrapper->Serialize();
+    auto copy_index_wrapper =
         std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
-    ASSERT_NO_THROW(copy_index->Load(binary->data.data(), binary->data.size()));
-    ASSERT_EQ(copy_index->dim(), copy_index->dim());
-    auto copy_binary = copy_index->Serialize();
+    ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size()));
+    ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim());
+    auto copy_binary = copy_index_wrapper->Serialize();
 }
 
 TEST(BinFlatWrapper, Build) {
@@ -416,17 +416,17 @@ TEST_P(IndexWrapperTest, BuildWithoutIds) {
 }
 
 TEST_P(IndexWrapperTest, Codec) {
-    auto index =
+    auto index_wrapper =
         std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
 
-    ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
+    ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset));
 
-    auto binary = index->Serialize();
-    auto copy_index =
+    auto binary = index_wrapper->Serialize();
+    auto copy_index_wrapper =
         std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str.c_str(), index_params_str.c_str());
-    ASSERT_NO_THROW(copy_index->Load(binary->data.data(), binary->data.size()));
-    ASSERT_EQ(copy_index->dim(), copy_index->dim());
-    auto copy_binary = copy_index->Serialize();
+    ASSERT_NO_THROW(copy_index_wrapper->Load(binary->data.data(), binary->data.size()));
+    ASSERT_EQ(copy_index_wrapper->dim(), copy_index_wrapper->dim());
+    auto copy_binary = copy_index_wrapper->Serialize();
     if (!milvus::indexbuilder::is_in_nm_list(index_type)) {
         // binary may be not same due to uncertain internal map order
         ASSERT_EQ(binary->data.size(), copy_binary->data.size());

internal/indexnode/index.go

@@ -20,12 +20,13 @@ package indexnode
 
 #cgo CFLAGS: -I${SRCDIR}/../core/output/include
 
-#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath,"${SRCDIR}/../core/output/lib"
-#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib
+#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath,"${SRCDIR}/../core/output/lib"
+#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib
 
 #include <stdlib.h>	// free
 #include "segcore/collection_c.h"
 #include "indexbuilder/index_c.h"
+#include "common/vector_index_c.h"
 
 */
 import "C"
@@ -33,6 +34,7 @@ import "C"
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"runtime"
 	"unsafe"
 
@@ -63,7 +65,75 @@ type CIndex struct {
 	close    bool
 }
 
+func GetBinarySetKeys(cBinarySet C.CBinarySet) ([]string, error) {
+	size := int(C.GetBinarySetSize(cBinarySet))
+	if size == 0 {
+		return nil, fmt.Errorf("BinarySet size is zero!")
+	}
+	datas := make([]unsafe.Pointer, size)
+
+	C.GetBinarySetKeys(cBinarySet, unsafe.Pointer(&datas[0]))
+	ret := make([]string, size)
+	for i := 0; i < size; i++ {
+		ret[i] = C.GoString((*C.char)(datas[i]))
+	}
+
+	return ret, nil
+}
+
+func GetBinarySetValue(cBinarySet C.CBinarySet, key string) ([]byte, error) {
+	cIndexKey := C.CString(key)
+	defer C.free(unsafe.Pointer(cIndexKey))
+	ret := C.GetBinarySetValueSize(cBinarySet, cIndexKey)
+	size := int(ret)
+	if size == 0 {
+		return nil, fmt.Errorf("GetBinarySetValueSize size is zero!")
+	}
+	value := make([]byte, size)
+	status := C.CopyBinarySetValue(unsafe.Pointer(&value[0]), cIndexKey, cBinarySet)
+
+	if err := HandleCStatus(&status, "CopyBinarySetValue failed"); err != nil {
+		return nil, err
+	}
+
+	return value, nil
+}
+
+func (index *CIndex) Serialize() ([]*Blob, error) {
+	var cBinarySet C.CBinarySet
+
+	status := C.SerializeToBinarySet(index.indexPtr, &cBinarySet)
+	defer func() {
+		if cBinarySet != nil {
+			C.DeleteBinarySet(cBinarySet)
+		}
+	}()
+	if err := HandleCStatus(&status, "SerializeToBinarySet failed"); err != nil {
+		return nil, err
+	}
+
+	keys, err := GetBinarySetKeys(cBinarySet)
+	if err != nil {
+		return nil, err
+	}
+	ret := make([]*Blob, 0)
+	for _, key := range keys {
+		value, err := GetBinarySetValue(cBinarySet, key)
+		if err != nil {
+			return nil, err
+		}
+		blob := &Blob{
+			Key:   key,
+			Value: value,
+		}
+		ret = append(ret, blob)
+	}
+
+	return ret, nil
+}
+
 // Serialize serializes vector data into bytes data so that it can be accessed in 'C'.
+/*
 func (index *CIndex) Serialize() ([]*Blob, error) {
 	var cBinary C.CBinary
 
@@ -94,8 +164,36 @@ func (index *CIndex) Serialize() ([]*Blob, error) {
 
 	return ret, nil
 }
+*/
+
+func (index *CIndex) Load(blobs []*Blob) error {
+	var cBinarySet C.CBinarySet
+	status := C.NewBinarySet(&cBinarySet)
+	defer C.DeleteBinarySet(cBinarySet)
+
+	if err := HandleCStatus(&status, "CIndex Load2 NewBinarySet failed"); err != nil {
+		return err
+	}
+	for _, blob := range blobs {
+		key := blob.Key
+		byteIndex := blob.Value
+		indexPtr := unsafe.Pointer(&byteIndex[0])
+		indexLen := C.int64_t(len(byteIndex))
+		binarySetKey := filepath.Base(key)
+		log.Debug("", zap.String("index key", binarySetKey))
+		indexKey := C.CString(binarySetKey)
+		status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey)
+		C.free(unsafe.Pointer(indexKey))
+		if err := HandleCStatus(&status, "CIndex Load AppendIndexBinary failed"); err != nil {
+			return err
+		}
+	}
+	status = C.LoadFromBinarySet(index.indexPtr, cBinarySet)
+	return HandleCStatus(&status, "AppendIndex failed")
+}
 
 // Load loads data from 'C'.
+/*
 func (index *CIndex) Load(blobs []*Blob) error {
 	binarySet := &indexcgopb.BinarySet{Datas: make([]*indexcgopb.Binary, 0)}
 	for _, blob := range blobs {
@@ -107,13 +205,10 @@ func (index *CIndex) Load(blobs []*Blob) error {
 		return err2
 	}
 
-	/*
-		CStatus
-		LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size);
-	*/
 	status := C.LoadFromSlicedBuffer(index.indexPtr, (*C.char)(unsafe.Pointer(&datas[0])), (C.int32_t)(len(datas)))
 	return HandleCStatus(&status, "LoadFromSlicedBuffer failed")
 }
+*/
 
 // BuildFloatVecIndexWithoutIds builds indexes for float vector.
 func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error {

internal/querynode/load_index_info.go

@@ -18,13 +18,15 @@ package querynode
 
 /*
 #cgo CFLAGS: -I${SRCDIR}/../core/output/include
-#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath,"${SRCDIR}/../core/output/lib"
-#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib
+#cgo darwin LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_segcore -Wl,-rpath,"${SRCDIR}/../core/output/lib"
+#cgo linux LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_common -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib
 
 #include "segcore/load_index_c.h"
+#include "common/vector_index_c.h"
 
 */
 import "C"
+
 import (
 	"path/filepath"
 	"unsafe"
@@ -108,9 +110,9 @@ func (li *LoadIndexInfo) appendIndexData(bytesIndex [][]byte, indexKeys []string
 		binarySetKey := filepath.Base(indexKeys[i])
 		log.Debug("", zap.String("index key", binarySetKey))
 		indexKey := C.CString(binarySetKey)
-		status = C.AppendBinaryIndex(cBinarySet, indexPtr, indexLen, indexKey)
+		status = C.AppendIndexBinary(cBinarySet, indexPtr, indexLen, indexKey)
 		C.free(unsafe.Pointer(indexKey))
-		if err := HandleCStatus(&status, "AppendBinaryIndex failed"); err != nil {
+		if err := HandleCStatus(&status, "LoadIndexInfo AppendIndexBinary failed"); err != nil {
 			return err
 		}
 	}