merge code

Former-commit-id: 213b4823192af7e340ac5a250c363d0a32a3c3aa

cpp/src/core/CMakeLists.txt

@@ -93,7 +93,7 @@ endif()
 set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
 
 if(BUILD_UNIT_TEST STREQUAL "ON")
-    add_subdirectory(test)
+#    add_subdirectory(test)
 endif()
 
 config_summary()

cpp/src/core/knowhere/CMakeLists.txt

@@ -29,26 +29,32 @@ if(NOT TARGET SPTAGLibStatic)
 endif()
 
 set(external_srcs
-        knowhere/adapter/sptag.cpp
-        knowhere/adapter/structure.cpp
-        knowhere/adapter/arrow.cpp
-        knowhere/common/exception.cpp
-        knowhere/common/timer.cpp
+        knowhere/adapter/SptagAdapter.cpp
+        knowhere/adapter/Structure.cpp
+        knowhere/adapter/ArrowAdapter.cpp
+        knowhere/common/Exception.cpp
+        knowhere/common/Timer.cpp
         )
 
 set(index_srcs
-        knowhere/index/preprocessor/normalize.cpp
-        knowhere/index/vector_index/cpu_kdt_rng.cpp
-        knowhere/index/vector_index/idmap.cpp
-        knowhere/index/vector_index/ivf.cpp
-        knowhere/index/vector_index/gpu_ivf.cpp
-        knowhere/index/vector_index/kdt_parameters.cpp
-        knowhere/index/vector_index/nsg_index.cpp
-        knowhere/index/vector_index/nsg/nsg.cpp
-        knowhere/index/vector_index/nsg/nsg_io.cpp
-        knowhere/index/vector_index/nsg/utils.cpp
-        knowhere/index/vector_index/cloner.cpp
-        knowhere/index/vector_index/FaissGpuResourceMgr.cpp
+        knowhere/index/preprocessor/Normalize.cpp
+        knowhere/index/vector_index/IndexKDT.cpp
+        knowhere/index/vector_index/IndexIDMAP.cpp
+        knowhere/index/vector_index/IndexIVF.cpp
+        knowhere/index/vector_index/IndexGPUIVF.cpp
+        knowhere/index/vector_index/helpers/KDTParameterMgr.cpp
+        knowhere/index/vector_index/IndexNSG.cpp
+        knowhere/index/vector_index/nsg/NSG.cpp
+        knowhere/index/vector_index/nsg/NSGIO.cpp
+        knowhere/index/vector_index/nsg/NSGHelper.cpp
+        knowhere/index/vector_index/helpers/Cloner.cpp
+        knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
+        knowhere/index/vector_index/IndexIVFSQ.cpp
+        knowhere/index/vector_index/IndexGPUIVFSQ.cpp
+        knowhere/index/vector_index/IndexIVFPQ.cpp
+        knowhere/index/vector_index/IndexGPUIVFPQ.cpp
+        knowhere/index/vector_index/FaissBaseIndex.cpp
+        knowhere/index/vector_index/helpers/FaissIO.cpp
         )
 
 set(depend_libs

cpp/src/core/knowhere/knowhere/adapter/arrow.cpp → cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp

@@ -16,7 +16,7 @@
 // under the License.
 
 
-#include "arrow.h"
+#include "ArrowAdapter.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/adapter/arrow.h → cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "knowhere/common/array.h"
+#include "knowhere/common/Array.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/sptag.cpp → cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp

@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "knowhere/index/vector_index/definitions.h"
-#include "sptag.h"
-#include "structure.h"
+#include "knowhere/index/vector_index/helpers/Definitions.h"
+#include "SptagAdapter.h"
+#include "Structure.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/sptag.h → cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h

@@ -22,7 +22,7 @@
 
 #include <SPTAG/AnnService/inc/Core/VectorIndex.h>
 
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/adapter/structure.cpp → cpp/src/core/knowhere/knowhere/adapter/Structure.cpp

@@ -16,7 +16,7 @@
 // under the License.
 
 
-#include "structure.h"
+#include "Structure.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/structure.h → cpp/src/core/knowhere/knowhere/adapter/Structure.h

@@ -19,7 +19,7 @@
 #pragma once
 
 #include <memory>
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/array.h → cpp/src/core/knowhere/knowhere/common/Array.h

@@ -18,8 +18,9 @@
 
 #pragma once
 
-#include "arrow/array.h"
-#include "schema.h"
+#include <arrow/array.h>
+
+#include "Schema.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/binary_set.h → cpp/src/core/knowhere/knowhere/common/BinarySet.h

@@ -23,7 +23,7 @@
 #include <vector>
 #include <memory>
 
-#include "id.h"
+#include "Id.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/buffer.h → cpp/src/core/knowhere/knowhere/common/Buffer.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/buffer.h"
+#include <arrow/buffer.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/dataset.h → cpp/src/core/knowhere/knowhere/common/Dataset.h

@@ -21,12 +21,12 @@
 #include <vector>
 #include <memory>
 
-#include "array.h"
-#include "buffer.h"
-#include "tensor.h"
-#include "schema.h"
-#include "config.h"
-#include "knowhere/adapter/arrow.h"
+#include "Array.h"
+#include "Buffer.h"
+#include "Tensor.h"
+#include "Schema.h"
+#include "Config.h"
+#include "knowhere/adapter/ArrowAdapter.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/exception.cpp → cpp/src/core/knowhere/knowhere/common/Exception.cpp

@@ -18,7 +18,7 @@
 
 #include <cstdio>
 
-#include "exception.h"
+#include "Exception.h"
 #include "Log.h"
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/Log.h

@@ -22,6 +22,7 @@
 
 namespace zilliz {
 namespace knowhere {
+
 #define KNOWHERE_DOMAIN_NAME "[KNOWHERE] "
 #define KNOWHERE_ERROR_TEXT "KNOWHERE Error:"
 
@@ -31,5 +32,6 @@ namespace knowhere {
 #define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME
 #define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME
 #define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME
+
 } // namespace knowhere
 } // namespace zilliz
\ No newline at end of file

cpp/src/core/knowhere/knowhere/common/schema.h → cpp/src/core/knowhere/knowhere/common/Schema.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/type.h"
+#include <arrow/type.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/tensor.h → cpp/src/core/knowhere/knowhere/common/Tensor.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/tensor.h"
+#include <arrow/tensor.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/timer.cpp → cpp/src/core/knowhere/knowhere/common/Timer.cpp

@@ -18,7 +18,7 @@
 
 #include <iostream> // TODO(linxj): using Log instead
 
-#include "timer.h"
+#include "Timer.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/index/index.h → cpp/src/core/knowhere/knowhere/index/Index.h

@@ -20,11 +20,11 @@
 
 #include <memory>
 
-#include "knowhere/common/binary_set.h"
-#include "knowhere/common/dataset.h"
-#include "index_type.h"
-#include "index_model.h"
-#include "knowhere/index/preprocessor/preprocessor.h"
+#include "knowhere/common/BinarySet.h"
+#include "knowhere/common/Dataset.h"
+#include "IndexType.h"
+#include "IndexModel.h"
+#include "knowhere/index/preprocessor/Preprocessor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/index_model.h → cpp/src/core/knowhere/knowhere/index/IndexModel.h

@@ -19,7 +19,7 @@
 #pragma once
 
 #include <memory>
-#include "knowhere/common/binary_set.h"
+#include "knowhere/common/BinarySet.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h → cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/index_io.h>
+#include <faiss/IndexIVF.h>
+
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "FaissBaseIndex.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+FaissBaseIndex::FaissBaseIndex(std::shared_ptr<faiss::Index> index) : index_(std::move(index)) {}
+
+BinarySet FaissBaseIndex::SerializeImpl() {
+    try {
+        faiss::Index *index = index_.get();
+
+        SealImpl();
+
+        MemoryIOWriter writer;
+        faiss::write_index(index, &writer);
+        auto data = std::make_shared<uint8_t>();
+        data.reset(writer.data_);
+
+        BinarySet res_set;
+        // TODO(linxj): use virtual func Name() instead of raw string.
+        res_set.Append("IVF", data, writer.rp);
+        return res_set;
+    } catch (std::exception &e) {
+        KNOWHERE_THROW_MSG(e.what());
+    }
+}
+
+void FaissBaseIndex::LoadImpl(const BinarySet &index_binary) {
+    auto binary = index_binary.GetByName("IVF");
+
+    MemoryIOReader reader;
+    reader.total = binary->size;
+    reader.data_ = binary->data.get();
+
+    faiss::Index *index = faiss::read_index(&reader);
+
+    index_.reset(index);
+}
+
+void FaissBaseIndex::SealImpl() {
+// TODO(linxj): enable
+//#ifdef ZILLIZ_FAISS
+    faiss::Index *index = index_.get();
+    auto idx = dynamic_cast<faiss::IndexIVF *>(index);
+    if (idx != nullptr) {
+        idx->to_readonly();
+    }
+    //else {
+    //    KNOHWERE_ERROR_MSG("Seal failed");
+    //}
+//#endif
+}
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <memory>
+
+#include <faiss/Index.h>
+
+#include "knowhere/common/BinarySet.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class FaissBaseIndex {
+ protected:
+    explicit FaissBaseIndex(std::shared_ptr<faiss::Index> index);
+
+    virtual BinarySet
+    SerializeImpl();
+
+    virtual void
+    LoadImpl(const BinarySet &index_binary);
+
+    virtual void
+    SealImpl();
+
+ protected:
+    std::shared_ptr<faiss::Index> index_ = nullptr;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp

@@ -20,18 +20,17 @@
 #include <faiss/gpu/GpuIndexFlat.h>
 #include <faiss/gpu/GpuIndexIVF.h>
 #include <faiss/gpu/GpuIndexIVFFlat.h>
-#include <faiss/gpu/GpuIndexIVFPQ.h>
 #include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/index_io.h>
 
 
-#include "knowhere/common/exception.h"
-#include "cloner.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "gpu_ivf.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/Cloner.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexGPUIVF.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
 
-#include <algorithm>
 
 namespace zilliz {
 namespace knowhere {
@@ -169,7 +168,7 @@ VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) {
 
 VectorIndexPtr GPUIVF::Clone() {
     auto cpu_idx = CopyGpuToCpu(Config());
-    return ::zilliz::knowhere::CopyCpuToGpu(cpu_idx, gpu_id_, Config());
+    return ::zilliz::knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config());
 }
 
 VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &config) {
@@ -187,84 +186,6 @@ void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) {
     }
 }
 
-IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto M = config["M"].as<size_t>();        // number of subquantizers(subvectors)
-    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
-    auto gpu_num = config.get_with_default("gpu_id", gpu_id_);
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support.
-
-    GETTENSOR(dataset)
-
-    // TODO(linxj): set device here.
-    // TODO(linxj): set gpu resource here.
-    faiss::gpu::StandardGpuResources res;
-    faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type);
-    device_index.train(rows, (float *) p_data);
-
-    std::shared_ptr<faiss::Index> host_index = nullptr;
-    host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index));
-
-    return std::make_shared<IVFIndexModel>(host_index);
-}
-
-std::shared_ptr<faiss::IVFSearchParameters> GPUIVFPQ::GenParams(const Config &config) {
-    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
-    params->nprobe = config.get_with_default("nprobe", size_t(1));
-    //params->scan_table_threshold = 0;
-    //params->polysemous_ht = 0;
-    //params->max_codes = 0;
-
-    return params;
-}
-
-VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) {
-    KNOWHERE_THROW_MSG("not support yet");
-}
-
-IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj):  gpu only support SQ4 SQ8 SQ16
-    gpu_id_ = config.get_with_default("gpu_id", gpu_id_);
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    std::stringstream index_type;
-    index_type << "IVF" << nlist << "," << "SQ" << nbits;
-    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
-
-    auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
-    if (temp_resource != nullptr) {
-        ResScope rs(temp_resource, gpu_id_, true);
-        auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index);
-        device_index->train(rows, (float *) p_data);
-
-        std::shared_ptr<faiss::Index> host_index = nullptr;
-        host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index));
-
-        delete device_index;
-        delete build_index;
-
-        return std::make_shared<IVFIndexModel>(host_index);
-    } else {
-        KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource");
-    }
-}
-
-VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) {
-    std::lock_guard<std::mutex> lk(mutex_);
-
-    faiss::Index *device_index = index_.get();
-    faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
-
-    std::shared_ptr<faiss::Index> new_index;
-    new_index.reset(host_index);
-    return std::make_shared<IVFSQ>(new_index);
-}
-
 void GPUIndex::SetGpuDevice(const int &gpu_id) {
     gpu_id_ = gpu_id;
 }

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+
+#include "IndexIVF.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIndex {
+public:
+     explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {}
+
+     GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource) {}
+
+     virtual VectorIndexPtr
+     CopyGpuToCpu(const Config &config) = 0;
+
+     virtual VectorIndexPtr
+     CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0;
+
+     void
+     SetGpuDevice(const int &gpu_id);
+
+     const int64_t &
+     GetGpuDevice();
+
+protected:
+     int64_t gpu_id_;
+     ResWPtr res_;
+};
+
+class GPUIVF : public IVF, public GPUIndex {
+public:
+     explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {}
+
+     explicit GPUIVF(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
+          : IVF(std::move(index)), GPUIndex(device_id, resource) {};
+
+     IndexModelPtr
+     Train(const DatasetPtr &dataset, const Config &config) override;
+
+     void
+     Add(const DatasetPtr &dataset, const Config &config) override;
+
+     void
+     set_index_model(IndexModelPtr model) override;
+
+     //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
+     VectorIndexPtr
+     CopyGpuToCpu(const Config &config) override;
+
+     VectorIndexPtr
+     CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
+
+     VectorIndexPtr
+     Clone() final;
+
+     // TODO(linxj): Deprecated
+     virtual IVFIndexPtr Copy_index_gpu_to_cpu();
+
+protected:
+     void
+     search_impl(int64_t n,
+                 const float *data,
+                 int64_t k,
+                 float *distances,
+                 int64_t *labels,
+                 const Config &cfg) override;
+
+     BinarySet
+     SerializeImpl() override;
+
+     void
+     LoadImpl(const BinarySet &index_binary) override;
+};
+
+} // namespace knowhere
+} // namespace zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuIndexIVFPQ.h>
+#include <faiss/gpu/GpuAutoTune.h>
+#include <faiss/IndexIVFPQ.h>
+
+#include "IndexGPUIVFPQ.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto M = config["M"].as<size_t>();        // number of subquantizers(subvectors)
+    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
+    auto gpu_num = config.get_with_default("gpu_id", gpu_id_);
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support.
+
+    GETTENSOR(dataset)
+
+    // TODO(linxj): set device here.
+    // TODO(linxj): set gpu resource here.
+    faiss::gpu::StandardGpuResources res;
+    faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type);
+    device_index.train(rows, (float *) p_data);
+
+    std::shared_ptr<faiss::Index> host_index = nullptr;
+    host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index));
+
+    return std::make_shared<IVFIndexModel>(host_index);
+}
+
+std::shared_ptr<faiss::IVFSearchParameters> GPUIVFPQ::GenParams(const Config &config) {
+    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
+    params->nprobe = config.get_with_default("nprobe", size_t(1));
+    //params->scan_table_threshold = 0;
+    //params->polysemous_ht = 0;
+    //params->max_codes = 0;
+
+    return params;
+}
+
+VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) {
+    KNOWHERE_THROW_MSG("not support yet");
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexGPUIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIVFPQ : public GPUIVF {
+public:
+    explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {}
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+public:
+    VectorIndexPtr
+    CopyGpuToCpu(const Config &config) override;
+
+protected:
+    // TODO(linxj): remove GenParams.
+    std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuAutoTune.h>
+
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/common/Exception.h"
+#include "IndexGPUIVFSQ.h"
+#include "IndexIVFSQ.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+    IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
+        auto nlist = config["nlist"].as<size_t>();
+        auto nbits = config["nbits"].as<size_t>(); // TODO(linxj):  gpu only support SQ4 SQ8 SQ16
+        gpu_id_ = config.get_with_default("gpu_id", gpu_id_);
+        auto metric_type = config["metric_type"].as_string() == "L2" ?
+                           faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+        GETTENSOR(dataset)
+
+        std::stringstream index_type;
+        index_type << "IVF" << nlist << "," << "SQ" << nbits;
+        auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
+
+        auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
+        if (temp_resource != nullptr) {
+            ResScope rs(temp_resource, gpu_id_, true);
+            auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index);
+            device_index->train(rows, (float *) p_data);
+
+            std::shared_ptr<faiss::Index> host_index = nullptr;
+            host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index));
+
+            delete device_index;
+            delete build_index;
+
+            return std::make_shared<IVFIndexModel>(host_index);
+        } else {
+            KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource");
+        }
+    }
+
+    VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) {
+        std::lock_guard<std::mutex> lk(mutex_);
+
+        faiss::Index *device_index = index_.get();
+        faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
+
+        std::shared_ptr<faiss::Index> new_index;
+        new_index.reset(host_index);
+        return std::make_shared<IVFSQ>(new_index);
+    }
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexGPUIVF.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIVFSQ : public GPUIVF {
+public:
+    explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {}
+
+    explicit GPUIVFSQ(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
+            : GPUIVF(std::move(index), device_id, resource) {};
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    VectorIndexPtr
+    CopyGpuToCpu(const Config &config) override;
+};
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp

@@ -23,9 +23,10 @@
 #include <faiss/gpu/GpuAutoTune.h>
 
 
-#include "knowhere/common/exception.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "idmap.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "IndexIDMAP.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h

@@ -18,17 +18,17 @@
 
 #pragma once
 
-#include "ivf.h"
-#include "gpu_ivf.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
 
 
 namespace zilliz {
 namespace knowhere {
 
-class IDMAP : public VectorIndex, public BasicIndex {
+class IDMAP : public VectorIndex, public FaissBaseIndex {
  public:
-    IDMAP() : BasicIndex(nullptr) {};
-    explicit IDMAP(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {};
+    IDMAP() : FaissBaseIndex(nullptr) {};
+    explicit IDMAP(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {};
     BinarySet Serialize() override;
     void Load(const BinarySet &index_binary) override;
     void Train(const Config &config);

cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp

@@ -23,14 +23,13 @@
 #include <faiss/IVFlib.h>
 #include <faiss/AuxIndexStructures.h>
 #include <faiss/index_io.h>
-#include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/gpu/GpuAutoTune.h>
 
 
-#include "knowhere/common/exception.h"
-#include "ivf.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "gpu_ivf.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
 
 
 namespace zilliz {
@@ -239,7 +238,7 @@ void IVF::Seal() {
 }
 
 
-IVFIndexModel::IVFIndexModel(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {}
+IVFIndexModel::IVFIndexModel(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {}
 
 BinarySet IVFIndexModel::Serialize() {
     if (!index_ || !index_->is_trained) {
@@ -258,159 +257,8 @@ void IVFIndexModel::SealImpl() {
     // do nothing
 }
 
-IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    std::stringstream index_type;
-    index_type << "IVF" << nlist << "," << "SQ" << nbits;
-    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
-    build_index->train(rows, (float *) p_data);
-
-    std::shared_ptr<faiss::Index> ret_index;
-    ret_index.reset(build_index);
-    return std::make_shared<IVFIndexModel>(ret_index);
-}
-
-VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
-    return std::make_shared<IVFSQ>(index);
-}
-
-VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
-    if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){
-        ResScope rs(res, device_id, false);
-        faiss::gpu::GpuClonerOptions option;
-        option.allInGpu = true;
-
-        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
-
-        std::shared_ptr<faiss::Index> device_index;
-        device_index.reset(gpu_index);
-        return std::make_shared<GPUIVFSQ>(device_index, device_id, res);
-    } else {
-        KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
-    }
-}
-
-IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto M = config["M"].as<size_t>();        // number of subquantizers(subvector)
-    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
-    auto index = std::make_shared<faiss::IndexIVFPQ>(coarse_quantizer, dim, nlist, M, nbits);
-    index->train(rows, (float *) p_data);
-
-    return std::make_shared<IVFIndexModel>(index);
-}
 
-std::shared_ptr<faiss::IVFSearchParameters> IVFPQ::GenParams(const Config &config) {
-    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
-    params->nprobe = config.get_with_default("nprobe", size_t(1));
-    //params->scan_table_threshold = 0;
-    //params->polysemous_ht = 0;
-    //params->max_codes = 0;
-
-    return params;
-}
-
-VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
-    return std::make_shared<IVFPQ>(index);
-}
-
-BasicIndex::BasicIndex(std::shared_ptr<faiss::Index> index) : index_(std::move(index)) {}
-
-BinarySet BasicIndex::SerializeImpl() {
-    try {
-        faiss::Index *index = index_.get();
-
-        SealImpl();
-
-        MemoryIOWriter writer;
-        faiss::write_index(index, &writer);
-        auto data = std::make_shared<uint8_t>();
-        data.reset(writer.data_);
-
-        BinarySet res_set;
-        // TODO(linxj): use virtual func Name() instead of raw string.
-        res_set.Append("IVF", data, writer.rp);
-        return res_set;
-    } catch (std::exception &e) {
-        KNOWHERE_THROW_MSG(e.what());
-    }
-}
-
-void BasicIndex::LoadImpl(const BinarySet &index_binary) {
-    auto binary = index_binary.GetByName("IVF");
-
-    MemoryIOReader reader;
-    reader.total = binary->size;
-    reader.data_ = binary->data.get();
 
-    faiss::Index *index = faiss::read_index(&reader);
-
-    index_.reset(index);
-}
-
-void BasicIndex::SealImpl() {
-// TODO(linxj): enable
-//#ifdef ZILLIZ_FAISS
-    faiss::Index *index = index_.get();
-    auto idx = dynamic_cast<faiss::IndexIVF *>(index);
-    if (idx != nullptr) {
-        idx->to_readonly();
-    }
-    //else {
-    //    KNOHWERE_ERROR_MSG("Seal failed");
-    //}
-//#endif
-}
-
-// TODO(linxj): Get From Config File
-static size_t magic_num = 2;
-size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) {
-    auto total_need = size * nitems + rp;
-
-    if (!data_) { // data == nullptr
-        total = total_need * magic_num;
-        rp = size * nitems;
-        data_ = new uint8_t[total];
-        memcpy((void *) (data_), ptr, rp);
-    }
-
-    if (total_need > total) {
-        total = total_need * magic_num;
-        auto new_data = new uint8_t[total];
-        memcpy((void *) new_data, (void *) data_, rp);
-        delete data_;
-        data_ = new_data;
-
-        memcpy((void *) (data_ + rp), ptr, size * nitems);
-        rp = total_need;
-    } else {
-        memcpy((void *) (data_ + rp), ptr, size * nitems);
-        rp = total_need;
-    }
-
-    return nitems;
-}
-
-size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) {
-    if (rp >= total) return 0;
-    size_t nremain = (total - rp) / size;
-    if (nremain < nitems) nitems = nremain;
-    memcpy(ptr, (void *) (data_ + rp), size * nitems);
-    rp += size * nitems;
-    return nitems;
-}
 
 
 }

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+
+#include "VectorIndex.h"
+#include "FaissBaseIndex.h"
+#include "faiss/IndexIVF.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+using Graph = std::vector<std::vector<int64_t>>;
+
+class IVF : public VectorIndex, protected FaissBaseIndex {
+ public:
+    IVF() : FaissBaseIndex(nullptr) {};
+
+    explicit IVF(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {}
+
+    VectorIndexPtr
+    Clone() override;;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    set_index_model(IndexModelPtr model) override;
+
+    void
+    Add(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    AddWithoutIds(const DatasetPtr &dataset, const Config &config);
+
+    DatasetPtr
+    Search(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config);
+
+    BinarySet
+    Serialize() override;
+
+    void
+    Load(const BinarySet &index_binary) override;
+
+    int64_t
+    Count() override;
+
+    int64_t
+    Dimension() override;
+
+    void
+    Seal() override;
+
+    virtual VectorIndexPtr
+    CopyCpuToGpu(const int64_t &device_id, const Config &config);
+
+
+ protected:
+    virtual std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config);
+
+    virtual VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index);
+
+    virtual void
+    search_impl(int64_t n,
+                const float *data,
+                int64_t k,
+                float *distances,
+                int64_t *labels,
+                const Config &cfg);
+
+ protected:
+    std::mutex mutex_;
+};
+
+using IVFIndexPtr = std::shared_ptr<IVF>;
+
+class GPUIVF;
+class IVFIndexModel : public IndexModel, public FaissBaseIndex {
+    friend IVF;
+    friend GPUIVF;
+
+ public:
+    explicit IVFIndexModel(std::shared_ptr<faiss::Index> index);
+
+    IVFIndexModel() : FaissBaseIndex(nullptr) {};
+
+    BinarySet
+    Serialize() override;
+
+    void
+    Load(const BinarySet &binary) override;
+
+ protected:
+    void
+    SealImpl() override;
+
+ protected:
+    std::mutex mutex_;
+};
+using IVFIndexModelPtr = std::shared_ptr<IVFIndexModel>;
+
+}
+}
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexIVFPQ.h>
+
+#include "IndexIVFPQ.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto M = config["M"].as<size_t>();        // number of subquantizers(subvector)
+    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+    GETTENSOR(dataset)
+
+    faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
+    auto index = std::make_shared<faiss::IndexIVFPQ>(coarse_quantizer, dim, nlist, M, nbits);
+    index->train(rows, (float *) p_data);
+
+    return std::make_shared<IVFIndexModel>(index);
+}
+
+std::shared_ptr<faiss::IVFSearchParameters> IVFPQ::GenParams(const Config &config) {
+    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
+    params->nprobe = config.get_with_default("nprobe", size_t(1));
+    //params->scan_table_threshold = 0;
+    //params->polysemous_ht = 0;
+    //params->max_codes = 0;
+
+    return params;
+}
+
+VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
+    return std::make_shared<IVFPQ>(index);
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class IVFPQ : public IVF {
+public:
+    explicit IVFPQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
+
+    IVFPQ() = default;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+protected:
+    std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config) override;
+
+    VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuAutoTune.h>
+
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexIVFSQ.h"
+#include "IndexGPUIVFSQ.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+    GETTENSOR(dataset)
+
+    std::stringstream index_type;
+    index_type << "IVF" << nlist << "," << "SQ" << nbits;
+    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
+    build_index->train(rows, (float *) p_data);
+
+    std::shared_ptr<faiss::Index> ret_index;
+    ret_index.reset(build_index);
+    return std::make_shared<IVFIndexModel>(ret_index);
+}
+
+VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
+    return std::make_shared<IVFSQ>(index);
+}
+
+VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
+    if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){
+        ResScope rs(res, device_id, false);
+        faiss::gpu::GpuClonerOptions option;
+        option.allInGpu = true;
+
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
+
+        std::shared_ptr<faiss::Index> device_index;
+        device_index.reset(gpu_index);
+        return std::make_shared<GPUIVFSQ>(device_index, device_id, res);
+    } else {
+        KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
+    }
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class IVFSQ : public IVF {
+public:
+    explicit IVFSQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
+
+    IVFSQ() = default;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    VectorIndexPtr
+    CopyCpuToGpu(const int64_t &device_id, const Config &config) override;
+
+protected:
+    VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
+};
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp

@@ -24,12 +24,12 @@
 
 #undef mkdir
 
-#include "cpu_kdt_rng.h"
-#include "definitions.h"
+#include "IndexKDT.h"
+#include "knowhere/index/vector_index/helpers/Definitions.h"
 //#include "knowhere/index/preprocessor/normalize.h"
-#include "kdt_parameters.h"
-#include "knowhere/adapter/sptag.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h"
+#include "knowhere/adapter/SptagAdapter.h"
+#include "knowhere/common/Exception.h"
 
 
 namespace zilliz {
@@ -117,7 +117,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) {
 
 void
 CPUKDTRNG::SetParameters(const Config &config) {
-    for (auto &para : KDTParameterManagement::GetInstance().GetKDTParameters()) {
+    for (auto &para : KDTParameterMgr::GetInstance().GetKDTParameters()) {
         auto value = config.get_with_default(para.first, para.second);
         index_ptr_->SetParameter(para.first, value);
     }

cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h

@@ -20,8 +20,8 @@
 
 #include <cstdint>
 #include <memory>
-#include "vector_index.h"
-#include "knowhere/index/index_model.h"
+#include "VectorIndex.h"
+#include "knowhere/index/IndexModel.h"
 #include <SPTAG/AnnService/inc/Core/VectorIndex.h>
 
 

cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp

@@ -16,15 +16,15 @@
 // under the License.
 
 
-#include "nsg_index.h"
-#include "knowhere/index/vector_index/nsg/nsg.h"
-#include "knowhere/index/vector_index/nsg/nsg_io.h"
-#include "idmap.h"
-#include "ivf.h"
-#include "gpu_ivf.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
+#include "IndexNSG.h"
+#include "knowhere/index/vector_index/nsg/NSG.h"
+#include "knowhere/index/vector_index/nsg/NSGIO.h"
+#include "IndexIDMAP.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "vector_index.h"
+#include "VectorIndex.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h → cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h

@@ -21,10 +21,10 @@
 
 #include <memory>
 
-#include "knowhere/common/config.h"
-#include "knowhere/common/dataset.h"
-#include "knowhere/index/index.h"
-#include "knowhere/index/preprocessor/preprocessor.h"
+#include "knowhere/common/Config.h"
+#include "knowhere/common/Dataset.h"
+#include "knowhere/index/Index.h"
+#include "knowhere/index/preprocessor/Preprocessor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-
-#include "ivf.h"
-#include "FaissGpuResourceMgr.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-class GPUIndex {
- public:
-    explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {}
-    GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){}
-
-    virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0;
-    virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0;
-
-    void SetGpuDevice(const int &gpu_id);
-    const int64_t &GetGpuDevice();
-
- protected:
-    int64_t gpu_id_;
-    ResWPtr res_;
-};
-
-class GPUIVF : public IVF, public GPUIndex {
- public:
-    explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {}
-    explicit GPUIVF(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
-        : IVF(std::move(index)), GPUIndex(device_id, resource) {};
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    void Add(const DatasetPtr &dataset, const Config &config) override;
-    void set_index_model(IndexModelPtr model) override;
-    //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-    VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
-    VectorIndexPtr Clone() final;
-
-    // TODO(linxj): Deprecated
-    virtual IVFIndexPtr Copy_index_gpu_to_cpu();
-
- protected:
-    void search_impl(int64_t n,
-                     const float *data,
-                     int64_t k,
-                     float *distances,
-                     int64_t *labels,
-                     const Config &cfg) override;
-    BinarySet SerializeImpl() override;
-    void LoadImpl(const BinarySet &index_binary) override;
-};
-
-class GPUIVFSQ : public GPUIVF {
- public:
-    explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {}
-    explicit GPUIVFSQ(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
-        : GPUIVF(std::move(index), device_id, resource) {};
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-
- public:
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-};
-
-class GPUIVFPQ : public GPUIVF {
- public:
-    explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {}
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-
- public:
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-
- protected:
-    // TODO(linxj): remove GenParams.
-    std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config) override;
-};
-
-
-} // namespace knowhere
-} // namespace zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp

@@ -16,15 +16,18 @@
 // under the License.
 
 
-#include "knowhere/common/exception.h"
-#include "cloner.h"
-#include "ivf.h"
-#include "gpu_ivf.h"
-#include "idmap.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/index/vector_index/IndexIVFSQ.h"
+#include "knowhere/index/vector_index/IndexIVFPQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "Cloner.h"
 
 
 namespace zilliz {
 namespace knowhere {
+namespace cloner {
 
 VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) {
     if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {
@@ -42,7 +45,7 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i
     if (auto cpu_index = std::dynamic_pointer_cast<IVFSQ>(index)) {
         return cpu_index->CopyCpuToGpu(device_id, config);
     } else if (auto cpu_index = std::dynamic_pointer_cast<IVFPQ>(index)) {
-        KNOWHERE_THROW_MSG("IVFPQ not support tranfer to gpu");
+        KNOWHERE_THROW_MSG("IVFPQ not support transfer to gpu");
     } else if (auto cpu_index = std::dynamic_pointer_cast<IVF>(index)) {
         return cpu_index->CopyCpuToGpu(device_id, config);
     } else if (auto cpu_index = std::dynamic_pointer_cast<IDMAP>(index)) {
@@ -52,5 +55,6 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i
     }
 }
 
+} // cloner
 }
 }

cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h

@@ -18,11 +18,12 @@
 
 #pragma once
 
-#include "vector_index.h"
+#include "knowhere/index/vector_index/VectorIndex.h"
 
 
 namespace zilliz {
 namespace knowhere {
+namespace cloner {
 
 // TODO(linxj): rename CopyToGpu
 extern VectorIndexPtr
@@ -31,5 +32,6 @@ CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config
 extern VectorIndexPtr
 CopyGpuToCpu(const VectorIndexPtr &index, const Config &config);
 
-}
-}
\ No newline at end of file
+} // cloner
+} // knowhere
+} // zilliz
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h

@@ -18,15 +18,15 @@
 
 #pragma once
 
-#include <string>
-
 
 namespace zilliz {
 namespace knowhere {
+namespace definition {
 
 #define META_ROWS ("rows")
 #define META_DIM ("dimension")
 #define META_K ("k")
 
-} // namespace knowhere
-} // namespace zilliz
+} // definition
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <cstring>
+
+#include "FaissIO.h"
+
+namespace zilliz {
+namespace knowhere {
+
+// TODO(linxj): Get From Config File
+static size_t magic_num = 2;
+size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) {
+    auto total_need = size * nitems + rp;
+
+    if (!data_) { // data == nullptr
+        total = total_need * magic_num;
+        rp = size * nitems;
+        data_ = new uint8_t[total];
+        memcpy((void *) (data_), ptr, rp);
+    }
+
+    if (total_need > total) {
+        total = total_need * magic_num;
+        auto new_data = new uint8_t[total];
+        memcpy((void *) new_data, (void *) data_, rp);
+        delete data_;
+        data_ = new_data;
+
+        memcpy((void *) (data_ + rp), ptr, size * nitems);
+        rp = total_need;
+    } else {
+        memcpy((void *) (data_ + rp), ptr, size * nitems);
+        rp = total_need;
+    }
+
+    return nitems;
+}
+
+size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) {
+    if (rp >= total) return 0;
+    size_t nremain = (total - rp) / size;
+    if (nremain < nitems) nitems = nremain;
+    memcpy(ptr, (void *) (data_ + rp), size * nitems);
+    rp += size * nitems;
+    return nitems;
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <faiss/AuxIndexStructures.h>
+
+namespace zilliz {
+namespace knowhere {
+
+struct MemoryIOWriter : public faiss::IOWriter {
+    uint8_t *data_ = nullptr;
+    size_t total = 0;
+    size_t rp = 0;
+
+    size_t
+    operator()(const void *ptr, size_t size, size_t nitems) override;
+};
+
+struct MemoryIOReader : public faiss::IOReader {
+    uint8_t *data_;
+    size_t rp = 0;
+    size_t total = 0;
+
+    size_t
+    operator()(void *ptr, size_t size, size_t nitems) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp

@@ -18,18 +18,18 @@
 
 #include <mutex>
 
-#include "kdt_parameters.h"
+#include "KDTParameterMgr.h"
 
 
 namespace zilliz {
 namespace knowhere {
 
 const std::vector<KDTParameter> &
-KDTParameterManagement::GetKDTParameters() {
+KDTParameterMgr::GetKDTParameters() {
     return kdt_parameters_;
 }
 
-KDTParameterManagement::KDTParameterManagement() {
+KDTParameterMgr::KDTParameterMgr() {
     kdt_parameters_ = std::vector<KDTParameter>{
         {"KDTNumber", "1"},
         {"NumTopDimensionKDTSplit", "5"},

cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h

@@ -27,22 +27,22 @@ namespace knowhere {
 
 using KDTParameter = std::pair<std::string, std::string>;
 
-class KDTParameterManagement {
+class KDTParameterMgr {
  public:
     const std::vector<KDTParameter> &
     GetKDTParameters();
 
  public:
-    static KDTParameterManagement &
+    static KDTParameterMgr &
     GetInstance() {
-        static KDTParameterManagement instance;
+        static KDTParameterMgr instance;
         return instance;
     }
 
-    KDTParameterManagement(const KDTParameterManagement &) = delete;
-    KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
+    KDTParameterMgr(const KDTParameterMgr &) = delete;
+    KDTParameterMgr &operator=(const KDTParameterMgr &) = delete;
  private:
-    KDTParameterManagement();
+    KDTParameterMgr();
 
  private:
     std::vector<KDTParameter> kdt_parameters_;

cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include <memory>
-#include <mutex>
-
-#include <faiss/IndexIVF.h>
-#include <faiss/AuxIndexStructures.h>
-#include <faiss/Index.h>
-
-#include "vector_index.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-class BasicIndex {
- protected:
-    explicit BasicIndex(std::shared_ptr<faiss::Index> index);
-    virtual BinarySet SerializeImpl();
-    virtual void LoadImpl(const BinarySet &index_binary);
-    virtual void
-    SealImpl();
-
- protected:
-    std::shared_ptr<faiss::Index> index_ = nullptr;
-};
-
-using Graph = std::vector<std::vector<int64_t>>;
-
-class IVF : public VectorIndex, protected BasicIndex {
- public:
-    IVF() : BasicIndex(nullptr) {};
-    explicit IVF(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {}
-    VectorIndexPtr Clone() override;;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    void set_index_model(IndexModelPtr model) override;
-    void Add(const DatasetPtr &dataset, const Config &config) override;
-    void AddWithoutIds(const DatasetPtr &dataset, const Config &config);
-    DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
-    void GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config);
-    BinarySet Serialize() override;
-    void Load(const BinarySet &index_binary) override;
-    int64_t Count() override;
-    int64_t Dimension() override;
-
-    void
-    Seal() override;
-
-    virtual VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config);
-
-
- protected:
-    virtual std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config);
-
-    virtual VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index);
-
-    virtual void search_impl(int64_t n,
-                             const float *data,
-                             int64_t k,
-                             float *distances,
-                             int64_t *labels,
-                             const Config &cfg);
-
- protected:
-    std::mutex mutex_;
-};
-
-using IVFIndexPtr = std::shared_ptr<IVF>;
-
-class IVFSQ : public IVF {
- public:
-    explicit IVFSQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
-    IVFSQ() = default;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config) override;
- protected:
-    VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
-};
-
-class IVFPQ : public IVF {
- public:
-    explicit IVFPQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
-    IVFPQ() = default;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
- protected:
-    std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config) override;
-    VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
-};
-
-
-//class OPQIVFPQ : public IVFPQ {
-// public:
-//    PreprocessorPtr BuildPreprocessor(const Dataset &dataset, const Config &config) override;
-//};
-
-
-class GPUIVF;
-
-
-struct MemoryIOWriter : public faiss::IOWriter {
-    uint8_t *data_ = nullptr;
-    size_t total = 0;
-    size_t rp = 0;
-
-    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
-};
-
-
-struct MemoryIOReader : public faiss::IOReader {
-    uint8_t *data_;
-    size_t rp = 0;
-    size_t total = 0;
-
-    size_t operator()(void *ptr, size_t size, size_t nitems) override;
-
-};
-
-
-class IVFIndexModel : public IndexModel, public BasicIndex {
-    friend IVF;
-    friend GPUIVF;
-
- public:
-    explicit IVFIndexModel(std::shared_ptr<faiss::Index> index);
-    IVFIndexModel() : BasicIndex(nullptr) {};
-    BinarySet Serialize() override;
- protected:
-    void SealImpl() override;
- public:
-    void Load(const BinarySet &binary) override;
-
- protected:
-    std::mutex mutex_;
-};
-
-using IVFIndexModelPtr = std::shared_ptr<IVFIndexModel>;
-
-}
-}
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp

@@ -22,10 +22,10 @@
 #include <stack>
 #include <omp.h>
 
-#include "nsg.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
-#include "utils.h"
+#include "NSG.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
+#include "NSGHelper.h"
 
 // TODO: enable macro
 //#include <gperftools/profiler.h>

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h

@@ -23,7 +23,7 @@
 #include <mutex>
 
 #include <boost/dynamic_bitset.hpp>
-#include "neighbor.h"
+#include "Neighbor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp

@@ -19,7 +19,7 @@
 #include <cstring>
 #include <fstream>
 
-#include "utils.h"
+#include "NSGHelper.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h

@@ -23,8 +23,8 @@
 
 #include <faiss/AutoTune.h>
 
-#include "nsg.h"
-#include "knowhere/common/config.h"
+#include "NSG.h"
+#include "knowhere/common/Config.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp

@@ -18,7 +18,7 @@
 
 #include <cstring>
 
-#include "nsg_io.h"
+#include "NSGIO.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h

@@ -18,8 +18,9 @@
 
 #pragma once
 
-#include "nsg.h"
-#include "knowhere/index/vector_index/ivf.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "NSG.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include "nsg.h"
-
-namespace zilliz {
-namespace knowhere {
-namespace algo {
-
-void read_from_file(NsgIndex* index, const char *filename);
-void write_to_file(NsgIndex* index, const char *filename);
-
-}
-}
-}

cpp/src/core/test/test_idmap.cpp

@@ -21,10 +21,10 @@
 #include <iostream>
 #include <sstream>
 
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/adapter/structure.h"
-#include "knowhere/index/vector_index/cloner.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/adapter/Structure.h"
+#include "knowhere/index/vector_index/utils/Cloner.h"
+#include "knowhere/common/Exception.h"
 
 #include "utils.h"
 

cpp/src/core/test/test_ivf.cpp

@@ -25,12 +25,12 @@
 #include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/gpu/GpuIndexIVFFlat.h>
 
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/ivf.h"
-#include "knowhere/adapter/structure.h"
-#include "knowhere/index/vector_index/cloner.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/adapter/Structure.h"
+#include "knowhere/index/vector_index/utils/Cloner.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
 
 #include "utils.h"
 

cpp/src/core/test/test_kdt.cpp

@@ -20,12 +20,12 @@
 
 #include <iostream>
 #include <sstream>
-#include "knowhere/common/exception.h"
+#include "knowhere/common/Exception.h"
 
-#include "knowhere/index/vector_index/cpu_kdt_rng.h"
-#include "knowhere/index/vector_index/definitions.h"
-#include "knowhere/adapter/sptag.h"
-#include "knowhere/adapter/structure.h"
+#include "knowhere/index/vector_index/IndexKDT.h"
+#include "knowhere/index/vector_index/utils/Definitions.h"
+#include "knowhere/adapter/SptagAdapter.h"
+#include "knowhere/adapter/Structure.h"
 
 #include "utils.h"
 

cpp/src/core/test/utils.h

@@ -23,7 +23,7 @@
 #include <cstdio>
 #include <fstream>
 
-#include "knowhere/adapter/structure.h"
+#include "knowhere/adapter/Structure.h"
 #include "knowhere/common/Log.h"
 
 class DataGen  {

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -25,7 +25,7 @@
 
 #include "src/wrapper/vec_index.h"
 #include "src/wrapper/vec_impl.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/common/Exception.h"
 
 #include <stdexcept>
 

cpp/src/scheduler/SchedInst.cpp

@@ -19,7 +19,7 @@
 #include "SchedInst.h"
 #include "server/ServerConfig.h"
 #include "ResourceFactory.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
 #include "Utils.h"
 
 

cpp/src/wrapper/KnowhereResource.cpp

@@ -17,7 +17,7 @@
 
 
 #include "KnowhereResource.h"
-#include "knowhere/index/vector_index/FaissGpuResourceMgr.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 #include "server/ServerConfig.h"
 
 #include <map>

cpp/src/wrapper/data_transfer.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "knowhere/adapter/structure.h"
+#include "knowhere/adapter/Structure.h"
 
 
 namespace zilliz {

cpp/src/wrapper/vec_impl.cpp

@@ -17,10 +17,10 @@
 
 
 #include "utils/Log.h"
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/index/vector_index/cloner.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/Cloner.h"
 
 #include "vec_impl.h"
 #include "data_transfer.h"
@@ -153,7 +153,7 @@ IndexType VecIndexImpl::GetType() {
 
 VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) {
     // TODO(linxj): exception handle
-    auto gpu_index = zilliz::knowhere::CopyCpuToGpu(index_, device_id, cfg);
+    auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
     auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
     new_index->dim = dim;
     return new_index;
@@ -161,7 +161,7 @@ VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg)
 
 VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) {
     // TODO(linxj): exception handle
-    auto cpu_index = zilliz::knowhere::CopyGpuToCpu(index_, cfg);
+    auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg);
     auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
     new_index->dim = dim;
     return new_index;

cpp/src/wrapper/vec_impl.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "knowhere/index/vector_index/vector_index.h"
+#include "knowhere/index/vector_index/VectorIndex.h"
 
 #include "vec_index.h"
 

cpp/src/wrapper/vec_index.cpp

@@ -15,12 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "knowhere/index/vector_index/ivf.h"
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/cpu_kdt_rng.h"
-#include "knowhere/index/vector_index/nsg_index.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIVFSQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
+#include "knowhere/index/vector_index/IndexIVFPQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/index/vector_index/IndexKDT.h"
+#include "knowhere/index/vector_index/IndexNSG.h"
+#include "knowhere/common/Exception.h"
 
 #include "vec_index.h"
 #include "vec_impl.h"

cpp/src/wrapper/vec_index.h

@@ -23,8 +23,8 @@
 
 #include "utils/Error.h"
 
-#include "knowhere/common/config.h"
-#include "knowhere/common/binary_set.h"
+#include "knowhere/common/Config.h"
+#include "knowhere/common/BinarySet.h"
 
 
 namespace zilliz {

cpp/unittest/db/utils.cpp

@@ -25,8 +25,7 @@
 #include "cache/CpuCacheMgr.h"
 #include "db/DBFactory.h"
 #include "db/Options.h"
-#include "server/ServerConfig.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 
 INITIALIZE_EASYLOGGINGPP
 

cpp/unittest/wrapper/wrapper_test.cpp

@@ -15,10 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 #include "utils/easylogging++.h"
-#include "wrapper/vec_index.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "src/wrapper/vec_index.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 #include "utils.h"
 
 #include <gtest/gtest.h>