Merge branch 'branch-0.5.0' of http://192.168.1.105:6060/megasearch/milvus into branch-0.5.0

Former-commit-id: d978a598f209139e8d3936a60eb34ae2fa3b4d42

cpp/CHANGELOG.md

@@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-562 - Add JobMgr and TaskCreator in Scheduler
 - MS-566 - Refactor cmake
 - MS-555 - Remove old scheduler
+- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data
 
 ## New Feature
 

cpp/README.md

@@ -68,16 +68,10 @@ $ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
 ###### Step 2 Build
 
 ```shell
-TBD
-cd [Milvus sourcecode path]/cpp/thirdparty
-git clone git@192.168.1.105:megasearch/knowhere.git
-cd knowhere
-./build.sh -t Debug
-or ./build.sh -t Release
-
-cd [sourcecode path]/cpp
-./build.sh -t Debug
-or ./build.sh -t Release
+$ cd [Milvus sourcecode path]/cpp
+$ ./build.sh -t Debug
+or 
+$ ./build.sh -t Release
 ```
 
 When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`.

cpp/src/core/CMakeLists.txt

@@ -93,7 +93,7 @@ endif()
 set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
 
 if(BUILD_UNIT_TEST STREQUAL "ON")
-    add_subdirectory(test)
+#    add_subdirectory(test)
 endif()
 
 config_summary()

cpp/src/core/knowhere/CMakeLists.txt

@@ -29,26 +29,32 @@ if(NOT TARGET SPTAGLibStatic)
 endif()
 
 set(external_srcs
-        knowhere/adapter/sptag.cpp
-        knowhere/adapter/structure.cpp
-        knowhere/adapter/arrow.cpp
-        knowhere/common/exception.cpp
-        knowhere/common/timer.cpp
+        knowhere/adapter/SptagAdapter.cpp
+        knowhere/adapter/Structure.cpp
+        knowhere/adapter/ArrowAdapter.cpp
+        knowhere/common/Exception.cpp
+        knowhere/common/Timer.cpp
         )
 
 set(index_srcs
-        knowhere/index/preprocessor/normalize.cpp
-        knowhere/index/vector_index/cpu_kdt_rng.cpp
-        knowhere/index/vector_index/idmap.cpp
-        knowhere/index/vector_index/ivf.cpp
-        knowhere/index/vector_index/gpu_ivf.cpp
-        knowhere/index/vector_index/kdt_parameters.cpp
-        knowhere/index/vector_index/nsg_index.cpp
-        knowhere/index/vector_index/nsg/nsg.cpp
-        knowhere/index/vector_index/nsg/nsg_io.cpp
-        knowhere/index/vector_index/nsg/utils.cpp
-        knowhere/index/vector_index/cloner.cpp
-        knowhere/index/vector_index/FaissGpuResourceMgr.cpp
+        knowhere/index/preprocessor/Normalize.cpp
+        knowhere/index/vector_index/IndexKDT.cpp
+        knowhere/index/vector_index/IndexIDMAP.cpp
+        knowhere/index/vector_index/IndexIVF.cpp
+        knowhere/index/vector_index/IndexGPUIVF.cpp
+        knowhere/index/vector_index/helpers/KDTParameterMgr.cpp
+        knowhere/index/vector_index/IndexNSG.cpp
+        knowhere/index/vector_index/nsg/NSG.cpp
+        knowhere/index/vector_index/nsg/NSGIO.cpp
+        knowhere/index/vector_index/nsg/NSGHelper.cpp
+        knowhere/index/vector_index/helpers/Cloner.cpp
+        knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
+        knowhere/index/vector_index/IndexIVFSQ.cpp
+        knowhere/index/vector_index/IndexGPUIVFSQ.cpp
+        knowhere/index/vector_index/IndexIVFPQ.cpp
+        knowhere/index/vector_index/IndexGPUIVFPQ.cpp
+        knowhere/index/vector_index/FaissBaseIndex.cpp
+        knowhere/index/vector_index/helpers/FaissIO.cpp
         )
 
 set(depend_libs

cpp/src/core/knowhere/knowhere/adapter/arrow.cpp → cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.cpp

@@ -16,7 +16,7 @@
 // under the License.
 
 
-#include "arrow.h"
+#include "ArrowAdapter.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/adapter/arrow.h → cpp/src/core/knowhere/knowhere/adapter/ArrowAdapter.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "knowhere/common/array.h"
+#include "knowhere/common/Array.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/sptag.cpp → cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.cpp

@@ -15,9 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "knowhere/index/vector_index/definitions.h"
-#include "sptag.h"
-#include "structure.h"
+#include "knowhere/index/vector_index/helpers/Definitions.h"
+#include "SptagAdapter.h"
+#include "Structure.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/sptag.h → cpp/src/core/knowhere/knowhere/adapter/SptagAdapter.h

@@ -22,7 +22,7 @@
 
 #include <SPTAG/AnnService/inc/Core/VectorIndex.h>
 
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/adapter/structure.cpp → cpp/src/core/knowhere/knowhere/adapter/Structure.cpp

@@ -16,7 +16,7 @@
 // under the License.
 
 
-#include "structure.h"
+#include "Structure.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/adapter/structure.h → cpp/src/core/knowhere/knowhere/adapter/Structure.h

@@ -19,7 +19,7 @@
 #pragma once
 
 #include <memory>
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/array.h → cpp/src/core/knowhere/knowhere/common/Array.h

@@ -18,8 +18,9 @@
 
 #pragma once
 
-#include "arrow/array.h"
-#include "schema.h"
+#include <arrow/array.h>
+
+#include "Schema.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/binary_set.h → cpp/src/core/knowhere/knowhere/common/BinarySet.h

@@ -23,7 +23,7 @@
 #include <vector>
 #include <memory>
 
-#include "id.h"
+#include "Id.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/buffer.h → cpp/src/core/knowhere/knowhere/common/Buffer.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/buffer.h"
+#include <arrow/buffer.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/dataset.h → cpp/src/core/knowhere/knowhere/common/Dataset.h

@@ -21,12 +21,12 @@
 #include <vector>
 #include <memory>
 
-#include "array.h"
-#include "buffer.h"
-#include "tensor.h"
-#include "schema.h"
-#include "config.h"
-#include "knowhere/adapter/arrow.h"
+#include "Array.h"
+#include "Buffer.h"
+#include "Tensor.h"
+#include "Schema.h"
+#include "Config.h"
+#include "knowhere/adapter/ArrowAdapter.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/exception.cpp → cpp/src/core/knowhere/knowhere/common/Exception.cpp

@@ -18,7 +18,7 @@
 
 #include <cstdio>
 
-#include "exception.h"
+#include "Exception.h"
 #include "Log.h"
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/Log.h

@@ -22,6 +22,7 @@
 
 namespace zilliz {
 namespace knowhere {
+
 #define KNOWHERE_DOMAIN_NAME "[KNOWHERE] "
 #define KNOWHERE_ERROR_TEXT "KNOWHERE Error:"
 
@@ -31,5 +32,6 @@ namespace knowhere {
 #define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME
 #define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME
 #define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME
+
 } // namespace knowhere
 } // namespace zilliz
\ No newline at end of file

cpp/src/core/knowhere/knowhere/common/schema.h → cpp/src/core/knowhere/knowhere/common/Schema.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/type.h"
+#include <arrow/type.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/tensor.h → cpp/src/core/knowhere/knowhere/common/Tensor.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "arrow/tensor.h"
+#include <arrow/tensor.h>
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/common/timer.cpp → cpp/src/core/knowhere/knowhere/common/Timer.cpp

@@ -18,7 +18,7 @@
 
 #include <iostream> // TODO(linxj): using Log instead
 
-#include "timer.h"
+#include "Timer.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/index/index.h → cpp/src/core/knowhere/knowhere/index/Index.h

@@ -20,11 +20,11 @@
 
 #include <memory>
 
-#include "knowhere/common/binary_set.h"
-#include "knowhere/common/dataset.h"
-#include "index_type.h"
-#include "index_model.h"
-#include "knowhere/index/preprocessor/preprocessor.h"
+#include "knowhere/common/BinarySet.h"
+#include "knowhere/common/Dataset.h"
+#include "IndexType.h"
+#include "IndexModel.h"
+#include "knowhere/index/preprocessor/Preprocessor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/index_model.h → cpp/src/core/knowhere/knowhere/index/IndexModel.h

@@ -19,7 +19,7 @@
 #pragma once
 
 #include <memory>
-#include "knowhere/common/binary_set.h"
+#include "knowhere/common/BinarySet.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/knowhere/knowhere/index/preprocessor/preprocessor.h → cpp/src/core/knowhere/knowhere/index/preprocessor/Preprocessor.h

@@ -20,7 +20,7 @@
 
 #include <memory>
 
-#include "knowhere/common/dataset.h"
+#include "knowhere/common/Dataset.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/index_io.h>
+#include <faiss/IndexIVF.h>
+
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "FaissBaseIndex.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+FaissBaseIndex::FaissBaseIndex(std::shared_ptr<faiss::Index> index) : index_(std::move(index)) {}
+
+BinarySet FaissBaseIndex::SerializeImpl() {
+    try {
+        faiss::Index *index = index_.get();
+
+        SealImpl();
+
+        MemoryIOWriter writer;
+        faiss::write_index(index, &writer);
+        auto data = std::make_shared<uint8_t>();
+        data.reset(writer.data_);
+
+        BinarySet res_set;
+        // TODO(linxj): use virtual func Name() instead of raw string.
+        res_set.Append("IVF", data, writer.rp);
+        return res_set;
+    } catch (std::exception &e) {
+        KNOWHERE_THROW_MSG(e.what());
+    }
+}
+
+void FaissBaseIndex::LoadImpl(const BinarySet &index_binary) {
+    auto binary = index_binary.GetByName("IVF");
+
+    MemoryIOReader reader;
+    reader.total = binary->size;
+    reader.data_ = binary->data.get();
+
+    faiss::Index *index = faiss::read_index(&reader);
+
+    index_.reset(index);
+}
+
+void FaissBaseIndex::SealImpl() {
+// TODO(linxj): enable
+//#ifdef ZILLIZ_FAISS
+    faiss::Index *index = index_.get();
+    auto idx = dynamic_cast<faiss::IndexIVF *>(index);
+    if (idx != nullptr) {
+        idx->to_readonly();
+    }
+    //else {
+    //    KNOHWERE_ERROR_MSG("Seal failed");
+    //}
+//#endif
+}
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/FaissBaseIndex.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <memory>
+
+#include <faiss/Index.h>
+
+#include "knowhere/common/BinarySet.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class FaissBaseIndex {
+ protected:
+    explicit FaissBaseIndex(std::shared_ptr<faiss::Index> index);
+
+    virtual BinarySet
+    SerializeImpl();
+
+    virtual void
+    LoadImpl(const BinarySet &index_binary);
+
+    virtual void
+    SealImpl();
+
+ protected:
+    std::shared_ptr<faiss::Index> index_ = nullptr;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.cpp

@@ -20,18 +20,17 @@
 #include <faiss/gpu/GpuIndexFlat.h>
 #include <faiss/gpu/GpuIndexIVF.h>
 #include <faiss/gpu/GpuIndexIVFFlat.h>
-#include <faiss/gpu/GpuIndexIVFPQ.h>
 #include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/IndexIVFPQ.h>
 #include <faiss/index_io.h>
 
 
-#include "knowhere/common/exception.h"
-#include "cloner.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "gpu_ivf.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/Cloner.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexGPUIVF.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
 
-#include <algorithm>
 
 namespace zilliz {
 namespace knowhere {
@@ -169,7 +168,7 @@ VectorIndexPtr GPUIVF::CopyGpuToCpu(const Config &config) {
 
 VectorIndexPtr GPUIVF::Clone() {
     auto cpu_idx = CopyGpuToCpu(Config());
-    return ::zilliz::knowhere::CopyCpuToGpu(cpu_idx, gpu_id_, Config());
+    return ::zilliz::knowhere::cloner::CopyCpuToGpu(cpu_idx, gpu_id_, Config());
 }
 
 VectorIndexPtr GPUIVF::CopyGpuToGpu(const int64_t &device_id, const Config &config) {
@@ -187,84 +186,6 @@ void GPUIVF::Add(const DatasetPtr &dataset, const Config &config) {
     }
 }
 
-IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto M = config["M"].as<size_t>();        // number of subquantizers(subvectors)
-    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
-    auto gpu_num = config.get_with_default("gpu_id", gpu_id_);
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support.
-
-    GETTENSOR(dataset)
-
-    // TODO(linxj): set device here.
-    // TODO(linxj): set gpu resource here.
-    faiss::gpu::StandardGpuResources res;
-    faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type);
-    device_index.train(rows, (float *) p_data);
-
-    std::shared_ptr<faiss::Index> host_index = nullptr;
-    host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index));
-
-    return std::make_shared<IVFIndexModel>(host_index);
-}
-
-std::shared_ptr<faiss::IVFSearchParameters> GPUIVFPQ::GenParams(const Config &config) {
-    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
-    params->nprobe = config.get_with_default("nprobe", size_t(1));
-    //params->scan_table_threshold = 0;
-    //params->polysemous_ht = 0;
-    //params->max_codes = 0;
-
-    return params;
-}
-
-VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) {
-    KNOWHERE_THROW_MSG("not support yet");
-}
-
-IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj):  gpu only support SQ4 SQ8 SQ16
-    gpu_id_ = config.get_with_default("gpu_id", gpu_id_);
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    std::stringstream index_type;
-    index_type << "IVF" << nlist << "," << "SQ" << nbits;
-    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
-
-    auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
-    if (temp_resource != nullptr) {
-        ResScope rs(temp_resource, gpu_id_, true);
-        auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index);
-        device_index->train(rows, (float *) p_data);
-
-        std::shared_ptr<faiss::Index> host_index = nullptr;
-        host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index));
-
-        delete device_index;
-        delete build_index;
-
-        return std::make_shared<IVFIndexModel>(host_index);
-    } else {
-        KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource");
-    }
-}
-
-VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) {
-    std::lock_guard<std::mutex> lk(mutex_);
-
-    faiss::Index *device_index = index_.get();
-    faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
-
-    std::shared_ptr<faiss::Index> new_index;
-    new_index.reset(host_index);
-    return std::make_shared<IVFSQ>(new_index);
-}
-
 void GPUIndex::SetGpuDevice(const int &gpu_id) {
     gpu_id_ = gpu_id;
 }

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVF.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+
+#include "IndexIVF.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIndex {
+public:
+     explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {}
+
+     GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource) {}
+
+     virtual VectorIndexPtr
+     CopyGpuToCpu(const Config &config) = 0;
+
+     virtual VectorIndexPtr
+     CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0;
+
+     void
+     SetGpuDevice(const int &gpu_id);
+
+     const int64_t &
+     GetGpuDevice();
+
+protected:
+     int64_t gpu_id_;
+     ResWPtr res_;
+};
+
+class GPUIVF : public IVF, public GPUIndex {
+public:
+     explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {}
+
+     explicit GPUIVF(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
+          : IVF(std::move(index)), GPUIndex(device_id, resource) {};
+
+     IndexModelPtr
+     Train(const DatasetPtr &dataset, const Config &config) override;
+
+     void
+     Add(const DatasetPtr &dataset, const Config &config) override;
+
+     void
+     set_index_model(IndexModelPtr model) override;
+
+     //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
+     VectorIndexPtr
+     CopyGpuToCpu(const Config &config) override;
+
+     VectorIndexPtr
+     CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
+
+     VectorIndexPtr
+     Clone() final;
+
+     // TODO(linxj): Deprecated
+     virtual IVFIndexPtr Copy_index_gpu_to_cpu();
+
+protected:
+     void
+     search_impl(int64_t n,
+                 const float *data,
+                 int64_t k,
+                 float *distances,
+                 int64_t *labels,
+                 const Config &cfg) override;
+
+     BinarySet
+     SerializeImpl() override;
+
+     void
+     LoadImpl(const BinarySet &index_binary) override;
+};
+
+} // namespace knowhere
+} // namespace zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuIndexIVFPQ.h>
+#include <faiss/gpu/GpuAutoTune.h>
+#include <faiss/IndexIVFPQ.h>
+
+#include "IndexGPUIVFPQ.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr GPUIVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto M = config["M"].as<size_t>();        // number of subquantizers(subvectors)
+    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
+    auto gpu_num = config.get_with_default("gpu_id", gpu_id_);
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_L2; // IP not support.
+
+    GETTENSOR(dataset)
+
+    // TODO(linxj): set device here.
+    // TODO(linxj): set gpu resource here.
+    faiss::gpu::StandardGpuResources res;
+    faiss::gpu::GpuIndexIVFPQ device_index(&res, dim, nlist, M, nbits, metric_type);
+    device_index.train(rows, (float *) p_data);
+
+    std::shared_ptr<faiss::Index> host_index = nullptr;
+    host_index.reset(faiss::gpu::index_gpu_to_cpu(&device_index));
+
+    return std::make_shared<IVFIndexModel>(host_index);
+}
+
+std::shared_ptr<faiss::IVFSearchParameters> GPUIVFPQ::GenParams(const Config &config) {
+    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
+    params->nprobe = config.get_with_default("nprobe", size_t(1));
+    //params->scan_table_threshold = 0;
+    //params->polysemous_ht = 0;
+    //params->max_codes = 0;
+
+    return params;
+}
+
+VectorIndexPtr GPUIVFPQ::CopyGpuToCpu(const Config &config) {
+    KNOWHERE_THROW_MSG("not support yet");
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFPQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexGPUIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIVFPQ : public GPUIVF {
+public:
+    explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {}
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+public:
+    VectorIndexPtr
+    CopyGpuToCpu(const Config &config) override;
+
+protected:
+    // TODO(linxj): remove GenParams.
+    std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuAutoTune.h>
+
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/common/Exception.h"
+#include "IndexGPUIVFSQ.h"
+#include "IndexIVFSQ.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+    IndexModelPtr GPUIVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
+        auto nlist = config["nlist"].as<size_t>();
+        auto nbits = config["nbits"].as<size_t>(); // TODO(linxj):  gpu only support SQ4 SQ8 SQ16
+        gpu_id_ = config.get_with_default("gpu_id", gpu_id_);
+        auto metric_type = config["metric_type"].as_string() == "L2" ?
+                           faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+        GETTENSOR(dataset)
+
+        std::stringstream index_type;
+        index_type << "IVF" << nlist << "," << "SQ" << nbits;
+        auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
+
+        auto temp_resource = FaissGpuResourceMgr::GetInstance().GetRes(gpu_id_);
+        if (temp_resource != nullptr) {
+            ResScope rs(temp_resource, gpu_id_, true);
+            auto device_index = faiss::gpu::index_cpu_to_gpu(temp_resource->faiss_res.get(), gpu_id_, build_index);
+            device_index->train(rows, (float *) p_data);
+
+            std::shared_ptr<faiss::Index> host_index = nullptr;
+            host_index.reset(faiss::gpu::index_gpu_to_cpu(device_index));
+
+            delete device_index;
+            delete build_index;
+
+            return std::make_shared<IVFIndexModel>(host_index);
+        } else {
+            KNOWHERE_THROW_MSG("Build IVFSQ can't get gpu resource");
+        }
+    }
+
+    VectorIndexPtr GPUIVFSQ::CopyGpuToCpu(const Config &config) {
+        std::lock_guard<std::mutex> lk(mutex_);
+
+        faiss::Index *device_index = index_.get();
+        faiss::Index *host_index = faiss::gpu::index_gpu_to_cpu(device_index);
+
+        std::shared_ptr<faiss::Index> new_index;
+        new_index.reset(host_index);
+        return std::make_shared<IVFSQ>(new_index);
+    }
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexGPUIVFSQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexGPUIVF.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+class GPUIVFSQ : public GPUIVF {
+public:
+    explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {}
+
+    explicit GPUIVFSQ(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
+            : GPUIVF(std::move(index), device_id, resource) {};
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    VectorIndexPtr
+    CopyGpuToCpu(const Config &config) override;
+};
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/idmap.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp

@@ -23,9 +23,10 @@
 #include <faiss/gpu/GpuAutoTune.h>
 
 
-#include "knowhere/common/exception.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "idmap.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "IndexIDMAP.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/idmap.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIDMAP.h

@@ -18,17 +18,17 @@
 
 #pragma once
 
-#include "ivf.h"
-#include "gpu_ivf.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
 
 
 namespace zilliz {
 namespace knowhere {
 
-class IDMAP : public VectorIndex, public BasicIndex {
+class IDMAP : public VectorIndex, public FaissBaseIndex {
  public:
-    IDMAP() : BasicIndex(nullptr) {};
-    explicit IDMAP(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {};
+    IDMAP() : FaissBaseIndex(nullptr) {};
+    explicit IDMAP(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {};
     BinarySet Serialize() override;
     void Load(const BinarySet &index_binary) override;
     void Train(const Config &config);

cpp/src/core/knowhere/knowhere/index/vector_index/ivf.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.cpp

@@ -23,14 +23,13 @@
 #include <faiss/IVFlib.h>
 #include <faiss/AuxIndexStructures.h>
 #include <faiss/index_io.h>
-#include <faiss/gpu/StandardGpuResources.h>
 #include <faiss/gpu/GpuAutoTune.h>
 
 
-#include "knowhere/common/exception.h"
-#include "ivf.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "gpu_ivf.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
 
 
 namespace zilliz {
@@ -239,7 +238,7 @@ void IVF::Seal() {
 }
 
 
-IVFIndexModel::IVFIndexModel(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {}
+IVFIndexModel::IVFIndexModel(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {}
 
 BinarySet IVFIndexModel::Serialize() {
     if (!index_ || !index_->is_trained) {
@@ -258,159 +257,8 @@ void IVFIndexModel::SealImpl() {
     // do nothing
 }
 
-IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    std::stringstream index_type;
-    index_type << "IVF" << nlist << "," << "SQ" << nbits;
-    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
-    build_index->train(rows, (float *) p_data);
-
-    std::shared_ptr<faiss::Index> ret_index;
-    ret_index.reset(build_index);
-    return std::make_shared<IVFIndexModel>(ret_index);
-}
-
-VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
-    return std::make_shared<IVFSQ>(index);
-}
-
-VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
-    if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){
-        ResScope rs(res, device_id, false);
-        faiss::gpu::GpuClonerOptions option;
-        option.allInGpu = true;
-
-        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
-
-        std::shared_ptr<faiss::Index> device_index;
-        device_index.reset(gpu_index);
-        return std::make_shared<GPUIVFSQ>(device_index, device_id, res);
-    } else {
-        KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
-    }
-}
-
-IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
-    auto nlist = config["nlist"].as<size_t>();
-    auto M = config["M"].as<size_t>();        // number of subquantizers(subvector)
-    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
-    auto metric_type = config["metric_type"].as_string() == "L2" ?
-                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
-
-    GETTENSOR(dataset)
-
-    faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
-    auto index = std::make_shared<faiss::IndexIVFPQ>(coarse_quantizer, dim, nlist, M, nbits);
-    index->train(rows, (float *) p_data);
-
-    return std::make_shared<IVFIndexModel>(index);
-}
 
-std::shared_ptr<faiss::IVFSearchParameters> IVFPQ::GenParams(const Config &config) {
-    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
-    params->nprobe = config.get_with_default("nprobe", size_t(1));
-    //params->scan_table_threshold = 0;
-    //params->polysemous_ht = 0;
-    //params->max_codes = 0;
-
-    return params;
-}
-
-VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
-    return std::make_shared<IVFPQ>(index);
-}
-
-BasicIndex::BasicIndex(std::shared_ptr<faiss::Index> index) : index_(std::move(index)) {}
-
-BinarySet BasicIndex::SerializeImpl() {
-    try {
-        faiss::Index *index = index_.get();
-
-        SealImpl();
-
-        MemoryIOWriter writer;
-        faiss::write_index(index, &writer);
-        auto data = std::make_shared<uint8_t>();
-        data.reset(writer.data_);
-
-        BinarySet res_set;
-        // TODO(linxj): use virtual func Name() instead of raw string.
-        res_set.Append("IVF", data, writer.rp);
-        return res_set;
-    } catch (std::exception &e) {
-        KNOWHERE_THROW_MSG(e.what());
-    }
-}
-
-void BasicIndex::LoadImpl(const BinarySet &index_binary) {
-    auto binary = index_binary.GetByName("IVF");
-
-    MemoryIOReader reader;
-    reader.total = binary->size;
-    reader.data_ = binary->data.get();
 
-    faiss::Index *index = faiss::read_index(&reader);
-
-    index_.reset(index);
-}
-
-void BasicIndex::SealImpl() {
-// TODO(linxj): enable
-//#ifdef ZILLIZ_FAISS
-    faiss::Index *index = index_.get();
-    auto idx = dynamic_cast<faiss::IndexIVF *>(index);
-    if (idx != nullptr) {
-        idx->to_readonly();
-    }
-    //else {
-    //    KNOHWERE_ERROR_MSG("Seal failed");
-    //}
-//#endif
-}
-
-// TODO(linxj): Get From Config File
-static size_t magic_num = 2;
-size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) {
-    auto total_need = size * nitems + rp;
-
-    if (!data_) { // data == nullptr
-        total = total_need * magic_num;
-        rp = size * nitems;
-        data_ = new uint8_t[total];
-        memcpy((void *) (data_), ptr, rp);
-    }
-
-    if (total_need > total) {
-        total = total_need * magic_num;
-        auto new_data = new uint8_t[total];
-        memcpy((void *) new_data, (void *) data_, rp);
-        delete data_;
-        data_ = new_data;
-
-        memcpy((void *) (data_ + rp), ptr, size * nitems);
-        rp = total_need;
-    } else {
-        memcpy((void *) (data_ + rp), ptr, size * nitems);
-        rp = total_need;
-    }
-
-    return nitems;
-}
-
-size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) {
-    if (rp >= total) return 0;
-    size_t nremain = (total - rp) / size;
-    if (nremain < nitems) nitems = nremain;
-    memcpy(ptr, (void *) (data_ + rp), size * nitems);
-    rp += size * nitems;
-    return nitems;
-}
 
 
 }

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVF.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+
+#include "VectorIndex.h"
+#include "FaissBaseIndex.h"
+#include "faiss/IndexIVF.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+using Graph = std::vector<std::vector<int64_t>>;
+
+class IVF : public VectorIndex, protected FaissBaseIndex {
+ public:
+    IVF() : FaissBaseIndex(nullptr) {};
+
+    explicit IVF(std::shared_ptr<faiss::Index> index) : FaissBaseIndex(std::move(index)) {}
+
+    VectorIndexPtr
+    Clone() override;;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    set_index_model(IndexModelPtr model) override;
+
+    void
+    Add(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    AddWithoutIds(const DatasetPtr &dataset, const Config &config);
+
+    DatasetPtr
+    Search(const DatasetPtr &dataset, const Config &config) override;
+
+    void
+    GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config);
+
+    BinarySet
+    Serialize() override;
+
+    void
+    Load(const BinarySet &index_binary) override;
+
+    int64_t
+    Count() override;
+
+    int64_t
+    Dimension() override;
+
+    void
+    Seal() override;
+
+    virtual VectorIndexPtr
+    CopyCpuToGpu(const int64_t &device_id, const Config &config);
+
+
+ protected:
+    virtual std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config);
+
+    virtual VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index);
+
+    virtual void
+    search_impl(int64_t n,
+                const float *data,
+                int64_t k,
+                float *distances,
+                int64_t *labels,
+                const Config &cfg);
+
+ protected:
+    std::mutex mutex_;
+};
+
+using IVFIndexPtr = std::shared_ptr<IVF>;
+
+class GPUIVF;
+class IVFIndexModel : public IndexModel, public FaissBaseIndex {
+    friend IVF;
+    friend GPUIVF;
+
+ public:
+    explicit IVFIndexModel(std::shared_ptr<faiss::Index> index);
+
+    IVFIndexModel() : FaissBaseIndex(nullptr) {};
+
+    BinarySet
+    Serialize() override;
+
+    void
+    Load(const BinarySet &binary) override;
+
+ protected:
+    void
+    SealImpl() override;
+
+ protected:
+    std::mutex mutex_;
+};
+using IVFIndexModelPtr = std::shared_ptr<IVFIndexModel>;
+
+}
+}
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/IndexFlat.h>
+#include <faiss/IndexIVFPQ.h>
+
+#include "IndexIVFPQ.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/adapter/VectorAdapter.h"
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr IVFPQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto M = config["M"].as<size_t>();        // number of subquantizers(subvector)
+    auto nbits = config["nbits"].as<size_t>();// number of bit per subvector index
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+    GETTENSOR(dataset)
+
+    faiss::Index *coarse_quantizer = new faiss::IndexFlat(dim, metric_type);
+    auto index = std::make_shared<faiss::IndexIVFPQ>(coarse_quantizer, dim, nlist, M, nbits);
+    index->train(rows, (float *) p_data);
+
+    return std::make_shared<IVFIndexModel>(index);
+}
+
+std::shared_ptr<faiss::IVFSearchParameters> IVFPQ::GenParams(const Config &config) {
+    auto params = std::make_shared<faiss::IVFPQSearchParameters>();
+    params->nprobe = config.get_with_default("nprobe", size_t(1));
+    //params->scan_table_threshold = 0;
+    //params->polysemous_ht = 0;
+    //params->max_codes = 0;
+
+    return params;
+}
+
+VectorIndexPtr IVFPQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
+    return std::make_shared<IVFPQ>(index);
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFPQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class IVFPQ : public IVF {
+public:
+    explicit IVFPQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
+
+    IVFPQ() = default;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+protected:
+    std::shared_ptr<faiss::IVFSearchParameters>
+    GenParams(const Config &config) override;
+
+    VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <faiss/gpu/GpuAutoTune.h>
+
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "IndexIVFSQ.h"
+#include "IndexGPUIVFSQ.h"
+
+
+namespace zilliz {
+namespace knowhere {
+
+IndexModelPtr IVFSQ::Train(const DatasetPtr &dataset, const Config &config) {
+    auto nlist = config["nlist"].as<size_t>();
+    auto nbits = config["nbits"].as<size_t>(); // TODO(linxj): only support SQ4 SQ6 SQ8 SQ16
+    auto metric_type = config["metric_type"].as_string() == "L2" ?
+                       faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
+
+    GETTENSOR(dataset)
+
+    std::stringstream index_type;
+    index_type << "IVF" << nlist << "," << "SQ" << nbits;
+    auto build_index = faiss::index_factory(dim, index_type.str().c_str(), metric_type);
+    build_index->train(rows, (float *) p_data);
+
+    std::shared_ptr<faiss::Index> ret_index;
+    ret_index.reset(build_index);
+    return std::make_shared<IVFIndexModel>(ret_index);
+}
+
+VectorIndexPtr IVFSQ::Clone_impl(const std::shared_ptr<faiss::Index> &index) {
+    return std::make_shared<IVFSQ>(index);
+}
+
+VectorIndexPtr IVFSQ::CopyCpuToGpu(const int64_t &device_id, const Config &config) {
+    if (auto res = FaissGpuResourceMgr::GetInstance().GetRes(device_id)){
+        ResScope rs(res, device_id, false);
+        faiss::gpu::GpuClonerOptions option;
+        option.allInGpu = true;
+
+        auto gpu_index = faiss::gpu::index_cpu_to_gpu(res->faiss_res.get(), device_id, index_.get(), &option);
+
+        std::shared_ptr<faiss::Index> device_index;
+        device_index.reset(gpu_index);
+        return std::make_shared<GPUIVFSQ>(device_index, device_id, res);
+    } else {
+        KNOWHERE_THROW_MSG("CopyCpuToGpu Error, can't get gpu_resource");
+    }
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/IndexIVFSQ.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "IndexIVF.h"
+
+namespace zilliz {
+namespace knowhere {
+
+class IVFSQ : public IVF {
+public:
+    explicit IVFSQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
+
+    IVFSQ() = default;
+
+    IndexModelPtr
+    Train(const DatasetPtr &dataset, const Config &config) override;
+
+    VectorIndexPtr
+    CopyCpuToGpu(const int64_t &device_id, const Config &config) override;
+
+protected:
+    VectorIndexPtr
+    Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
+};
+
+} // knowhere
+} // zilliz
+

cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.cpp

@@ -24,12 +24,12 @@
 
 #undef mkdir
 
-#include "cpu_kdt_rng.h"
-#include "definitions.h"
+#include "IndexKDT.h"
+#include "knowhere/index/vector_index/helpers/Definitions.h"
 //#include "knowhere/index/preprocessor/normalize.h"
-#include "kdt_parameters.h"
-#include "knowhere/adapter/sptag.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/helpers/KDTParameterMgr.h"
+#include "knowhere/adapter/SptagAdapter.h"
+#include "knowhere/common/Exception.h"
 
 
 namespace zilliz {
@@ -117,7 +117,7 @@ CPUKDTRNG::Add(const DatasetPtr &origin, const Config &add_config) {
 
 void
 CPUKDTRNG::SetParameters(const Config &config) {
-    for (auto &para : KDTParameterManagement::GetInstance().GetKDTParameters()) {
+    for (auto &para : KDTParameterMgr::GetInstance().GetKDTParameters()) {
         auto value = config.get_with_default(para.first, para.second);
         index_ptr_->SetParameter(para.first, value);
     }

cpp/src/core/knowhere/knowhere/index/vector_index/cpu_kdt_rng.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexKDT.h

@@ -20,8 +20,8 @@
 
 #include <cstdint>
 #include <memory>
-#include "vector_index.h"
-#include "knowhere/index/index_model.h"
+#include "VectorIndex.h"
+#include "knowhere/index/IndexModel.h"
 #include <SPTAG/AnnService/inc/Core/VectorIndex.h>
 
 

cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.cpp

@@ -16,15 +16,15 @@
 // under the License.
 
 
-#include "nsg_index.h"
-#include "knowhere/index/vector_index/nsg/nsg.h"
-#include "knowhere/index/vector_index/nsg/nsg_io.h"
-#include "idmap.h"
-#include "ivf.h"
-#include "gpu_ivf.h"
-#include "knowhere/adapter/faiss_adopt.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
+#include "IndexNSG.h"
+#include "knowhere/index/vector_index/nsg/NSG.h"
+#include "knowhere/index/vector_index/nsg/NSGIO.h"
+#include "IndexIDMAP.h"
+#include "IndexIVF.h"
+#include "IndexGPUIVF.h"
+#include "knowhere/adapter/VectorAdapter.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg_index.h → cpp/src/core/knowhere/knowhere/index/vector_index/IndexNSG.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "vector_index.h"
+#include "VectorIndex.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/vector_index.h → cpp/src/core/knowhere/knowhere/index/vector_index/VectorIndex.h

@@ -21,10 +21,10 @@
 
 #include <memory>
 
-#include "knowhere/common/config.h"
-#include "knowhere/common/dataset.h"
-#include "knowhere/index/index.h"
-#include "knowhere/index/preprocessor/preprocessor.h"
+#include "knowhere/common/Config.h"
+#include "knowhere/common/Dataset.h"
+#include "knowhere/index/Index.h"
+#include "knowhere/index/preprocessor/Preprocessor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/gpu_ivf.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-
-#include "ivf.h"
-#include "FaissGpuResourceMgr.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-class GPUIndex {
- public:
-    explicit GPUIndex(const int &device_id) : gpu_id_(device_id) {}
-    GPUIndex(const int& device_id, const ResPtr& resource): gpu_id_(device_id), res_(resource){}
-
-    virtual VectorIndexPtr CopyGpuToCpu(const Config &config) = 0;
-    virtual VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) = 0;
-
-    void SetGpuDevice(const int &gpu_id);
-    const int64_t &GetGpuDevice();
-
- protected:
-    int64_t gpu_id_;
-    ResWPtr res_;
-};
-
-class GPUIVF : public IVF, public GPUIndex {
- public:
-    explicit GPUIVF(const int &device_id) : IVF(), GPUIndex(device_id) {}
-    explicit GPUIVF(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
-        : IVF(std::move(index)), GPUIndex(device_id, resource) {};
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    void Add(const DatasetPtr &dataset, const Config &config) override;
-    void set_index_model(IndexModelPtr model) override;
-    //DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-    VectorIndexPtr CopyGpuToGpu(const int64_t &device_id, const Config &config) override;
-    VectorIndexPtr Clone() final;
-
-    // TODO(linxj): Deprecated
-    virtual IVFIndexPtr Copy_index_gpu_to_cpu();
-
- protected:
-    void search_impl(int64_t n,
-                     const float *data,
-                     int64_t k,
-                     float *distances,
-                     int64_t *labels,
-                     const Config &cfg) override;
-    BinarySet SerializeImpl() override;
-    void LoadImpl(const BinarySet &index_binary) override;
-};
-
-class GPUIVFSQ : public GPUIVF {
- public:
-    explicit GPUIVFSQ(const int &device_id) : GPUIVF(device_id) {}
-    explicit GPUIVFSQ(std::shared_ptr<faiss::Index> index, const int64_t &device_id, ResPtr &resource)
-        : GPUIVF(std::move(index), device_id, resource) {};
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-
- public:
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-};
-
-class GPUIVFPQ : public GPUIVF {
- public:
-    explicit GPUIVFPQ(const int &device_id) : GPUIVF(device_id) {}
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-
- public:
-    VectorIndexPtr CopyGpuToCpu(const Config &config) override;
-
- protected:
-    // TODO(linxj): remove GenParams.
-    std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config) override;
-};
-
-
-} // namespace knowhere
-} // namespace zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/cloner.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp

@@ -16,15 +16,18 @@
 // under the License.
 
 
-#include "knowhere/common/exception.h"
-#include "cloner.h"
-#include "ivf.h"
-#include "gpu_ivf.h"
-#include "idmap.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/index/vector_index/IndexIVFSQ.h"
+#include "knowhere/index/vector_index/IndexIVFPQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "Cloner.h"
 
 
 namespace zilliz {
 namespace knowhere {
+namespace cloner {
 
 VectorIndexPtr CopyGpuToCpu(const VectorIndexPtr &index, const Config &config) {
     if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {
@@ -42,7 +45,7 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i
     if (auto cpu_index = std::dynamic_pointer_cast<IVFSQ>(index)) {
         return cpu_index->CopyCpuToGpu(device_id, config);
     } else if (auto cpu_index = std::dynamic_pointer_cast<IVFPQ>(index)) {
-        KNOWHERE_THROW_MSG("IVFPQ not support tranfer to gpu");
+        KNOWHERE_THROW_MSG("IVFPQ not support transfer to gpu");
     } else if (auto cpu_index = std::dynamic_pointer_cast<IVF>(index)) {
         return cpu_index->CopyCpuToGpu(device_id, config);
     } else if (auto cpu_index = std::dynamic_pointer_cast<IDMAP>(index)) {
@@ -52,5 +55,6 @@ VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_i
     }
 }
 
+} // cloner
 }
 }

cpp/src/core/knowhere/knowhere/index/vector_index/cloner.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Cloner.h

@@ -18,11 +18,12 @@
 
 #pragma once
 
-#include "vector_index.h"
+#include "knowhere/index/vector_index/VectorIndex.h"
 
 
 namespace zilliz {
 namespace knowhere {
+namespace cloner {
 
 // TODO(linxj): rename CopyToGpu
 extern VectorIndexPtr
@@ -31,5 +32,6 @@ CopyCpuToGpu(const VectorIndexPtr &index, const int64_t &device_id, const Config
 extern VectorIndexPtr
 CopyGpuToCpu(const VectorIndexPtr &index, const Config &config);
 
-}
-}
\ No newline at end of file
+} // cloner
+} // knowhere
+} // zilliz
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/definitions.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/Definitions.h

@@ -18,15 +18,15 @@
 
 #pragma once
 
-#include <string>
-
 
 namespace zilliz {
 namespace knowhere {
+namespace definition {
 
 #define META_ROWS ("rows")
 #define META_DIM ("dimension")
 #define META_K ("k")
 
-} // namespace knowhere
-} // namespace zilliz
+} // definition
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <cstring>
+
+#include "FaissIO.h"
+
+namespace zilliz {
+namespace knowhere {
+
+// TODO(linxj): Get From Config File
+static size_t magic_num = 2;
+size_t MemoryIOWriter::operator()(const void *ptr, size_t size, size_t nitems) {
+    auto total_need = size * nitems + rp;
+
+    if (!data_) { // data == nullptr
+        total = total_need * magic_num;
+        rp = size * nitems;
+        data_ = new uint8_t[total];
+        memcpy((void *) (data_), ptr, rp);
+    }
+
+    if (total_need > total) {
+        total = total_need * magic_num;
+        auto new_data = new uint8_t[total];
+        memcpy((void *) new_data, (void *) data_, rp);
+        delete data_;
+        data_ = new_data;
+
+        memcpy((void *) (data_ + rp), ptr, size * nitems);
+        rp = total_need;
+    } else {
+        memcpy((void *) (data_ + rp), ptr, size * nitems);
+        rp = total_need;
+    }
+
+    return nitems;
+}
+
+size_t MemoryIOReader::operator()(void *ptr, size_t size, size_t nitems) {
+    if (rp >= total) return 0;
+    size_t nremain = (total - rp) / size;
+    if (nremain < nitems) nitems = nremain;
+    memcpy(ptr, (void *) (data_ + rp), size * nitems);
+    rp += size * nitems;
+    return nitems;
+}
+
+} // knowhere
+} // zilliz

cpp/src/core/knowhere/knowhere/index/vector_index/helpers/FaissIO.h

+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include <faiss/AuxIndexStructures.h>
+
+namespace zilliz {
+namespace knowhere {
+
+struct MemoryIOWriter : public faiss::IOWriter {
+    uint8_t *data_ = nullptr;
+    size_t total = 0;
+    size_t rp = 0;
+
+    size_t
+    operator()(const void *ptr, size_t size, size_t nitems) override;
+};
+
+struct MemoryIOReader : public faiss::IOReader {
+    uint8_t *data_;
+    size_t rp = 0;
+    size_t total = 0;
+
+    size_t
+    operator()(void *ptr, size_t size, size_t nitems) override;
+};
+
+} // knowhere
+} // zilliz
+
+
+

cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.cpp

@@ -18,18 +18,18 @@
 
 #include <mutex>
 
-#include "kdt_parameters.h"
+#include "KDTParameterMgr.h"
 
 
 namespace zilliz {
 namespace knowhere {
 
 const std::vector<KDTParameter> &
-KDTParameterManagement::GetKDTParameters() {
+KDTParameterMgr::GetKDTParameters() {
     return kdt_parameters_;
 }
 
-KDTParameterManagement::KDTParameterManagement() {
+KDTParameterMgr::KDTParameterMgr() {
     kdt_parameters_ = std::vector<KDTParameter>{
         {"KDTNumber", "1"},
         {"NumTopDimensionKDTSplit", "5"},

cpp/src/core/knowhere/knowhere/index/vector_index/kdt_parameters.h → cpp/src/core/knowhere/knowhere/index/vector_index/helpers/KDTParameterMgr.h

@@ -27,22 +27,22 @@ namespace knowhere {
 
 using KDTParameter = std::pair<std::string, std::string>;
 
-class KDTParameterManagement {
+class KDTParameterMgr {
  public:
     const std::vector<KDTParameter> &
     GetKDTParameters();
 
  public:
-    static KDTParameterManagement &
+    static KDTParameterMgr &
     GetInstance() {
-        static KDTParameterManagement instance;
+        static KDTParameterMgr instance;
         return instance;
     }
 
-    KDTParameterManagement(const KDTParameterManagement &) = delete;
-    KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
+    KDTParameterMgr(const KDTParameterMgr &) = delete;
+    KDTParameterMgr &operator=(const KDTParameterMgr &) = delete;
  private:
-    KDTParameterManagement();
+    KDTParameterMgr();
 
  private:
     std::vector<KDTParameter> kdt_parameters_;

cpp/src/core/knowhere/knowhere/index/vector_index/ivf.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include <memory>
-#include <mutex>
-
-#include <faiss/IndexIVF.h>
-#include <faiss/AuxIndexStructures.h>
-#include <faiss/Index.h>
-
-#include "vector_index.h"
-
-
-namespace zilliz {
-namespace knowhere {
-
-class BasicIndex {
- protected:
-    explicit BasicIndex(std::shared_ptr<faiss::Index> index);
-    virtual BinarySet SerializeImpl();
-    virtual void LoadImpl(const BinarySet &index_binary);
-    virtual void
-    SealImpl();
-
- protected:
-    std::shared_ptr<faiss::Index> index_ = nullptr;
-};
-
-using Graph = std::vector<std::vector<int64_t>>;
-
-class IVF : public VectorIndex, protected BasicIndex {
- public:
-    IVF() : BasicIndex(nullptr) {};
-    explicit IVF(std::shared_ptr<faiss::Index> index) : BasicIndex(std::move(index)) {}
-    VectorIndexPtr Clone() override;;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    void set_index_model(IndexModelPtr model) override;
-    void Add(const DatasetPtr &dataset, const Config &config) override;
-    void AddWithoutIds(const DatasetPtr &dataset, const Config &config);
-    DatasetPtr Search(const DatasetPtr &dataset, const Config &config) override;
-    void GenGraph(const int64_t &k, Graph &graph, const DatasetPtr &dataset, const Config &config);
-    BinarySet Serialize() override;
-    void Load(const BinarySet &index_binary) override;
-    int64_t Count() override;
-    int64_t Dimension() override;
-
-    void
-    Seal() override;
-
-    virtual VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config);
-
-
- protected:
-    virtual std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config);
-
-    virtual VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index);
-
-    virtual void search_impl(int64_t n,
-                             const float *data,
-                             int64_t k,
-                             float *distances,
-                             int64_t *labels,
-                             const Config &cfg);
-
- protected:
-    std::mutex mutex_;
-};
-
-using IVFIndexPtr = std::shared_ptr<IVF>;
-
-class IVFSQ : public IVF {
- public:
-    explicit IVFSQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
-    IVFSQ() = default;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
-    VectorIndexPtr CopyCpuToGpu(const int64_t &device_id, const Config &config) override;
- protected:
-    VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
-};
-
-class IVFPQ : public IVF {
- public:
-    explicit IVFPQ(std::shared_ptr<faiss::Index> index) : IVF(std::move(index)) {}
-    IVFPQ() = default;
-    IndexModelPtr Train(const DatasetPtr &dataset, const Config &config) override;
- protected:
-    std::shared_ptr<faiss::IVFSearchParameters> GenParams(const Config &config) override;
-    VectorIndexPtr Clone_impl(const std::shared_ptr<faiss::Index> &index) override;
-};
-
-
-//class OPQIVFPQ : public IVFPQ {
-// public:
-//    PreprocessorPtr BuildPreprocessor(const Dataset &dataset, const Config &config) override;
-//};
-
-
-class GPUIVF;
-
-
-struct MemoryIOWriter : public faiss::IOWriter {
-    uint8_t *data_ = nullptr;
-    size_t total = 0;
-    size_t rp = 0;
-
-    size_t operator()(const void *ptr, size_t size, size_t nitems) override;
-};
-
-
-struct MemoryIOReader : public faiss::IOReader {
-    uint8_t *data_;
-    size_t rp = 0;
-    size_t total = 0;
-
-    size_t operator()(void *ptr, size_t size, size_t nitems) override;
-
-};
-
-
-class IVFIndexModel : public IndexModel, public BasicIndex {
-    friend IVF;
-    friend GPUIVF;
-
- public:
-    explicit IVFIndexModel(std::shared_ptr<faiss::Index> index);
-    IVFIndexModel() : BasicIndex(nullptr) {};
-    BinarySet Serialize() override;
- protected:
-    void SealImpl() override;
- public:
-    void Load(const BinarySet &binary) override;
-
- protected:
-    std::mutex mutex_;
-};
-
-using IVFIndexModelPtr = std::shared_ptr<IVFIndexModel>;
-
-}
-}
\ No newline at end of file

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.cpp

@@ -22,10 +22,10 @@
 #include <stack>
 #include <omp.h>
 
-#include "nsg.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
-#include "utils.h"
+#include "NSG.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
+#include "NSGHelper.h"
 
 // TODO: enable macro
 //#include <gperftools/profiler.h>

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSG.h

@@ -23,7 +23,7 @@
 #include <mutex>
 
 #include <boost/dynamic_bitset.hpp>
-#include "neighbor.h"
+#include "Neighbor.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.cpp

@@ -19,7 +19,7 @@
 #include <cstring>
 #include <fstream>
 
-#include "utils.h"
+#include "NSGHelper.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/utils.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGHelper.h

@@ -23,8 +23,8 @@
 
 #include <faiss/AutoTune.h>
 
-#include "nsg.h"
-#include "knowhere/common/config.h"
+#include "NSG.h"
+#include "knowhere/common/Config.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.cpp → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.cpp

@@ -18,7 +18,7 @@
 
 #include <cstring>
 
-#include "nsg_io.h"
+#include "NSGIO.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/nsg_io.h → cpp/src/core/knowhere/knowhere/index/vector_index/nsg/NSGIO.h

@@ -18,8 +18,9 @@
 
 #pragma once
 
-#include "nsg.h"
-#include "knowhere/index/vector_index/ivf.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
+#include "NSG.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
 
 
 namespace zilliz {

cpp/src/core/knowhere/knowhere/index/vector_index/nsg/index_io.h

-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include "nsg.h"
-
-namespace zilliz {
-namespace knowhere {
-namespace algo {
-
-void read_from_file(NsgIndex* index, const char *filename);
-void write_to_file(NsgIndex* index, const char *filename);
-
-}
-}
-}

cpp/src/core/test/test_idmap.cpp

@@ -21,10 +21,10 @@
 #include <iostream>
 #include <sstream>
 
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/adapter/structure.h"
-#include "knowhere/index/vector_index/cloner.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/adapter/Structure.h"
+#include "knowhere/index/vector_index/utils/Cloner.h"
+#include "knowhere/common/Exception.h"
 
 #include "utils.h"
 

cpp/src/core/test/test_ivf.cpp

@@ -25,12 +25,12 @@
 #include <faiss/gpu/GpuAutoTune.h>
 #include <faiss/gpu/GpuIndexIVFFlat.h>
 
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/ivf.h"
-#include "knowhere/adapter/structure.h"
-#include "knowhere/index/vector_index/cloner.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/common/timer.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/adapter/Structure.h"
+#include "knowhere/index/vector_index/utils/Cloner.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/common/Timer.h"
 
 #include "utils.h"
 

cpp/src/core/test/test_kdt.cpp

@@ -20,12 +20,12 @@
 
 #include <iostream>
 #include <sstream>
-#include "knowhere/common/exception.h"
+#include "knowhere/common/Exception.h"
 
-#include "knowhere/index/vector_index/cpu_kdt_rng.h"
-#include "knowhere/index/vector_index/definitions.h"
-#include "knowhere/adapter/sptag.h"
-#include "knowhere/adapter/structure.h"
+#include "knowhere/index/vector_index/IndexKDT.h"
+#include "knowhere/index/vector_index/utils/Definitions.h"
+#include "knowhere/adapter/SptagAdapter.h"
+#include "knowhere/adapter/Structure.h"
 
 #include "utils.h"
 

cpp/src/core/test/utils.h

@@ -23,7 +23,7 @@
 #include <cstdio>
 #include <fstream>
 
-#include "knowhere/adapter/structure.h"
+#include "knowhere/adapter/Structure.h"
 #include "knowhere/common/Log.h"
 
 class DataGen  {

cpp/src/db/Options.cpp

@@ -17,7 +17,7 @@
 
 #include "Options.h"
 #include "utils/Exception.h"
-#include "utils/easylogging++.h"
+#include "utils/Log.h"
 
 #include <stdlib.h>
 #include <assert.h>
@@ -56,11 +56,11 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
         std::vector<std::string> kv;
         boost::algorithm::split(kv, token, boost::is_any_of(":"));
         if (kv.size() != 2) {
-            LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
+            ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
             continue;
         }
         if (kv[0] != "disk" && kv[0] != "days") {
-            LOG(WARNING) << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
+            ENGINE_LOG_WARNING << "Invalid ArchiveConf Criterias: " << token << " Ignore!";
             continue;
         }
         try {
@@ -68,20 +68,22 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
             criterias_[kv[0]] = value;
         }
         catch (std::out_of_range&){
-            LOG(ERROR) << "Out of range: '" << kv[1] << "'";
-            throw OutOfRangeException();
+            std::string msg = "Out of range: '" + kv[1] + "'";
+            ENGINE_LOG_ERROR << msg;
+            throw InvalidArgumentException(msg);
         }
         catch (...){
-            LOG(ERROR) << "Invalid argument: '" << kv[1] << "'";
-            throw InvalidArgumentException();
+            std::string msg = "Invalid argument: '" + kv[1] + "'";
+            ENGINE_LOG_ERROR << msg;
+            throw InvalidArgumentException(msg);
         }
     }
 }
 
 void ArchiveConf::ParseType(const std::string& type) {
     if (type != "delete" && type != "swap") {
-        LOG(ERROR) << "Invalid argument: type='" << type << "'";
-        throw InvalidArgumentException();
+        std::string msg = "Invalid argument: type='" + type + "'";
+        throw InvalidArgumentException(msg);
     }
     type_ = type;
 }

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -25,7 +25,7 @@
 
 #include "src/wrapper/vec_index.h"
 #include "src/wrapper/vec_impl.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/common/Exception.h"
 
 #include <stdexcept>
 
@@ -45,14 +45,18 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
       nlist_(nlist) {
 
     index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
-    if (!index_) throw Exception("Create Empty VecIndex");
+    if (!index_) {
+        throw Exception(DB_ERROR, "Could not create VecIndex");
+    }
 
     Config build_cfg;
     build_cfg["dim"] = dimension;
     build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
     AutoGenParams(index_->GetType(), 0, build_cfg);
     auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
-    if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
+    if (ec != KNOWHERE_SUCCESS) {
+        throw Exception(DB_ERROR, "Build index error");
+    }
 }
 
 ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
@@ -273,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
 
     auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
-        throw Exception("Create Empty VecIndex");
+        throw Exception(DB_ERROR, "Could not create VecIndex");
     }
 
     Config build_cfg;
@@ -287,7 +291,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
                                  from_index->GetRawVectors(),
                                  from_index->GetRawIds(),
                                  build_cfg);
-    if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
+    if (ec != KNOWHERE_SUCCESS) { throw Exception(DB_ERROR, "Build index error"); }
 
     return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
 }

cpp/src/db/meta/Meta.h

@@ -31,6 +31,9 @@ namespace milvus {
 namespace engine {
 namespace meta {
 
+static const char* META_TABLES = "Tables";
+static const char* META_TABLEFILES = "TableFiles";
+
 class Meta {
  public:
     virtual ~Meta() = default;

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -19,6 +19,7 @@
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
 #include "utils/Log.h"
+#include "utils/Exception.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 
@@ -55,7 +56,7 @@ Status HandleException(const std::string &desc, const char* what = nullptr) {
 
 inline auto StoragePrototype(const std::string &path) {
     return make_storage(path,
-                        make_table("Tables",
+                        make_table(META_TABLES,
                                    make_column("id", &TableSchema::id_, primary_key()),
                                    make_column("table_id", &TableSchema::table_id_, unique()),
                                    make_column("state", &TableSchema::state_),
@@ -66,7 +67,7 @@ inline auto StoragePrototype(const std::string &path) {
                                    make_column("engine_type", &TableSchema::engine_type_),
                                    make_column("nlist", &TableSchema::nlist_),
                                    make_column("metric_type", &TableSchema::metric_type_)),
-                        make_table("TableFiles",
+                        make_table(META_TABLEFILES,
                                    make_column("id", &TableFileSchema::id_, primary_key()),
                                    make_column("table_id", &TableFileSchema::table_id_),
                                    make_column("engine_type", &TableFileSchema::engine_type_),
@@ -122,6 +123,17 @@ Status SqliteMetaImpl::Initialize() {
 
     ConnectorPtr = std::make_unique<ConnectorT>(StoragePrototype(options_.path + "/meta.sqlite"));
 
+    //old meta could be recreated since schema changed, throw exception if meta schema is not compatible
+    auto ret = ConnectorPtr->sync_schema_simulate();
+    if(ret.find(META_TABLES) != ret.end()
+        && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLES]) {
+            throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+    }
+    if(ret.find(META_TABLEFILES) != ret.end()
+       && sqlite_orm::sync_schema_result::dropped_and_recreated == ret[META_TABLEFILES]) {
+        throw Exception(DB_INCOMPATIB_META, "Meta schema is created by Milvus old version");
+    }
+
     ConnectorPtr->sync_schema();
     ConnectorPtr->open_forever(); // thread safe option
     ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
@@ -1246,8 +1258,8 @@ Status SqliteMetaImpl::DropAll() {
     ENGINE_LOG_DEBUG << "Drop all sqlite meta";
 
     try {
-        ConnectorPtr->drop_table("Tables");
-        ConnectorPtr->drop_table("TableFiles");
+        ConnectorPtr->drop_table(META_TABLES);
+        ConnectorPtr->drop_table(META_TABLEFILES);
     } catch (std::exception &e) {
         return HandleException("Encounter exception when drop all meta", e.what());
     }

cpp/src/main.cpp

@@ -102,10 +102,6 @@ main(int argc, char *argv[]) {
         }
     }
 
-    server::Server &server = server::Server::Instance();
-    server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
-    server.Start();
-
     /* Handle Signal */
     signal(SIGHUP, server::SignalUtil::HandleSignal);
     signal(SIGINT, server::SignalUtil::HandleSignal);
@@ -114,6 +110,10 @@ main(int argc, char *argv[]) {
     signal(SIGUSR2, server::SignalUtil::HandleSignal);
     signal(SIGTERM, server::SignalUtil::HandleSignal);
 
+    server::Server &server = server::Server::Instance();
+    server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
+    server.Start();
+
     /* wait signal */
     pause();
 

cpp/src/scheduler/SchedInst.cpp

@@ -19,7 +19,7 @@
 #include "SchedInst.h"
 #include "server/ServerConfig.h"
 #include "ResourceFactory.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
 #include "Utils.h"
 
 

cpp/src/server/DBWrapper.cpp

@@ -109,15 +109,10 @@ Status DBWrapper::StartService() {
     }
 
     //create db instance
-    std::string msg = opt.meta.path;
     try {
         db_ = engine::DBFactory::Build(opt);
     } catch(std::exception& ex) {
-        msg = ex.what();
-    }
-
-    if(db_ == nullptr) {
-        std::cerr << "ERROR! Failed to open database: " << msg << std::endl;
+        std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl;
         kill(0, SIGUSR1);
     }
 

cpp/src/server/Server.cpp

@@ -196,8 +196,8 @@ Server::Start() {
         server::Metrics::GetInstance().Init();
         server::SystemInfo::GetInstance().Init();
 
-        std::cout << "Milvus server start successfully." << std::endl;
         StartService();
+        std::cout << "Milvus server start successfully." << std::endl;
 
     } catch (std::exception &ex) {
         std::cerr << "Milvus server encounter exception: " << ex.what();

cpp/src/utils/Error.h

@@ -86,6 +86,7 @@ constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
 constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
 constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
 constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
+constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
 
 //knowhere error code
 constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);

cpp/src/utils/Exception.h

@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "utils/Error.h"
+
 #include <exception>
 #include <string>
 
@@ -25,13 +27,14 @@ namespace milvus {
 
 class Exception : public std::exception {
 public:
-    Exception(const std::string& message)
-        : message_(message) {
+    Exception(ErrorCode code, const std::string& message)
+        : code_(code_),
+          message_(message) {
         }
 
-    Exception()
-        : message_() {
-        }
+    ErrorCode code() const throw() {
+        return code_;
+    }
 
     virtual const char* what() const throw() {
         if (message_.empty()) {
@@ -44,20 +47,21 @@ public:
     virtual ~Exception() throw() {};
 
 protected:
-
+    ErrorCode code_;
     std::string message_;
 };
 
 class InvalidArgumentException : public Exception {
 public:
-    InvalidArgumentException() : Exception("Invalid Argument"){};
-    InvalidArgumentException(const std::string& message) : Exception(message) {};
-};
+    InvalidArgumentException()
+        : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
+
+    };
+    InvalidArgumentException(const std::string& message)
+        : Exception(SERVER_INVALID_ARGUMENT, message) {
+
+    };
 
-class OutOfRangeException : public Exception {
-public:
-    OutOfRangeException() : Exception("Out Of Range"){};
-    OutOfRangeException(const std::string& message) : Exception(message) {};
 };
 
 } // namespace milvus

cpp/src/utils/Log.h

@@ -24,7 +24,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define SERVER_DOMAIN_NAME "[SERVER] "
-#define SERVER_ERROR_TEXT "SERVER Error:"
 
 #define SERVER_LOG_TRACE LOG(TRACE) << SERVER_DOMAIN_NAME
 #define SERVER_LOG_DEBUG LOG(DEBUG) << SERVER_DOMAIN_NAME
@@ -35,7 +34,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define ENGINE_DOMAIN_NAME "[ENGINE] "
-#define ENGINE_ERROR_TEXT "ENGINE Error:"
 
 #define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
 #define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
@@ -46,7 +44,6 @@ namespace milvus {
 
 /////////////////////////////////////////////////////////////////////////////////////////////////
 #define WRAPPER_DOMAIN_NAME "[WRAPPER] "
-#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
 
 #define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
 #define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME

cpp/src/utils/LogUtil.cpp

@@ -17,10 +17,8 @@
 
 #include "LogUtil.h"
 #include "server/ServerConfig.h"
-#include "easylogging++.h"
 
 #include <ctype.h>
-
 #include <string>
 #include <libgen.h>
 

cpp/src/wrapper/KnowhereResource.cpp

@@ -17,7 +17,7 @@
 
 
 #include "KnowhereResource.h"
-#include "knowhere/index/vector_index/FaissGpuResourceMgr.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 #include "server/ServerConfig.h"
 
 #include <map>

cpp/src/wrapper/data_transfer.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "knowhere/adapter/structure.h"
+#include "knowhere/adapter/Structure.h"
 
 
 namespace zilliz {

cpp/src/wrapper/vec_impl.cpp

@@ -17,10 +17,10 @@
 
 
 #include "utils/Log.h"
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/common/exception.h"
-#include "knowhere/index/vector_index/cloner.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/common/Exception.h"
+#include "knowhere/index/vector_index/helpers/Cloner.h"
 
 #include "vec_impl.h"
 #include "data_transfer.h"
@@ -153,7 +153,7 @@ IndexType VecIndexImpl::GetType() {
 
 VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg) {
     // TODO(linxj): exception handle
-    auto gpu_index = zilliz::knowhere::CopyCpuToGpu(index_, device_id, cfg);
+    auto gpu_index = zilliz::knowhere::cloner::CopyCpuToGpu(index_, device_id, cfg);
     auto new_index = std::make_shared<VecIndexImpl>(gpu_index, ConvertToGpuIndexType(type));
     new_index->dim = dim;
     return new_index;
@@ -161,7 +161,7 @@ VecIndexPtr VecIndexImpl::CopyToGpu(const int64_t &device_id, const Config &cfg)
 
 VecIndexPtr VecIndexImpl::CopyToCpu(const Config &cfg) {
     // TODO(linxj): exception handle
-    auto cpu_index = zilliz::knowhere::CopyGpuToCpu(index_, cfg);
+    auto cpu_index = zilliz::knowhere::cloner::CopyGpuToCpu(index_, cfg);
     auto new_index = std::make_shared<VecIndexImpl>(cpu_index, ConvertToCpuIndexType(type));
     new_index->dim = dim;
     return new_index;

cpp/src/wrapper/vec_impl.h

@@ -18,7 +18,7 @@
 
 #pragma once
 
-#include "knowhere/index/vector_index/vector_index.h"
+#include "knowhere/index/vector_index/VectorIndex.h"
 
 #include "vec_index.h"
 

cpp/src/wrapper/vec_index.cpp

@@ -15,12 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "knowhere/index/vector_index/ivf.h"
-#include "knowhere/index/vector_index/idmap.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/cpu_kdt_rng.h"
-#include "knowhere/index/vector_index/nsg_index.h"
-#include "knowhere/common/exception.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
+#include "knowhere/index/vector_index/IndexGPUIVF.h"
+#include "knowhere/index/vector_index/IndexIVFSQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVFSQ.h"
+#include "knowhere/index/vector_index/IndexIVFPQ.h"
+#include "knowhere/index/vector_index/IndexGPUIVFPQ.h"
+#include "knowhere/index/vector_index/IndexIDMAP.h"
+#include "knowhere/index/vector_index/IndexKDT.h"
+#include "knowhere/index/vector_index/IndexNSG.h"
+#include "knowhere/common/Exception.h"
 
 #include "vec_index.h"
 #include "vec_impl.h"

cpp/src/wrapper/vec_index.h

@@ -23,8 +23,8 @@
 
 #include "utils/Error.h"
 
-#include "knowhere/common/config.h"
-#include "knowhere/common/binary_set.h"
+#include "knowhere/common/Config.h"
+#include "knowhere/common/BinarySet.h"
 
 
 namespace zilliz {

cpp/unittest/db/db_tests.cpp

@@ -24,10 +24,7 @@
 #include "utils/CommonUtil.h"
 
 #include <gtest/gtest.h>
-#include "utils/easylogging++.h"
-
 #include <boost/filesystem.hpp>
-
 #include <thread>
 #include <random>
 

cpp/unittest/db/meta_tests.cpp

@@ -15,17 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <stdlib.h>
-#include <time.h>
-
 #include "utils.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/Utils.h"
 #include "db/meta/MetaConsts.h"
 
+#include <gtest/gtest.h>
+#include <thread>
+#include <stdlib.h>
+#include <time.h>
+
 using namespace zilliz::milvus;
 using namespace zilliz::milvus::engine;
 

cpp/unittest/db/misc_test.cpp

@@ -21,7 +21,6 @@
 #include "db/Utils.h"
 #include "utils/Status.h"
 #include "utils/Exception.h"
-#include "utils/easylogging++.h"
 
 #include <gtest/gtest.h>
 #include <thread>
@@ -31,13 +30,13 @@
 using namespace zilliz::milvus;
 
 TEST(DBMiscTest, EXCEPTION_TEST) {
-    Exception ex1("");
+    Exception ex1(100, "error");
     std::string what = ex1.what();
-    ASSERT_FALSE(what.empty());
+    ASSERT_EQ(what, "error");
+    ASSERT_EQ(ex1.code(), 100);
 
-    OutOfRangeException ex2;
-    what = ex2.what();
-    ASSERT_FALSE(what.empty());
+    InvalidArgumentException ex2;
+    ASSERT_EQ(ex2.code(), SERVER_INVALID_ARGUMENT);
 }
 
 TEST(DBMiscTest, OPTIONS_TEST) {

cpp/unittest/db/mysql_db_test.cpp

@@ -21,7 +21,6 @@
 #include "db/meta/MetaConsts.h"
 
 #include <gtest/gtest.h>
-#include "utils/easylogging++.h"
 #include <boost/filesystem.hpp>
 
 #include <thread>

cpp/unittest/db/mysql_meta_test.cpp

@@ -15,20 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <stdlib.h>
-#include <time.h>
-
 #include "utils.h"
 #include "db/meta/MySQLMetaImpl.h"
 #include "db/Utils.h"
 #include "db/meta/MetaConsts.h"
 
-#include "mysql++/mysql++.h"
-
 #include <iostream>
+#include <thread>
+#include <stdlib.h>
+#include <time.h>
+#include <gtest/gtest.h>
+#include <mysql++/mysql++.h>
 
 using namespace zilliz::milvus;
 using namespace zilliz::milvus::engine;

cpp/unittest/db/utils.cpp

@@ -25,8 +25,7 @@
 #include "cache/CpuCacheMgr.h"
 #include "db/DBFactory.h"
 #include "db/Options.h"
-#include "server/ServerConfig.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
 
 INITIALIZE_EASYLOGGINGPP
 

cpp/unittest/main.cpp

@@ -15,13 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <gmock/gmock.h>
 #include "utils/easylogging++.h"
-
 #include "server/ServerConfig.h"
 #include "utils/CommonUtil.h"
 
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
 INITIALIZE_EASYLOGGINGPP
 
 using namespace zilliz::milvus;