update...

Former-commit-id: 6366c5292e1483be5009e46bd273fdb00433fea6

cpp/src/CMakeLists.txt

@@ -10,7 +10,7 @@ aux_source_directory(config config_files)
 aux_source_directory(server server_files)
 aux_source_directory(utils utils_files)
 aux_source_directory(db db_files)
-aux_source_directory(wrapper wrapper_files)
+#aux_source_directory(wrapper wrapper_files)
 aux_source_directory(metrics metrics_files)
 aux_source_directory(wrapper/knowhere knowhere_files)
 

cpp/src/cache/DataObj.h

@@ -28,7 +28,7 @@ public:
             return 0;
         }
 
-        return index_->ntotal*(index_->dim*4);
+        return index_->Count() * index_->Dimension() * sizeof(float);
     }
 
 private:

cpp/src/db/EngineFactory.cpp

@@ -4,7 +4,7 @@
  * Proprietary and confidential.
  ******************************************************************************/
 #include "EngineFactory.h"
-#include "FaissExecutionEngine.h"
+//#include "FaissExecutionEngine.h"
 #include "ExecutionEngineImpl.h"
 #include "Log.h"
 

cpp/src/db/ExecutionEngineImpl.cpp

@@ -3,13 +3,14 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited.
  * Proprietary and confidential.
  ******************************************************************************/
-#include "ExecutionEngineImpl.h"
+#include <src/server/ServerConfig.h>
 #include "Log.h"
 
+#include "src/cache/CpuCacheMgr.h"
+#include "ExecutionEngineImpl.h"
+#include "wrapper/knowhere/vec_index.h"
 #include "wrapper/knowhere/vec_impl.h"
-#include "knowhere/index/vector_index/ivf.h"
-#include "knowhere/index/vector_index/gpu_ivf.h"
-#include "knowhere/index/vector_index/cpu_kdt_rng.h"
+
 
 namespace zilliz {
 namespace milvus {
@@ -17,95 +18,151 @@ namespace engine {
 
 
 ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
-        const std::string& location,
-        EngineType type)
-    : location_(location) {
-    index_ = CreatetVecIndex(type);
+                                         const std::string &location,
+                                         EngineType type)
+    : location_(location), dim(dimension), build_type(type) {
+    index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
+    std::static_pointer_cast<BFIndex>(index_)->Build(dimension);
 }
 
-vecwise::engine::VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
-    std::shared_ptr<zilliz::knowhere::VectorIndex> index;
-    switch(type) {
-        case EngineType::FAISS_IDMAP: {
+ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
+                                         const std::string &location,
+                                         EngineType type)
+    : index_(std::move(index)), location_(location), build_type(type) {
+}
 
+VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
+    std::shared_ptr<VecIndex> index;
+    switch (type) {
+        case EngineType::FAISS_IDMAP: {
+            index = GetVecIndexFactory(IndexType::FAISS_IDMAP);
             break;
         }
         case EngineType::FAISS_IVFFLAT_GPU: {
-            index = std::make_shared<zilliz::knowhere::GPUIVF>(0);
+            index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_GPU);
             break;
         }
         case EngineType::FAISS_IVFFLAT_CPU: {
-            index = std::make_shared<zilliz::knowhere::IVF>();
+            index = GetVecIndexFactory(IndexType::FAISS_IVFFLAT_CPU);
             break;
         }
         case EngineType::SPTAG_KDT_RNT_CPU: {
-            index = std::make_shared<zilliz::knowhere::CPUKDTRNG>();
+            index = GetVecIndexFactory(IndexType::SPTAG_KDT_RNT_CPU);
             break;
         }
-        default:{
+        default: {
             ENGINE_LOG_ERROR << "Invalid engine type";
             return nullptr;
         }
     }
-
-    return std::make_shared<vecwise::engine::VecIndexImpl>(index);
+    return index;
 }
 
 Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) {
-
+    index_->Add(n, xdata, xids, Config::object{{"dim", dim}});
     return Status::OK();
 }
 
 size_t ExecutionEngineImpl::Count() const {
-    return 0;
+    return index_->Count();
 }
 
 size_t ExecutionEngineImpl::Size() const {
-    return 0;
+    return (size_t) (Count() * Dimension()) * sizeof(float);
 }
 
 size_t ExecutionEngineImpl::Dimension() const {
-    return 0;
+    return index_->Dimension();
 }
 
 size_t ExecutionEngineImpl::PhysicalSize() const {
-    return 0;
+    return (size_t) (Count() * Dimension()) * sizeof(float);
 }
 
 Status ExecutionEngineImpl::Serialize() {
+    // TODO(groot):
+    auto binaryset = index_->Serialize();
     return Status::OK();
 }
 
 Status ExecutionEngineImpl::Load() {
-
+    // TODO(groot):
     return Status::OK();
 }
 
-Status ExecutionEngineImpl::Merge(const std::string& location) {
+VecIndexPtr ExecutionEngineImpl::Load(const std::string &location) {
+    // TODO(groot): dev func in Fake code
+    // pseude code
+    //auto data = read_file(location);
+    //auto index_type = get_index_type(data);
+    //auto binaryset = get_index_binary(data);
+    /////
+
+    //return LoadVecIndex(index_type, binaryset);
+    return nullptr;
+}
+
+Status ExecutionEngineImpl::Merge(const std::string &location) {
+    if (location == location_) {
+        return Status::Error("Cannot Merge Self");
+    }
+    ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_;
 
+    auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location);
+    if (!to_merge) {
+        to_merge = Load(location);
+    }
+
+    auto file_index = std::dynamic_pointer_cast<BFIndex>(index_);
+    index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds());
     return Status::OK();
 }
 
+// TODO(linxj): add config
 ExecutionEnginePtr
-ExecutionEngineImpl::BuildIndex(const std::string& location) {
-    return nullptr;
+ExecutionEngineImpl::BuildIndex(const std::string &location) {
+    ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
+
+    auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
+    auto to_index = CreatetVecIndex(build_type);
+    to_index->BuildAll(Count(),
+                       from_index->GetRawVectors(),
+                       from_index->GetRawIds(),
+                       Config::object{{"dim", Dimension()}, {"gpu_id", gpu_num}});
+
+    return std::make_shared<ExecutionEngineImpl>(to_index, location, build_type);
 }
 
 Status ExecutionEngineImpl::Search(long n,
-                                    const float *data,
-                                    long k,
-                                    float *distances,
-                                    long *labels) const {
-
+                                   const float *data,
+                                   long k,
+                                   float *distances,
+                                   long *labels) const {
+    index_->Search(n, data, distances, labels, Config::object{{"k", k}, {"nprobe", nprobe_}});
     return Status::OK();
 }
 
 Status ExecutionEngineImpl::Cache() {
+    zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, index_);
 
     return Status::OK();
 }
 
 Status ExecutionEngineImpl::Init() {
+    using namespace zilliz::milvus::server;
+    ServerConfig &config = ServerConfig::GetInstance();
+    ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
+    gpu_num = server_config.GetInt32Value("gpu_index", 0);
+
+    switch (build_type) {
+        case EngineType::FAISS_IVFFLAT_GPU: {
+        }
+        case EngineType::FAISS_IVFFLAT_CPU: {
+            ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
+            nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
+            break;
+        }
+    }
 
     return Status::OK();
 }

cpp/src/db/ExecutionEngineImpl.h

@@ -11,17 +11,22 @@
 #include <memory>
 #include <string>
 
+
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
 
 class ExecutionEngineImpl : public ExecutionEngine {
-public:
+ public:
 
     ExecutionEngineImpl(uint16_t dimension,
-            const std::string& location,
-            EngineType type);
+                        const std::string &location,
+                        EngineType type);
+
+    ExecutionEngineImpl(VecIndexPtr index,
+                        const std::string &location,
+                        EngineType type);
 
     Status AddWithIds(long n, const float *xdata, const long *xids) override;
 
@@ -37,7 +42,7 @@ public:
 
     Status Load() override;
 
-    Status Merge(const std::string& location) override;
+    Status Merge(const std::string &location) override;
 
     Status Search(long n,
                   const float *data,
@@ -45,21 +50,26 @@ public:
                   float *distances,
                   long *labels) const override;
 
-    ExecutionEnginePtr BuildIndex(const std::string&) override;
+    ExecutionEnginePtr BuildIndex(const std::string &) override;
 
     Status Cache() override;
 
     Status Init() override;
 
-private:
-    vecwise::engine::VecIndexPtr CreatetVecIndex(EngineType type);
+ private:
+    VecIndexPtr CreatetVecIndex(EngineType type);
+
+    VecIndexPtr Load(const std::string &location);
 
-protected:
-    vecwise::engine::VecIndexPtr index_;
+ protected:
+    VecIndexPtr index_ = nullptr;
+    EngineType build_type;
 
+    int64_t dim;
     std::string location_;
 
     size_t nprobe_ = 0;
+    int64_t gpu_num = 0;
 };
 
 

cpp/src/db/FaissExecutionEngine.cpp

@@ -3,6 +3,7 @@
  * Unauthorized copying of this file, via any medium is strictly prohibited.
  * Proprietary and confidential.
  ******************************************************************************/
+#if 0
 #include "FaissExecutionEngine.h"
 #include "Log.h"
 
@@ -181,3 +182,4 @@ Status FaissExecutionEngine::Init() {
 } // namespace engine
 } // namespace milvus
 } // namespace zilliz
+#endif

cpp/src/db/FaissExecutionEngine.h

@@ -5,6 +5,7 @@
  ******************************************************************************/
 #pragma once
 
+#if 0
 #include "ExecutionEngine.h"
 #include "faiss/Index.h"
 
@@ -71,3 +72,4 @@ protected:
 } // namespace engine
 } // namespace milvus
 } // namespace zilliz
+#endif

cpp/src/wrapper/Index.cpp

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 // TODO: maybe support static search
 #ifdef GPU_VERSION
 #include "faiss/gpu/GpuAutoTune.h"
@@ -80,3 +81,4 @@ Index_ptr read_index(const std::string &file_name) {
 }
 }
 }
+#endif

cpp/src/wrapper/Index.h

@@ -6,23 +6,29 @@
 
 #pragma once
 
-#include <vector>
-#include <string>
-#include <unordered_map>
-#include <memory>
-#include <fstream>
+//#include <vector>
+//#include <string>
+//#include <unordered_map>
+//#include <memory>
+//#include <fstream>
+//
+//#include "faiss/AutoTune.h"
+//#include "faiss/index_io.h"
+//
+//#include "Operand.h"
 
-#include "faiss/AutoTune.h"
-#include "faiss/index_io.h"
+#include "knowhere/vec_index.h"
 
-#include "Operand.h"
 
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
-class Index;
-using Index_ptr = std::shared_ptr<Index>;
+using Index_ptr = VecIndexPtr;
+
+#if 0
+//class Index;
+//using Index_ptr = std::shared_ptr<Index>;
 
 class Index {
     typedef long idx_t;
@@ -75,6 +81,7 @@ private:
 void write_index(const Index_ptr &index, const std::string &file_name);
 
 extern Index_ptr read_index(const std::string &file_name);
+#endif
 
 
 }

cpp/src/wrapper/IndexBuilder.cpp

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 #include "mutex"
 
 
@@ -128,10 +129,8 @@ Index_ptr BgCpuBuilder::build_all(const long &nb, const float *xb, const long *i
     return std::make_shared<Index>(index);
 }
 
-// TODO: Be Factory pattern later
 IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd) {
     if (opd->index_type == "IDMap") {
-        // TODO: fix hardcode
         IndexBuilderPtr index = nullptr;
         return std::make_shared<BgCpuBuilder>(opd);
     }
@@ -142,3 +141,4 @@ IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd) {
 }
 }
 }
+#endif

cpp/src/wrapper/IndexBuilder.h

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 #pragma once
 
 #include "faiss/Index.h"
@@ -64,3 +65,4 @@ extern IndexBuilderPtr GetIndexBuilder(const Operand_ptr &opd);
 }
 }
 }
+#endif

cpp/src/wrapper/Operand.cpp

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 #include "Operand.h"
 
 
@@ -90,3 +91,4 @@ Operand_ptr str_to_operand(const std::string &input) {
 }
 }
 }
+#endif

cpp/src/wrapper/Operand.h

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
+#if 0
 #pragma once
 
 #include <string>
@@ -42,3 +43,4 @@ extern Operand_ptr str_to_operand(const std::string &input);
 }
 }
 }
+#endif

cpp/src/wrapper/knowhere/data_transfer.cpp

@@ -8,7 +8,7 @@
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 using namespace zilliz::knowhere;

cpp/src/wrapper/knowhere/data_transfer.h

@@ -10,7 +10,7 @@
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 extern zilliz::knowhere::DatasetPtr

cpp/src/wrapper/knowhere/vec_impl.cpp

@@ -4,18 +4,14 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 
-#include "knowhere/index/index.h"
-#include "knowhere/index/index_model.h"
-#include "knowhere/index/index_type.h"
-#include "knowhere/adapter/sptag.h"
-#include "knowhere/common/tensor.h"
+#include "knowhere/index/vector_index/idmap.h"
 
 #include "vec_impl.h"
 #include "data_transfer.h"
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 using namespace zilliz::knowhere;
@@ -26,8 +22,8 @@ void VecIndexImpl::BuildAll(const long &nb,
                             const Config &cfg,
                             const long &nt,
                             const float *xt) {
-    auto d = cfg["dim"].as<int>();
-    auto dataset = GenDatasetWithIds(nb, d, xb, ids);
+    dim = cfg["dim"].as<int>();
+    auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
 
     auto preprocessor = index_->BuildPreprocessor(dataset, cfg);
     index_->set_preprocessor(preprocessor);
@@ -39,7 +35,7 @@ void VecIndexImpl::BuildAll(const long &nb,
 void VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) {
     // TODO(linxj): Assert index is trained;
 
-    auto d = cfg["dim"].as<int>();
+    auto d = cfg.get_with_default("dim", dim);
     auto dataset = GenDatasetWithIds(nb, d, xb, ids);
 
     index_->Add(dataset, cfg);
@@ -48,8 +44,8 @@ void VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const C
 void VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) {
     // TODO: Assert index is trained;
 
-    auto d = cfg["dim"].as<int>();
     auto k = cfg["k"].as<int>();
+    auto d = cfg.get_with_default("dim", dim);
     auto dataset = GenDataset(nq, d, xq);
 
     Config search_cfg;
@@ -90,6 +86,27 @@ void VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) {
     index_->Load(index_binary);
 }
 
+int64_t VecIndexImpl::Dimension() {
+    return index_->Dimension();
+}
+
+int64_t VecIndexImpl::Count() {
+    return index_->Count();
+}
+
+float *BFIndex::GetRawVectors() {
+    return std::static_pointer_cast<IDMAP>(index_)->GetRawVectors();
+}
+
+int64_t *BFIndex::GetRawIds() {
+    return std::static_pointer_cast<IDMAP>(index_)->GetRawIds();
+}
+
+void BFIndex::Build(const int64_t &d) {
+    dim = d;
+    std::static_pointer_cast<IDMAP>(index_)->Train(dim);
+}
+
 }
 }
 }

cpp/src/wrapper/knowhere/vec_impl.h

@@ -12,7 +12,7 @@
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 class VecIndexImpl : public VecIndex {
@@ -24,15 +24,26 @@ class VecIndexImpl : public VecIndex {
                   const Config &cfg,
                   const long &nt,
                   const float *xt) override;
+    int64_t Dimension() override;
+    int64_t Count() override;
     void Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override;
     zilliz::knowhere::BinarySet Serialize() override;
     void Load(const zilliz::knowhere::BinarySet &index_binary) override;
     void Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override;
 
- private:
+ protected:
+    int64_t dim;
     std::shared_ptr<zilliz::knowhere::VectorIndex> index_ = nullptr;
 };
 
+class BFIndex : public VecIndexImpl {
+ public:
+    explicit BFIndex(std::shared_ptr<zilliz::knowhere::VectorIndex> index) : VecIndexImpl(std::move(index)) {};
+    void Build(const int64_t& d);
+    float* GetRawVectors();
+    int64_t* GetRawIds();
+};
+
 }
 }
 }

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -4,6 +4,7 @@
 // Proprietary and confidential.
 ////////////////////////////////////////////////////////////////////////////////
 #include "knowhere/index/vector_index/ivf.h"
+#include "knowhere/index/vector_index/idmap.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/index/vector_index/cpu_kdt_rng.h"
 
@@ -12,27 +13,42 @@
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 // TODO(linxj): index_type => enum struct
-VecIndexPtr GetVecIndexFactory(const std::string &index_type) {
+VecIndexPtr GetVecIndexFactory(const IndexType &type) {
     std::shared_ptr<zilliz::knowhere::VectorIndex> index;
-    if (index_type == "IVF") {
-        index = std::make_shared<zilliz::knowhere::IVF>();
-    } else if (index_type == "GPUIVF") {
-        index = std::make_shared<zilliz::knowhere::GPUIVF>(0);
-    } else if (index_type == "SPTAG") {
-        index = std::make_shared<zilliz::knowhere::CPUKDTRNG>();
+    switch (type) {
+        case IndexType::FAISS_IDMAP: {
+            index = std::make_shared<zilliz::knowhere::IDMAP>();
+            return std::make_shared<BFIndex>(index);
+        }
+        case IndexType::FAISS_IVFFLAT_CPU: {
+            index = std::make_shared<zilliz::knowhere::IVF>();
+            break;
+        }
+        case IndexType::FAISS_IVFFLAT_GPU: {
+            index = std::make_shared<zilliz::knowhere::GPUIVF>(0);
+            break;
+        }
+        case IndexType::SPTAG_KDT_RNT_CPU: {
+            index = std::make_shared<zilliz::knowhere::CPUKDTRNG>();
+            break;
+        }
+            //// TODO(linxj): Support NSG
+            //case IndexType ::NSG: {
+            //    index = std::make_shared<zilliz::knowhere::NSG>();
+            //    break;
+            //}
+        default: {
+            return nullptr;
+        }
     }
-    // TODO(linxj): Support NSG
-    //else if (index_type == "NSG") {
-    //    index = std::make_shared<zilliz::knowhere::NSG>();
-    //}
     return std::make_shared<VecIndexImpl>(index);
 }
 
-VecIndexPtr LoadVecIndex(const std::string &index_type, const zilliz::knowhere::BinarySet &index_binary) {
+VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary) {
     auto index = GetVecIndexFactory(index_type);
     index->Load(index_binary);
     return index;

cpp/src/wrapper/knowhere/vec_index.h

@@ -14,7 +14,7 @@
 
 
 namespace zilliz {
-namespace vecwise {
+namespace milvus {
 namespace engine {
 
 // TODO(linxj): jsoncons => rapidjson or other.
@@ -40,6 +40,10 @@ class VecIndex {
                         long *ids,
                         const Config &cfg = Config()) = 0;
 
+    virtual int64_t Dimension() = 0;
+
+    virtual int64_t Count() = 0;
+
     virtual zilliz::knowhere::BinarySet Serialize() = 0;
 
     virtual void Load(const zilliz::knowhere::BinarySet &index_binary) = 0;
@@ -47,9 +51,18 @@ class VecIndex {
 
 using VecIndexPtr = std::shared_ptr<VecIndex>;
 
-extern VecIndexPtr GetVecIndexFactory(const std::string &index_type);
+enum class IndexType {
+    INVALID = 0,
+    FAISS_IDMAP = 1,
+    FAISS_IVFFLAT_GPU,
+    FAISS_IVFFLAT_CPU,
+    SPTAG_KDT_RNT_CPU,
+    NSG,
+};
+
+extern VecIndexPtr GetVecIndexFactory(const IndexType &type);
 
-extern VecIndexPtr LoadVecIndex(const std::string &index_type, const zilliz::knowhere::BinarySet &index_binary);
+extern VecIndexPtr LoadVecIndex(const IndexType &index_type, const zilliz::knowhere::BinarySet &index_binary);
 
 }
 }

knowhere @ c0df7662

-Subproject commit 2d543bfab655398f30113681f348519acac40ab5
+Subproject commit c0df766214d7fa288ffedd77cd06a8ba8620c8df