new engine

Former-commit-id: 178888a85b0cc0394d429dfeac4300971f65079f

cpp/src/db/EngineFactory.cpp

@@ -5,13 +5,14 @@
  ******************************************************************************/
 #include "EngineFactory.h"
 #include "FaissExecutionEngine.h"
+#include "ExecutionEngineImpl.h"
 #include "Log.h"
 
-
 namespace zilliz {
 namespace milvus {
 namespace engine {
 
+#if 0
 ExecutionEnginePtr
 EngineFactory::Build(uint16_t dimension,
                      const std::string &location,
@@ -26,7 +27,7 @@ EngineFactory::Build(uint16_t dimension,
             break;
         }
 
-        case EngineType::FAISS_IVFFLAT: {
+        case EngineType::FAISS_IVFFLAT_GPU: {
             execution_engine_ptr =
                 ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
             break;
@@ -41,6 +42,24 @@ EngineFactory::Build(uint16_t dimension,
     execution_engine_ptr->Init();
     return execution_engine_ptr;
 }
+#else
+ExecutionEnginePtr
+EngineFactory::Build(uint16_t dimension,
+                     const std::string &location,
+                     EngineType type) {
+
+    if(type == EngineType::INVALID) {
+        ENGINE_LOG_ERROR << "Unsupported engine type";
+        return nullptr;
+    }
+
+    ExecutionEnginePtr execution_engine_ptr =
+            std::make_shared<ExecutionEngineImpl>(dimension, location, type);
+
+    execution_engine_ptr->Init();
+    return execution_engine_ptr;
+}
+#endif
 
 }
 }

cpp/src/db/ExecutionEngine.h

@@ -17,7 +17,9 @@ namespace engine {
 enum class EngineType {
     INVALID = 0,
     FAISS_IDMAP = 1,
-    FAISS_IVFFLAT,
+    FAISS_IVFFLAT_GPU,
+    FAISS_IVFFLAT_CPU,
+    SPTAG_KDT_RNT_CPU,
 };
 
 class ExecutionEngine {

cpp/src/db/ExecutionEngineImpl.cpp

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+#include "ExecutionEngineImpl.h"
+#include "Log.h"
+
+#include "wrapper/knowhere/vec_impl.h"
+#include "knowhere/index/vector_index/ivf.h"
+#include "knowhere/index/vector_index/gpu_ivf.h"
+#include "knowhere/index/vector_index/cpu_kdt_rng.h"
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+
+ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
+        const std::string& location,
+        EngineType type)
+    : location_(location) {
+    index_ = CreatetVecIndex(type);
+}
+
+vecwise::engine::VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
+    std::shared_ptr<zilliz::knowhere::VectorIndex> index;
+    switch(type) {
+        case EngineType::FAISS_IDMAP: {
+
+            break;
+        }
+        case EngineType::FAISS_IVFFLAT_GPU: {
+            index = std::make_shared<zilliz::knowhere::GPUIVF>(0);
+            break;
+        }
+        case EngineType::FAISS_IVFFLAT_CPU: {
+            index = std::make_shared<zilliz::knowhere::IVF>();
+            break;
+        }
+        case EngineType::SPTAG_KDT_RNT_CPU: {
+            index = std::make_shared<zilliz::knowhere::CPUKDTRNG>();
+            break;
+        }
+        default:{
+            ENGINE_LOG_ERROR << "Invalid engine type";
+            return nullptr;
+        }
+    }
+
+    return std::make_shared<vecwise::engine::VecIndexImpl>(index);
+}
+
+Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) {
+
+    return Status::OK();
+}
+
+size_t ExecutionEngineImpl::Count() const {
+    return 0;
+}
+
+size_t ExecutionEngineImpl::Size() const {
+    return 0;
+}
+
+size_t ExecutionEngineImpl::Dimension() const {
+    return 0;
+}
+
+size_t ExecutionEngineImpl::PhysicalSize() const {
+    return 0;
+}
+
+Status ExecutionEngineImpl::Serialize() {
+    return Status::OK();
+}
+
+Status ExecutionEngineImpl::Load() {
+
+    return Status::OK();
+}
+
+Status ExecutionEngineImpl::Merge(const std::string& location) {
+
+    return Status::OK();
+}
+
+ExecutionEnginePtr
+ExecutionEngineImpl::BuildIndex(const std::string& location) {
+    return nullptr;
+}
+
+Status ExecutionEngineImpl::Search(long n,
+                                    const float *data,
+                                    long k,
+                                    float *distances,
+                                    long *labels) const {
+
+    return Status::OK();
+}
+
+Status ExecutionEngineImpl::Cache() {
+
+    return Status::OK();
+}
+
+Status ExecutionEngineImpl::Init() {
+
+    return Status::OK();
+}
+
+
+} // namespace engine
+} // namespace milvus
+} // namespace zilliz

cpp/src/db/ExecutionEngineImpl.h

+/*******************************************************************************
+ * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+ * Unauthorized copying of this file, via any medium is strictly prohibited.
+ * Proprietary and confidential.
+ ******************************************************************************/
+#pragma once
+
+#include "ExecutionEngine.h"
+#include "wrapper/knowhere/vec_index.h"
+
+#include <memory>
+#include <string>
+
+namespace zilliz {
+namespace milvus {
+namespace engine {
+
+
+class ExecutionEngineImpl : public ExecutionEngine {
+public:
+
+    ExecutionEngineImpl(uint16_t dimension,
+            const std::string& location,
+            EngineType type);
+
+    Status AddWithIds(long n, const float *xdata, const long *xids) override;
+
+    size_t Count() const override;
+
+    size_t Size() const override;
+
+    size_t Dimension() const override;
+
+    size_t PhysicalSize() const override;
+
+    Status Serialize() override;
+
+    Status Load() override;
+
+    Status Merge(const std::string& location) override;
+
+    Status Search(long n,
+                  const float *data,
+                  long k,
+                  float *distances,
+                  long *labels) const override;
+
+    ExecutionEnginePtr BuildIndex(const std::string&) override;
+
+    Status Cache() override;
+
+    Status Init() override;
+
+private:
+    vecwise::engine::VecIndexPtr CreatetVecIndex(EngineType type);
+
+protected:
+    vecwise::engine::VecIndexPtr index_;
+
+    std::string location_;
+
+    size_t nprobe_ = 0;
+};
+
+
+} // namespace engine
+} // namespace milvus
+} // namespace zilliz

cpp/src/server/RequestTask.cpp

@@ -29,7 +29,7 @@ namespace {
         static std::map<int, engine::EngineType> map_type = {
                 {0, engine::EngineType::INVALID},
                 {1, engine::EngineType::FAISS_IDMAP},
-                {2, engine::EngineType::FAISS_IVFFLAT},
+                {2, engine::EngineType::FAISS_IVFFLAT_GPU},
         };
 
         if(map_type.find(type) == map_type.end()) {
@@ -43,7 +43,7 @@ namespace {
         static std::map<engine::EngineType, int> map_type = {
                 {engine::EngineType::INVALID, 0},
                 {engine::EngineType::FAISS_IDMAP, 1},
-                {engine::EngineType::FAISS_IVFFLAT, 2},
+                {engine::EngineType::FAISS_IVFFLAT_GPU, 2},
         };
 
         if(map_type.find(type) == map_type.end()) {

cpp/unittest/db/misc_test.cpp

@@ -46,7 +46,7 @@ TEST(DBMiscTest, ENGINE_API_TEST) {
     auto engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::INVALID);
     ASSERT_EQ(engine_ptr, nullptr);
 
-    engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT);
+    engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT_GPU);
     ASSERT_NE(engine_ptr, nullptr);
 
     engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IDMAP);