Merge branch 'branch-0.3.1-yuncong' into 'branch-0.3.1-yuncong'

MS-208 Add buildinde interface for C++ SDK

See merge request megasearch/milvus!190

Former-commit-id: 2aab4376fcffd10e48b15f46d2cd5fc7bc09ba1b

cpp/CHANGELOG.md

@@ -17,6 +17,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl
 - MS-204 - Support multi db_path
 - MS-206 - Support SQ8 index type
+- MS-208 - Add buildinde interface for C++ SDK
 
 ## New Feature
 - MS-195 - Add nlist and use_blas_threshold conf

cpp/src/db/EngineFactory.cpp

@@ -22,19 +22,19 @@ EngineFactory::Build(uint16_t dimension,
     switch (type) {
         case EngineType::FAISS_IDMAP: {
             execution_engine_ptr =
-                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat"));
+                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat"));
             break;
         }
 
         case EngineType::FAISS_IVFFLAT: {
             execution_engine_ptr =
-                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
+                ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat"));
             break;
         }
 
         case EngineType::FAISS_IVFSQ8: {
             execution_engine_ptr =
-                    ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat"));
+                    ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat"));
             break;
         }
 

cpp/src/db/FaissExecutionEngine.cpp

@@ -162,7 +162,8 @@ Status FaissExecutionEngine::Cache() {
 
 Status FaissExecutionEngine::Init() {
 
-    if(build_index_type_ == "IVF") {
+    if(build_index_type_ == BUILD_INDEX_TYPE_IVF ||
+        build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) {
 
         using namespace zilliz::milvus::server;
         ServerConfig &config = ServerConfig::GetInstance();
@@ -170,7 +171,7 @@ Status FaissExecutionEngine::Init() {
         nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
         nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384);
 
-    } else if(build_index_type_ == "IDMap") {
+    } else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) {
         ;
     } else {
         return Status::Error("Wrong index type: ", build_index_type_);

cpp/src/db/FaissExecutionEngine.h

@@ -15,6 +15,9 @@ namespace zilliz {
 namespace milvus {
 namespace engine {
 
+const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap";
+const static std::string BUILD_INDEX_TYPE_IVF = "IVF";
+const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8";
 
 class FaissExecutionEngine : public ExecutionEngine {
 public:

cpp/src/sdk/examples/simple/src/ClientTest.cpp

@@ -8,6 +8,7 @@
 
 #include <iostream>
 #include <time.h>
+#include <chrono>
 #include <unistd.h>
 
 using namespace ::milvus;
@@ -126,6 +127,43 @@ namespace {
         std::cout << "Waiting " << seconds << " seconds ..." << std::endl;
         sleep(seconds);
     }
+
+    class TimeRecorder {
+    public:
+        TimeRecorder(const std::string& title)
+        : title_(title) {
+            start_ = std::chrono::system_clock::now();
+        }
+
+        ~TimeRecorder() {
+            std::chrono::system_clock::time_point end = std::chrono::system_clock::now();
+            long span = (std::chrono::duration_cast<std::chrono::milliseconds> (end - start_)).count();
+            std::cout << title_ << " totally cost: " << span << " ms" << std::endl;
+        }
+
+    private:
+        std::string title_;
+        std::chrono::system_clock::time_point start_;
+    };
+
+    void DoSearch(std::shared_ptr<Connection> conn,
+                  const std::vector<RowRecord>& record_array,
+                  const std::string& phase_name) {
+        std::vector<Range> query_range_array;
+        Range rg;
+        rg.start_value = CurrentTmDate();
+        rg.end_value = CurrentTmDate();
+        query_range_array.emplace_back(rg);
+        std::vector<TopKQueryResult> topk_query_result_array;
+
+        {
+            TimeRecorder rc(phase_name);
+            Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
+            std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
+        }
+
+        PrintSearchResult(topk_query_result_array);
+    }
 }
 
 void
@@ -188,21 +226,22 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         PrintRecordIdArray(record_ids);
     }
 
-    {//search vectors
+    std::vector<RowRecord> search_record_array;
+    BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, search_record_array);
+
+    {//search vectors without index
         Sleep(2);
+        DoSearch(conn, search_record_array, "Search without index");
+    }
 
-        std::vector<RowRecord> record_array;
-        BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, record_array);
+    {//wait unit build index finish
+        std::cout << "Wait until build all index done" << std::endl;
+        Status stat = conn->BuildIndex(TABLE_NAME);
+        std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
+    }
 
-        std::vector<Range> query_range_array;
-        Range rg;
-        rg.start_value = CurrentTmDate();
-        rg.end_value = CurrentTmDate();
-        query_range_array.emplace_back(rg);
-        std::vector<TopKQueryResult> topk_query_result_array;
-        Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
-        std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
-        PrintSearchResult(topk_query_result_array);
+    {//search vectors after build index finish
+        DoSearch(conn, search_record_array, "Search after build index finish");
     }
 
     {//delete table

cpp/src/sdk/include/MilvusApi.h

@@ -181,6 +181,17 @@ public:
     virtual Status DeleteTable(const std::string &table_name) = 0;
 
 
+    /**
+     * @brief Build index method
+     *
+     * This method is used to build index for whole table
+     *
+     * @param table_name, table name is going to be build index.
+     *
+     * @return Indicate if build index successfully.
+     */
+    virtual Status BuildIndex(const std::string &table_name) = 0;
+
     /**
      * @brief Add vector to table
      *

cpp/src/sdk/src/client/ClientProxy.cpp

@@ -126,6 +126,22 @@ ClientProxy::DeleteTable(const std::string &table_name) {
     return Status::OK();
 }
 
+Status
+ClientProxy::BuildIndex(const std::string &table_name) {
+    if(!IsConnected()) {
+        return Status(StatusCode::NotConnected, "not connected to server");
+    }
+
+    try {
+        ClientPtr()->interface()->BuildIndex(table_name);
+
+    }  catch ( std::exception& ex) {
+        return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
+    }
+
+    return Status::OK();
+}
+
 Status
 ClientProxy::AddVector(const std::string &table_name,
                           const std::vector<RowRecord> &record_array,

cpp/src/sdk/src/client/ClientProxy.h

@@ -27,6 +27,8 @@ public:
 
     virtual Status DeleteTable(const std::string &table_name) override;
 
+    virtual Status BuildIndex(const std::string &table_name) override;
+
     virtual Status AddVector(const std::string &table_name,
                              const std::vector<RowRecord> &record_array,
                              std::vector<int64_t> &id_array) override;

cpp/src/sdk/src/interface/ConnectionImpl.cpp

@@ -66,6 +66,11 @@ ConnectionImpl::DeleteTable(const std::string &table_name) {
     return client_proxy_->DeleteTable(table_name);
 }
 
+Status
+ConnectionImpl::BuildIndex(const std::string &table_name) {
+    return client_proxy_->BuildIndex(table_name);
+}
+
 Status
 ConnectionImpl::AddVector(const std::string &table_name,
                           const std::vector<RowRecord> &record_array,

cpp/src/sdk/src/interface/ConnectionImpl.h

@@ -29,6 +29,8 @@ public:
 
     virtual Status DeleteTable(const std::string &table_name) override;
 
+    virtual Status BuildIndex(const std::string &table_name) override;
+
     virtual Status AddVector(const std::string &table_name,
                              const std::vector<RowRecord> &record_array,
                              std::vector<int64_t> &id_array) override;