diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 845ec2cedabd7aeb68679eef5ec2f326750795bf..6ea62b4efb1036064db1563faeaf029f9a9b3a62 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -17,6 +17,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl - MS-204 - Support multi db_path - MS-206 - Support SQ8 index type +- MS-208 - Add buildinde interface for C++ SDK ## New Feature - MS-195 - Add nlist and use_blas_threshold conf diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index f45fb103b9f33eee2229c3d036213cca7f01b799..2ca4b24aa1bd513fdfac85eac4deeeba7e0c9408 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -22,19 +22,19 @@ EngineFactory::Build(uint16_t dimension, switch (type) { case EngineType::FAISS_IDMAP: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IDMAP, "IDMap,Flat")); break; } case EngineType::FAISS_IVFFLAT: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVF, "IDMap,Flat")); break; } case EngineType::FAISS_IVFSQ8: { execution_engine_ptr = - ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVFSQ8", "IDMap,Flat")); + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, BUILD_INDEX_TYPE_IVFSQ8, "IDMap,Flat")); break; } diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 26f144ff19f9ff42518f8815c2c1d449371f0311..dd22f9cb07811aba1ad453024642cea07bd84d59 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -162,7 +162,8 @@ Status FaissExecutionEngine::Cache() { Status FaissExecutionEngine::Init() { - if(build_index_type_ == "IVF") { + if(build_index_type_ == BUILD_INDEX_TYPE_IVF || + build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); @@ -170,7 +171,7 @@ Status FaissExecutionEngine::Init() { nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); nlist_ = engine_config.GetInt32Value(CONFIG_NLIST,16384); - } else if(build_index_type_ == "IDMap") { + } else if(build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ; } else { return Status::Error("Wrong index type: ", build_index_type_); diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index eda0e4d70e45eb3f1e75e174f7ece8d38b2c6fa8..7005061ff697147ec02f1f9f0ad0ea5ce1e73c5e 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -15,6 +15,9 @@ namespace zilliz { namespace milvus { namespace engine { +const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap"; +const static std::string BUILD_INDEX_TYPE_IVF = "IVF"; +const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8"; class FaissExecutionEngine : public ExecutionEngine { public: diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index f2513ac2c3d22d4bd935358c9e6b25014153bcd8..b2f1c56ba4755d37a06103be5f636902dbb4982f 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -8,6 +8,7 @@ #include #include +#include #include using namespace ::milvus; @@ -126,6 +127,43 @@ namespace { std::cout << "Waiting " << seconds << " seconds ..." << std::endl; sleep(seconds); } + + class TimeRecorder { + public: + TimeRecorder(const std::string& title) + : title_(title) { + start_ = std::chrono::system_clock::now(); + } + + ~TimeRecorder() { + std::chrono::system_clock::time_point end = std::chrono::system_clock::now(); + long span = (std::chrono::duration_cast (end - start_)).count(); + std::cout << title_ << " totally cost: " << span << " ms" << std::endl; + } + + private: + std::string title_; + std::chrono::system_clock::time_point start_; + }; + + void DoSearch(std::shared_ptr conn, + const std::vector& record_array, + const std::string& phase_name) { + std::vector query_range_array; + Range rg; + rg.start_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(); + query_range_array.emplace_back(rg); + std::vector topk_query_result_array; + + { + TimeRecorder rc(phase_name); + Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); + std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; + } + + PrintSearchResult(topk_query_result_array); + } } void @@ -188,21 +226,22 @@ ClientTest::Test(const std::string& address, const std::string& port) { PrintRecordIdArray(record_ids); } - {//search vectors + std::vector search_record_array; + BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, search_record_array); + + {//search vectors without index Sleep(2); + DoSearch(conn, search_record_array, "Search without index"); + } - std::vector record_array; - BuildVectors(SEARCH_TARGET, SEARCH_TARGET + NQ, record_array); + {//wait unit build index finish + std::cout << "Wait until build all index done" << std::endl; + Status stat = conn->BuildIndex(TABLE_NAME); + std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; + } - std::vector query_range_array; - Range rg; - rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); - query_range_array.emplace_back(rg); - std::vector topk_query_result_array; - Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array); - std::cout << "SearchVector function call status: " << stat.ToString() << std::endl; - PrintSearchResult(topk_query_result_array); + {//search vectors after build index finish + DoSearch(conn, search_record_array, "Search after build index finish"); } {//delete table diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index ec146219818927770cd80681f88ba9d4ae568f07..a7529d3e00a97632c23d01c61a2d46db41ba0a68 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -181,6 +181,17 @@ public: virtual Status DeleteTable(const std::string &table_name) = 0; + /** + * @brief Build index method + * + * This method is used to build index for whole table + * + * @param table_name, table name is going to be build index. + * + * @return Indicate if build index successfully. + */ + virtual Status BuildIndex(const std::string &table_name) = 0; + /** * @brief Add vector to table * diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index d26007cd0be64829e611eee36d255df20d1bf970..81dc8d28e42c56e7a436fb08a5d0ffbe45a5046c 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -126,6 +126,22 @@ ClientProxy::DeleteTable(const std::string &table_name) { return Status::OK(); } +Status +ClientProxy::BuildIndex(const std::string &table_name) { + if(!IsConnected()) { + return Status(StatusCode::NotConnected, "not connected to server"); + } + + try { + ClientPtr()->interface()->BuildIndex(table_name); + + } catch ( std::exception& ex) { + return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what())); + } + + return Status::OK(); +} + Status ClientProxy::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index a2ede77c407262a004e82c39b344c6baa2de4560..3e4850ef0761a020df48025c76271d1b11520c68 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -27,6 +27,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override; diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index efb5f61b1bf293f018e40523002711236252bffc..737cd7d83b1b999210b0b37324600d6dcf2e8288 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -66,6 +66,11 @@ ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); } +Status +ConnectionImpl::BuildIndex(const std::string &table_name) { + return client_proxy_->BuildIndex(table_name); +} + Status ConnectionImpl::AddVector(const std::string &table_name, const std::vector &record_array, diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 0f9cd14e39c7be5c54158c9ee7935550b1fbff83..60c6cd9b2d42a3d93ce7046add476046488d62ff 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -29,6 +29,8 @@ public: virtual Status DeleteTable(const std::string &table_name) override; + virtual Status BuildIndex(const std::string &table_name) override; + virtual Status AddVector(const std::string &table_name, const std::vector &record_array, std::vector &id_array) override;