add new interface definition

Former-commit-id: ca855d7705452a546df73e50626156931d2c4db3

cpp/build.sh

@@ -11,7 +11,7 @@ PROFILING="OFF"
 BUILD_FAISS_WITH_MKL="OFF"
 USE_JFROG_CACHE="OFF"
 KNOWHERE_OPTS=""
-MILVUS_WITH_THRIFT="ON"
+MILVUS_WITH_THRIFT="OFF"
 
 while getopts "p:d:t:uhlrcgmj" arg
 do

cpp/cmake/DefineOptions.cmake

@@ -90,7 +90,7 @@ define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
 
 define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
 
-define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" ON)
+define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" OFF)
 
 define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
 

cpp/cmake/ThirdPartyPackages.cmake

@@ -2679,7 +2679,7 @@ macro(build_grpc)
     add_dependencies(grpc_protoc grpc_ep)
 endmacro()
 
-#if(NOT MILVUS_WITH_THRIFT STREQUAL "ON")
+if(NOT MILVUS_WITH_THRIFT STREQUAL "ON")
     resolve_dependency(GRPC)
 
     get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
@@ -2690,4 +2690,4 @@ endmacro()
     include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
     link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
 
-#endif()
+endif()

cpp/src/CMakeLists.txt

@@ -35,6 +35,7 @@ set(license_generator_files
         license/LicenseLibrary.cpp
         )
 
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
 set(thrift_service_files
         thrift/gen-cpp/MilvusService.cpp
         thrift/gen-cpp/milvus_constants.cpp
@@ -44,7 +45,7 @@ set(thrift_service_files
         server/thrift_impl/ThreadPoolServer.cpp
         server/thrift_impl/ThreadPoolServer.h
         )
-
+else()
 set(grpc_service_files
         metrics/SystemInfo.cpp
         metrics/SystemInfo.h
@@ -53,6 +54,7 @@ set(grpc_service_files
         grpc/gen-status/status.grpc.pb.cc
         grpc/gen-status/status.pb.cc
         )
+endif()
 
 set(db_files
         ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
@@ -77,17 +79,17 @@ include_directories(/usr/include/mysql)
 include_directories(grpc/gen-status)
 include_directories(grpc/gen-milvus)
 
-#if (MILVUS_WITH_THRIFT STREQUAL "ON")
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
     set(client_thrift_lib
             thrift)
-#else()
+else()
     set(client_grpc_lib
             grpcpp_channelz
             grpc++
             grpc
             grpc_protobuf
             grpc_protoc)
-#endif()
+endif()
 
 set(third_party_libs
         knowhere
@@ -197,34 +199,25 @@ set(knowhere_libs
         tbb
         )
 
-#if (MILVUS_WITH_THRIFT STREQUAL "ON")
-#    add_executable(milvus_thrift_server
-#            ${config_files}
-#            ${server_files}
-#            ${thriftserver_files}
-#            ${utils_files}
-#            ${thrift_service_files}
-#            ${metrics_files}
-#            )
-#else()
-#    add_executable(milvus_grpc_server
-#            ${config_files}
-#            ${server_files}
-#            ${grpcserver_files}
-#            ${utils_files}
-#            ${grpc_service_files}
-#            ${metrics_files}
-#            )
-#endif()
-add_executable(milvus_server
-        ${config_files}
-        ${server_files}
-        ${thriftserver_files}
-        ${grpcserver_files}
-        ${utils_files}
-        ${thrift_service_files}
-        ${grpc_service_files}
-        ${metrics_files})
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
+    add_executable(milvus_thrift_server
+            ${config_files}
+            ${server_files}
+            ${thriftserver_files}
+            ${utils_files}
+            ${thrift_service_files}
+            ${metrics_files}
+            )
+else()
+    add_executable(milvus_grpc_server
+            ${config_files}
+            ${server_files}
+            ${grpcserver_files}
+            ${utils_files}
+            ${grpc_service_files}
+            ${metrics_files}
+            )
+endif()
 
 if (ENABLE_LICENSE STREQUAL "ON")
 	add_executable(get_sys_info ${get_sys_info_files})
@@ -233,28 +226,25 @@ if (ENABLE_LICENSE STREQUAL "ON")
 	target_link_libraries(get_sys_info ${license_libs} license_check ${third_party_libs})
 	target_link_libraries(license_generator ${license_libs} ${third_party_libs})
 
-#    if(MILVUS_WITH_THRIFT STREQUAL "ON")
-#        target_link_libraries(milvus_thrift_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
-#    else()
-#        target_link_libraries(milvus_grpc_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
-#    endif()
-    target_link_libraries(milvus_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
+    if(MILVUS_WITH_THRIFT STREQUAL "ON")
+        target_link_libraries(milvus_thrift_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
+    else()
+        target_link_libraries(milvus_grpc_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
+    endif()
 
 else ()
-#    if(MILVUS_WITH_THRIFT STREQUAL "ON")
-#        target_link_libraries(milvus_thrift_server ${server_libs} ${knowhere_libs} ${third_party_libs})
-#    else()
-#        target_link_libraries(milvus_grpc_server ${server_libs} ${knowhere_libs} ${third_party_libs})
-#    endif()
-        target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
+    if(MILVUS_WITH_THRIFT STREQUAL "ON")
+        target_link_libraries(milvus_thrift_server ${server_libs} ${knowhere_libs} ${third_party_libs})
+    else()
+        target_link_libraries(milvus_grpc_server ${server_libs} ${knowhere_libs} ${third_party_libs})
+    endif()
 endif()
 
-#if (MILVUS_WITH_THRIFT STREQUAL "ON")
-#    install(TARGETS milvus_thrift_server DESTINATION bin)
-#else()
-#    install(TARGETS milvus_grpc_server DESTINATION bin)
-#endif()
-    install(TARGETS milvus_server DESTINATION bin)
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
+    install(TARGETS milvus_thrift_server DESTINATION bin)
+else()
+    install(TARGETS milvus_grpc_server DESTINATION bin)
+endif()
 
 install(FILES
         ${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}

cpp/src/db/DBImpl.cpp

@@ -89,6 +89,7 @@ DBImpl::DBImpl(const Options& options)
     meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode);
     mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_);
     if (options.mode != Options::MODE::READ_ONLY) {
+        std::cout << "StartTimerTasks\n";
         ENGINE_LOG_TRACE << "StartTimerTasks";
         StartTimerTasks();
     }

cpp/src/grpc/cpp_gen.sh

 #!/bin/bash
 
-protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` status.proto
+/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" status.proto
 
-protoc -I . --cpp_out=./gen-status status.proto
+/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-status status.proto
 
-protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` milvus.proto
+/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" milvus.proto
 
-protoc -I . --cpp_out=./gen-milvus milvus.proto
\ No newline at end of file
+/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto
\ No newline at end of file

cpp/src/grpc/milvus.proto

@@ -38,9 +38,9 @@ message RowRecord {
 }
 
 /**
- * @brief Infos to be inserted
+ * @brief params to be inserted
  */
-message InsertInfos {
+message InsertParam {
     string table_name = 1;
     repeated RowRecord row_record_array = 2;
 }
@@ -54,9 +54,9 @@ message VectorIds {
 }
 
 /**
- * @brief Infos for searching vector
+ * @brief params for searching vector
  */
-message SearchVectorInfos {
+message SearchParam {
     string table_name = 1;
     repeated RowRecord query_record_array = 2;
     repeated Range query_range_array = 3;
@@ -64,15 +64,15 @@ message SearchVectorInfos {
 }
 
 /**
- * @brief Infos for searching vector in files
+ * @brief params for searching vector in files
  */
-message SearchVectorInFilesInfos {
+message SearchInFilesParam {
     repeated string file_id_array = 1;
-    SearchVectorInfos search_vector_infos = 2;
+    SearchParam search_param = 2;
 }
 
 /**
- * @brief Query result infos
+ * @brief Query result params
  */
 message QueryResult {
     int64 id = 1;
@@ -119,11 +119,28 @@ message Command {
 }
 
 /**
- * @brief Give Server Command
+ * @brief Index
  */
-message ServerStatus{
-    Status status = 1;
-    string info = 2;
+message Index {
+    int32 index_type = 1;
+    int64 nlist = 2;
+    int32 index_file_size = 3;
+}
+
+/**
+ * @brief Index params
+ */
+message IndexParam {
+    TableName table_name = 1;
+    Index index = 2;
+}
+
+/**
+ * @brief table name and range for DeleteByRange
+ */
+message DeleteByRangeParam {
+    Range range = 1;
+    string table_name = 2;
 }
 
 service MilvusService {
@@ -165,7 +182,7 @@ service MilvusService {
      * @param table_name, table is going to be built index.
      *
      */
-    rpc BuildIndex(TableName) returns (Status) {}
+    rpc CreateIndex(IndexParam) returns (Status) {}
 
     /**
      * @brief Add vector array to table
@@ -177,7 +194,7 @@ service MilvusService {
      *
      * @return vector id array
      */
-    rpc InsertVector(InsertInfos) returns (VectorIds) {}
+    rpc Insert(InsertParam) returns (VectorIds) {}
 
     /**
      * @brief Query vector
@@ -191,7 +208,7 @@ service MilvusService {
      *
      * @return query result array.
      */
-    rpc SearchVector(SearchVectorInfos) returns (stream TopKQueryResult) {}
+    rpc Search(SearchParam) returns (stream TopKQueryResult) {}
 
     /**
      * @brief Internal use query interface
@@ -205,7 +222,7 @@ service MilvusService {
      *
      * @return query result array.
      */
-    rpc SearchVectorInFiles(SearchVectorInFilesInfos) returns (stream TopKQueryResult) {}
+    rpc SearchInFiles(SearchInFilesParam) returns (stream TopKQueryResult) {}
 
     /**
      * @brief Get table schema
@@ -227,7 +244,7 @@ service MilvusService {
      *
      * @return table schema
      */
-    rpc GetTableRowCount(TableName) returns (TableRowCount) {}
+    rpc CountTable(TableName) returns (TableRowCount) {}
 
     /**
      * @brief List all tables in database
@@ -246,5 +263,42 @@ service MilvusService {
      *
      * @return Server status.
      */
-    rpc Ping(Command) returns (ServerStatus) {}
+    rpc Cmd(Command) returns (StringReply) {}
+
+     /**
+     * @brief delete table by range
+     *
+     * This method is used to delete vector by range
+     *
+     * @return rpc status.
+     */
+     rpc DeleteByRange(DeleteByRangeParam) returns (Status) {}
+
+     /**
+      * @brief preload table
+      *
+      * This method is used to preload table
+      *
+      * @return Status.
+      */
+     rpc PreloadTable(TableName) returns (Status) {}
+
+     /**
+      * @brief describe index
+      *
+      * This method is used to describe index
+      *
+      * @return Status.
+      */
+     rpc DescribeIndex(TableName) returns (IndexParam) {}
+
+     /**
+      * @brief drop index
+      *
+      * This method is used to drop index
+      *
+      * @return Status.
+      */
+     rpc DropIndex(TableName) returns (Status) {}
+
 }

cpp/src/sdk/CMakeLists.txt

@@ -12,7 +12,7 @@ include_directories(/usr/include)
 include_directories(include)
 include_directories(/usr/local/include)
 
-#if (MILVUS_WITH_THRIFT STREQUAL "ON")
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
     aux_source_directory(thrift thrift_client_files)
     include_directories(thrift)
     include_directories(${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp)
@@ -34,7 +34,7 @@ include_directories(/usr/local/include)
             ${third_party_libs}
             )
     install(TARGETS milvus_thrift_sdk DESTINATION lib)
-#else()
+else()
     aux_source_directory(grpc grpc_client_files)
 
     include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
@@ -58,6 +58,6 @@ include_directories(/usr/local/include)
             ${third_party_libs}
             )
     install(TARGETS milvus_grpc_sdk DESTINATION lib)
-#endif()
+endif()
 
 add_subdirectory(examples)

cpp/src/sdk/examples/CMakeLists.txt

@@ -4,8 +4,8 @@
 # Proprietary and confidential.
 #-------------------------------------------------------------------------------
 
-#if (MILVUS_WITH_THRIFT STREQUAL "ON")
+if (MILVUS_WITH_THRIFT STREQUAL "ON")
     add_subdirectory(thriftsimple)
-#else()
+else()
     add_subdirectory(grpcsimple)
-#endif()
\ No newline at end of file
+endif()
\ No newline at end of file

cpp/src/sdk/examples/grpcsimple/main.cpp

@@ -25,7 +25,7 @@ main(int argc, char *argv[]) {
                                            {NULL, 0, 0, 0}};
 
     int option_index = 0;
-    std::string address = "127.0.0.1", port = "19531";
+    std::string address = "127.0.0.1", port = "19530";
     app_name = argv[0];
 
     int value;

cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp

@@ -174,7 +174,7 @@ namespace {
         std::vector<TopKQueryResult> topk_query_result_array;
         {
             TimeRecorder rc(phase_name);
-            Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
+            Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
             std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
         }
 
@@ -211,7 +211,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
         for(auto& table : tables) {
             int64_t row_count = 0;
 //            conn->DropTable(table);
-            stat = conn->GetTableRowCount(table, row_count);
+            stat = conn->CountTable(table, row_count);
             std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
         }
     }
@@ -290,7 +290,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
 
             auto start = std::chrono::high_resolution_clock::now();
 
-            Status stat = conn->InsertVector(TABLE_NAME, record_array, record_ids);
+            Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
             auto finish = std::chrono::high_resolution_clock::now();
             std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
 
@@ -311,9 +311,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
     }
 
     {//wait unit build index finish
-        std::cout << "Wait until build all index done" << std::endl;
-        Status stat = conn->BuildIndex(TABLE_NAME);
-        std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
+//        std::cout << "Wait until build all index done" << std::endl;
+//        Status stat = conn->CreateIndex();
+//        std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
     }
 
     {//search vectors after build index finish

cpp/src/sdk/grpc/ClientProxy.cpp

@@ -54,7 +54,7 @@ Status
 ClientProxy::Connected() const {
     try {
         std::string info;
-        return client_ptr_->Ping(info, "");
+        return client_ptr_->Cmd(info, "");
     } catch (std::exception &ex) {
         return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
     }
@@ -101,11 +101,6 @@ ClientProxy::HasTable(const std::string &table_name) {
     return result;
 }
 
-Status
-ClientProxy::DeleteTable(const std::string &table_name) {
-    return this->DropTable(table_name);
-}
-
 Status
 ClientProxy::DropTable(const std::string &table_name) {
     try {
@@ -118,11 +113,13 @@ ClientProxy::DropTable(const std::string &table_name) {
 }
 
 Status
-ClientProxy::BuildIndex(const std::string &table_name) {
+ClientProxy::CreateIndex(const IndexParam &index_param) {
     try {
-        ::milvus::grpc::TableName grpc_table_name;
-        grpc_table_name.set_table_name(table_name);
-        return client_ptr_->BuildIndex(grpc_table_name);
+        //TODO:add index params
+        ::milvus::grpc::IndexParam grpc_index_param;
+        grpc_index_param.mutable_table_name()->set_table_name(
+                index_param.table_name);
+        return client_ptr_->CreateIndex(grpc_index_param);
 
     } catch (std::exception &ex) {
         return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
@@ -130,14 +127,7 @@ ClientProxy::BuildIndex(const std::string &table_name) {
 }
 
 Status
-ClientProxy::AddVector(const std::string &table_name,
-                          const std::vector<RowRecord> &record_array,
-                          std::vector<int64_t> &id_array) {
-    return InsertVector(table_name, record_array, id_array);
-}
-
-Status
-ClientProxy::InsertVector(const std::string &table_name,
+ClientProxy::Insert(const std::string &table_name,
                           const std::vector<RowRecord> &record_array,
                           std::vector<int64_t> &id_array) {
     Status status = Status::OK();
@@ -187,11 +177,11 @@ ClientProxy::InsertVector(const std::string &table_name,
             }
         }
 #else
-        ::milvus::grpc::InsertInfos insert_infos;
-        insert_infos.set_table_name(table_name);
+        ::milvus::grpc::InsertParam insert_param;
+        insert_param.set_table_name(table_name);
 
         for (auto &record : record_array) {
-            ::milvus::grpc::RowRecord *grpc_record = insert_infos.add_row_record_array();
+            ::milvus::grpc::RowRecord *grpc_record = insert_param.add_row_record_array();
             for (size_t i = 0; i < record.data.size(); i++) {
                 grpc_record->add_vector_data(record.data[i]);
             }
@@ -200,7 +190,7 @@ ClientProxy::InsertVector(const std::string &table_name,
         ::milvus::grpc::VectorIds vector_ids;
 
         //Single thread
-        client_ptr_->InsertVector(vector_ids, insert_infos, status);
+        client_ptr_->Insert(vector_ids, insert_param, status);
         auto finish = std::chrono::high_resolution_clock::now();
 
         for (size_t i = 0; i < vector_ids.vector_id_array_size(); i++) {
@@ -216,18 +206,18 @@ ClientProxy::InsertVector(const std::string &table_name,
 }
 
 Status
-ClientProxy::SearchVector(const std::string &table_name,
+ClientProxy::Search(const std::string &table_name,
                           const std::vector<RowRecord> &query_record_array,
                           const std::vector<Range> &query_range_array,
                           int64_t topk,
                           std::vector<TopKQueryResult> &topk_query_result_array) {
     try {
         //step 1: convert vectors data
-        ::milvus::grpc::SearchVectorInfos search_vector_infos;
-        search_vector_infos.set_table_name(table_name);
-        search_vector_infos.set_topk(topk);
+        ::milvus::grpc::SearchParam search_param;
+        search_param.set_table_name(table_name);
+        search_param.set_topk(topk);
         for (auto &record : query_record_array) {
-            ::milvus::grpc::RowRecord *row_record = search_vector_infos.add_query_record_array();
+            ::milvus::grpc::RowRecord *row_record = search_param.add_query_record_array();
             for (auto &rec : record.data) {
                 row_record->add_vector_data(rec);
             }
@@ -235,14 +225,14 @@ ClientProxy::SearchVector(const std::string &table_name,
 
         //step 2: convert range array
         for (auto &range : query_range_array) {
-            ::milvus::grpc::Range *grpc_range = search_vector_infos.add_query_range_array();
+            ::milvus::grpc::Range *grpc_range = search_param.add_query_range_array();
             grpc_range->set_start_value(range.start_value);
             grpc_range->set_end_value(range.end_value);
         }
 
         //step 3: search vectors
         std::vector<::milvus::grpc::TopKQueryResult> result_array;
-        Status status = client_ptr_->SearchVector(result_array, search_vector_infos);
+        Status status = client_ptr_->Search(result_array, search_param);
 
         //step 4: convert result array
         for (auto &grpc_topk_result : result_array) {
@@ -284,10 +274,10 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
 }
 
 Status
-ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
+ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) {
     try {
         Status status;
-        row_count = client_ptr_->GetTableRowCount(table_name, status);
+        row_count = client_ptr_->CountTable(table_name, status);
         return status;
     } catch (std::exception &ex) {
         return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what()));
@@ -309,7 +299,7 @@ ClientProxy::ServerVersion() const {
     Status status = Status::OK();
     try {
         std::string version;
-        Status status = client_ptr_->Ping(version, "version");
+        Status status = client_ptr_->Cmd(version, "version");
         return version;
     } catch (std::exception &ex) {
         return "";
@@ -324,11 +314,31 @@ ClientProxy::ServerStatus() const {
 
     try {
         std::string dummy;
-        Status status = client_ptr_->Ping(dummy, "");
+        Status status = client_ptr_->Cmd(dummy, "");
         return "server alive";
     } catch (std::exception &ex) {
         return "connection lost";
     }
 }
 
+Status
+ClientProxy::DeleteByRange(milvus::Range &range, const std::string &table_name) {
+
+}
+
+Status
+ClientProxy::PreloadTable(const std::string &table_name) const {
+
+}
+
+IndexParam
+ClientProxy::DescribeIndex(const std::string &table_name) const {
+
+}
+
+Status
+ClientProxy::DropIndex(const std::string &table_name) const {
+
+}
+
 }