提交 468120bb 编写于 作者: P peng.xu

Merge branch 'branch-0.4.0' into 'branch-0.4.0'

add new interface definition and move milvus_server to milvus_grpc_server

See merge request megasearch/milvus!335

Former-commit-id: 4cfb155aff68852ca68cbc11dda129749d5bed20
......@@ -11,7 +11,7 @@ PROFILING="OFF"
BUILD_FAISS_WITH_MKL="OFF"
USE_JFROG_CACHE="OFF"
KNOWHERE_OPTS=""
MILVUS_WITH_THRIFT="ON"
MILVUS_WITH_THRIFT="OFF"
while getopts "p:d:t:uhlrcgmj" arg
do
......
......@@ -90,7 +90,7 @@ define_option(MILVUS_WITH_SQLITE_ORM "Build with SQLite ORM library" ON)
define_option(MILVUS_WITH_MYSQLPP "Build with MySQL++" ON)
define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" ON)
define_option(MILVUS_WITH_THRIFT "Build with Apache Thrift library" OFF)
define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
......
......@@ -2679,7 +2679,7 @@ macro(build_grpc)
add_dependencies(grpc_protoc grpc_ep)
endmacro()
#if(NOT MILVUS_WITH_THRIFT STREQUAL "ON")
if(NOT MILVUS_WITH_THRIFT STREQUAL "ON")
resolve_dependency(GRPC)
get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
......@@ -2690,4 +2690,4 @@ endmacro()
include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
#endif()
endif()
#!/bin/bash
../bin/milvus_server -c ../conf/server_config.yaml -l ../conf/log_config.conf
../bin/milvus_grpc_server -c ../conf/server_config.yaml -l ../conf/log_config.conf
......@@ -36,6 +36,7 @@ set(license_generator_files
license/LicenseLibrary.cpp
)
if (MILVUS_WITH_THRIFT STREQUAL "ON")
set(thrift_service_files
thrift/gen-cpp/MilvusService.cpp
thrift/gen-cpp/milvus_constants.cpp
......@@ -45,7 +46,7 @@ set(thrift_service_files
server/thrift_impl/ThreadPoolServer.cpp
server/thrift_impl/ThreadPoolServer.h
)
else()
set(grpc_service_files
metrics/SystemInfo.cpp
metrics/SystemInfo.h
......@@ -54,6 +55,7 @@ set(grpc_service_files
grpc/gen-status/status.grpc.pb.cc
grpc/gen-status/status.pb.cc
)
endif()
set(db_files
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
......@@ -79,17 +81,17 @@ include_directories(/usr/include/mysql)
include_directories(grpc/gen-status)
include_directories(grpc/gen-milvus)
#if (MILVUS_WITH_THRIFT STREQUAL "ON")
if (MILVUS_WITH_THRIFT STREQUAL "ON")
set(client_thrift_lib
thrift)
#else()
else()
set(client_grpc_lib
grpcpp_channelz
grpc++
grpc
grpc_protobuf
grpc_protoc)
#endif()
endif()
set(third_party_libs
knowhere
......@@ -199,34 +201,25 @@ set(knowhere_libs
tbb
)
#if (MILVUS_WITH_THRIFT STREQUAL "ON")
# add_executable(milvus_thrift_server
# ${config_files}
# ${server_files}
# ${thriftserver_files}
# ${utils_files}
# ${thrift_service_files}
# ${metrics_files}
# )
#else()
# add_executable(milvus_grpc_server
# ${config_files}
# ${server_files}
# ${grpcserver_files}
# ${utils_files}
# ${grpc_service_files}
# ${metrics_files}
# )
#endif()
add_executable(milvus_server
${config_files}
${server_files}
${thriftserver_files}
${grpcserver_files}
${utils_files}
${thrift_service_files}
${grpc_service_files}
${metrics_files})
if (MILVUS_WITH_THRIFT STREQUAL "ON")
add_executable(milvus_thrift_server
${config_files}
${server_files}
${thriftserver_files}
${utils_files}
${thrift_service_files}
${metrics_files}
)
else()
add_executable(milvus_grpc_server
${config_files}
${server_files}
${grpcserver_files}
${utils_files}
${grpc_service_files}
${metrics_files}
)
endif()
if (ENABLE_LICENSE STREQUAL "ON")
add_executable(get_sys_info ${get_sys_info_files})
......@@ -235,28 +228,25 @@ if (ENABLE_LICENSE STREQUAL "ON")
target_link_libraries(get_sys_info ${license_libs} license_check ${third_party_libs})
target_link_libraries(license_generator ${license_libs} ${third_party_libs})
# if(MILVUS_WITH_THRIFT STREQUAL "ON")
# target_link_libraries(milvus_thrift_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
# else()
# target_link_libraries(milvus_grpc_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
# endif()
target_link_libraries(milvus_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
if(MILVUS_WITH_THRIFT STREQUAL "ON")
target_link_libraries(milvus_thrift_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
else()
target_link_libraries(milvus_grpc_server ${server_libs} license_check ${knowhere_libs} ${third_party_libs})
endif()
else ()
# if(MILVUS_WITH_THRIFT STREQUAL "ON")
# target_link_libraries(milvus_thrift_server ${server_libs} ${knowhere_libs} ${third_party_libs})
# else()
# target_link_libraries(milvus_grpc_server ${server_libs} ${knowhere_libs} ${third_party_libs})
# endif()
target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
if(MILVUS_WITH_THRIFT STREQUAL "ON")
target_link_libraries(milvus_thrift_server ${server_libs} ${knowhere_libs} ${third_party_libs})
else()
target_link_libraries(milvus_grpc_server ${server_libs} ${knowhere_libs} ${third_party_libs})
endif()
endif()
#if (MILVUS_WITH_THRIFT STREQUAL "ON")
# install(TARGETS milvus_thrift_server DESTINATION bin)
#else()
# install(TARGETS milvus_grpc_server DESTINATION bin)
#endif()
install(TARGETS milvus_server DESTINATION bin)
if (MILVUS_WITH_THRIFT STREQUAL "ON")
install(TARGETS milvus_thrift_server DESTINATION bin)
else()
install(TARGETS milvus_grpc_server DESTINATION bin)
endif()
install(FILES
${KNOWHERE_BUILD_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}tbb${CMAKE_SHARED_LIBRARY_SUFFIX}
......
#!/bin/bash
protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` status.proto
/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-status --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" status.proto
protoc -I . --cpp_out=./gen-status status.proto
/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-status status.proto
protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` milvus.proto
/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --grpc_out=./gen-milvus --plugin=protoc-gen-grpc="/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/grpc_cpp_plugin" milvus.proto
protoc -I . --cpp_out=./gen-milvus milvus.proto
\ No newline at end of file
/home/yukun/test/milvus/cpp/cmake-build-debug/grpc_ep-prefix/src/grpc_ep/bins/opt/protobuf/protoc -I . --cpp_out=./gen-milvus milvus.proto
\ No newline at end of file
此差异已折叠。
......@@ -38,9 +38,9 @@ message RowRecord {
}
/**
* @brief Infos to be inserted
* @brief params to be inserted
*/
message InsertInfos {
message InsertParam {
string table_name = 1;
repeated RowRecord row_record_array = 2;
}
......@@ -54,9 +54,9 @@ message VectorIds {
}
/**
* @brief Infos for searching vector
* @brief params for searching vector
*/
message SearchVectorInfos {
message SearchParam {
string table_name = 1;
repeated RowRecord query_record_array = 2;
repeated Range query_range_array = 3;
......@@ -64,15 +64,15 @@ message SearchVectorInfos {
}
/**
* @brief Infos for searching vector in files
* @brief params for searching vector in files
*/
message SearchVectorInFilesInfos {
message SearchInFilesParam {
repeated string file_id_array = 1;
SearchVectorInfos search_vector_infos = 2;
SearchParam search_param = 2;
}
/**
* @brief Query result infos
* @brief Query result params
*/
message QueryResult {
int64 id = 1;
......@@ -119,11 +119,28 @@ message Command {
}
/**
* @brief Give Server Command
* @brief Index
*/
message ServerStatus{
Status status = 1;
string info = 2;
message Index {
int32 index_type = 1;
int64 nlist = 2;
int32 index_file_size = 3;
}
/**
* @brief Index params
*/
message IndexParam {
TableName table_name = 1;
Index index = 2;
}
/**
* @brief table name and range for DeleteByRange
*/
message DeleteByRangeParam {
Range range = 1;
string table_name = 2;
}
service MilvusService {
......@@ -165,7 +182,7 @@ service MilvusService {
* @param table_name, table is going to be built index.
*
*/
rpc BuildIndex(TableName) returns (Status) {}
rpc CreateIndex(IndexParam) returns (Status) {}
/**
* @brief Add vector array to table
......@@ -177,7 +194,7 @@ service MilvusService {
*
* @return vector id array
*/
rpc InsertVector(InsertInfos) returns (VectorIds) {}
rpc Insert(InsertParam) returns (VectorIds) {}
/**
* @brief Query vector
......@@ -191,7 +208,7 @@ service MilvusService {
*
* @return query result array.
*/
rpc SearchVector(SearchVectorInfos) returns (stream TopKQueryResult) {}
rpc Search(SearchParam) returns (stream TopKQueryResult) {}
/**
* @brief Internal use query interface
......@@ -205,7 +222,7 @@ service MilvusService {
*
* @return query result array.
*/
rpc SearchVectorInFiles(SearchVectorInFilesInfos) returns (stream TopKQueryResult) {}
rpc SearchInFiles(SearchInFilesParam) returns (stream TopKQueryResult) {}
/**
* @brief Get table schema
......@@ -227,7 +244,7 @@ service MilvusService {
*
* @return table schema
*/
rpc GetTableRowCount(TableName) returns (TableRowCount) {}
rpc CountTable(TableName) returns (TableRowCount) {}
/**
* @brief List all tables in database
......@@ -246,5 +263,42 @@ service MilvusService {
*
* @return Server status.
*/
rpc Ping(Command) returns (ServerStatus) {}
rpc Cmd(Command) returns (StringReply) {}
/**
* @brief delete table by range
*
* This method is used to delete vector by range
*
* @return rpc status.
*/
rpc DeleteByRange(DeleteByRangeParam) returns (Status) {}
/**
* @brief preload table
*
* This method is used to preload table
*
* @return Status.
*/
rpc PreloadTable(TableName) returns (Status) {}
/**
* @brief describe index
*
* This method is used to describe index
*
* @return Status.
*/
rpc DescribeIndex(TableName) returns (IndexParam) {}
/**
* @brief drop index
*
* This method is used to drop index
*
* @return Status.
*/
rpc DropIndex(TableName) returns (Status) {}
}
......@@ -12,7 +12,7 @@ include_directories(/usr/include)
include_directories(include)
include_directories(/usr/local/include)
#if (MILVUS_WITH_THRIFT STREQUAL "ON")
if (MILVUS_WITH_THRIFT STREQUAL "ON")
aux_source_directory(thrift thrift_client_files)
include_directories(thrift)
include_directories(${CMAKE_SOURCE_DIR}/src/thrift/gen-cpp)
......@@ -34,7 +34,7 @@ include_directories(/usr/local/include)
${third_party_libs}
)
install(TARGETS milvus_thrift_sdk DESTINATION lib)
#else()
else()
aux_source_directory(grpc grpc_client_files)
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
......@@ -58,6 +58,6 @@ include_directories(/usr/local/include)
${third_party_libs}
)
install(TARGETS milvus_grpc_sdk DESTINATION lib)
#endif()
endif()
add_subdirectory(examples)
......@@ -4,8 +4,8 @@
# Proprietary and confidential.
#-------------------------------------------------------------------------------
#if (MILVUS_WITH_THRIFT STREQUAL "ON")
if (MILVUS_WITH_THRIFT STREQUAL "ON")
add_subdirectory(thriftsimple)
#else()
else()
add_subdirectory(grpcsimple)
#endif()
\ No newline at end of file
endif()
\ No newline at end of file
......@@ -25,7 +25,7 @@ main(int argc, char *argv[]) {
{NULL, 0, 0, 0}};
int option_index = 0;
std::string address = "127.0.0.1", port = "19531";
std::string address = "127.0.0.1", port = "19530";
app_name = argv[0];
int value;
......
......@@ -23,7 +23,7 @@ namespace {
constexpr int64_t NQ = 10;
constexpr int64_t TOP_K = 10;
constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different
constexpr int64_t ADD_VECTOR_LOOP = 1;
constexpr int64_t ADD_VECTOR_LOOP = 5;
constexpr int64_t SECONDS_EACH_HOUR = 3600;
#define BLOCK_SPLITER std::cout << "===========================================" << std::endl;
......@@ -174,7 +174,7 @@ namespace {
std::vector<TopKQueryResult> topk_query_result_array;
{
TimeRecorder rc(phase_name);
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
}
......@@ -211,7 +211,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
for(auto& table : tables) {
int64_t row_count = 0;
// conn->DropTable(table);
stat = conn->GetTableRowCount(table, row_count);
stat = conn->CountTable(table, row_count);
std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl;
}
}
......@@ -290,7 +290,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
auto start = std::chrono::high_resolution_clock::now();
Status stat = conn->InsertVector(TABLE_NAME, record_array, record_ids);
Status stat = conn->Insert(TABLE_NAME, record_array, record_ids);
auto finish = std::chrono::high_resolution_clock::now();
std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
......@@ -311,9 +311,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
}
{//wait unit build index finish
std::cout << "Wait until build all index done" << std::endl;
Status stat = conn->BuildIndex(TABLE_NAME);
std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
// std::cout << "Wait until build all index done" << std::endl;
// Status stat = conn->CreateIndex();
// std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;
}
{//search vectors after build index finish
......
......@@ -54,7 +54,7 @@ Status
ClientProxy::Connected() const {
try {
std::string info;
return client_ptr_->Ping(info, "");
return client_ptr_->Cmd(info, "");
} catch (std::exception &ex) {
return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what()));
}
......@@ -101,11 +101,6 @@ ClientProxy::HasTable(const std::string &table_name) {
return result;
}
Status
ClientProxy::DeleteTable(const std::string &table_name) {
return this->DropTable(table_name);
}
Status
ClientProxy::DropTable(const std::string &table_name) {
try {
......@@ -118,11 +113,13 @@ ClientProxy::DropTable(const std::string &table_name) {
}
Status
ClientProxy::BuildIndex(const std::string &table_name) {
ClientProxy::CreateIndex(const IndexParam &index_param) {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
return client_ptr_->BuildIndex(grpc_table_name);
//TODO:add index params
::milvus::grpc::IndexParam grpc_index_param;
grpc_index_param.mutable_table_name()->set_table_name(
index_param.table_name);
return client_ptr_->CreateIndex(grpc_index_param);
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "failed to build index: " + std::string(ex.what()));
......@@ -130,14 +127,7 @@ ClientProxy::BuildIndex(const std::string &table_name) {
}
Status
ClientProxy::AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) {
return InsertVector(table_name, record_array, id_array);
}
Status
ClientProxy::InsertVector(const std::string &table_name,
ClientProxy::Insert(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) {
Status status = Status::OK();
......@@ -187,11 +177,11 @@ ClientProxy::InsertVector(const std::string &table_name,
}
}
#else
::milvus::grpc::InsertInfos insert_infos;
insert_infos.set_table_name(table_name);
::milvus::grpc::InsertParam insert_param;
insert_param.set_table_name(table_name);
for (auto &record : record_array) {
::milvus::grpc::RowRecord *grpc_record = insert_infos.add_row_record_array();
::milvus::grpc::RowRecord *grpc_record = insert_param.add_row_record_array();
for (size_t i = 0; i < record.data.size(); i++) {
grpc_record->add_vector_data(record.data[i]);
}
......@@ -200,7 +190,7 @@ ClientProxy::InsertVector(const std::string &table_name,
::milvus::grpc::VectorIds vector_ids;
//Single thread
client_ptr_->InsertVector(vector_ids, insert_infos, status);
client_ptr_->Insert(vector_ids, insert_param, status);
auto finish = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < vector_ids.vector_id_array_size(); i++) {
......@@ -216,18 +206,18 @@ ClientProxy::InsertVector(const std::string &table_name,
}
Status
ClientProxy::SearchVector(const std::string &table_name,
ClientProxy::Search(const std::string &table_name,
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
try {
//step 1: convert vectors data
::milvus::grpc::SearchVectorInfos search_vector_infos;
search_vector_infos.set_table_name(table_name);
search_vector_infos.set_topk(topk);
::milvus::grpc::SearchParam search_param;
search_param.set_table_name(table_name);
search_param.set_topk(topk);
for (auto &record : query_record_array) {
::milvus::grpc::RowRecord *row_record = search_vector_infos.add_query_record_array();
::milvus::grpc::RowRecord *row_record = search_param.add_query_record_array();
for (auto &rec : record.data) {
row_record->add_vector_data(rec);
}
......@@ -235,14 +225,14 @@ ClientProxy::SearchVector(const std::string &table_name,
//step 2: convert range array
for (auto &range : query_range_array) {
::milvus::grpc::Range *grpc_range = search_vector_infos.add_query_range_array();
::milvus::grpc::Range *grpc_range = search_param.add_query_range_array();
grpc_range->set_start_value(range.start_value);
grpc_range->set_end_value(range.end_value);
}
//step 3: search vectors
std::vector<::milvus::grpc::TopKQueryResult> result_array;
Status status = client_ptr_->SearchVector(result_array, search_vector_infos);
Status status = client_ptr_->Search(result_array, search_param);
//step 4: convert result array
for (auto &grpc_topk_result : result_array) {
......@@ -284,10 +274,10 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
}
Status
ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
ClientProxy::CountTable(const std::string &table_name, int64_t &row_count) {
try {
Status status;
row_count = client_ptr_->GetTableRowCount(table_name, status);
row_count = client_ptr_->CountTable(table_name, status);
return status;
} catch (std::exception &ex) {
return Status(StatusCode::UnknownError, "fail to show tables: " + std::string(ex.what()));
......@@ -309,7 +299,7 @@ ClientProxy::ServerVersion() const {
Status status = Status::OK();
try {
std::string version;
Status status = client_ptr_->Ping(version, "version");
Status status = client_ptr_->Cmd(version, "version");
return version;
} catch (std::exception &ex) {
return "";
......@@ -324,11 +314,31 @@ ClientProxy::ServerStatus() const {
try {
std::string dummy;
Status status = client_ptr_->Ping(dummy, "");
Status status = client_ptr_->Cmd(dummy, "");
return "server alive";
} catch (std::exception &ex) {
return "connection lost";
}
}
Status
ClientProxy::DeleteByRange(milvus::Range &range, const std::string &table_name) {
}
Status
ClientProxy::PreloadTable(const std::string &table_name) const {
}
IndexParam
ClientProxy::DescribeIndex(const std::string &table_name) const {
}
Status
ClientProxy::DropIndex(const std::string &table_name) const {
}
}
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册