Skip to content

milvus
milvus

BaiXuePrincess / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 7
Star 4
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 6c227a35 编写于 作者: Q quicksilver
浏览文件
操作

Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0

上级 e476af44 49ec7e1a
展开全部 隐藏空白更改
内联 并排
Showing with 1044 addition and 1678 deletion
CHANGELOG.md
浏览文件 @ 6c227a35
# Changelog
Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.6.0 (TODO)
## Bug
- \#228 - memory usage increased slowly during searching vectors
- \#246 - Exclude src/external folder from code coverage for jenkin ci
- \#248 - Reside src/external in thirdparty
- \#316 - Some files not merged after vectors added
## Feature
- \#12 - Pure CPU version for Milvus
......@@ -19,11 +20,27 @@ Please mark all change in change log and use the ticket from JIRA.
- \#260 - C++ SDK README
- \#314 - add Find FAISS in CMake
- \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
- \#322 - Add option to enable / disable prometheus
## Task
# Milvus 0.5.3 (2019-11-13)
## Bug
- \#258 - Bytes type in proto cause big-endian/little-endian problem
## Feature
## Improvement
- \#204 - improve grpc performance in search
- \#207 - Add more unittest for config set/get
- \#208 - optimize unittest to support run single test more easily
- \#284 - Change C++ SDK to shared library
- \#260 - C++ SDK README
## Task
# Milvus 0.5.2 (TODO)
# Milvus 0.5.2 (2019-11-07)
## Bug
- \#194 - Search faild: message="Table file doesn't exist"
......@@ -31,7 +48,7 @@ Please mark all change in change log and use the ticket from JIRA.
## Feature
## Improvement
- \#190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.52
- \#190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.5.2
## Task
......
core/.gitignore
浏览文件 @ 6c227a35
......@@ -2,7 +2,7 @@ milvus/
conf/server_config.yaml
conf/log_config.conf
src/config.h
version.h
src/version.h
lcov_out/
base.info
output.info
......
core/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}")
MACRO(GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND sh "-c" "git log --decorate | head -n 1 | sed 's/.*(\\(.*\\))/\\1/' | sed 's/.* \\(.*\\),.*/\\1/' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
ENDMACRO(GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
......@@ -67,7 +67,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL "")
message(WARNING "Failed to determine Milvus version from git branch name")
set(MILVUS_VERSION "0.6.0")
endif ()
endif()
message(STATUS "Build version = ${MILVUS_VERSION}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/config.h @ONLY)
......@@ -117,17 +117,17 @@ include(DefineOptions)
include(BuildUtils)
include(ThirdPartyPackages)
if(MILVUS_USE_CCACHE)
find_program(CCACHE_FOUND ccache)
if(CCACHE_FOUND)
message(STATUS "Using ccache: ${CCACHE_FOUND}")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set(ENV{CCACHE_COMMENTS} "1")
endif(CCACHE_FOUND)
endif()
if (MILVUS_USE_CCACHE)
find_program(CCACHE_FOUND ccache)
if (CCACHE_FOUND)
message(STATUS "Using ccache: ${CCACHE_FOUND}")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set(ENV{CCACHE_COMMENTS} "1")
endif (CCACHE_FOUND)
endif ()
set(MILVUS_CPU_VERSION false)
if (MILVUS_GPU_VERSION)
......@@ -142,6 +142,10 @@ else ()
add_compile_definitions("MILVUS_CPU_VERSION")
endif ()
if (MILVUS_WITH_PROMETHEUS)
add_compile_definitions("MILVUS_WITH_PROMETHEUS")
endif ()
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
if (MILVUS_GPU_VERSION)
......@@ -176,9 +180,9 @@ endif ()
if (MILVUS_GPU_VERSION)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_gpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
else()
else ()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_cpu_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
endif()
endif ()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
......
core/build.sh
浏览文件 @ 6c227a35
......@@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
GPU_VERSION="OFF" #defaults to CPU version
WITH_MKL="OFF"
FAISS_ROOT=""
FAISS_ROOT="" #FAISS root path
FAISS_SOURCE="BUNDLED"
WITH_PROMETHEUS="ON"
while getopts "p:d:t:f:ulrcgjhxzm" arg
while getopts "p:d:t:f:ulrcgjhxzme" arg
do
case $arg in
p)
......@@ -63,7 +64,10 @@ do
;;
m)
WITH_MKL="ON"
;;
;;
e)
WITH_PROMETHEUS="OFF"
;;
h) # help
echo "
......@@ -80,10 +84,11 @@ parameter:
-j: use jfrog cache build directory(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-e: build without prometheus
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
"
exit 0
;;
......@@ -118,6 +123,7 @@ CMAKE_CMD="cmake \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DFAISS_WITH_MKL=${WITH_MKL} \
-DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \
../"
echo ${CMAKE_CMD}
${CMAKE_CMD}
......
core/src/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -37,6 +37,7 @@ endforeach ()
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
......@@ -91,6 +92,11 @@ set(engine_files
${wrapper_files}
)
if (MILVUS_WITH_PROMETHEUS)
set(engine_files ${engine_files}
${metrics_prometheus_files})
endif ()
set(client_grpc_lib
grpcpp_channelz
grpc++
......@@ -115,7 +121,6 @@ set(third_party_libs
sqlite
${client_grpc_lib}
yaml-cpp
${prometheus_lib}
mysqlpp
zlib
${boost_lib}
......@@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION)
)
endif ()
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
if (MILVUS_ENABLE_PROFILING)
set(third_party_libs ${third_party_libs}
gperftools
libunwind
)
endif ()
if (MILVUS_WITH_PROMETHEUS)
set(third_party_libs ${third_party_libs}
${prometheus_lib}
)
endif ()
set(engine_libs
pthread
libgomp.a
......@@ -166,13 +177,22 @@ target_link_libraries(milvus_engine
${engine_libs}
)
add_library(metrics STATIC ${metrics_files})
if (MILVUS_WITH_PROMETHEUS)
add_library(metrics STATIC ${metrics_files} ${metrics_prometheus_files})
else ()
add_library(metrics STATIC ${metrics_files})
endif ()
set(metrics_lib
yaml-cpp
${prometheus_lib}
)
if (MILVUS_WITH_PROMETHEUS)
set(metrics_lib ${metrics_lib}
${prometheus_lib}
)
endif ()
target_link_libraries(metrics ${metrics_lib})
set(server_libs
......
core/src/db/DBImpl.cpp
浏览文件 @ 6c227a35
......@@ -619,6 +619,18 @@ DBImpl::StartCompactionTask() {
{
std::lock_guard<std::mutex> lck(compact_result_mutex_);
if (compact_thread_results_.empty()) {
// collect merge files for all tables(if compact_table_ids_ is empty) for two reasons:
// 1. other tables may still has un-merged files
// 2. server may be closed unexpected, these un-merge files need to be merged when server restart
if (compact_table_ids_.empty()) {
std::vector<meta::TableSchema> table_schema_array;
meta_ptr_->AllTables(table_schema_array);
for (auto& schema : table_schema_array) {
compact_table_ids_.insert(schema.table_id_);
}
}
// start merge file thread
compact_thread_results_.push_back(
compact_thread_pool_.enqueue(&DBImpl::BackgroundCompaction, this, compact_table_ids_));
compact_table_ids_.clear();
......@@ -717,7 +729,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
for (auto& kv : raw_files) {
auto files = kv.second;
if (files.size() < options_.merge_trigger_number_) {
ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
ENGINE_LOG_TRACE << "Files number not greater equal than merge trigger number, skip merge action";
continue;
}
......@@ -734,7 +746,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
void
DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread start";
ENGINE_LOG_TRACE << "Background compaction thread start";
Status status;
for (auto& table_id : table_ids) {
......@@ -757,7 +769,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
}
meta_ptr_->CleanUpFilesWithTTL(ttl);
ENGINE_LOG_TRACE << " Background compaction thread exit";
ENGINE_LOG_TRACE << "Background compaction thread exit";
}
void
......
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 6c227a35
......@@ -1392,6 +1392,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
} // Scoped Connection
Status ret;
int64_t to_merge_files = 0;
for (auto& resRow : res) {
TableFileSchema table_file;
table_file.file_size_ = resRow["file_size"];
......@@ -1420,13 +1421,14 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date_] = TableFilesSchema();
to_merge_files++;
}
files[table_file.date_].push_back(table_file);
}
if (res.size() > 0) {
ENGINE_LOG_DEBUG << "Collect " << res.size() << " to-merge files";
if (to_merge_files > 0) {
ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files";
}
return ret;
} catch (std::exception& e) {
......@@ -1809,6 +1811,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
mysqlpp::StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
int64_t remove_tables = 0;
if (!res.empty()) {
std::stringstream idsToDeleteSS;
for (auto& resRow : res) {
......@@ -1817,7 +1820,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
resRow["table_id"].to_string(table_id);
utils::DeleteTablePath(options_, table_id, false); // only delete empty folder
remove_tables++;
idsToDeleteSS << "id = " << std::to_string(id) << " OR ";
}
std::string idsToDeleteStr = idsToDeleteSS.str();
......@@ -1832,8 +1835,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
}
if (res.size() > 0) {
ENGINE_LOG_DEBUG << "Remove " << res.size() << " tables from meta";
if (remove_tables > 0) {
ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables from meta";
}
} // Scoped Connection
} catch (std::exception& e) {
......
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @ 6c227a35
......@@ -971,6 +971,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
order_by(&TableFileSchema::file_size_).desc());
Status result;
int64_t to_merge_files = 0;
for (auto& file : selected) {
TableFileSchema table_file;
table_file.file_size_ = std::get<4>(file);
......@@ -999,11 +1000,13 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
if (dateItr == files.end()) {
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date_].push_back(table_file);
to_merge_files++;
}
if (selected.size() > 0) {
ENGINE_LOG_DEBUG << "Collect " << selected.size() << " to-merge files";
if (to_merge_files > 0) {
ENGINE_LOG_TRACE << "Collect " << to_merge_files << " to-merge files";
}
return result;
} catch (std::exception& e) {
......@@ -1313,16 +1316,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
try {
server::MetricCollector metric;
int64_t remove_tables = 0;
for (auto& table_id : table_ids) {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::file_id_),
where(c(&TableFileSchema::table_id_) == table_id));
if (selected.size() == 0) {
utils::DeleteTablePath(options_, table_id);
remove_tables++;
}
}
if (table_ids.size() > 0) {
ENGINE_LOG_DEBUG << "Remove " << table_ids.size() << " tables folder";
if (remove_tables) {
ENGINE_LOG_DEBUG << "Remove " << remove_tables << " tables folder";
}
} catch (std::exception& e) {
return HandleException("Encounter exception when delete table folder", e.what());
......
core/src/grpc/README.md 0 → 100644
浏览文件 @ 6c227a35
We manually change two APIs in "milvus.pd.h":
add_vector_data()
add_row_id_array()
add_ids()
add_distances()
If proto files need be generated again, remember to re-change above APIs.
\ No newline at end of file
core/src/grpc/cpp_gen.sh 100644 → 100755
浏览文件 @ 6c227a35
文件模式从 100644 更改为 100755
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
浏览文件 @ 6c227a35
......@@ -431,60 +431,60 @@ void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* cont
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::VectorIds>::Create(channel_.get(), cq, rpcmethod_Insert_, context, request, false);
}
::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResultList* response) {
::grpc::Status MilvusService::Stub::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::milvus::grpc::TopKQueryResult* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Search_, context, request, response);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
void MilvusService::Stub::experimental_async::Search(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_Search_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_Search_, context, request, false);
}
::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResultList* response) {
::grpc::Status MilvusService::Stub::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::milvus::grpc::TopKQueryResult* response) {
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_SearchInFiles_, context, request, response);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, std::function<void(::grpc::Status)> f) {
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, std::function<void(::grpc::Status)> f) {
::grpc_impl::internal::CallbackUnaryCall(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, std::move(f));
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResultList* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
void MilvusService::Stub::experimental_async::SearchInFiles(::grpc::ClientContext* context, const ::grpc::ByteBuffer* request, ::milvus::grpc::TopKQueryResult* response, ::grpc::experimental::ClientUnaryReactor* reactor) {
::grpc_impl::internal::ClientCallbackUnaryFactory::Create(stub_->channel_.get(), stub_->rpcmethod_SearchInFiles_, context, request, response, reactor);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::AsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, true);
}
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResultList>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResultList>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false);
::grpc::ClientAsyncResponseReader< ::milvus::grpc::TopKQueryResult>* MilvusService::Stub::PrepareAsyncSearchInFilesRaw(::grpc::ClientContext* context, const ::milvus::grpc::SearchInFilesParam& request, ::grpc::CompletionQueue* cq) {
return ::grpc_impl::internal::ClientAsyncResponseReaderFactory< ::milvus::grpc::TopKQueryResult>::Create(channel_.get(), cq, rpcmethod_SearchInFiles_, context, request, false);
}
::grpc::Status MilvusService::Stub::Cmd(::grpc::ClientContext* context, const ::milvus::grpc::Command& request, ::milvus::grpc::StringReply* response) {
......@@ -640,12 +640,12 @@ MilvusService::Service::Service() {
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[13],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResultList>(
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchParam, ::milvus::grpc::TopKQueryResult>(
std::mem_fn(&MilvusService::Service::Search), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[14],
::grpc::internal::RpcMethod::NORMAL_RPC,
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResultList>(
new ::grpc::internal::RpcMethodHandler< MilvusService::Service, ::milvus::grpc::SearchInFilesParam, ::milvus::grpc::TopKQueryResult>(
std::mem_fn(&MilvusService::Service::SearchInFiles), this)));
AddMethod(new ::grpc::internal::RpcServiceMethod(
MilvusService_method_names[15],
......@@ -758,14 +758,14 @@ MilvusService::Service::~Service() {
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResultList* response) {
::grpc::Status MilvusService::Service::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request, ::milvus::grpc::TopKQueryResult* response) {
(void) context;
(void) request;
(void) response;
return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED, "");
}
::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResultList* response) {
::grpc::Status MilvusService::Service::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request, ::milvus::grpc::TopKQueryResult* response) {
(void) context;
(void) request;
(void) response;
......
core/src/grpc/milvus.proto
浏览文件 @ 6c227a35
......@@ -110,24 +110,11 @@ message SearchInFilesParam {
/**
* @brief Query result params
*/
message QueryResult {
int64 id = 1;
double distance = 2;
}
/**
* @brief TopK query result
*/
message TopKQueryResult {
repeated QueryResult query_result_arrays = 1;
}
/**
* @brief List of topK query result
*/
message TopKQueryResultList {
Status status = 1;
repeated TopKQueryResult topk_query_result = 2;
int64 row_num = 2;
repeated int64 ids = 3;
repeated float distances = 4;
}
/**
......@@ -311,18 +298,18 @@ service MilvusService {
*
* @param SearchParam, search parameters.
*
* @return TopKQueryResultList
* @return TopKQueryResult
*/
rpc Search(SearchParam) returns (TopKQueryResultList) {}
rpc Search(SearchParam) returns (TopKQueryResult) {}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInFilesParam, search in files paremeters.
*
* @return TopKQueryResultList
* @return TopKQueryResult
*/
rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResultList) {}
rpc SearchInFiles(SearchInFilesParam) returns (TopKQueryResult) {}
/**
* @brief This method is used to give the server status.
......
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
浏览文件 @ 6c227a35
......@@ -89,34 +89,35 @@ ConvertToDataset(std::vector<SPTAG::QueryResult> query_results) {
}
}
auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);
// TODO: magic
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
// auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
// auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());
auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());
std::vector<FieldPtr> fields{field_id, field_dist};
auto schema = std::make_shared<Schema>(fields);
return std::make_shared<Dataset>(array, schema);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);
//
// // TODO: magic
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
// // auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// // auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
//
// // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());
// auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());
// std::vector<FieldPtr> fields{field_id, field_dist};
// auto schema = std::make_shared<Schema>(fields);
//
// return std::make_shared<Dataset>(array, schema);
return std::make_shared<Dataset>((void*)p_id, (void*)p_dist);
}
} // namespace knowhere
core/src/index/knowhere/knowhere/common/Dataset.h
浏览文件 @ 6c227a35
......@@ -54,6 +54,9 @@ class Dataset {
: tensor_(std::move(tensor)), tensor_schema_(std::move(tensor_schema)) {
}
Dataset(void* ids, void* dists) : ids_(ids), dists_(dists) {
}
Dataset(const Dataset&) = delete;
Dataset&
operator=(const Dataset&) = delete;
......@@ -128,6 +131,16 @@ class Dataset {
tensor_schema_ = std::move(tensor_schema);
}
void*
ids() {
return ids_;
}
void*
dist() {
return dists_;
}
// const Config &
// meta() const { return meta_; }
......@@ -141,6 +154,9 @@ class Dataset {
SchemaPtr array_schema_;
std::vector<TensorPtr> tensor_;
SchemaPtr tensor_schema_;
// TODO(yukun): using smart pointer
void* ids_;
void* dists_;
// Config meta_;
};
......
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
浏览文件 @ 6c227a35
......@@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) {
search_impl(rows, (float*)p_data, config->k, res_dis, res_ids, Config());
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>(array, nullptr);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return std::make_shared<Dataset>((void*)res_ids, (void*)res_dis);
}
void
......
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
浏览文件 @ 6c227a35
......@@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
// std::cout << ss_res_id.str() << std::endl;
// std::cout << ss_res_dist.str() << std::endl << std::endl;
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>(array, nullptr);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>((void*)res_ids, (void*)res_dis);
}
void
......
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
浏览文件 @ 6c227a35
......@@ -88,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) {
s_params.search_length = build_cfg->search_length;
index_->Search((float*)p_data, rows, dim, build_cfg->k, res_dis, res_ids, s_params);
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
auto int64_type = std::make_shared<arrow::Int64Type>();
auto float_type = std::make_shared<arrow::FloatType>();
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
std::vector<ArrayPtr> array{ids, dists};
return std::make_shared<Dataset>(array, nullptr);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return std::make_shared<Dataset>((void*)res_ids, (void*)res_dis);
}
IndexModelPtr
......
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
浏览文件 @ 6c227a35
......@@ -202,14 +202,14 @@ parse_ann_test_name(const std::string& ann_test_name, size_t& dim, faiss::Metric
int32_t
GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::idx_t* index, size_t ground_k, size_t k,
size_t nq, int32_t index_add_loops) {
assert(ground_k <= k);
size_t min_k = std::min(ground_k, k);
int hit = 0;
for (int i = 0; i < nq; i++) {
// count the num of results exist in ground truth result set
// each result replicates INDEX_ADD_LOOPS times
for (int j_c = 0; j_c < k; j_c++) {
int r_c = index[i * k + j_c];
for (int j_g = 0; j_g < ground_k / index_add_loops; j_g++) {
for (int j_g = 0; j_g < min_k / index_add_loops; j_g++) {
if (ground_index[i * ground_k + j_g] == r_c) {
hit++;
continue;
......@@ -283,7 +283,11 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
// add index multiple times to get ~1G data set
for (int i = 0; i < index_add_loops; i++) {
printf("[%.3f s] No.%d Indexing database, size %ld*%ld\n", elapsed() - t0, i, nb, d);
gpu_index->add(nb, xb);
std::vector<faiss::Index::idx_t> xids(nb);
for (int t = 0; t < nb; t++) {
xids[t] = i * nb + t;
}
gpu_index->add_with_ids(nb, xb, xids.data());
}
printf("[%.3f s] Coping GPU index to CPU\n", elapsed() - t0);
......@@ -359,13 +363,14 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
const int32_t search_loops) {
double t0 = elapsed();
const size_t NQ = 1000, NQ_START = 10, NQ_STEP = 10;
const size_t K = 1000, K_START = 100, K_STEP = 10;
const std::vector<size_t> NQ = {10, 100};
const std::vector<size_t> K = {10, 100, 1000};
const size_t GK = 100; // topk of ground truth
std::unordered_map<size_t, std::string> mode_str_map = {
{MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}};
double copy_time = 0.0;
faiss::Index *gpu_index, *index;
if (query_mode != MODE_CPU) {
faiss::gpu::GpuClonerOptions option;
......@@ -375,7 +380,6 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
index_composition.index = cpu_index;
index_composition.quantizer = nullptr;
double copy_time;
switch (query_mode) {
case MODE_MIX: {
index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data
......@@ -420,34 +424,39 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
}
for (auto nprobe : nprobes) {
switch (query_mode) {
case MODE_CPU:
case MODE_MIX: {
faiss::ParameterSpace params;
std::string nprobe_str = "nprobe=" + std::to_string(nprobe);
params.set_index_parameters(index, nprobe_str.c_str());
break;
}
case MODE_GPU: {
faiss::gpu::GpuIndexIVF* gpu_index_ivf = dynamic_cast<faiss::gpu::GpuIndexIVF*>(index);
gpu_index_ivf->setNumProbes(nprobe);
// brute-force need not set nprobe
if (index_key.find("IDMap") == std::string::npos) {
switch (query_mode) {
case MODE_CPU:
case MODE_MIX: {
faiss::ParameterSpace params;
std::string nprobe_str = "nprobe=" + std::to_string(nprobe);
params.set_index_parameters(index, nprobe_str.c_str());
break;
}
case MODE_GPU: {
faiss::gpu::GpuIndexIVF* gpu_index_ivf = dynamic_cast<faiss::gpu::GpuIndexIVF*>(index);
gpu_index_ivf->setNumProbes(nprobe);
}
}
}
// output buffers
faiss::Index::idx_t* I = new faiss::Index::idx_t[NQ * K];
faiss::Index::distance_t* D = new faiss::Index::distance_t[NQ * K];
faiss::Index::idx_t* I = new faiss::Index::idx_t[NQ.back() * K.back()];
faiss::Index::distance_t* D = new faiss::Index::distance_t[NQ.back() * K.back()];
printf("\n%s | %s - %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(),
mode_str_map[query_mode].c_str(), nprobe);
printf("======================================================================================\n");
for (size_t t_nq = NQ_START; t_nq <= NQ; t_nq *= NQ_STEP) { // nq = {10, 100, 1000}
for (size_t t_k = K_START; t_k <= K; t_k *= K_STEP) { // k = {100, 1000}
for (size_t j = 0; j < K.size(); j++) {
size_t t_k = K[j];
for (size_t i = 0; i < NQ.size(); i++) {
size_t t_nq = NQ[i];
faiss::indexIVF_stats.quantization_time = 0.0;
faiss::indexIVF_stats.search_time = 0.0;
double t_start = elapsed(), t_end;
for (int i = 0; i < search_loops; i++) {
for (int s = 0; s < search_loops; s++) {
index->search(t_nq, xq, t_k, D, I);
}
t_end = elapsed();
......@@ -466,7 +475,7 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, t_k,
(t_end - t_start) / search_loops, faiss::indexIVF_stats.quantization_time / 1000 / search_loops,
faiss::indexIVF_stats.search_time / 1000 / search_loops,
(hit / float(t_nq * GK / index_add_loops)));
(hit / float(t_nq * std::min(GK, t_k) / index_add_loops)));
}
}
printf("======================================================================================\n");
......@@ -479,8 +488,9 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
}
void
test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, const QueryMode query_mode,
int32_t index_add_loops, const std::vector<size_t>& nprobes, int32_t search_loops) {
test_ann_hdf5(const std::string& ann_test_name, const std::string& cluster_type, const std::string& index_type,
const QueryMode query_mode, int32_t index_add_loops, const std::vector<size_t>& nprobes,
int32_t search_loops) {
double t0 = elapsed();
faiss::gpu::StandardGpuResources res;
......@@ -493,7 +503,7 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
return;
}
std::string index_key = "IVF16384," + index_type;
std::string index_key = cluster_type + "," + index_type;
if (!parse_ann_test_name(ann_test_name, dim, metric_type)) {
printf("Invalid ann test name: %s\n", ann_test_name.c_str());
......@@ -526,7 +536,7 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
*
* Dataset Dimensions Train_size Test_size Neighbors Distance Download
* Fashion-
* MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB)
MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB)
* GIST 960 1,000,000 1,000 100 Euclidean HDF5 (3.6GB)
* GloVe 100 1,183,514 10,000 100 Angular HDF5 (463MB)
* GloVe 200 1,183,514 10,000 100 Angular HDF5 (918MB)
......@@ -542,30 +552,39 @@ TEST(FAISSTEST, BENCHMARK) {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const int32_t SIFT_INSERT_LOOPS = 2; // insert twice to get ~1G data set
test_ann_hdf5("sift-128-euclidean", "Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IDMap", "Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IDMap", "Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#ifdef CUSTOMIZATION
test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("sift-128-euclidean", "IVF16384", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
#endif
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const int32_t GLOVE_INSERT_LOOPS = 1;
test_ann_hdf5("glove-200-angular", "Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
#ifdef CUSTOMIZATION
test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
test_ann_hdf5("glove-200-angular", "IVF16384", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes,
SEARCH_LOOPS);
#endif
}
core/src/index/unittest/test_ivf.cpp
浏览文件 @ 6c227a35
......@@ -181,11 +181,13 @@ TEST_P(IVFTest, clone_test) {
// PrintResult(result, nq, k);
auto AssertEqual = [&](knowhere::DatasetPtr p1, knowhere::DatasetPtr p2) {
auto ids_p1 = p1->array()[0];
auto ids_p2 = p2->array()[0];
auto ids_p1 = p1->ids();
auto ids_p2 = p2->ids();
for (int i = 0; i < nq * k; ++i) {
EXPECT_EQ(*(ids_p2->data()->GetValues<int64_t>(1, i)), *(ids_p1->data()->GetValues<int64_t>(1, i)));
EXPECT_EQ(*((int64_t*)(ids_p2) + i), *((int64_t*)(ids_p1) + i));
// EXPECT_EQ(*(ids_p2->data()->GetValues<int64_t>(1, i)), *(ids_p1->data()->GetValues<int64_t>(1,
// i)));
}
};
......
core/src/index/unittest/test_kdt.cpp
浏览文件 @ 6c227a35
......@@ -66,15 +66,19 @@ TEST_F(KDTTest, kdt_basic) {
AssertAnns(result, nq, k);
{
auto ids = result->array()[0];
auto dists = result->array()[1];
// auto ids = result->array()[0];
// auto dists = result->array()[1];
auto ids = result->ids();
auto dists = result->dist();
std::stringstream ss_id;
std::stringstream ss_dist;
for (auto i = 0; i < nq; i++) {
for (auto j = 0; j < k; ++j) {
ss_id << *ids->data()->GetValues<int64_t>(1, i * k + j) << " ";
ss_dist << *dists->data()->GetValues<float>(1, i * k + j) << " ";
ss_id << *((int64_t*)(ids) + i * k + j) << " ";
ss_dist << *((float*)(dists) + i * k + j) << " ";
// ss_id << *ids->data()->GetValues<int64_t>(1, i * k + j) << " ";
// ss_dist << *dists->data()->GetValues<float>(1, i * k + j) << " ";
}
ss_id << std::endl;
ss_dist << std::endl;
......
core/src/index/unittest/utils.cpp
浏览文件 @ 6c227a35
......@@ -151,9 +151,10 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) {
void
AssertAnns(const knowhere::DatasetPtr& result, const int& nq, const int& k) {
auto ids = result->array()[0];
auto ids = result->ids();
for (auto i = 0; i < nq; i++) {
EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
EXPECT_EQ(i, *((int64_t*)(ids) + i * k));
// EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
}
}
......
core/src/metrics/Metrics.cpp
浏览文件 @ 6c227a35
......@@ -16,8 +16,10 @@
// under the License.
#include "metrics/Metrics.h"
#include "PrometheusMetrics.h"
#include "server/Config.h"
#ifdef MILVUS_WITH_PROMETHEUS
#include "metrics/prometheus/PrometheusMetrics.h"
#endif
#include <string>
......@@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() {
config.GetMetricConfigCollector(collector_type_str);
#ifdef MILVUS_WITH_PROMETHEUS
if (collector_type_str == "prometheus") {
return PrometheusMetrics::GetInstance();
} else {
return MetricsBase::GetInstance();
}
#else
return MetricsBase::GetInstance();
#endif
}
} // namespace server
......
core/src/metrics/PrometheusMetrics.cpp → core/src/metrics/prometheus/PrometheusMetrics.cpp
浏览文件 @ 6c227a35
......@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "SystemInfo.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include "cache/GpuCacheMgr.h"
#include "metrics/SystemInfo.h"
#include "server/Config.h"
#include "utils/Log.h"
......
core/src/metrics/PrometheusMetrics.h → core/src/metrics/prometheus/PrometheusMetrics.h
浏览文件 @ 6c227a35
......@@ -24,7 +24,7 @@
#include <string>
#include <vector>
#include "MetricBase.h"
#include "metrics/MetricBase.h"
#include "utils/Error.h"
#define METRICS_NOW_TIME std::chrono::system_clock::now()
......
core/src/sdk/examples/partition/src/ClientTest.cpp
浏览文件 @ 6c227a35
......@@ -133,18 +133,18 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors
std::cout << "Search in correct partition" << std::endl;
std::vector<std::string> partiton_tags = {std::to_string(TARGET_PARTITION)};
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
std::cout << "Search in wrong partition" << std::endl;
partiton_tags = {"0"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
std::cout << "Search by regex matched partition tag" << std::endl;
partiton_tags = {"\\d"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
}
{ // wait unit build index finish
......@@ -182,9 +182,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors
std::cout << "Search in whole table" << std::endl;
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
}
{ // drop index
......
core/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @ 6c227a35
......@@ -144,9 +144,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
milvus_sdk::Utils::Sleep(3);
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
}
{ // wait unit build index finish
......@@ -169,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<milvus::TopKQueryResult> topk_query_result_array;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
topk_query_result_array);
topk_query_result);
}
{ // drop index
......
core/src/sdk/examples/utils/Utils.cpp
浏览文件 @ 6c227a35
......@@ -155,32 +155,34 @@ Utils::BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& ve
void
Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
const milvus::TopKQueryResult& topk_query_result) {
BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result_array.size() << std::endl;
size_t nq = topk_query_result.row_num;
size_t topk = topk_query_result.ids.size() / nq;
std::cout << "Returned result count: " << nq * topk << std::endl;
int32_t index = 0;
for (auto& result : topk_query_result_array) {
for (size_t i = 0; i < nq; i++) {
auto search_id = search_record_array[index].first;
index++;
std::cout << "No." << std::to_string(index) << " vector " << std::to_string(search_id) << " top "
<< std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl;
for (auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance);
std::cout << std::endl;
std::cout << "No." << index << " vector " << search_id << " top " << topk << " search result:" << std::endl;
for (size_t j = 0; j < topk; j++) {
size_t idx = i * nq + j;
std::cout << "\t" << topk_query_result.ids[idx] << "\t" << topk_query_result.distances[idx] << std::endl;
}
}
BLOCK_SPLITER
}
void
Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
const milvus::TopKQueryResult& topk_query_result) {
BLOCK_SPLITER
size_t nq = topk_query_result.row_num;
size_t result_k = topk_query_result.ids.size() / nq;
int64_t index = 0;
for (auto& result : topk_query_result_array) {
auto result_id = result.query_result_arrays[0].id;
for (size_t i = 0; i < nq; i++) {
auto result_id = topk_query_result.ids[i * result_k];
auto search_id = search_record_array[index++].first;
if (result_id != search_id) {
std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl;
......@@ -195,8 +197,10 @@ void
Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
std::vector<milvus::TopKQueryResult>& topk_query_result_array) {
topk_query_result_array.clear();
milvus::TopKQueryResult& topk_query_result) {
topk_query_result.distances.clear();
topk_query_result.ids.clear();
topk_query_result.row_num = 0;
std::vector<milvus::Range> query_range_array;
milvus::Range rg;
......@@ -212,14 +216,14 @@ Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& tab
{
BLOCK_SPLITER
milvus_sdk::TimeRecorder rc("search");
milvus::Status stat = conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe,
topk_query_result_array);
milvus::Status stat =
conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe, topk_query_result);
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
BLOCK_SPLITER
}
PrintSearchResult(search_record_array, topk_query_result_array);
CheckSearchResult(search_record_array, topk_query_result_array);
PrintSearchResult(search_record_array, topk_query_result);
CheckSearchResult(search_record_array, topk_query_result);
}
} // namespace milvus_sdk
core/src/sdk/examples/utils/Utils.h
浏览文件 @ 6c227a35
......@@ -61,17 +61,17 @@ class Utils {
static void
PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array);
const milvus::TopKQueryResult& topk_query_result);
static void
CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
const std::vector<milvus::TopKQueryResult>& topk_query_result_array);
const milvus::TopKQueryResult& topk_query_result);
static void
DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
std::vector<milvus::TopKQueryResult>& topk_query_result_array);
milvus::TopKQueryResult& topk_query_result);
};
} // namespace milvus_sdk
core/src/sdk/grpc/ClientProxy.cpp
浏览文件 @ 6c227a35
......@@ -32,6 +32,13 @@ UriCheck(const std::string& uri) {
return (index != std::string::npos);
}
void
CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) {
auto vector_data = target->mutable_vector_data();
vector_data->Resize(static_cast<int>(src.data.size()), 0.0);
memcpy(vector_data->mutable_data(), src.data.data(), src.data.size() * sizeof(float));
}
Status
ClientProxy::Connect(const ConnectParam& param) {
std::string uri = param.ip_address + ":" + param.port;
......@@ -189,23 +196,22 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
for (size_t i = 0; i < record.data.size(); i++) {
grpc_record->add_vector_data(record.data[i]);
}
CopyRowRecord(grpc_record, record);
}
// Single thread
::milvus::grpc::VectorIds vector_ids;
if (!id_array.empty()) {
for (auto i = 0; i < id_array.size(); i++) {
insert_param.add_row_id_array(id_array[i]);
}
/* set user's ids */
auto row_ids = insert_param.mutable_row_id_array();
row_ids->Reserve(static_cast<int>(id_array.size()));
memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t));
client_ptr_->Insert(vector_ids, insert_param, status);
} else {
client_ptr_->Insert(vector_ids, insert_param, status);
for (size_t i = 0; i < vector_ids.vector_id_array_size(); i++) {
id_array.push_back(vector_ids.vector_id_array(i));
}
/* return Milvus generated ids back to user */
id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end());
}
#endif
} catch (std::exception& ex) {
......@@ -218,7 +224,7 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_
Status
ClientProxy::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) {
int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) {
try {
// step 1: convert vectors data
::milvus::grpc::SearchParam search_param;
......@@ -230,9 +236,7 @@ ClientProxy::Search(const std::string& table_name, const std::vector<std::string
}
for (auto& record : query_record_array) {
::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array();
for (auto& rec : record.data) {
row_record->add_vector_data(rec);
}
CopyRowRecord(row_record, record);
}
// step 2: convert range array
......@@ -243,21 +247,17 @@ ClientProxy::Search(const std::string& table_name, const std::vector<std::string
}
// step 3: search vectors
::milvus::grpc::TopKQueryResultList topk_query_result_list;
Status status = client_ptr_->Search(topk_query_result_list, search_param);
::milvus::grpc::TopKQueryResult result;
Status status = client_ptr_->Search(result, search_param);
// step 4: convert result array
for (uint64_t i = 0; i < topk_query_result_list.topk_query_result_size(); ++i) {
TopKQueryResult result;
for (uint64_t j = 0; j < topk_query_result_list.topk_query_result(i).query_result_arrays_size(); ++j) {
QueryResult query_result;
query_result.id = topk_query_result_list.topk_query_result(i).query_result_arrays(j).id();
query_result.distance = topk_query_result_list.topk_query_result(i).query_result_arrays(j).distance();
result.query_result_arrays.emplace_back(query_result);
}
topk_query_result.row_num = result.row_num();
topk_query_result.ids.resize(result.ids().size());
memcpy(topk_query_result.ids.data(), result.ids().data(), result.ids().size() * sizeof(int64_t));
topk_query_result.distances.resize(result.distances().size());
memcpy(topk_query_result.distances.data(), result.distances().data(),
result.distances().size() * sizeof(float));
topk_query_result_array.emplace_back(result);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "fail to search vectors: " + std::string(ex.what()));
......
core/src/sdk/grpc/ClientProxy.h
浏览文件 @ 6c227a35
......@@ -60,7 +60,7 @@ class ClientProxy : public Connection {
Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) override;
int64_t nprobe, TopKQueryResult& topk_query_result) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
......
core/src/sdk/grpc/GrpcClient.cpp
浏览文件 @ 6c227a35
......@@ -134,20 +134,20 @@ GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::
}
Status
GrpcClient::Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list,
GrpcClient::Search(::milvus::grpc::TopKQueryResult& topk_query_result,
const ::milvus::grpc::SearchParam& search_param) {
::milvus::grpc::TopKQueryResult query_result;
ClientContext context;
::grpc::Status grpc_status = stub_->Search(&context, search_param, &topk_query_result_list);
::grpc::Status grpc_status = stub_->Search(&context, search_param, &topk_query_result);
if (!grpc_status.ok()) {
std::cerr << "SearchVector rpc failed!" << std::endl;
std::cerr << grpc_status.error_message() << std::endl;
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (topk_query_result_list.status().error_code() != grpc::SUCCESS) {
std::cerr << topk_query_result_list.status().reason() << std::endl;
return Status(StatusCode::ServerFailed, topk_query_result_list.status().reason());
if (topk_query_result.status().error_code() != grpc::SUCCESS) {
std::cerr << topk_query_result.status().reason() << std::endl;
return Status(StatusCode::ServerFailed, topk_query_result.status().reason());
}
return Status::OK();
......
core/src/sdk/grpc/GrpcClient.h
浏览文件 @ 6c227a35
......@@ -57,7 +57,7 @@ class GrpcClient {
Insert(grpc::VectorIds& vector_ids, const grpc::InsertParam& insert_param, Status& status);
Status
Search(::milvus::grpc::TopKQueryResultList& topk_query_result_list, const grpc::SearchParam& search_param);
Search(::milvus::grpc::TopKQueryResult& topk_query_result, const grpc::SearchParam& search_param);
Status
DescribeTable(grpc::TableSchema& grpc_schema, const std::string& table_name);
......
core/src/sdk/include/MilvusApi.h
浏览文件 @ 6c227a35
......@@ -78,19 +78,13 @@ struct RowRecord {
std::vector<float> data; ///< Vector raw data
};
/**
* @brief Query result
*/
struct QueryResult {
int64_t id; ///< Output result
double distance; ///< Vector similarity distance
};
/**
* @brief TopK query result
*/
struct TopKQueryResult {
std::vector<QueryResult> query_result_arrays; ///< TopK query result
int64_t row_num;
std::vector<int64_t> ids;
std::vector<float> distances;
};
/**
......@@ -274,7 +268,7 @@ class Connection {
virtual Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) = 0;
int64_t nprobe, TopKQueryResult& topk_query_result) = 0;
/**
* @brief Show table description
......
core/src/sdk/interface/ConnectionImpl.cpp
浏览文件 @ 6c227a35
......@@ -91,9 +91,9 @@ ConnectionImpl::Insert(const std::string& table_name, const std::string& partiti
Status
ConnectionImpl::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) {
int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) {
return client_proxy_->Search(table_name, partiton_tags, query_record_array, query_range_array, topk, nprobe,
topk_query_result_array);
topk_query_result);
}
Status
......
core/src/sdk/interface/ConnectionImpl.h
浏览文件 @ 6c227a35
......@@ -62,7 +62,7 @@ class ConnectionImpl : public Connection {
Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, std::vector<TopKQueryResult>& topk_query_result_array) override;
int64_t nprobe, TopKQueryResult& topk_query_result) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
......
core/src/server/Config.h
浏览文件 @ 6c227a35
......@@ -93,11 +93,13 @@ static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER = ",";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu";
#else
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT = "cpu,gpu0";
#endif
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
#ifdef MILVUS_CPU_VERSION
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "cpu";
......
core/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @ 6c227a35
......@@ -75,7 +75,7 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc:
::grpc::Status
GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request,
::milvus::grpc::TopKQueryResultList* response) {
::milvus::grpc::TopKQueryResult* response) {
std::vector<std::string> file_id_array;
BaseTaskPtr task_ptr = SearchTask::Create(request, file_id_array, response);
::milvus::grpc::Status grpc_status;
......@@ -87,7 +87,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc:
::grpc::Status
GrpcRequestHandler::SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::TopKQueryResultList* response) {
::milvus::grpc::TopKQueryResult* response) {
std::vector<std::string> file_id_array;
for (int i = 0; i < request->file_id_array_size(); i++) {
file_id_array.push_back(request->file_id_array(i));
......
core/src/server/grpc_impl/GrpcRequestHandler.h
浏览文件 @ 6c227a35
......@@ -153,7 +153,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
// @return TopKQueryResultList
::grpc::Status
Search(::grpc::ServerContext* context, const ::milvus::grpc::SearchParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
::milvus::grpc::TopKQueryResult* response) override;
// *
// @brief This method is used to query vector in specified files.
//
......@@ -162,7 +163,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
// @return TopKQueryResultList
::grpc::Status
SearchInFiles(::grpc::ServerContext* context, const ::milvus::grpc::SearchInFilesParam* request,
::milvus::grpc::TopKQueryResultList* response) override;
::milvus::grpc::TopKQueryResult* response) override;
// *
// @brief This method is used to give the server status.
//
......
core/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @ 6c227a35
......@@ -20,6 +20,7 @@
#include <string.h>
#include <map>
#include <string>
#include <utility>
#include <vector>
//#include <gperftools/profiler.h>
......@@ -542,16 +543,16 @@ InsertTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SearchTask::SearchTask(const ::milvus::grpc::SearchParam* search_vector_infos,
const std::vector<std::string>& file_id_array, ::milvus::grpc::TopKQueryResultList* response)
const std::vector<std::string>& file_id_array, ::milvus::grpc::TopKQueryResult* response)
: GrpcBaseTask(DQL_TASK_GROUP),
search_param_(search_vector_infos),
file_id_array_(file_id_array),
topk_result_list(response) {
topk_result_(response) {
}
BaseTaskPtr
SearchTask::Create(const ::milvus::grpc::SearchParam* search_vector_infos,
const std::vector<std::string>& file_id_array, ::milvus::grpc::TopKQueryResultList* response) {
const std::vector<std::string>& file_id_array, ::milvus::grpc::TopKQueryResult* response) {
if (search_vector_infos == nullptr) {
SERVER_LOG_ERROR << "grpc input is null!";
return nullptr;
......@@ -679,18 +680,13 @@ SearchTask::OnExecute() {
return Status::OK(); // empty table
}
size_t result_k = result_ids.size() / record_count;
// step 7: construct result array
for (size_t i = 0; i < record_count; i++) {
::milvus::grpc::TopKQueryResult* topk_query_result = topk_result_list->add_topk_query_result();
for (size_t j = 0; j < result_k; j++) {
::milvus::grpc::QueryResult* grpc_result = topk_query_result->add_query_result_arrays();
size_t idx = i * result_k + j;
grpc_result->set_id(result_ids[idx]);
grpc_result->set_distance(result_distances[idx]);
}
}
topk_result_->set_row_num(record_count);
topk_result_->mutable_ids()->Resize(static_cast<int>(result_ids.size()), 0);
memcpy(topk_result_->mutable_ids()->mutable_data(), result_ids.data(), result_ids.size() * sizeof(int64_t));
topk_result_->mutable_distances()->Resize(static_cast<int>(result_distances.size()), 0.0);
memcpy(topk_result_->mutable_distances()->mutable_data(), result_distances.data(),
result_distances.size() * sizeof(float));
// step 8: print time cost percent
rc.RecordSection("construct result and send");
......
core/src/server/grpc_impl/GrpcRequestTask.h
浏览文件 @ 6c227a35
......@@ -153,11 +153,11 @@ class SearchTask : public GrpcBaseTask {
public:
static BaseTaskPtr
Create(const ::milvus::grpc::SearchParam* search_param, const std::vector<std::string>& file_id_array,
::milvus::grpc::TopKQueryResultList* response);
::milvus::grpc::TopKQueryResult* response);
protected:
SearchTask(const ::milvus::grpc::SearchParam* search_param, const std::vector<std::string>& file_id_array,
::milvus::grpc::TopKQueryResultList* response);
::milvus::grpc::TopKQueryResult* response);
Status
OnExecute() override;
......@@ -165,7 +165,7 @@ class SearchTask : public GrpcBaseTask {
private:
const ::milvus::grpc::SearchParam* search_param_;
std::vector<std::string> file_id_array_;
::milvus::grpc::TopKQueryResultList* topk_result_list;
::milvus::grpc::TopKQueryResult* topk_result_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
......
core/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @ 6c227a35
......@@ -93,7 +93,7 @@ GrpcServer::StartService() {
builder.SetCompressionAlgorithmSupportStatus(GRPC_COMPRESS_STREAM_GZIP, true);
builder.SetDefaultCompressionAlgorithm(GRPC_COMPRESS_STREAM_GZIP);
builder.SetDefaultCompressionLevel(GRPC_COMPRESS_LEVEL_HIGH);
builder.SetDefaultCompressionLevel(GRPC_COMPRESS_LEVEL_NONE);
GrpcRequestHandler service;
......
core/src/wrapper/VecImpl.cpp
浏览文件 @ 6c227a35
......@@ -84,8 +84,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
Config search_cfg = cfg;
auto res = index_->Search(dataset, search_cfg);
auto ids_array = res->array()[0];
auto dis_array = res->array()[1];
// auto ids_array = res->array()[0];
// auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
......@@ -104,12 +104,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
// auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy(ids, p_ids, sizeof(int64_t) * nq * k);
memcpy(dist, p_dist, sizeof(float) * nq * k);
memcpy(ids, res->ids(), sizeof(int64_t) * nq * k);
memcpy(dist, res->dist(), sizeof(float) * nq * k);
free(res->ids());
free(res->dist());
} catch (knowhere::KnowhereException& e) {
WRAPPER_LOG_ERROR << e.what();
return Status(KNOWHERE_UNEXPECTED_ERROR, e.what());
......
core/unittest/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -110,12 +110,18 @@ set(unittest_libs
pthread
metrics
gfortran
prometheus-cpp-pull
prometheus-cpp-push
prometheus-cpp-core
dl
z
)
if (MILVUS_WITH_PROMETHEUS)
set(unittest_libs ${unittest_libs}
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
)
endif ()
set(unittest_libs ${unittest_libs}
dl
z
)
if (MILVUS_GPU_VERSION)
include_directories("${CUDA_INCLUDE_DIRS}")
......@@ -135,4 +141,4 @@ add_subdirectory(db)
add_subdirectory(wrapper)
add_subdirectory(metrics)
add_subdirectory(scheduler)
add_subdirectory(server)
\ No newline at end of file
add_subdirectory(server)
core/unittest/db/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -17,8 +17,16 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
set(test_files
${CMAKE_CURRENT_SOURCE_DIR}/test_db.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_db_mysql.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_engine.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_mem.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_meta.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_meta_mysql.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_misc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_search.cpp
${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp)
add_executable(test_db
${common_files}
......
core/unittest/metrics/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -17,7 +17,16 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
set(test_files
test_metricbase.cpp
test_metrics.cpp
utils.cpp
)
if (MILVUS_WITH_PROMETHEUS)
set(test_files ${test_files}
test_prometheus.cpp)
endif ()
add_executable(test_metrics
${common_files}
......
core/unittest/metrics/test_prometheus.cpp
浏览文件 @ 6c227a35
......@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "server/Config.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include <gtest/gtest.h>
#include <iostream>
......
core/unittest/scheduler/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -17,7 +17,17 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
set(test_files
${CMAKE_CURRENT_SOURCE_DIR}/test_algorithm.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_event.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_node.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_normal.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_resource.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_resource_factory.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_resource_mgr.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_scheduler.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_task.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_tasktable.cpp)
add_executable(test_scheduler
${common_files}
......
core/unittest/server/CMakeLists.txt
浏览文件 @ 6c227a35
......@@ -17,7 +17,12 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} test_files)
set(test_files
${CMAKE_CURRENT_SOURCE_DIR}/test_cache.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_config.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_rpc.cpp
${CMAKE_CURRENT_SOURCE_DIR}/test_util.cpp
${CMAKE_CURRENT_SOURCE_DIR}/utils.cpp)
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
......
core/unittest/server/test_config.cpp
浏览文件 @ 6c227a35
......@@ -415,6 +415,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
s = config.SetResourceConfigSearchResources("gpu10");
ASSERT_FALSE(s.ok());
s = config.SetResourceConfigSearchResources("cpu");
ASSERT_TRUE(s.ok());
s = config.SetResourceConfigIndexBuildDevice("gup2");
ASSERT_FALSE(s.ok());
s = config.SetResourceConfigIndexBuildDevice("gpu16");
......@@ -435,3 +438,4 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());
}
core/unittest/server/test_rpc.cpp
浏览文件 @ 6c227a35
......@@ -43,6 +43,12 @@ static constexpr int64_t VECTOR_COUNT = 1000;
static constexpr int64_t INSERT_LOOP = 10;
constexpr int64_t SECONDS_EACH_HOUR = 3600;
void CopyRowRecord(::milvus::grpc::RowRecord* target, const std::vector<float>& src) {
auto vector_data = target->mutable_vector_data();
vector_data->Resize(static_cast<int>(src.size()), 0.0);
memcpy(vector_data->mutable_data(), src.data(), src.size()* sizeof(float));
}
class RpcHandlerTest : public testing::Test {
protected:
void
......@@ -201,9 +207,7 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
::milvus::grpc::VectorIds vector_ids;
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
CopyRowRecord(grpc_record, record);
}
handler->Insert(&context, &request, &vector_ids);
ASSERT_EQ(vector_ids.vector_id_array_size(), VECTOR_COUNT);
......@@ -212,8 +216,8 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
TEST_F(RpcHandlerTest, SEARCH_TEST) {
::grpc::ServerContext context;
::milvus::grpc::SearchParam request;
::milvus::grpc::TopKQueryResultList response;
// test null input
::milvus::grpc::TopKQueryResult response;
//test null input
handler->Search(&context, nullptr, &response);
// test invalid table name
......@@ -240,22 +244,17 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
::milvus::grpc::InsertParam insert_param;
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
CopyRowRecord(grpc_record, record);
}
// insert vectors
insert_param.set_table_name(TABLE_NAME);
::milvus::grpc::VectorIds vector_ids;
handler->Insert(&context, &insert_param, &vector_ids);
sleep(7);
BuildVectors(0, 10, record_array);
for (auto& record : record_array) {
::milvus::grpc::RowRecord* row_record = request.add_query_record_array();
for (auto& rec : record) {
row_record->add_vector_data(rec);
}
CopyRowRecord(row_record, record);
}
handler->Search(&context, &request, &response);
......@@ -324,9 +323,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
CopyRowRecord(grpc_record, record);
}
// test vector_id size not equal to row record size
vector_ids.clear_vector_id_array();
......@@ -341,9 +338,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
vector_ids.clear_vector_id_array();
for (uint64_t i = 0; i < 10; ++i) {
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t j = 0; j < 10; j++) {
grpc_record->add_vector_data(record_array[i][j]);
}
CopyRowRecord(grpc_record, record_array[i]);
}
handler->Insert(&context, &request, &vector_ids);
......
tests/milvus-java-test/pom.xml
浏览文件 @ 6c227a35
......@@ -99,7 +99,7 @@
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.3.0</version>
</dependency>
<!-- <dependency>-->
......@@ -134,4 +134,4 @@
</dependencies>
</project>
\ No newline at end of file
</project>
tests/milvus_python_test/test_ping.py
浏览文件 @ 6c227a35
import logging
import pytest
__version__ = '0.5.1'
__version__ = '0.6.0'
class TestPing:
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或