Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
6c227a35
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
6c227a35
编写于
11月 15, 2019
作者:
Q
quicksilver
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0
上级
e476af44
49ec7e1a
变更
56
展开全部
隐藏空白更改
内联
并排
Showing
56 changed file
with
1044 addition
and
1678 deletion
+1044
-1678
CHANGELOG.md
CHANGELOG.md
+20
-3
core/.gitignore
core/.gitignore
+1
-1
core/CMakeLists.txt
core/CMakeLists.txt
+19
-15
core/build.sh
core/build.sh
+10
-4
core/src/CMakeLists.txt
core/src/CMakeLists.txt
+24
-4
core/src/db/DBImpl.cpp
core/src/db/DBImpl.cpp
+15
-3
core/src/db/meta/MySQLMetaImpl.cpp
core/src/db/meta/MySQLMetaImpl.cpp
+8
-5
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+9
-4
core/src/grpc/README.md
core/src/grpc/README.md
+6
-0
core/src/grpc/cpp_gen.sh
core/src/grpc/cpp_gen.sh
+0
-0
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
+22
-22
core/src/grpc/gen-milvus/milvus.grpc.pb.h
core/src/grpc/gen-milvus/milvus.grpc.pb.h
+80
-80
core/src/grpc/gen-milvus/milvus.pb.cc
core/src/grpc/gen-milvus/milvus.pb.cc
+346
-841
core/src/grpc/gen-milvus/milvus.pb.h
core/src/grpc/gen-milvus/milvus.pb.h
+119
-406
core/src/grpc/milvus.proto
core/src/grpc/milvus.proto
+7
-20
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
+29
-28
core/src/index/knowhere/knowhere/common/Dataset.h
core/src/index/knowhere/knowhere/common/Dataset.h
+16
-0
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
...index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
+18
-17
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
...c/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
+17
-17
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
...c/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
+18
-17
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
...c/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
+60
-41
core/src/index/unittest/test_ivf.cpp
core/src/index/unittest/test_ivf.cpp
+5
-3
core/src/index/unittest/test_kdt.cpp
core/src/index/unittest/test_kdt.cpp
+8
-4
core/src/index/unittest/utils.cpp
core/src/index/unittest/utils.cpp
+3
-2
core/src/metrics/Metrics.cpp
core/src/metrics/Metrics.cpp
+7
-1
core/src/metrics/prometheus/PrometheusMetrics.cpp
core/src/metrics/prometheus/PrometheusMetrics.cpp
+2
-2
core/src/metrics/prometheus/PrometheusMetrics.h
core/src/metrics/prometheus/PrometheusMetrics.h
+1
-1
core/src/sdk/examples/partition/src/ClientTest.cpp
core/src/sdk/examples/partition/src/ClientTest.cpp
+6
-6
core/src/sdk/examples/simple/src/ClientTest.cpp
core/src/sdk/examples/simple/src/ClientTest.cpp
+4
-4
core/src/sdk/examples/utils/Utils.cpp
core/src/sdk/examples/utils/Utils.cpp
+22
-18
core/src/sdk/examples/utils/Utils.h
core/src/sdk/examples/utils/Utils.h
+3
-3
core/src/sdk/grpc/ClientProxy.cpp
core/src/sdk/grpc/ClientProxy.cpp
+25
-25
core/src/sdk/grpc/ClientProxy.h
core/src/sdk/grpc/ClientProxy.h
+1
-1
core/src/sdk/grpc/GrpcClient.cpp
core/src/sdk/grpc/GrpcClient.cpp
+5
-5
core/src/sdk/grpc/GrpcClient.h
core/src/sdk/grpc/GrpcClient.h
+1
-1
core/src/sdk/include/MilvusApi.h
core/src/sdk/include/MilvusApi.h
+4
-10
core/src/sdk/interface/ConnectionImpl.cpp
core/src/sdk/interface/ConnectionImpl.cpp
+2
-2
core/src/sdk/interface/ConnectionImpl.h
core/src/sdk/interface/ConnectionImpl.h
+1
-1
core/src/server/Config.h
core/src/server/Config.h
+2
-0
core/src/server/grpc_impl/GrpcRequestHandler.cpp
core/src/server/grpc_impl/GrpcRequestHandler.cpp
+2
-2
core/src/server/grpc_impl/GrpcRequestHandler.h
core/src/server/grpc_impl/GrpcRequestHandler.h
+4
-2
core/src/server/grpc_impl/GrpcRequestTask.cpp
core/src/server/grpc_impl/GrpcRequestTask.cpp
+10
-14
core/src/server/grpc_impl/GrpcRequestTask.h
core/src/server/grpc_impl/GrpcRequestTask.h
+3
-3
core/src/server/grpc_impl/GrpcServer.cpp
core/src/server/grpc_impl/GrpcServer.cpp
+1
-1
core/src/wrapper/VecImpl.cpp
core/src/wrapper/VecImpl.cpp
+8
-6
core/unittest/CMakeLists.txt
core/unittest/CMakeLists.txt
+12
-6
core/unittest/db/CMakeLists.txt
core/unittest/db/CMakeLists.txt
+10
-2
core/unittest/metrics/CMakeLists.txt
core/unittest/metrics/CMakeLists.txt
+10
-1
core/unittest/metrics/test_prometheus.cpp
core/unittest/metrics/test_prometheus.cpp
+1
-1
core/unittest/scheduler/CMakeLists.txt
core/unittest/scheduler/CMakeLists.txt
+11
-1
core/unittest/scheduler/test_task.cpp
core/unittest/scheduler/test_task.cpp
+0
-0
core/unittest/server/CMakeLists.txt
core/unittest/server/CMakeLists.txt
+6
-1
core/unittest/server/test_config.cpp
core/unittest/server/test_config.cpp
+4
-0
core/unittest/server/test_rpc.cpp
core/unittest/server/test_rpc.cpp
+13
-18
tests/milvus-java-test/pom.xml
tests/milvus-java-test/pom.xml
+2
-2
tests/milvus_python_test/test_ping.py
tests/milvus_python_test/test_ping.py
+1
-1
未找到文件。
CHANGELOG.md
浏览文件 @
6c227a35
# Changelog
Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.6.0 (TODO)
## Bug
-
\#
228 - memory usage increased slowly during searching vectors
-
\#
246 - Exclude src/external folder from code coverage for jenkin ci
-
\#
248 - Reside src/external in thirdparty
-
\#
316 - Some files not merged after vectors added
## Feature
-
\#
12 - Pure CPU version for Milvus
...
...
@@ -19,11 +20,27 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
260 - C++ SDK README
-
\#
314 - add Find FAISS in CMake
-
\#
310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
-
\#
322 - Add option to enable / disable prometheus
## Task
# Milvus 0.5.3 (2019-11-13)
## Bug
-
\#
258 - Bytes type in proto cause big-endian/little-endian problem
## Feature
## Improvement
-
\#
204 - improve grpc performance in search
-
\#
207 - Add more unittest for config set/get
-
\#
208 - optimize unittest to support run single test more easily
-
\#
284 - Change C++ SDK to shared library
-
\#
260 - C++ SDK README
## Task
# Milvus 0.5.2 (
TODO
)
# Milvus 0.5.2 (
2019-11-07
)
## Bug
-
\#
194 - Search faild: message="Table file doesn't exist"
...
...
@@ -31,7 +48,7 @@ Please mark all change in change log and use the ticket from JIRA.
## Feature
## Improvement
-
\#
190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.52
-
\#
190 - Update default config:use_blas_threshold to 1100 and server version printout to 0.5
.
2
## Task
...
...
core/.gitignore
浏览文件 @
6c227a35
...
...
@@ -2,7 +2,7 @@ milvus/
conf/server_config.yaml
conf/log_config.conf
src/config.h
version.h
src/
version.h
lcov_out/
base.info
output.info
...
...
core/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}")
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
execute_process
(
COMMAND sh
"-c"
"git log --decorate | head -n 1 | sed 's/.*(
\\
(.*
\\
))/
\\
1/' | sed 's/.*
\\
(.*
\\
),.*/
\\
1/' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
GET_GIT_BRANCH_NAME
(
GIT_BRANCH_NAME
)
...
...
@@ -67,7 +67,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL
""
)
message
(
WARNING
"Failed to determine Milvus version from git branch name"
)
set
(
MILVUS_VERSION
"0.6.0"
)
endif
()
endif
()
message
(
STATUS
"Build version =
${
MILVUS_VERSION
}
"
)
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/src/config.h.in
${
CMAKE_CURRENT_SOURCE_DIR
}
/src/config.h @ONLY
)
...
...
@@ -117,17 +117,17 @@ include(DefineOptions)
include
(
BuildUtils
)
include
(
ThirdPartyPackages
)
if
(
MILVUS_USE_CCACHE
)
find_program
(
CCACHE_FOUND ccache
)
if
(
CCACHE_FOUND
)
message
(
STATUS
"Using ccache:
${
CCACHE_FOUND
}
"
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_COMPILE
${
CCACHE_FOUND
}
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_LINK
${
CCACHE_FOUND
}
)
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set
(
ENV{CCACHE_COMMENTS}
"1"
)
endif
(
CCACHE_FOUND
)
endif
()
if
(
MILVUS_USE_CCACHE
)
find_program
(
CCACHE_FOUND ccache
)
if
(
CCACHE_FOUND
)
message
(
STATUS
"Using ccache:
${
CCACHE_FOUND
}
"
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_COMPILE
${
CCACHE_FOUND
}
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_LINK
${
CCACHE_FOUND
}
)
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set
(
ENV{CCACHE_COMMENTS}
"1"
)
endif
(
CCACHE_FOUND
)
endif
()
set
(
MILVUS_CPU_VERSION false
)
if
(
MILVUS_GPU_VERSION
)
...
...
@@ -142,6 +142,10 @@ else ()
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
endif
()
if
(
MILVUS_WITH_PROMETHEUS
)
add_compile_definitions
(
"MILVUS_WITH_PROMETHEUS"
)
endif
()
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -DELPP_THREAD_SAFE -fopenmp"
)
if
(
MILVUS_GPU_VERSION
)
...
...
@@ -176,9 +180,9 @@ endif ()
if
(
MILVUS_GPU_VERSION
)
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_gpu_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_config.yaml
)
else
()
else
()
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_cpu_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_config.yaml
)
endif
()
endif
()
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/log_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/log_config.conf
)
...
...
core/build.sh
浏览文件 @
6c227a35
...
...
@@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER
=
/usr/local/cuda/bin/nvcc
GPU_VERSION
=
"OFF"
#defaults to CPU version
WITH_MKL
=
"OFF"
FAISS_ROOT
=
""
FAISS_ROOT
=
""
#FAISS root path
FAISS_SOURCE
=
"BUNDLED"
WITH_PROMETHEUS
=
"ON"
while
getopts
"p:d:t:f:ulrcgjhxzm"
arg
while
getopts
"p:d:t:f:ulrcgjhxzm
e
"
arg
do
case
$arg
in
p
)
...
...
@@ -63,7 +64,10 @@ do
;;
m
)
WITH_MKL
=
"ON"
;;
;;
e
)
WITH_PROMETHEUS
=
"OFF"
;;
h
)
# help
echo
"
...
...
@@ -80,10 +84,11 @@ parameter:
-j: use jfrog cache build directory(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-e: build without prometheus
-h: help
usage:
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} -f
\$
{FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} -f
\$
{FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-
e] [-
h]
"
exit
0
;;
...
...
@@ -118,6 +123,7 @@ CMAKE_CMD="cmake \
-DCUSTOMIZATION=
${
CUSTOMIZATION
}
\
-DMILVUS_GPU_VERSION=
${
GPU_VERSION
}
\
-DFAISS_WITH_MKL=
${
WITH_MKL
}
\
-DMILVUS_WITH_PROMETHEUS=
${
WITH_PROMETHEUS
}
\
../"
echo
${
CMAKE_CMD
}
${
CMAKE_CMD
}
...
...
core/src/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -37,6 +37,7 @@ endforeach ()
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/cache cache_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/config config_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/metrics metrics_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/metrics/prometheus metrics_prometheus_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db db_main_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db/engine db_engine_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db/insert db_insert_files
)
...
...
@@ -91,6 +92,11 @@ set(engine_files
${
wrapper_files
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
engine_files
${
engine_files
}
${
metrics_prometheus_files
}
)
endif
()
set
(
client_grpc_lib
grpcpp_channelz
grpc++
...
...
@@ -115,7 +121,6 @@ set(third_party_libs
sqlite
${
client_grpc_lib
}
yaml-cpp
${
prometheus_lib
}
mysqlpp
zlib
${
boost_lib
}
...
...
@@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION)
)
endif
()
if
(
MILVUS_ENABLE_PROFILING
STREQUAL
"ON"
)
if
(
MILVUS_ENABLE_PROFILING
)
set
(
third_party_libs
${
third_party_libs
}
gperftools
libunwind
)
endif
()
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
third_party_libs
${
third_party_libs
}
${
prometheus_lib
}
)
endif
()
set
(
engine_libs
pthread
libgomp.a
...
...
@@ -166,13 +177,22 @@ target_link_libraries(milvus_engine
${
engine_libs
}
)
add_library
(
metrics STATIC
${
metrics_files
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
add_library
(
metrics STATIC
${
metrics_files
}
${
metrics_prometheus_files
}
)
else
()
add_library
(
metrics STATIC
${
metrics_files
}
)
endif
()
set
(
metrics_lib
yaml-cpp
${
prometheus_lib
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
metrics_lib
${
metrics_lib
}
${
prometheus_lib
}
)
endif
()
target_link_libraries
(
metrics
${
metrics_lib
}
)
set
(
server_libs
...
...
core/src/db/DBImpl.cpp
浏览文件 @
6c227a35
...
...
@@ -619,6 +619,18 @@ DBImpl::StartCompactionTask() {
{
std
::
lock_guard
<
std
::
mutex
>
lck
(
compact_result_mutex_
);
if
(
compact_thread_results_
.
empty
())
{
// collect merge files for all tables(if compact_table_ids_ is empty) for two reasons:
// 1. other tables may still has un-merged files
// 2. server may be closed unexpected, these un-merge files need to be merged when server restart
if
(
compact_table_ids_
.
empty
())
{
std
::
vector
<
meta
::
TableSchema
>
table_schema_array
;
meta_ptr_
->
AllTables
(
table_schema_array
);
for
(
auto
&
schema
:
table_schema_array
)
{
compact_table_ids_
.
insert
(
schema
.
table_id_
);
}
}
// start merge file thread
compact_thread_results_
.
push_back
(
compact_thread_pool_
.
enqueue
(
&
DBImpl
::
BackgroundCompaction
,
this
,
compact_table_ids_
));
compact_table_ids_
.
clear
();
...
...
@@ -717,7 +729,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
for
(
auto
&
kv
:
raw_files
)
{
auto
files
=
kv
.
second
;
if
(
files
.
size
()
<
options_
.
merge_trigger_number_
)
{
ENGINE_LOG_
DEBUG
<<
"Files number not greater equal than merge trigger number, skip merge action"
;
ENGINE_LOG_
TRACE
<<
"Files number not greater equal than merge trigger number, skip merge action"
;
continue
;
}
...
...
@@ -734,7 +746,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
void
DBImpl
::
BackgroundCompaction
(
std
::
set
<
std
::
string
>
table_ids
)
{
ENGINE_LOG_TRACE
<<
"
Background compaction thread start"
;
ENGINE_LOG_TRACE
<<
"Background compaction thread start"
;
Status
status
;
for
(
auto
&
table_id
:
table_ids
)
{
...
...
@@ -757,7 +769,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
}
meta_ptr_
->
CleanUpFilesWithTTL
(
ttl
);
ENGINE_LOG_TRACE
<<
"
Background compaction thread exit"
;
ENGINE_LOG_TRACE
<<
"Background compaction thread exit"
;
}
void
...
...
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
6c227a35
...
...
@@ -1392,6 +1392,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
}
// Scoped Connection
Status
ret
;
int64_t
to_merge_files
=
0
;
for
(
auto
&
resRow
:
res
)
{
TableFileSchema
table_file
;
table_file
.
file_size_
=
resRow
[
"file_size"
];
...
...
@@ -1420,13 +1421,14 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
auto
dateItr
=
files
.
find
(
table_file
.
date_
);
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date_
]
=
TableFilesSchema
();
to_merge_files
++
;
}
files
[
table_file
.
date_
].
push_back
(
table_file
);
}
if
(
res
.
size
()
>
0
)
{
ENGINE_LOG_
DEBUG
<<
"Collect "
<<
res
.
size
()
<<
" to-merge files"
;
if
(
to_merge_files
>
0
)
{
ENGINE_LOG_
TRACE
<<
"Collect "
<<
to_merge_files
<<
" to-merge files"
;
}
return
ret
;
}
catch
(
std
::
exception
&
e
)
{
...
...
@@ -1809,6 +1811,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
mysqlpp
::
StoreQueryResult
res
=
cleanUpFilesWithTTLQuery
.
store
();
int64_t
remove_tables
=
0
;
if
(
!
res
.
empty
())
{
std
::
stringstream
idsToDeleteSS
;
for
(
auto
&
resRow
:
res
)
{
...
...
@@ -1817,7 +1820,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
resRow
[
"table_id"
].
to_string
(
table_id
);
utils
::
DeleteTablePath
(
options_
,
table_id
,
false
);
// only delete empty folder
remove_tables
++
;
idsToDeleteSS
<<
"id = "
<<
std
::
to_string
(
id
)
<<
" OR "
;
}
std
::
string
idsToDeleteStr
=
idsToDeleteSS
.
str
();
...
...
@@ -1832,8 +1835,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
}
if
(
re
s
.
size
()
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
re
s
.
size
()
<<
" tables from meta"
;
if
(
re
move_tables
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
re
move_tables
<<
" tables from meta"
;
}
}
// Scoped Connection
}
catch
(
std
::
exception
&
e
)
{
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
6c227a35
...
...
@@ -971,6 +971,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
order_by
(
&
TableFileSchema
::
file_size_
).
desc
());
Status
result
;
int64_t
to_merge_files
=
0
;
for
(
auto
&
file
:
selected
)
{
TableFileSchema
table_file
;
table_file
.
file_size_
=
std
::
get
<
4
>
(
file
);
...
...
@@ -999,11 +1000,13 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date_
]
=
TableFilesSchema
();
}
files
[
table_file
.
date_
].
push_back
(
table_file
);
to_merge_files
++
;
}
if
(
selected
.
size
()
>
0
)
{
ENGINE_LOG_
DEBUG
<<
"Collect "
<<
selected
.
size
()
<<
" to-merge files"
;
if
(
to_merge_files
>
0
)
{
ENGINE_LOG_
TRACE
<<
"Collect "
<<
to_merge_files
<<
" to-merge files"
;
}
return
result
;
}
catch
(
std
::
exception
&
e
)
{
...
...
@@ -1313,16 +1316,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
try
{
server
::
MetricCollector
metric
;
int64_t
remove_tables
=
0
;
for
(
auto
&
table_id
:
table_ids
)
{
auto
selected
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
file_id_
),
where
(
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
));
if
(
selected
.
size
()
==
0
)
{
utils
::
DeleteTablePath
(
options_
,
table_id
);
remove_tables
++
;
}
}
if
(
table_ids
.
size
()
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
table_ids
.
size
()
<<
" tables folder"
;
if
(
remove_tables
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
remove_tables
<<
" tables folder"
;
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table folder"
,
e
.
what
());
...
...
core/src/grpc/README.md
0 → 100644
浏览文件 @
6c227a35
We manually change two APIs in "milvus.pd.h":
add_vector_data()
add_row_id_array()
add_ids()
add_distances()
If proto files need be generated again, remember to re-change above APIs.
\ No newline at end of file
core/src/grpc/cpp_gen.sh
100644 → 100755
浏览文件 @
6c227a35
文件模式从 100644 更改为 100755
core/src/grpc/gen-milvus/milvus.grpc.pb.cc
浏览文件 @
6c227a35
...
...
@@ -431,60 +431,60 @@ void MilvusService::Stub::experimental_async::Insert(::grpc::ClientContext* cont
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
VectorIds
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_Insert_
,
context
,
request
,
false
);
}
::
grpc
::
Status
MilvusService
::
Stub
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
grpc
::
Status
MilvusService
::
Stub
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
return
::
grpc
::
internal
::
BlockingUnaryCall
(
channel_
.
get
(),
rpcmethod_Search_
,
context
,
request
,
response
);
}
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
::
grpc_impl
::
internal
::
CallbackUnaryCall
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_Search_
,
context
,
request
,
response
,
std
::
move
(
f
));
}
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
::
grpc_impl
::
internal
::
CallbackUnaryCall
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_Search_
,
context
,
request
,
response
,
std
::
move
(
f
));
}
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
::
grpc_impl
::
internal
::
ClientCallbackUnaryFactory
::
Create
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_Search_
,
context
,
request
,
response
,
reactor
);
}
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
void
MilvusService
::
Stub
::
experimental_async
::
Search
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
::
grpc_impl
::
internal
::
ClientCallbackUnaryFactory
::
Create
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_Search_
,
context
,
request
,
response
,
reactor
);
}
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
List
>*
MilvusService
::
Stub
::
AsyncSearchRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
List
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_Search_
,
context
,
request
,
true
);
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
>*
MilvusService
::
Stub
::
AsyncSearchRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_Search_
,
context
,
request
,
true
);
}
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
List
>*
MilvusService
::
Stub
::
PrepareAsyncSearchRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
List
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_Search_
,
context
,
request
,
false
);
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
>*
MilvusService
::
Stub
::
PrepareAsyncSearchRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_Search_
,
context
,
request
,
false
);
}
::
grpc
::
Status
MilvusService
::
Stub
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
grpc
::
Status
MilvusService
::
Stub
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
return
::
grpc
::
internal
::
BlockingUnaryCall
(
channel_
.
get
(),
rpcmethod_SearchInFiles_
,
context
,
request
,
response
);
}
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
::
grpc_impl
::
internal
::
CallbackUnaryCall
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_SearchInFiles_
,
context
,
request
,
response
,
std
::
move
(
f
));
}
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
std
::
function
<
void
(
::
grpc
::
Status
)
>
f
)
{
::
grpc_impl
::
internal
::
CallbackUnaryCall
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_SearchInFiles_
,
context
,
request
,
response
,
std
::
move
(
f
));
}
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
::
grpc_impl
::
internal
::
ClientCallbackUnaryFactory
::
Create
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_SearchInFiles_
,
context
,
request
,
response
,
reactor
);
}
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
void
MilvusService
::
Stub
::
experimental_async
::
SearchInFiles
(
::
grpc
::
ClientContext
*
context
,
const
::
grpc
::
ByteBuffer
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
,
::
grpc
::
experimental
::
ClientUnaryReactor
*
reactor
)
{
::
grpc_impl
::
internal
::
ClientCallbackUnaryFactory
::
Create
(
stub_
->
channel_
.
get
(),
stub_
->
rpcmethod_SearchInFiles_
,
context
,
request
,
response
,
reactor
);
}
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
List
>*
MilvusService
::
Stub
::
AsyncSearchInFilesRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
List
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_SearchInFiles_
,
context
,
request
,
true
);
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
>*
MilvusService
::
Stub
::
AsyncSearchInFilesRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_SearchInFiles_
,
context
,
request
,
true
);
}
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
List
>*
MilvusService
::
Stub
::
PrepareAsyncSearchInFilesRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
List
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_SearchInFiles_
,
context
,
request
,
false
);
::
grpc
::
ClientAsyncResponseReader
<
::
milvus
::
grpc
::
TopKQueryResult
>*
MilvusService
::
Stub
::
PrepareAsyncSearchInFilesRaw
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
&
request
,
::
grpc
::
CompletionQueue
*
cq
)
{
return
::
grpc_impl
::
internal
::
ClientAsyncResponseReaderFactory
<
::
milvus
::
grpc
::
TopKQueryResult
>::
Create
(
channel_
.
get
(),
cq
,
rpcmethod_SearchInFiles_
,
context
,
request
,
false
);
}
::
grpc
::
Status
MilvusService
::
Stub
::
Cmd
(
::
grpc
::
ClientContext
*
context
,
const
::
milvus
::
grpc
::
Command
&
request
,
::
milvus
::
grpc
::
StringReply
*
response
)
{
...
...
@@ -640,12 +640,12 @@ MilvusService::Service::Service() {
AddMethod
(
new
::
grpc
::
internal
::
RpcServiceMethod
(
MilvusService_method_names
[
13
],
::
grpc
::
internal
::
RpcMethod
::
NORMAL_RPC
,
new
::
grpc
::
internal
::
RpcMethodHandler
<
MilvusService
::
Service
,
::
milvus
::
grpc
::
SearchParam
,
::
milvus
::
grpc
::
TopKQueryResult
List
>
(
new
::
grpc
::
internal
::
RpcMethodHandler
<
MilvusService
::
Service
,
::
milvus
::
grpc
::
SearchParam
,
::
milvus
::
grpc
::
TopKQueryResult
>
(
std
::
mem_fn
(
&
MilvusService
::
Service
::
Search
),
this
)));
AddMethod
(
new
::
grpc
::
internal
::
RpcServiceMethod
(
MilvusService_method_names
[
14
],
::
grpc
::
internal
::
RpcMethod
::
NORMAL_RPC
,
new
::
grpc
::
internal
::
RpcMethodHandler
<
MilvusService
::
Service
,
::
milvus
::
grpc
::
SearchInFilesParam
,
::
milvus
::
grpc
::
TopKQueryResult
List
>
(
new
::
grpc
::
internal
::
RpcMethodHandler
<
MilvusService
::
Service
,
::
milvus
::
grpc
::
SearchInFilesParam
,
::
milvus
::
grpc
::
TopKQueryResult
>
(
std
::
mem_fn
(
&
MilvusService
::
Service
::
SearchInFiles
),
this
)));
AddMethod
(
new
::
grpc
::
internal
::
RpcServiceMethod
(
MilvusService_method_names
[
15
],
...
...
@@ -758,14 +758,14 @@ MilvusService::Service::~Service() {
return
::
grpc
::
Status
(
::
grpc
::
StatusCode
::
UNIMPLEMENTED
,
""
);
}
::
grpc
::
Status
MilvusService
::
Service
::
Search
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
grpc
::
Status
MilvusService
::
Service
::
Search
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
(
void
)
context
;
(
void
)
request
;
(
void
)
response
;
return
::
grpc
::
Status
(
::
grpc
::
StatusCode
::
UNIMPLEMENTED
,
""
);
}
::
grpc
::
Status
MilvusService
::
Service
::
SearchInFiles
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
grpc
::
Status
MilvusService
::
Service
::
SearchInFiles
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
(
void
)
context
;
(
void
)
request
;
(
void
)
response
;
...
...
core/src/grpc/gen-milvus/milvus.grpc.pb.h
浏览文件 @
6c227a35
此差异已折叠。
点击以展开。
core/src/grpc/gen-milvus/milvus.pb.cc
浏览文件 @
6c227a35
此差异已折叠。
点击以展开。
core/src/grpc/gen-milvus/milvus.pb.h
浏览文件 @
6c227a35
此差异已折叠。
点击以展开。
core/src/grpc/milvus.proto
浏览文件 @
6c227a35
...
...
@@ -110,24 +110,11 @@ message SearchInFilesParam {
/**
* @brief Query result params
*/
message
QueryResult
{
int64
id
=
1
;
double
distance
=
2
;
}
/**
* @brief TopK query result
*/
message
TopKQueryResult
{
repeated
QueryResult
query_result_arrays
=
1
;
}
/**
* @brief List of topK query result
*/
message
TopKQueryResultList
{
Status
status
=
1
;
repeated
TopKQueryResult
topk_query_result
=
2
;
int64
row_num
=
2
;
repeated
int64
ids
=
3
;
repeated
float
distances
=
4
;
}
/**
...
...
@@ -311,18 +298,18 @@ service MilvusService {
*
* @param SearchParam, search parameters.
*
* @return TopKQueryResult
List
* @return TopKQueryResult
*/
rpc
Search
(
SearchParam
)
returns
(
TopKQueryResult
List
)
{}
rpc
Search
(
SearchParam
)
returns
(
TopKQueryResult
)
{}
/**
* @brief This method is used to query vector in specified files.
*
* @param SearchInFilesParam, search in files paremeters.
*
* @return TopKQueryResult
List
* @return TopKQueryResult
*/
rpc
SearchInFiles
(
SearchInFilesParam
)
returns
(
TopKQueryResult
List
)
{}
rpc
SearchInFiles
(
SearchInFilesParam
)
returns
(
TopKQueryResult
)
{}
/**
* @brief This method is used to give the server status.
...
...
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
浏览文件 @
6c227a35
...
...
@@ -89,34 +89,35 @@ ConvertToDataset(std::vector<SPTAG::QueryResult> query_results) {
}
}
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
p_id
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
p_dist
,
sizeof
(
float
)
*
elems
);
// TODO: magic
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
// auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
// auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
auto
field_id
=
std
::
make_shared
<
Field
>
(
"id"
,
std
::
make_shared
<
arrow
::
Int64Type
>
());
auto
field_dist
=
std
::
make_shared
<
Field
>
(
"dist"
,
std
::
make_shared
<
arrow
::
FloatType
>
());
std
::
vector
<
FieldPtr
>
fields
{
field_id
,
field_dist
};
auto
schema
=
std
::
make_shared
<
Schema
>
(
fields
);
return
std
::
make_shared
<
Dataset
>
(
array
,
schema
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);
//
// // TODO: magic
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
// // auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// // auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
//
// // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());
// auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());
// std::vector<FieldPtr> fields{field_id, field_dist};
// auto schema = std::make_shared<Schema>(fields);
//
// return std::make_shared<Dataset>(array, schema);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
p_id
,
(
void
*
)
p_dist
);
}
}
// namespace knowhere
core/src/index/knowhere/knowhere/common/Dataset.h
浏览文件 @
6c227a35
...
...
@@ -54,6 +54,9 @@ class Dataset {
:
tensor_
(
std
::
move
(
tensor
)),
tensor_schema_
(
std
::
move
(
tensor_schema
))
{
}
Dataset
(
void
*
ids
,
void
*
dists
)
:
ids_
(
ids
),
dists_
(
dists
)
{
}
Dataset
(
const
Dataset
&
)
=
delete
;
Dataset
&
operator
=
(
const
Dataset
&
)
=
delete
;
...
...
@@ -128,6 +131,16 @@ class Dataset {
tensor_schema_
=
std
::
move
(
tensor_schema
);
}
void
*
ids
()
{
return
ids_
;
}
void
*
dist
()
{
return
dists_
;
}
// const Config &
// meta() const { return meta_; }
...
...
@@ -141,6 +154,9 @@ class Dataset {
SchemaPtr
array_schema_
;
std
::
vector
<
TensorPtr
>
tensor_
;
SchemaPtr
tensor_schema_
;
// TODO(yukun): using smart pointer
void
*
ids_
;
void
*
dists_
;
// Config meta_;
};
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
浏览文件 @
6c227a35
...
...
@@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) {
search_impl
(
rows
,
(
float
*
)
p_data
,
config
->
k
,
res_dis
,
res_ids
,
Config
());
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
void
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
浏览文件 @
6c227a35
...
...
@@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
// std::cout << ss_res_id.str() << std::endl;
// std::cout << ss_res_dist.str() << std::endl << std::endl;
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
//
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
//
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
//
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
//
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
//
auto int64_type = std::make_shared<arrow::Int64Type>();
//
auto float_type = std::make_shared<arrow::FloatType>();
//
//
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
//
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
//
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
//
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
//
std::vector<ArrayPtr> array{ids, dists};
return
std
::
make_shared
<
Dataset
>
(
(
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
void
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
浏览文件 @
6c227a35
...
...
@@ -88,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) {
s_params
.
search_length
=
build_cfg
->
search_length
;
index_
->
Search
((
float
*
)
p_data
,
rows
,
dim
,
build_cfg
->
k
,
res_dis
,
res_ids
,
s_params
);
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
IndexModelPtr
...
...
core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp
浏览文件 @
6c227a35
...
...
@@ -202,14 +202,14 @@ parse_ann_test_name(const std::string& ann_test_name, size_t& dim, faiss::Metric
int32_t
GetResultHitCount
(
const
faiss
::
Index
::
idx_t
*
ground_index
,
const
faiss
::
Index
::
idx_t
*
index
,
size_t
ground_k
,
size_t
k
,
size_t
nq
,
int32_t
index_add_loops
)
{
assert
(
ground_k
<=
k
);
size_t
min_k
=
std
::
min
(
ground_k
,
k
);
int
hit
=
0
;
for
(
int
i
=
0
;
i
<
nq
;
i
++
)
{
// count the num of results exist in ground truth result set
// each result replicates INDEX_ADD_LOOPS times
for
(
int
j_c
=
0
;
j_c
<
k
;
j_c
++
)
{
int
r_c
=
index
[
i
*
k
+
j_c
];
for
(
int
j_g
=
0
;
j_g
<
ground
_k
/
index_add_loops
;
j_g
++
)
{
for
(
int
j_g
=
0
;
j_g
<
min
_k
/
index_add_loops
;
j_g
++
)
{
if
(
ground_index
[
i
*
ground_k
+
j_g
]
==
r_c
)
{
hit
++
;
continue
;
...
...
@@ -283,7 +283,11 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std
// add index multiple times to get ~1G data set
for
(
int
i
=
0
;
i
<
index_add_loops
;
i
++
)
{
printf
(
"[%.3f s] No.%d Indexing database, size %ld*%ld
\n
"
,
elapsed
()
-
t0
,
i
,
nb
,
d
);
gpu_index
->
add
(
nb
,
xb
);
std
::
vector
<
faiss
::
Index
::
idx_t
>
xids
(
nb
);
for
(
int
t
=
0
;
t
<
nb
;
t
++
)
{
xids
[
t
]
=
i
*
nb
+
t
;
}
gpu_index
->
add_with_ids
(
nb
,
xb
,
xids
.
data
());
}
printf
(
"[%.3f s] Coping GPU index to CPU
\n
"
,
elapsed
()
-
t0
);
...
...
@@ -359,13 +363,14 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
const
int32_t
search_loops
)
{
double
t0
=
elapsed
();
const
s
ize_t
NQ
=
1000
,
NQ_START
=
10
,
NQ_STEP
=
10
;
const
s
ize_t
K
=
1000
,
K_START
=
100
,
K_STEP
=
10
;
const
s
td
::
vector
<
size_t
>
NQ
=
{
10
,
100
}
;
const
s
td
::
vector
<
size_t
>
K
=
{
10
,
100
,
1000
}
;
const
size_t
GK
=
100
;
// topk of ground truth
std
::
unordered_map
<
size_t
,
std
::
string
>
mode_str_map
=
{
{
MODE_CPU
,
"MODE_CPU"
},
{
MODE_MIX
,
"MODE_MIX"
},
{
MODE_GPU
,
"MODE_GPU"
}};
double
copy_time
=
0.0
;
faiss
::
Index
*
gpu_index
,
*
index
;
if
(
query_mode
!=
MODE_CPU
)
{
faiss
::
gpu
::
GpuClonerOptions
option
;
...
...
@@ -375,7 +380,6 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
index_composition
.
index
=
cpu_index
;
index_composition
.
quantizer
=
nullptr
;
double
copy_time
;
switch
(
query_mode
)
{
case
MODE_MIX
:
{
index_composition
.
mode
=
1
;
// 0: all data, 1: copy quantizer, 2: copy data
...
...
@@ -420,34 +424,39 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
}
for
(
auto
nprobe
:
nprobes
)
{
switch
(
query_mode
)
{
case
MODE_CPU
:
case
MODE_MIX
:
{
faiss
::
ParameterSpace
params
;
std
::
string
nprobe_str
=
"nprobe="
+
std
::
to_string
(
nprobe
);
params
.
set_index_parameters
(
index
,
nprobe_str
.
c_str
());
break
;
}
case
MODE_GPU
:
{
faiss
::
gpu
::
GpuIndexIVF
*
gpu_index_ivf
=
dynamic_cast
<
faiss
::
gpu
::
GpuIndexIVF
*>
(
index
);
gpu_index_ivf
->
setNumProbes
(
nprobe
);
// brute-force need not set nprobe
if
(
index_key
.
find
(
"IDMap"
)
==
std
::
string
::
npos
)
{
switch
(
query_mode
)
{
case
MODE_CPU
:
case
MODE_MIX
:
{
faiss
::
ParameterSpace
params
;
std
::
string
nprobe_str
=
"nprobe="
+
std
::
to_string
(
nprobe
);
params
.
set_index_parameters
(
index
,
nprobe_str
.
c_str
());
break
;
}
case
MODE_GPU
:
{
faiss
::
gpu
::
GpuIndexIVF
*
gpu_index_ivf
=
dynamic_cast
<
faiss
::
gpu
::
GpuIndexIVF
*>
(
index
);
gpu_index_ivf
->
setNumProbes
(
nprobe
);
}
}
}
// output buffers
faiss
::
Index
::
idx_t
*
I
=
new
faiss
::
Index
::
idx_t
[
NQ
*
K
];
faiss
::
Index
::
distance_t
*
D
=
new
faiss
::
Index
::
distance_t
[
NQ
*
K
];
faiss
::
Index
::
idx_t
*
I
=
new
faiss
::
Index
::
idx_t
[
NQ
.
back
()
*
K
.
back
()
];
faiss
::
Index
::
distance_t
*
D
=
new
faiss
::
Index
::
distance_t
[
NQ
.
back
()
*
K
.
back
()
];
printf
(
"
\n
%s | %s - %s | nprobe=%lu
\n
"
,
ann_test_name
.
c_str
(),
index_key
.
c_str
(),
mode_str_map
[
query_mode
].
c_str
(),
nprobe
);
printf
(
"======================================================================================
\n
"
);
for
(
size_t
t_nq
=
NQ_START
;
t_nq
<=
NQ
;
t_nq
*=
NQ_STEP
)
{
// nq = {10, 100, 1000}
for
(
size_t
t_k
=
K_START
;
t_k
<=
K
;
t_k
*=
K_STEP
)
{
// k = {100, 1000}
for
(
size_t
j
=
0
;
j
<
K
.
size
();
j
++
)
{
size_t
t_k
=
K
[
j
];
for
(
size_t
i
=
0
;
i
<
NQ
.
size
();
i
++
)
{
size_t
t_nq
=
NQ
[
i
];
faiss
::
indexIVF_stats
.
quantization_time
=
0.0
;
faiss
::
indexIVF_stats
.
search_time
=
0.0
;
double
t_start
=
elapsed
(),
t_end
;
for
(
int
i
=
0
;
i
<
search_loops
;
i
++
)
{
for
(
int
s
=
0
;
s
<
search_loops
;
s
++
)
{
index
->
search
(
t_nq
,
xq
,
t_k
,
D
,
I
);
}
t_end
=
elapsed
();
...
...
@@ -466,7 +475,7 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
printf
(
"nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f
\n
"
,
t_nq
,
t_k
,
(
t_end
-
t_start
)
/
search_loops
,
faiss
::
indexIVF_stats
.
quantization_time
/
1000
/
search_loops
,
faiss
::
indexIVF_stats
.
search_time
/
1000
/
search_loops
,
(
hit
/
float
(
t_nq
*
GK
/
index_add_loops
)));
(
hit
/
float
(
t_nq
*
std
::
min
(
GK
,
t_k
)
/
index_add_loops
)));
}
}
printf
(
"======================================================================================
\n
"
);
...
...
@@ -479,8 +488,9 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key
}
void
test_ann_hdf5
(
const
std
::
string
&
ann_test_name
,
const
std
::
string
&
index_type
,
const
QueryMode
query_mode
,
int32_t
index_add_loops
,
const
std
::
vector
<
size_t
>&
nprobes
,
int32_t
search_loops
)
{
test_ann_hdf5
(
const
std
::
string
&
ann_test_name
,
const
std
::
string
&
cluster_type
,
const
std
::
string
&
index_type
,
const
QueryMode
query_mode
,
int32_t
index_add_loops
,
const
std
::
vector
<
size_t
>&
nprobes
,
int32_t
search_loops
)
{
double
t0
=
elapsed
();
faiss
::
gpu
::
StandardGpuResources
res
;
...
...
@@ -493,7 +503,7 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
return
;
}
std
::
string
index_key
=
"IVF16384
,"
+
index_type
;
std
::
string
index_key
=
cluster_type
+
"
,"
+
index_type
;
if
(
!
parse_ann_test_name
(
ann_test_name
,
dim
,
metric_type
))
{
printf
(
"Invalid ann test name: %s
\n
"
,
ann_test_name
.
c_str
());
...
...
@@ -526,7 +536,7 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c
*
* Dataset Dimensions Train_size Test_size Neighbors Distance Download
* Fashion-
*
MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB)
MNIST 784 60,000 10,000 100 Euclidean HDF5 (217MB)
* GIST 960 1,000,000 1,000 100 Euclidean HDF5 (3.6GB)
* GloVe 100 1,183,514 10,000 100 Angular HDF5 (463MB)
* GloVe 200 1,183,514 10,000 100 Angular HDF5 (918MB)
...
...
@@ -542,30 +552,39 @@ TEST(FAISSTEST, BENCHMARK) {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const
int32_t
SIFT_INSERT_LOOPS
=
2
;
// insert twice to get ~1G data set
test_ann_hdf5
(
"sift-128-euclidean"
,
"Flat"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"Flat"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IDMap"
,
"Flat"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IDMap"
,
"Flat"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IVF16384"
,
"Flat"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IVF16384"
,
"Flat"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"
IVF16384"
,
"
SQ8"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"
IVF16384"
,
"
SQ8"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
#ifdef CUSTOMIZATION
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_MIX
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"SQ8Hybrid"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_CPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_MIX
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"sift-128-euclidean"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_GPU
,
SIFT_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
#endif
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
const
int32_t
GLOVE_INSERT_LOOPS
=
1
;
test_ann_hdf5
(
"glove-200-angular"
,
"Flat"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"Flat"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"
IVF16384"
,
"
Flat"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"
IVF16384"
,
"
Flat"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"
IVF16384"
,
"
SQ8"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"
IVF16384"
,
"
SQ8"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
#ifdef CUSTOMIZATION
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_MIX
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"SQ8Hybrid"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_CPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_MIX
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
test_ann_hdf5
(
"glove-200-angular"
,
"IVF16384"
,
"SQ8Hybrid"
,
MODE_GPU
,
GLOVE_INSERT_LOOPS
,
param_nprobes
,
SEARCH_LOOPS
);
#endif
}
core/src/index/unittest/test_ivf.cpp
浏览文件 @
6c227a35
...
...
@@ -181,11 +181,13 @@ TEST_P(IVFTest, clone_test) {
// PrintResult(result, nq, k);
auto
AssertEqual
=
[
&
](
knowhere
::
DatasetPtr
p1
,
knowhere
::
DatasetPtr
p2
)
{
auto
ids_p1
=
p1
->
array
()[
0
]
;
auto
ids_p2
=
p2
->
array
()[
0
]
;
auto
ids_p1
=
p1
->
ids
()
;
auto
ids_p2
=
p2
->
ids
()
;
for
(
int
i
=
0
;
i
<
nq
*
k
;
++
i
)
{
EXPECT_EQ
(
*
(
ids_p2
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
)),
*
(
ids_p1
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
)));
EXPECT_EQ
(
*
((
int64_t
*
)(
ids_p2
)
+
i
),
*
((
int64_t
*
)(
ids_p1
)
+
i
));
// EXPECT_EQ(*(ids_p2->data()->GetValues<int64_t>(1, i)), *(ids_p1->data()->GetValues<int64_t>(1,
// i)));
}
};
...
...
core/src/index/unittest/test_kdt.cpp
浏览文件 @
6c227a35
...
...
@@ -66,15 +66,19 @@ TEST_F(KDTTest, kdt_basic) {
AssertAnns
(
result
,
nq
,
k
);
{
auto
ids
=
result
->
array
()[
0
];
auto
dists
=
result
->
array
()[
1
];
// auto ids = result->array()[0];
// auto dists = result->array()[1];
auto
ids
=
result
->
ids
();
auto
dists
=
result
->
dist
();
std
::
stringstream
ss_id
;
std
::
stringstream
ss_dist
;
for
(
auto
i
=
0
;
i
<
nq
;
i
++
)
{
for
(
auto
j
=
0
;
j
<
k
;
++
j
)
{
ss_id
<<
*
ids
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
*
k
+
j
)
<<
" "
;
ss_dist
<<
*
dists
->
data
()
->
GetValues
<
float
>
(
1
,
i
*
k
+
j
)
<<
" "
;
ss_id
<<
*
((
int64_t
*
)(
ids
)
+
i
*
k
+
j
)
<<
" "
;
ss_dist
<<
*
((
float
*
)(
dists
)
+
i
*
k
+
j
)
<<
" "
;
// ss_id << *ids->data()->GetValues<int64_t>(1, i * k + j) << " ";
// ss_dist << *dists->data()->GetValues<float>(1, i * k + j) << " ";
}
ss_id
<<
std
::
endl
;
ss_dist
<<
std
::
endl
;
...
...
core/src/index/unittest/utils.cpp
浏览文件 @
6c227a35
...
...
@@ -151,9 +151,10 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) {
void
AssertAnns
(
const
knowhere
::
DatasetPtr
&
result
,
const
int
&
nq
,
const
int
&
k
)
{
auto
ids
=
result
->
array
()[
0
]
;
auto
ids
=
result
->
ids
()
;
for
(
auto
i
=
0
;
i
<
nq
;
i
++
)
{
EXPECT_EQ
(
i
,
*
(
ids
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
*
k
)));
EXPECT_EQ
(
i
,
*
((
int64_t
*
)(
ids
)
+
i
*
k
));
// EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
}
}
...
...
core/src/metrics/Metrics.cpp
浏览文件 @
6c227a35
...
...
@@ -16,8 +16,10 @@
// under the License.
#include "metrics/Metrics.h"
#include "PrometheusMetrics.h"
#include "server/Config.h"
#ifdef MILVUS_WITH_PROMETHEUS
#include "metrics/prometheus/PrometheusMetrics.h"
#endif
#include <string>
...
...
@@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() {
config
.
GetMetricConfigCollector
(
collector_type_str
);
#ifdef MILVUS_WITH_PROMETHEUS
if
(
collector_type_str
==
"prometheus"
)
{
return
PrometheusMetrics
::
GetInstance
();
}
else
{
return
MetricsBase
::
GetInstance
();
}
#else
return
MetricsBase
::
GetInstance
();
#endif
}
}
// namespace server
...
...
core/src/metrics/PrometheusMetrics.cpp
→
core/src/metrics/
prometheus/
PrometheusMetrics.cpp
浏览文件 @
6c227a35
...
...
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "SystemInfo.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include "cache/GpuCacheMgr.h"
#include "metrics/SystemInfo.h"
#include "server/Config.h"
#include "utils/Log.h"
...
...
core/src/metrics/PrometheusMetrics.h
→
core/src/metrics/
prometheus/
PrometheusMetrics.h
浏览文件 @
6c227a35
...
...
@@ -24,7 +24,7 @@
#include <string>
#include <vector>
#include "MetricBase.h"
#include "
metrics/
MetricBase.h"
#include "utils/Error.h"
#define METRICS_NOW_TIME std::chrono::system_clock::now()
...
...
core/src/sdk/examples/partition/src/ClientTest.cpp
浏览文件 @
6c227a35
...
...
@@ -133,18 +133,18 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{
// search vectors
std
::
cout
<<
"Search in correct partition"
<<
std
::
endl
;
std
::
vector
<
std
::
string
>
partiton_tags
=
{
std
::
to_string
(
TARGET_PARTITION
)};
std
::
vector
<
milvus
::
TopKQueryResult
>
topk_query_result_array
;
milvus
::
TopKQueryResult
topk_query_result
;
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
std
::
cout
<<
"Search in wrong partition"
<<
std
::
endl
;
partiton_tags
=
{
"0"
};
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
std
::
cout
<<
"Search by regex matched partition tag"
<<
std
::
endl
;
partiton_tags
=
{
"
\\
d"
};
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
}
{
// wait unit build index finish
...
...
@@ -182,9 +182,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{
// search vectors
std
::
cout
<<
"Search in whole table"
<<
std
::
endl
;
std
::
vector
<
std
::
string
>
partiton_tags
;
std
::
vector
<
milvus
::
TopKQueryResult
>
topk_query_result_array
;
milvus
::
TopKQueryResult
topk_query_result
;
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
}
{
// drop index
...
...
core/src/sdk/examples/simple/src/ClientTest.cpp
浏览文件 @
6c227a35
...
...
@@ -144,9 +144,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
milvus_sdk
::
Utils
::
Sleep
(
3
);
{
// search vectors
std
::
vector
<
std
::
string
>
partiton_tags
;
std
::
vector
<
milvus
::
TopKQueryResult
>
topk_query_result_array
;
milvus
::
TopKQueryResult
topk_query_result
;
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
}
{
// wait unit build index finish
...
...
@@ -169,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{
// search vectors
std
::
vector
<
std
::
string
>
partiton_tags
;
std
::
vector
<
milvus
::
TopKQueryResult
>
topk_query_result_array
;
milvus
::
TopKQueryResult
topk_query_result
;
milvus_sdk
::
Utils
::
DoSearch
(
conn
,
TABLE_NAME
,
partiton_tags
,
TOP_K
,
NPROBE
,
search_record_array
,
topk_query_result
_array
);
topk_query_result
);
}
{
// drop index
...
...
core/src/sdk/examples/utils/Utils.cpp
浏览文件 @
6c227a35
...
...
@@ -155,32 +155,34 @@ Utils::BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& ve
void
Utils
::
PrintSearchResult
(
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
const
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
)
{
const
milvus
::
TopKQueryResult
&
topk_query_result
)
{
BLOCK_SPLITER
std
::
cout
<<
"Returned result count: "
<<
topk_query_result_array
.
size
()
<<
std
::
endl
;
size_t
nq
=
topk_query_result
.
row_num
;
size_t
topk
=
topk_query_result
.
ids
.
size
()
/
nq
;
std
::
cout
<<
"Returned result count: "
<<
nq
*
topk
<<
std
::
endl
;
int32_t
index
=
0
;
for
(
auto
&
result
:
topk_query_result_array
)
{
for
(
size_t
i
=
0
;
i
<
nq
;
i
++
)
{
auto
search_id
=
search_record_array
[
index
].
first
;
index
++
;
std
::
cout
<<
"No."
<<
std
::
to_string
(
index
)
<<
" vector "
<<
std
::
to_string
(
search_id
)
<<
" top "
<<
std
::
to_string
(
result
.
query_result_arrays
.
size
())
<<
" search result:"
<<
std
::
endl
;
for
(
auto
&
item
:
result
.
query_result_arrays
)
{
std
::
cout
<<
"
\t
"
<<
std
::
to_string
(
item
.
id
)
<<
"
\t
distance:"
<<
std
::
to_string
(
item
.
distance
);
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
"No."
<<
index
<<
" vector "
<<
search_id
<<
" top "
<<
topk
<<
" search result:"
<<
std
::
endl
;
for
(
size_t
j
=
0
;
j
<
topk
;
j
++
)
{
size_t
idx
=
i
*
nq
+
j
;
std
::
cout
<<
"
\t
"
<<
topk_query_result
.
ids
[
idx
]
<<
"
\t
"
<<
topk_query_result
.
distances
[
idx
]
<<
std
::
endl
;
}
}
BLOCK_SPLITER
}
void
Utils
::
CheckSearchResult
(
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
const
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
)
{
const
milvus
::
TopKQueryResult
&
topk_query_result
)
{
BLOCK_SPLITER
size_t
nq
=
topk_query_result
.
row_num
;
size_t
result_k
=
topk_query_result
.
ids
.
size
()
/
nq
;
int64_t
index
=
0
;
for
(
auto
&
result
:
topk_query_result_array
)
{
auto
result_id
=
result
.
query_result_arrays
[
0
].
id
;
for
(
size_t
i
=
0
;
i
<
nq
;
i
++
)
{
auto
result_id
=
topk_query_result
.
ids
[
i
*
result_k
]
;
auto
search_id
=
search_record_array
[
index
++
].
first
;
if
(
result_id
!=
search_id
)
{
std
::
cout
<<
"The top 1 result is wrong: "
<<
result_id
<<
" vs. "
<<
search_id
<<
std
::
endl
;
...
...
@@ -195,8 +197,10 @@ void
Utils
::
DoSearch
(
std
::
shared_ptr
<
milvus
::
Connection
>
conn
,
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
int64_t
top_k
,
int64_t
nprobe
,
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
)
{
topk_query_result_array
.
clear
();
milvus
::
TopKQueryResult
&
topk_query_result
)
{
topk_query_result
.
distances
.
clear
();
topk_query_result
.
ids
.
clear
();
topk_query_result
.
row_num
=
0
;
std
::
vector
<
milvus
::
Range
>
query_range_array
;
milvus
::
Range
rg
;
...
...
@@ -212,14 +216,14 @@ Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& tab
{
BLOCK_SPLITER
milvus_sdk
::
TimeRecorder
rc
(
"search"
);
milvus
::
Status
stat
=
conn
->
Search
(
table_name
,
partiton_tags
,
record_array
,
query_range_array
,
top_k
,
nprobe
,
topk_query_result_array
);
milvus
::
Status
stat
=
conn
->
Search
(
table_name
,
partiton_tags
,
record_array
,
query_range_array
,
top_k
,
nprobe
,
topk_query_result
);
std
::
cout
<<
"SearchVector function call status: "
<<
stat
.
message
()
<<
std
::
endl
;
BLOCK_SPLITER
}
PrintSearchResult
(
search_record_array
,
topk_query_result
_array
);
CheckSearchResult
(
search_record_array
,
topk_query_result
_array
);
PrintSearchResult
(
search_record_array
,
topk_query_result
);
CheckSearchResult
(
search_record_array
,
topk_query_result
);
}
}
// namespace milvus_sdk
core/src/sdk/examples/utils/Utils.h
浏览文件 @
6c227a35
...
...
@@ -61,17 +61,17 @@ class Utils {
static
void
PrintSearchResult
(
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
const
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
);
const
milvus
::
TopKQueryResult
&
topk_query_result
);
static
void
CheckSearchResult
(
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
const
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
);
const
milvus
::
TopKQueryResult
&
topk_query_result
);
static
void
DoSearch
(
std
::
shared_ptr
<
milvus
::
Connection
>
conn
,
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
int64_t
top_k
,
int64_t
nprobe
,
const
std
::
vector
<
std
::
pair
<
int64_t
,
milvus
::
RowRecord
>>&
search_record_array
,
std
::
vector
<
milvus
::
TopKQueryResult
>&
topk_query_result_array
);
milvus
::
TopKQueryResult
&
topk_query_result
);
};
}
// namespace milvus_sdk
core/src/sdk/grpc/ClientProxy.cpp
浏览文件 @
6c227a35
...
...
@@ -32,6 +32,13 @@ UriCheck(const std::string& uri) {
return
(
index
!=
std
::
string
::
npos
);
}
void
CopyRowRecord
(
::
milvus
::
grpc
::
RowRecord
*
target
,
const
RowRecord
&
src
)
{
auto
vector_data
=
target
->
mutable_vector_data
();
vector_data
->
Resize
(
static_cast
<
int
>
(
src
.
data
.
size
()),
0.0
);
memcpy
(
vector_data
->
mutable_data
(),
src
.
data
.
data
(),
src
.
data
.
size
()
*
sizeof
(
float
));
}
Status
ClientProxy
::
Connect
(
const
ConnectParam
&
param
)
{
std
::
string
uri
=
param
.
ip_address
+
":"
+
param
.
port
;
...
...
@@ -189,23 +196,22 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_
for
(
auto
&
record
:
record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
grpc_record
=
insert_param
.
add_row_record_array
();
for
(
size_t
i
=
0
;
i
<
record
.
data
.
size
();
i
++
)
{
grpc_record
->
add_vector_data
(
record
.
data
[
i
]);
}
CopyRowRecord
(
grpc_record
,
record
);
}
// Single thread
::
milvus
::
grpc
::
VectorIds
vector_ids
;
if
(
!
id_array
.
empty
())
{
for
(
auto
i
=
0
;
i
<
id_array
.
size
();
i
++
)
{
insert_param
.
add_row_id_array
(
id_array
[
i
]);
}
/* set user's ids */
auto
row_ids
=
insert_param
.
mutable_row_id_array
();
row_ids
->
Reserve
(
static_cast
<
int
>
(
id_array
.
size
()));
memcpy
(
row_ids
->
mutable_data
(),
id_array
.
data
(),
id_array
.
size
()
*
sizeof
(
int64_t
));
client_ptr_
->
Insert
(
vector_ids
,
insert_param
,
status
);
}
else
{
client_ptr_
->
Insert
(
vector_ids
,
insert_param
,
status
);
for
(
size_t
i
=
0
;
i
<
vector_ids
.
vector_id_array_size
();
i
++
)
{
id_array
.
push_back
(
vector_ids
.
vector_id_array
(
i
));
}
/* return Milvus generated ids back to user */
id_array
.
insert
(
id_array
.
end
(),
vector_ids
.
vector_id_array
().
begin
(),
vector_ids
.
vector_id_array
().
end
());
}
#endif
}
catch
(
std
::
exception
&
ex
)
{
...
...
@@ -218,7 +224,7 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_
Status
ClientProxy
::
Search
(
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
const
std
::
vector
<
RowRecord
>&
query_record_array
,
const
std
::
vector
<
Range
>&
query_range_array
,
int64_t
topk
,
int64_t
nprobe
,
std
::
vector
<
TopKQueryResult
>&
topk_query_result_array
)
{
int64_t
topk
,
int64_t
nprobe
,
TopKQueryResult
&
topk_query_result
)
{
try
{
// step 1: convert vectors data
::
milvus
::
grpc
::
SearchParam
search_param
;
...
...
@@ -230,9 +236,7 @@ ClientProxy::Search(const std::string& table_name, const std::vector<std::string
}
for
(
auto
&
record
:
query_record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
row_record
=
search_param
.
add_query_record_array
();
for
(
auto
&
rec
:
record
.
data
)
{
row_record
->
add_vector_data
(
rec
);
}
CopyRowRecord
(
row_record
,
record
);
}
// step 2: convert range array
...
...
@@ -243,21 +247,17 @@ ClientProxy::Search(const std::string& table_name, const std::vector<std::string
}
// step 3: search vectors
::
milvus
::
grpc
::
TopKQueryResult
List
topk_query_result_lis
t
;
Status
status
=
client_ptr_
->
Search
(
topk_query_result_lis
t
,
search_param
);
::
milvus
::
grpc
::
TopKQueryResult
resul
t
;
Status
status
=
client_ptr_
->
Search
(
resul
t
,
search_param
);
// step 4: convert result array
for
(
uint64_t
i
=
0
;
i
<
topk_query_result_list
.
topk_query_result_size
();
++
i
)
{
TopKQueryResult
result
;
for
(
uint64_t
j
=
0
;
j
<
topk_query_result_list
.
topk_query_result
(
i
).
query_result_arrays_size
();
++
j
)
{
QueryResult
query_result
;
query_result
.
id
=
topk_query_result_list
.
topk_query_result
(
i
).
query_result_arrays
(
j
).
id
();
query_result
.
distance
=
topk_query_result_list
.
topk_query_result
(
i
).
query_result_arrays
(
j
).
distance
();
result
.
query_result_arrays
.
emplace_back
(
query_result
);
}
topk_query_result
.
row_num
=
result
.
row_num
();
topk_query_result
.
ids
.
resize
(
result
.
ids
().
size
());
memcpy
(
topk_query_result
.
ids
.
data
(),
result
.
ids
().
data
(),
result
.
ids
().
size
()
*
sizeof
(
int64_t
));
topk_query_result
.
distances
.
resize
(
result
.
distances
().
size
());
memcpy
(
topk_query_result
.
distances
.
data
(),
result
.
distances
().
data
(),
result
.
distances
().
size
()
*
sizeof
(
float
));
topk_query_result_array
.
emplace_back
(
result
);
}
return
status
;
}
catch
(
std
::
exception
&
ex
)
{
return
Status
(
StatusCode
::
UnknownError
,
"fail to search vectors: "
+
std
::
string
(
ex
.
what
()));
...
...
core/src/sdk/grpc/ClientProxy.h
浏览文件 @
6c227a35
...
...
@@ -60,7 +60,7 @@ class ClientProxy : public Connection {
Status
Search
(
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
const
std
::
vector
<
RowRecord
>&
query_record_array
,
const
std
::
vector
<
Range
>&
query_range_array
,
int64_t
topk
,
int64_t
nprobe
,
std
::
vector
<
TopKQueryResult
>&
topk_query_result_array
)
override
;
int64_t
nprobe
,
TopKQueryResult
&
topk_query_result
)
override
;
Status
DescribeTable
(
const
std
::
string
&
table_name
,
TableSchema
&
table_schema
)
override
;
...
...
core/src/sdk/grpc/GrpcClient.cpp
浏览文件 @
6c227a35
...
...
@@ -134,20 +134,20 @@ GrpcClient::Insert(::milvus::grpc::VectorIds& vector_ids, const ::milvus::grpc::
}
Status
GrpcClient
::
Search
(
::
milvus
::
grpc
::
TopKQueryResult
List
&
topk_query_result_lis
t
,
GrpcClient
::
Search
(
::
milvus
::
grpc
::
TopKQueryResult
&
topk_query_resul
t
,
const
::
milvus
::
grpc
::
SearchParam
&
search_param
)
{
::
milvus
::
grpc
::
TopKQueryResult
query_result
;
ClientContext
context
;
::
grpc
::
Status
grpc_status
=
stub_
->
Search
(
&
context
,
search_param
,
&
topk_query_result
_list
);
::
grpc
::
Status
grpc_status
=
stub_
->
Search
(
&
context
,
search_param
,
&
topk_query_result
);
if
(
!
grpc_status
.
ok
())
{
std
::
cerr
<<
"SearchVector rpc failed!"
<<
std
::
endl
;
std
::
cerr
<<
grpc_status
.
error_message
()
<<
std
::
endl
;
return
Status
(
StatusCode
::
RPCFailed
,
grpc_status
.
error_message
());
}
if
(
topk_query_result
_list
.
status
().
error_code
()
!=
grpc
::
SUCCESS
)
{
std
::
cerr
<<
topk_query_result
_list
.
status
().
reason
()
<<
std
::
endl
;
return
Status
(
StatusCode
::
ServerFailed
,
topk_query_result
_list
.
status
().
reason
());
if
(
topk_query_result
.
status
().
error_code
()
!=
grpc
::
SUCCESS
)
{
std
::
cerr
<<
topk_query_result
.
status
().
reason
()
<<
std
::
endl
;
return
Status
(
StatusCode
::
ServerFailed
,
topk_query_result
.
status
().
reason
());
}
return
Status
::
OK
();
...
...
core/src/sdk/grpc/GrpcClient.h
浏览文件 @
6c227a35
...
...
@@ -57,7 +57,7 @@ class GrpcClient {
Insert
(
grpc
::
VectorIds
&
vector_ids
,
const
grpc
::
InsertParam
&
insert_param
,
Status
&
status
);
Status
Search
(
::
milvus
::
grpc
::
TopKQueryResult
List
&
topk_query_result_lis
t
,
const
grpc
::
SearchParam
&
search_param
);
Search
(
::
milvus
::
grpc
::
TopKQueryResult
&
topk_query_resul
t
,
const
grpc
::
SearchParam
&
search_param
);
Status
DescribeTable
(
grpc
::
TableSchema
&
grpc_schema
,
const
std
::
string
&
table_name
);
...
...
core/src/sdk/include/MilvusApi.h
浏览文件 @
6c227a35
...
...
@@ -78,19 +78,13 @@ struct RowRecord {
std
::
vector
<
float
>
data
;
///< Vector raw data
};
/**
* @brief Query result
*/
struct
QueryResult
{
int64_t
id
;
///< Output result
double
distance
;
///< Vector similarity distance
};
/**
* @brief TopK query result
*/
struct
TopKQueryResult
{
std
::
vector
<
QueryResult
>
query_result_arrays
;
///< TopK query result
int64_t
row_num
;
std
::
vector
<
int64_t
>
ids
;
std
::
vector
<
float
>
distances
;
};
/**
...
...
@@ -274,7 +268,7 @@ class Connection {
virtual
Status
Search
(
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
const
std
::
vector
<
RowRecord
>&
query_record_array
,
const
std
::
vector
<
Range
>&
query_range_array
,
int64_t
topk
,
int64_t
nprobe
,
std
::
vector
<
TopKQueryResult
>&
topk_query_result_array
)
=
0
;
int64_t
nprobe
,
TopKQueryResult
&
topk_query_result
)
=
0
;
/**
* @brief Show table description
...
...
core/src/sdk/interface/ConnectionImpl.cpp
浏览文件 @
6c227a35
...
...
@@ -91,9 +91,9 @@ ConnectionImpl::Insert(const std::string& table_name, const std::string& partiti
Status
ConnectionImpl
::
Search
(
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
const
std
::
vector
<
RowRecord
>&
query_record_array
,
const
std
::
vector
<
Range
>&
query_range_array
,
int64_t
topk
,
int64_t
nprobe
,
std
::
vector
<
TopKQueryResult
>&
topk_query_result_array
)
{
int64_t
topk
,
int64_t
nprobe
,
TopKQueryResult
&
topk_query_result
)
{
return
client_proxy_
->
Search
(
table_name
,
partiton_tags
,
query_record_array
,
query_range_array
,
topk
,
nprobe
,
topk_query_result
_array
);
topk_query_result
);
}
Status
...
...
core/src/sdk/interface/ConnectionImpl.h
浏览文件 @
6c227a35
...
...
@@ -62,7 +62,7 @@ class ConnectionImpl : public Connection {
Status
Search
(
const
std
::
string
&
table_name
,
const
std
::
vector
<
std
::
string
>&
partiton_tags
,
const
std
::
vector
<
RowRecord
>&
query_record_array
,
const
std
::
vector
<
Range
>&
query_range_array
,
int64_t
topk
,
int64_t
nprobe
,
std
::
vector
<
TopKQueryResult
>&
topk_query_result_array
)
override
;
int64_t
nprobe
,
TopKQueryResult
&
topk_query_result
)
override
;
Status
DescribeTable
(
const
std
::
string
&
table_name
,
TableSchema
&
table_schema
)
override
;
...
...
core/src/server/Config.h
浏览文件 @
6c227a35
...
...
@@ -93,11 +93,13 @@ static const char* CONFIG_RESOURCE_MODE = "mode";
static
const
char
*
CONFIG_RESOURCE_MODE_DEFAULT
=
"simple"
;
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES
=
"search_resources"
;
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
=
","
;
#ifdef MILVUS_CPU_VERSION
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
=
"cpu"
;
#else
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
=
"cpu,gpu0"
;
#endif
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE
=
"index_build_device"
;
#ifdef MILVUS_CPU_VERSION
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT
=
"cpu"
;
...
...
core/src/server/grpc_impl/GrpcRequestHandler.cpp
浏览文件 @
6c227a35
...
...
@@ -75,7 +75,7 @@ GrpcRequestHandler::Insert(::grpc::ServerContext* context, const ::milvus::grpc:
::
grpc
::
Status
GrpcRequestHandler
::
Search
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
std
::
vector
<
std
::
string
>
file_id_array
;
BaseTaskPtr
task_ptr
=
SearchTask
::
Create
(
request
,
file_id_array
,
response
);
::
milvus
::
grpc
::
Status
grpc_status
;
...
...
@@ -87,7 +87,7 @@ GrpcRequestHandler::Search(::grpc::ServerContext* context, const ::milvus::grpc:
::
grpc
::
Status
GrpcRequestHandler
::
SearchInFiles
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
std
::
vector
<
std
::
string
>
file_id_array
;
for
(
int
i
=
0
;
i
<
request
->
file_id_array_size
();
i
++
)
{
file_id_array
.
push_back
(
request
->
file_id_array
(
i
));
...
...
core/src/server/grpc_impl/GrpcRequestHandler.h
浏览文件 @
6c227a35
...
...
@@ -153,7 +153,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
// @return TopKQueryResultList
::
grpc
::
Status
Search
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResultList
*
response
)
override
;
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
override
;
// *
// @brief This method is used to query vector in specified files.
//
...
...
@@ -162,7 +163,8 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
// @return TopKQueryResultList
::
grpc
::
Status
SearchInFiles
(
::
grpc
::
ServerContext
*
context
,
const
::
milvus
::
grpc
::
SearchInFilesParam
*
request
,
::
milvus
::
grpc
::
TopKQueryResultList
*
response
)
override
;
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
override
;
// *
// @brief This method is used to give the server status.
//
...
...
core/src/server/grpc_impl/GrpcRequestTask.cpp
浏览文件 @
6c227a35
...
...
@@ -20,6 +20,7 @@
#include <string.h>
#include <map>
#include <string>
#include <utility>
#include <vector>
//#include <gperftools/profiler.h>
...
...
@@ -542,16 +543,16 @@ InsertTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SearchTask
::
SearchTask
(
const
::
milvus
::
grpc
::
SearchParam
*
search_vector_infos
,
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
:
GrpcBaseTask
(
DQL_TASK_GROUP
),
search_param_
(
search_vector_infos
),
file_id_array_
(
file_id_array
),
topk_result_
list
(
response
)
{
topk_result_
(
response
)
{
}
BaseTaskPtr
SearchTask
::
Create
(
const
::
milvus
::
grpc
::
SearchParam
*
search_vector_infos
,
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
)
{
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
*
response
)
{
if
(
search_vector_infos
==
nullptr
)
{
SERVER_LOG_ERROR
<<
"grpc input is null!"
;
return
nullptr
;
...
...
@@ -679,18 +680,13 @@ SearchTask::OnExecute() {
return
Status
::
OK
();
// empty table
}
size_t
result_k
=
result_ids
.
size
()
/
record_count
;
// step 7: construct result array
for
(
size_t
i
=
0
;
i
<
record_count
;
i
++
)
{
::
milvus
::
grpc
::
TopKQueryResult
*
topk_query_result
=
topk_result_list
->
add_topk_query_result
();
for
(
size_t
j
=
0
;
j
<
result_k
;
j
++
)
{
::
milvus
::
grpc
::
QueryResult
*
grpc_result
=
topk_query_result
->
add_query_result_arrays
();
size_t
idx
=
i
*
result_k
+
j
;
grpc_result
->
set_id
(
result_ids
[
idx
]);
grpc_result
->
set_distance
(
result_distances
[
idx
]);
}
}
topk_result_
->
set_row_num
(
record_count
);
topk_result_
->
mutable_ids
()
->
Resize
(
static_cast
<
int
>
(
result_ids
.
size
()),
0
);
memcpy
(
topk_result_
->
mutable_ids
()
->
mutable_data
(),
result_ids
.
data
(),
result_ids
.
size
()
*
sizeof
(
int64_t
));
topk_result_
->
mutable_distances
()
->
Resize
(
static_cast
<
int
>
(
result_distances
.
size
()),
0.0
);
memcpy
(
topk_result_
->
mutable_distances
()
->
mutable_data
(),
result_distances
.
data
(),
result_distances
.
size
()
*
sizeof
(
float
));
// step 8: print time cost percent
rc
.
RecordSection
(
"construct result and send"
);
...
...
core/src/server/grpc_impl/GrpcRequestTask.h
浏览文件 @
6c227a35
...
...
@@ -153,11 +153,11 @@ class SearchTask : public GrpcBaseTask {
public:
static
BaseTaskPtr
Create
(
const
::
milvus
::
grpc
::
SearchParam
*
search_param
,
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
);
::
milvus
::
grpc
::
TopKQueryResult
*
response
);
protected:
SearchTask
(
const
::
milvus
::
grpc
::
SearchParam
*
search_param
,
const
std
::
vector
<
std
::
string
>&
file_id_array
,
::
milvus
::
grpc
::
TopKQueryResult
List
*
response
);
::
milvus
::
grpc
::
TopKQueryResult
*
response
);
Status
OnExecute
()
override
;
...
...
@@ -165,7 +165,7 @@ class SearchTask : public GrpcBaseTask {
private:
const
::
milvus
::
grpc
::
SearchParam
*
search_param_
;
std
::
vector
<
std
::
string
>
file_id_array_
;
::
milvus
::
grpc
::
TopKQueryResult
List
*
topk_result_list
;
::
milvus
::
grpc
::
TopKQueryResult
*
topk_result_
;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
...
...
core/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @
6c227a35
...
...
@@ -93,7 +93,7 @@ GrpcServer::StartService() {
builder
.
SetCompressionAlgorithmSupportStatus
(
GRPC_COMPRESS_STREAM_GZIP
,
true
);
builder
.
SetDefaultCompressionAlgorithm
(
GRPC_COMPRESS_STREAM_GZIP
);
builder
.
SetDefaultCompressionLevel
(
GRPC_COMPRESS_LEVEL_
HIGH
);
builder
.
SetDefaultCompressionLevel
(
GRPC_COMPRESS_LEVEL_
NONE
);
GrpcRequestHandler
service
;
...
...
core/src/wrapper/VecImpl.cpp
浏览文件 @
6c227a35
...
...
@@ -84,8 +84,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
Config
search_cfg
=
cfg
;
auto
res
=
index_
->
Search
(
dataset
,
search_cfg
);
auto
ids_array
=
res
->
array
()[
0
];
auto
dis_array
=
res
->
array
()[
1
];
//
auto ids_array = res->array()[0];
//
auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
...
...
@@ -104,12 +104,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto
p_ids
=
ids_array
->
data
()
->
GetValues
<
int64_t
>
(
1
,
0
);
auto
p_dist
=
dis_array
->
data
()
->
GetValues
<
float
>
(
1
,
0
);
//
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
//
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy
(
ids
,
p_ids
,
sizeof
(
int64_t
)
*
nq
*
k
);
memcpy
(
dist
,
p_dist
,
sizeof
(
float
)
*
nq
*
k
);
memcpy
(
ids
,
res
->
ids
(),
sizeof
(
int64_t
)
*
nq
*
k
);
memcpy
(
dist
,
res
->
dist
(),
sizeof
(
float
)
*
nq
*
k
);
free
(
res
->
ids
());
free
(
res
->
dist
());
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
Status
(
KNOWHERE_UNEXPECTED_ERROR
,
e
.
what
());
...
...
core/unittest/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -110,12 +110,18 @@ set(unittest_libs
pthread
metrics
gfortran
prometheus-cpp-pull
prometheus-cpp-push
prometheus-cpp-core
dl
z
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
unittest_libs
${
unittest_libs
}
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
)
endif
()
set
(
unittest_libs
${
unittest_libs
}
dl
z
)
if
(
MILVUS_GPU_VERSION
)
include_directories
(
"
${
CUDA_INCLUDE_DIRS
}
"
)
...
...
@@ -135,4 +141,4 @@ add_subdirectory(db)
add_subdirectory
(
wrapper
)
add_subdirectory
(
metrics
)
add_subdirectory
(
scheduler
)
add_subdirectory
(
server
)
\ No newline at end of file
add_subdirectory
(
server
)
core/unittest/db/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -17,8 +17,16 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory
(
${
CMAKE_CURRENT_SOURCE_DIR
}
test_files
)
set
(
test_files
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_db.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_db_mysql.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_engine.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_mem.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_meta.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_meta_mysql.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_misc.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_search.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/utils.cpp
)
add_executable
(
test_db
${
common_files
}
...
...
core/unittest/metrics/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -17,7 +17,16 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory
(
${
CMAKE_CURRENT_SOURCE_DIR
}
test_files
)
set
(
test_files
test_metricbase.cpp
test_metrics.cpp
utils.cpp
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
test_files
${
test_files
}
test_prometheus.cpp
)
endif
()
add_executable
(
test_metrics
${
common_files
}
...
...
core/unittest/metrics/test_prometheus.cpp
浏览文件 @
6c227a35
...
...
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "server/Config.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include <gtest/gtest.h>
#include <iostream>
...
...
core/unittest/scheduler/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -17,7 +17,17 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory
(
${
CMAKE_CURRENT_SOURCE_DIR
}
test_files
)
set
(
test_files
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_algorithm.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_event.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_node.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_normal.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_resource.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_resource_factory.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_resource_mgr.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_scheduler.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_task.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_tasktable.cpp
)
add_executable
(
test_scheduler
${
common_files
}
...
...
core/unittest/scheduler/t
ask_test
.cpp
→
core/unittest/scheduler/t
est_task
.cpp
浏览文件 @
6c227a35
文件已移动
core/unittest/server/CMakeLists.txt
浏览文件 @
6c227a35
...
...
@@ -17,7 +17,12 @@
# under the License.
#-------------------------------------------------------------------------------
aux_source_directory
(
${
CMAKE_CURRENT_SOURCE_DIR
}
test_files
)
set
(
test_files
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_cache.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_config.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_rpc.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_util.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/utils.cpp
)
include_directories
(
"
${
CUDA_TOOLKIT_ROOT_DIR
}
/include"
)
link_directories
(
"
${
CUDA_TOOLKIT_ROOT_DIR
}
/lib64"
)
...
...
core/unittest/server/test_config.cpp
浏览文件 @
6c227a35
...
...
@@ -415,6 +415,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
s
=
config
.
SetResourceConfigSearchResources
(
"gpu10"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetResourceConfigSearchResources
(
"cpu"
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"gup2"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"gpu16"
);
...
...
@@ -435,3 +438,4 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
s
=
config
.
ResetDefaultConfig
();
ASSERT_TRUE
(
s
.
ok
());
}
core/unittest/server/test_rpc.cpp
浏览文件 @
6c227a35
...
...
@@ -43,6 +43,12 @@ static constexpr int64_t VECTOR_COUNT = 1000;
static
constexpr
int64_t
INSERT_LOOP
=
10
;
constexpr
int64_t
SECONDS_EACH_HOUR
=
3600
;
void
CopyRowRecord
(
::
milvus
::
grpc
::
RowRecord
*
target
,
const
std
::
vector
<
float
>&
src
)
{
auto
vector_data
=
target
->
mutable_vector_data
();
vector_data
->
Resize
(
static_cast
<
int
>
(
src
.
size
()),
0.0
);
memcpy
(
vector_data
->
mutable_data
(),
src
.
data
(),
src
.
size
()
*
sizeof
(
float
));
}
class
RpcHandlerTest
:
public
testing
::
Test
{
protected:
void
...
...
@@ -201,9 +207,7 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
::
milvus
::
grpc
::
VectorIds
vector_ids
;
for
(
auto
&
record
:
record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
grpc_record
=
request
.
add_row_record_array
();
for
(
size_t
i
=
0
;
i
<
record
.
size
();
i
++
)
{
grpc_record
->
add_vector_data
(
record
[
i
]);
}
CopyRowRecord
(
grpc_record
,
record
);
}
handler
->
Insert
(
&
context
,
&
request
,
&
vector_ids
);
ASSERT_EQ
(
vector_ids
.
vector_id_array_size
(),
VECTOR_COUNT
);
...
...
@@ -212,8 +216,8 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
TEST_F
(
RpcHandlerTest
,
SEARCH_TEST
)
{
::
grpc
::
ServerContext
context
;
::
milvus
::
grpc
::
SearchParam
request
;
::
milvus
::
grpc
::
TopKQueryResult
List
response
;
//
test null input
::
milvus
::
grpc
::
TopKQueryResult
response
;
//test null input
handler
->
Search
(
&
context
,
nullptr
,
&
response
);
// test invalid table name
...
...
@@ -240,22 +244,17 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
::
milvus
::
grpc
::
InsertParam
insert_param
;
for
(
auto
&
record
:
record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
grpc_record
=
insert_param
.
add_row_record_array
();
for
(
size_t
i
=
0
;
i
<
record
.
size
();
i
++
)
{
grpc_record
->
add_vector_data
(
record
[
i
]);
}
CopyRowRecord
(
grpc_record
,
record
);
}
// insert vectors
insert_param
.
set_table_name
(
TABLE_NAME
);
::
milvus
::
grpc
::
VectorIds
vector_ids
;
handler
->
Insert
(
&
context
,
&
insert_param
,
&
vector_ids
);
sleep
(
7
);
BuildVectors
(
0
,
10
,
record_array
);
for
(
auto
&
record
:
record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
row_record
=
request
.
add_query_record_array
();
for
(
auto
&
rec
:
record
)
{
row_record
->
add_vector_data
(
rec
);
}
CopyRowRecord
(
row_record
,
record
);
}
handler
->
Search
(
&
context
,
&
request
,
&
response
);
...
...
@@ -324,9 +323,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
for
(
auto
&
record
:
record_array
)
{
::
milvus
::
grpc
::
RowRecord
*
grpc_record
=
request
.
add_row_record_array
();
for
(
size_t
i
=
0
;
i
<
record
.
size
();
i
++
)
{
grpc_record
->
add_vector_data
(
record
[
i
]);
}
CopyRowRecord
(
grpc_record
,
record
);
}
// test vector_id size not equal to row record size
vector_ids
.
clear_vector_id_array
();
...
...
@@ -341,9 +338,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
vector_ids
.
clear_vector_id_array
();
for
(
uint64_t
i
=
0
;
i
<
10
;
++
i
)
{
::
milvus
::
grpc
::
RowRecord
*
grpc_record
=
request
.
add_row_record_array
();
for
(
size_t
j
=
0
;
j
<
10
;
j
++
)
{
grpc_record
->
add_vector_data
(
record_array
[
i
][
j
]);
}
CopyRowRecord
(
grpc_record
,
record_array
[
i
]);
}
handler
->
Insert
(
&
context
,
&
request
,
&
vector_ids
);
...
...
tests/milvus-java-test/pom.xml
浏览文件 @
6c227a35
...
...
@@ -99,7 +99,7 @@
<dependency>
<groupId>
io.milvus
</groupId>
<artifactId>
milvus-sdk-java
</artifactId>
<version>
0.
2.0-SNAPSHOT
</version>
<version>
0.
3.0
</version>
</dependency>
<!-- <dependency>-->
...
...
@@ -134,4 +134,4 @@
</dependencies>
</project>
\ No newline at end of file
</project>
tests/milvus_python_test/test_ping.py
浏览文件 @
6c227a35
import
logging
import
pytest
__version__
=
'0.
5.1
'
__version__
=
'0.
6.0
'
class
TestPing
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录