Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
1e991d0f
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
Fork自
milvus / milvus
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
1e991d0f
编写于
11月 15, 2019
作者:
Z
zirui.chen
浏览文件
操作
浏览文件
下载
差异文件
format changelog and fix several problems
上级
98571d5f
49ec7e1a
变更
26
隐藏空白更改
内联
并排
Showing
26 changed file
with
250 addition
and
162 deletion
+250
-162
CHANGELOG.md
CHANGELOG.md
+6
-14
core/CMakeLists.txt
core/CMakeLists.txt
+18
-14
core/build.sh
core/build.sh
+10
-4
core/conf/server_gpu_config.template
core/conf/server_gpu_config.template
+1
-1
core/src/CMakeLists.txt
core/src/CMakeLists.txt
+24
-4
core/src/db/DBImpl.cpp
core/src/db/DBImpl.cpp
+15
-3
core/src/db/meta/MySQLMetaImpl.cpp
core/src/db/meta/MySQLMetaImpl.cpp
+8
-5
core/src/db/meta/SqliteMetaImpl.cpp
core/src/db/meta/SqliteMetaImpl.cpp
+9
-4
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
+29
-28
core/src/index/knowhere/knowhere/common/Dataset.h
core/src/index/knowhere/knowhere/common/Dataset.h
+16
-0
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
...index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
+18
-17
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
...c/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
+17
-17
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
...c/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
+18
-17
core/src/index/unittest/test_ivf.cpp
core/src/index/unittest/test_ivf.cpp
+5
-3
core/src/index/unittest/test_kdt.cpp
core/src/index/unittest/test_kdt.cpp
+8
-4
core/src/index/unittest/utils.cpp
core/src/index/unittest/utils.cpp
+3
-2
core/src/metrics/Metrics.cpp
core/src/metrics/Metrics.cpp
+7
-1
core/src/metrics/prometheus/PrometheusMetrics.cpp
core/src/metrics/prometheus/PrometheusMetrics.cpp
+2
-2
core/src/metrics/prometheus/PrometheusMetrics.h
core/src/metrics/prometheus/PrometheusMetrics.h
+1
-1
core/src/wrapper/VecImpl.cpp
core/src/wrapper/VecImpl.cpp
+8
-6
core/unittest/CMakeLists.txt
core/unittest/CMakeLists.txt
+12
-6
core/unittest/metrics/CMakeLists.txt
core/unittest/metrics/CMakeLists.txt
+9
-4
core/unittest/metrics/test_prometheus.cpp
core/unittest/metrics/test_prometheus.cpp
+1
-1
core/unittest/wrapper/utils.h
core/unittest/wrapper/utils.h
+2
-1
tests/milvus-java-test/pom.xml
tests/milvus-java-test/pom.xml
+2
-2
tests/milvus_python_test/test_ping.py
tests/milvus_python_test/test_ping.py
+1
-1
未找到文件。
CHANGELOG.md
浏览文件 @
1e991d0f
...
...
@@ -4,31 +4,23 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.6.0 (TODO)
## Bug
-
\#
228 - memory usage increased slowly during searching vectors
-
\#
246 - Exclude src/external folder from code coverage for jenkin ci
-
\#
248 - Reside src/external in thirdparty
##
## Feature
-
\#
12 - Pure CPU version for Milvus
-
\#
77 - Support table partition
-
\#
226 - Experimental shards middleware for Milvus
-
#127 - Support new Index type IVFPQ
##
-
\#
127 - Support new Index type IVFPQ
## Improvement
-
\#
275 - Rename C++ SDK IndexType
-
\#
284 - Change C++ SDK to shared library
-
\#
260 - C++ SDK README
-
\#
314 - add Find FAISS in CMake
-
\#
310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
##
## Task
# Milvus 0.5.3 (2019-11-13)
...
...
@@ -117,7 +109,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-658 - Fix SQ8 Hybrid can't search
-
MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
-
\#
9 - Change default gpu_cache_capacity to 4
-
\#
20 - C++ sdk example get grpc error
-
\#
20 - C++ sdk example get grpc error
-
\#
23 - Add unittest to improve code coverage
-
\#
31 - make clang-format failed after run build.sh -l
-
\#
39 - Create SQ8H index hang if using github server version
...
...
@@ -169,7 +161,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-635 - Add compile option to support customized faiss
-
MS-660 - add ubuntu_build_deps.sh
-
\#
18 - Add all test cases
# Milvus 0.4.0 (2019-09-12)
## Bug
...
...
@@ -378,11 +370,11 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-82 - Update server startup welcome message
-
MS-83 - Update vecwise to Milvus
-
MS-77 - Performance issue of post-search action
-
MS-22 - Enhancement for MemVector size control
-
MS-22 - Enhancement for MemVector size control
-
MS-92 - Unify behavior of debug and release build
-
MS-98 - Install all unit test to installation directory
-
MS-115 - Change is_startup of metric_config switch from true to on
-
MS-122 - Archive criteria config
-
MS-122 - Archive criteria config
-
MS-124 - HasTable interface
-
MS-126 - Add more error code
-
MS-128 - Change default db path
...
...
core/CMakeLists.txt
浏览文件 @
1e991d0f
...
...
@@ -33,7 +33,7 @@ message(STATUS "Build time = ${BUILD_TIME}")
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
execute_process
(
COMMAND sh
"-c"
"git log --decorate | head -n 1 | sed 's/.*(
\\
(.*
\\
))/
\\
1/' | sed 's/.*
\\
(.*
\\
),.*/
\\
1/' | sed 's=[a-zA-Z]*\/==g'"
OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
GET_GIT_BRANCH_NAME
(
GIT_BRANCH_NAME
)
...
...
@@ -117,17 +117,17 @@ include(DefineOptions)
include
(
BuildUtils
)
include
(
ThirdPartyPackages
)
if
(
MILVUS_USE_CCACHE
)
find_program
(
CCACHE_FOUND ccache
)
if
(
CCACHE_FOUND
)
message
(
STATUS
"Using ccache:
${
CCACHE_FOUND
}
"
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_COMPILE
${
CCACHE_FOUND
}
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_LINK
${
CCACHE_FOUND
}
)
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set
(
ENV{CCACHE_COMMENTS}
"1"
)
endif
(
CCACHE_FOUND
)
endif
()
if
(
MILVUS_USE_CCACHE
)
find_program
(
CCACHE_FOUND ccache
)
if
(
CCACHE_FOUND
)
message
(
STATUS
"Using ccache:
${
CCACHE_FOUND
}
"
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_COMPILE
${
CCACHE_FOUND
}
)
set_property
(
GLOBAL PROPERTY RULE_LAUNCH_LINK
${
CCACHE_FOUND
}
)
# let ccache preserve C++ comments, because some of them may be
# meaningful to the compiler
set
(
ENV{CCACHE_COMMENTS}
"1"
)
endif
(
CCACHE_FOUND
)
endif
()
set
(
MILVUS_CPU_VERSION false
)
if
(
MILVUS_GPU_VERSION
)
...
...
@@ -142,6 +142,10 @@ else ()
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
endif
()
if
(
MILVUS_WITH_PROMETHEUS
)
add_compile_definitions
(
"MILVUS_WITH_PROMETHEUS"
)
endif
()
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
set
(
CMAKE_CXX_FLAGS
"
${
CMAKE_CXX_FLAGS
}
-O3 -fPIC -DELPP_THREAD_SAFE -fopenmp"
)
if
(
MILVUS_GPU_VERSION
)
...
...
@@ -176,9 +180,9 @@ endif ()
if
(
MILVUS_GPU_VERSION
)
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_gpu_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_config.yaml
)
else
()
else
()
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_cpu_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/server_config.yaml
)
endif
()
endif
()
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/log_config.template
${
CMAKE_CURRENT_SOURCE_DIR
}
/conf/log_config.conf
)
...
...
core/build.sh
浏览文件 @
1e991d0f
...
...
@@ -14,10 +14,11 @@ CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER
=
/usr/local/cuda/bin/nvcc
GPU_VERSION
=
"OFF"
#defaults to CPU version
WITH_MKL
=
"OFF"
FAISS_ROOT
=
""
FAISS_ROOT
=
""
#FAISS root path
FAISS_SOURCE
=
"BUNDLED"
WITH_PROMETHEUS
=
"ON"
while
getopts
"p:d:t:f:ulrcgjhxzm"
arg
while
getopts
"p:d:t:f:ulrcgjhxzm
e
"
arg
do
case
$arg
in
p
)
...
...
@@ -63,7 +64,10 @@ do
;;
m
)
WITH_MKL
=
"ON"
;;
;;
e
)
WITH_PROMETHEUS
=
"OFF"
;;
h
)
# help
echo
"
...
...
@@ -80,10 +84,11 @@ parameter:
-j: use jfrog cache build directory(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-e: build without prometheus
-h: help
usage:
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} -f
\$
{FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-h]
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} -f
\$
{FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-
e] [-
h]
"
exit
0
;;
...
...
@@ -118,6 +123,7 @@ CMAKE_CMD="cmake \
-DCUSTOMIZATION=
${
CUSTOMIZATION
}
\
-DMILVUS_GPU_VERSION=
${
GPU_VERSION
}
\
-DFAISS_WITH_MKL=
${
WITH_MKL
}
\
-DMILVUS_WITH_PROMETHEUS=
${
WITH_PROMETHEUS
}
\
../"
echo
${
CMAKE_CMD
}
${
CMAKE_CMD
}
...
...
core/conf/server_gpu_config.template
浏览文件 @
1e991d0f
...
...
@@ -29,7 +29,7 @@ metric_config:
cache_config:
cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
gpu_cache_capacity:
3
# GB, GPU memory used for cache, must be a positive integer
gpu_cache_capacity:
4
# GB, GPU memory used for cache, must be a positive integer
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
...
...
core/src/CMakeLists.txt
浏览文件 @
1e991d0f
...
...
@@ -37,6 +37,7 @@ endforeach ()
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/cache cache_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/config config_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/metrics metrics_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/metrics/prometheus metrics_prometheus_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db db_main_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db/engine db_engine_files
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/db/insert db_insert_files
)
...
...
@@ -91,6 +92,11 @@ set(engine_files
${
wrapper_files
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
engine_files
${
engine_files
}
${
metrics_prometheus_files
}
)
endif
()
set
(
client_grpc_lib
grpcpp_channelz
grpc++
...
...
@@ -115,7 +121,6 @@ set(third_party_libs
sqlite
${
client_grpc_lib
}
yaml-cpp
${
prometheus_lib
}
mysqlpp
zlib
${
boost_lib
}
...
...
@@ -138,13 +143,19 @@ if (MILVUS_GPU_VERSION)
)
endif
()
if
(
MILVUS_ENABLE_PROFILING
STREQUAL
"ON"
)
if
(
MILVUS_ENABLE_PROFILING
)
set
(
third_party_libs
${
third_party_libs
}
gperftools
libunwind
)
endif
()
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
third_party_libs
${
third_party_libs
}
${
prometheus_lib
}
)
endif
()
set
(
engine_libs
pthread
libgomp.a
...
...
@@ -166,13 +177,22 @@ target_link_libraries(milvus_engine
${
engine_libs
}
)
add_library
(
metrics STATIC
${
metrics_files
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
add_library
(
metrics STATIC
${
metrics_files
}
${
metrics_prometheus_files
}
)
else
()
add_library
(
metrics STATIC
${
metrics_files
}
)
endif
()
set
(
metrics_lib
yaml-cpp
${
prometheus_lib
}
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
metrics_lib
${
metrics_lib
}
${
prometheus_lib
}
)
endif
()
target_link_libraries
(
metrics
${
metrics_lib
}
)
set
(
server_libs
...
...
core/src/db/DBImpl.cpp
浏览文件 @
1e991d0f
...
...
@@ -619,6 +619,18 @@ DBImpl::StartCompactionTask() {
{
std
::
lock_guard
<
std
::
mutex
>
lck
(
compact_result_mutex_
);
if
(
compact_thread_results_
.
empty
())
{
// collect merge files for all tables(if compact_table_ids_ is empty) for two reasons:
// 1. other tables may still has un-merged files
// 2. server may be closed unexpected, these un-merge files need to be merged when server restart
if
(
compact_table_ids_
.
empty
())
{
std
::
vector
<
meta
::
TableSchema
>
table_schema_array
;
meta_ptr_
->
AllTables
(
table_schema_array
);
for
(
auto
&
schema
:
table_schema_array
)
{
compact_table_ids_
.
insert
(
schema
.
table_id_
);
}
}
// start merge file thread
compact_thread_results_
.
push_back
(
compact_thread_pool_
.
enqueue
(
&
DBImpl
::
BackgroundCompaction
,
this
,
compact_table_ids_
));
compact_table_ids_
.
clear
();
...
...
@@ -717,7 +729,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
for
(
auto
&
kv
:
raw_files
)
{
auto
files
=
kv
.
second
;
if
(
files
.
size
()
<
options_
.
merge_trigger_number_
)
{
ENGINE_LOG_
DEBUG
<<
"Files number not greater equal than merge trigger number, skip merge action"
;
ENGINE_LOG_
TRACE
<<
"Files number not greater equal than merge trigger number, skip merge action"
;
continue
;
}
...
...
@@ -734,7 +746,7 @@ DBImpl::BackgroundMergeFiles(const std::string& table_id) {
void
DBImpl
::
BackgroundCompaction
(
std
::
set
<
std
::
string
>
table_ids
)
{
ENGINE_LOG_TRACE
<<
"
Background compaction thread start"
;
ENGINE_LOG_TRACE
<<
"Background compaction thread start"
;
Status
status
;
for
(
auto
&
table_id
:
table_ids
)
{
...
...
@@ -757,7 +769,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
}
meta_ptr_
->
CleanUpFilesWithTTL
(
ttl
);
ENGINE_LOG_TRACE
<<
"
Background compaction thread exit"
;
ENGINE_LOG_TRACE
<<
"Background compaction thread exit"
;
}
void
...
...
core/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @
1e991d0f
...
...
@@ -1392,6 +1392,7 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
}
// Scoped Connection
Status
ret
;
int64_t
to_merge_files
=
0
;
for
(
auto
&
resRow
:
res
)
{
TableFileSchema
table_file
;
table_file
.
file_size_
=
resRow
[
"file_size"
];
...
...
@@ -1420,13 +1421,14 @@ MySQLMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFiles
auto
dateItr
=
files
.
find
(
table_file
.
date_
);
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date_
]
=
TableFilesSchema
();
to_merge_files
++
;
}
files
[
table_file
.
date_
].
push_back
(
table_file
);
}
if
(
res
.
size
()
>
0
)
{
ENGINE_LOG_
DEBUG
<<
"Collect "
<<
res
.
size
()
<<
" to-merge files"
;
if
(
to_merge_files
>
0
)
{
ENGINE_LOG_
TRACE
<<
"Collect "
<<
to_merge_files
<<
" to-merge files"
;
}
return
ret
;
}
catch
(
std
::
exception
&
e
)
{
...
...
@@ -1809,6 +1811,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
mysqlpp
::
StoreQueryResult
res
=
cleanUpFilesWithTTLQuery
.
store
();
int64_t
remove_tables
=
0
;
if
(
!
res
.
empty
())
{
std
::
stringstream
idsToDeleteSS
;
for
(
auto
&
resRow
:
res
)
{
...
...
@@ -1817,7 +1820,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
resRow
[
"table_id"
].
to_string
(
table_id
);
utils
::
DeleteTablePath
(
options_
,
table_id
,
false
);
// only delete empty folder
remove_tables
++
;
idsToDeleteSS
<<
"id = "
<<
std
::
to_string
(
id
)
<<
" OR "
;
}
std
::
string
idsToDeleteStr
=
idsToDeleteSS
.
str
();
...
...
@@ -1832,8 +1835,8 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}
}
if
(
re
s
.
size
()
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
re
s
.
size
()
<<
" tables from meta"
;
if
(
re
move_tables
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
re
move_tables
<<
" tables from meta"
;
}
}
// Scoped Connection
}
catch
(
std
::
exception
&
e
)
{
...
...
core/src/db/meta/SqliteMetaImpl.cpp
浏览文件 @
1e991d0f
...
...
@@ -971,6 +971,7 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
order_by
(
&
TableFileSchema
::
file_size_
).
desc
());
Status
result
;
int64_t
to_merge_files
=
0
;
for
(
auto
&
file
:
selected
)
{
TableFileSchema
table_file
;
table_file
.
file_size_
=
std
::
get
<
4
>
(
file
);
...
...
@@ -999,11 +1000,13 @@ SqliteMetaImpl::FilesToMerge(const std::string& table_id, DatePartionedTableFile
if
(
dateItr
==
files
.
end
())
{
files
[
table_file
.
date_
]
=
TableFilesSchema
();
}
files
[
table_file
.
date_
].
push_back
(
table_file
);
to_merge_files
++
;
}
if
(
selected
.
size
()
>
0
)
{
ENGINE_LOG_
DEBUG
<<
"Collect "
<<
selected
.
size
()
<<
" to-merge files"
;
if
(
to_merge_files
>
0
)
{
ENGINE_LOG_
TRACE
<<
"Collect "
<<
to_merge_files
<<
" to-merge files"
;
}
return
result
;
}
catch
(
std
::
exception
&
e
)
{
...
...
@@ -1313,16 +1316,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
try
{
server
::
MetricCollector
metric
;
int64_t
remove_tables
=
0
;
for
(
auto
&
table_id
:
table_ids
)
{
auto
selected
=
ConnectorPtr
->
select
(
columns
(
&
TableFileSchema
::
file_id_
),
where
(
c
(
&
TableFileSchema
::
table_id_
)
==
table_id
));
if
(
selected
.
size
()
==
0
)
{
utils
::
DeleteTablePath
(
options_
,
table_id
);
remove_tables
++
;
}
}
if
(
table_ids
.
size
()
>
0
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
table_ids
.
size
()
<<
" tables folder"
;
if
(
remove_tables
)
{
ENGINE_LOG_DEBUG
<<
"Remove "
<<
remove_tables
<<
" tables folder"
;
}
}
catch
(
std
::
exception
&
e
)
{
return
HandleException
(
"Encounter exception when delete table folder"
,
e
.
what
());
...
...
core/src/index/knowhere/knowhere/adapter/SptagAdapter.cpp
浏览文件 @
1e991d0f
...
...
@@ -89,34 +89,35 @@ ConvertToDataset(std::vector<SPTAG::QueryResult> query_results) {
}
}
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
p_id
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
p_dist
,
sizeof
(
float
)
*
elems
);
// TODO: magic
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
// auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
// auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
auto
field_id
=
std
::
make_shared
<
Field
>
(
"id"
,
std
::
make_shared
<
arrow
::
Int64Type
>
());
auto
field_dist
=
std
::
make_shared
<
Field
>
(
"dist"
,
std
::
make_shared
<
arrow
::
FloatType
>
());
std
::
vector
<
FieldPtr
>
fields
{
field_id
,
field_dist
};
auto
schema
=
std
::
make_shared
<
Schema
>
(
fields
);
return
std
::
make_shared
<
Dataset
>
(
array
,
schema
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)p_id, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)p_dist, sizeof(float) * elems);
//
// // TODO: magic
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
// // auto id_array_data = std::make_shared<ArrayData>(int64_type, sizeof(int64_t) * elems, id_bufs);
// // auto dist_array_data = std::make_shared<ArrayData>(float_type, sizeof(float) * elems, dist_bufs);
//
// // auto ids = ConstructInt64Array((uint8_t*)p_id, sizeof(int64_t) * elems);
// // auto dists = ConstructFloatArray((uint8_t*)p_dist, sizeof(float) * elems);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// auto field_id = std::make_shared<Field>("id", std::make_shared<arrow::Int64Type>());
// auto field_dist = std::make_shared<Field>("dist", std::make_shared<arrow::FloatType>());
// std::vector<FieldPtr> fields{field_id, field_dist};
// auto schema = std::make_shared<Schema>(fields);
//
// return std::make_shared<Dataset>(array, schema);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
p_id
,
(
void
*
)
p_dist
);
}
}
// namespace knowhere
core/src/index/knowhere/knowhere/common/Dataset.h
浏览文件 @
1e991d0f
...
...
@@ -54,6 +54,9 @@ class Dataset {
:
tensor_
(
std
::
move
(
tensor
)),
tensor_schema_
(
std
::
move
(
tensor_schema
))
{
}
Dataset
(
void
*
ids
,
void
*
dists
)
:
ids_
(
ids
),
dists_
(
dists
)
{
}
Dataset
(
const
Dataset
&
)
=
delete
;
Dataset
&
operator
=
(
const
Dataset
&
)
=
delete
;
...
...
@@ -128,6 +131,16 @@ class Dataset {
tensor_schema_
=
std
::
move
(
tensor_schema
);
}
void
*
ids
()
{
return
ids_
;
}
void
*
dist
()
{
return
dists_
;
}
// const Config &
// meta() const { return meta_; }
...
...
@@ -141,6 +154,9 @@ class Dataset {
SchemaPtr
array_schema_
;
std
::
vector
<
TensorPtr
>
tensor_
;
SchemaPtr
tensor_schema_
;
// TODO(yukun): using smart pointer
void
*
ids_
;
void
*
dists_
;
// Config meta_;
};
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIDMAP.cpp
浏览文件 @
1e991d0f
...
...
@@ -80,23 +80,24 @@ IDMAP::Search(const DatasetPtr& dataset, const Config& config) {
search_impl
(
rows
,
(
float
*
)
p_data
,
config
->
k
,
res_dis
,
res_ids
,
Config
());
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
void
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp
浏览文件 @
1e991d0f
...
...
@@ -139,23 +139,23 @@ IVF::Search(const DatasetPtr& dataset, const Config& config) {
// std::cout << ss_res_id.str() << std::endl;
// std::cout << ss_res_dist.str() << std::endl << std::endl;
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
//
auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
//
auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
//
//
std::vector<BufferPtr> id_bufs{nullptr, id_buf};
//
std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
//
auto int64_type = std::make_shared<arrow::Int64Type>();
//
auto float_type = std::make_shared<arrow::FloatType>();
//
//
auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
//
auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
//
auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
//
auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
//
std::vector<ArrayPtr> array{ids, dists};
return
std
::
make_shared
<
Dataset
>
(
(
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
void
...
...
core/src/index/knowhere/knowhere/index/vector_index/IndexNSG.cpp
浏览文件 @
1e991d0f
...
...
@@ -88,23 +88,24 @@ NSG::Search(const DatasetPtr& dataset, const Config& config) {
s_params
.
search_length
=
build_cfg
->
search_length
;
index_
->
Search
((
float
*
)
p_data
,
rows
,
dim
,
build_cfg
->
k
,
res_dis
,
res_ids
,
s_params
);
auto
id_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_ids
,
sizeof
(
int64_t
)
*
elems
);
auto
dist_buf
=
MakeMutableBufferSmart
((
uint8_t
*
)
res_dis
,
sizeof
(
float
)
*
elems
);
std
::
vector
<
BufferPtr
>
id_bufs
{
nullptr
,
id_buf
};
std
::
vector
<
BufferPtr
>
dist_bufs
{
nullptr
,
dist_buf
};
auto
int64_type
=
std
::
make_shared
<
arrow
::
Int64Type
>
();
auto
float_type
=
std
::
make_shared
<
arrow
::
FloatType
>
();
auto
id_array_data
=
arrow
::
ArrayData
::
Make
(
int64_type
,
elems
,
id_bufs
);
auto
dist_array_data
=
arrow
::
ArrayData
::
Make
(
float_type
,
elems
,
dist_bufs
);
auto
ids
=
std
::
make_shared
<
NumericArray
<
arrow
::
Int64Type
>>
(
id_array_data
);
auto
dists
=
std
::
make_shared
<
NumericArray
<
arrow
::
FloatType
>>
(
dist_array_data
);
std
::
vector
<
ArrayPtr
>
array
{
ids
,
dists
};
return
std
::
make_shared
<
Dataset
>
(
array
,
nullptr
);
// auto id_buf = MakeMutableBufferSmart((uint8_t*)res_ids, sizeof(int64_t) * elems);
// auto dist_buf = MakeMutableBufferSmart((uint8_t*)res_dis, sizeof(float) * elems);
// std::vector<BufferPtr> id_bufs{nullptr, id_buf};
// std::vector<BufferPtr> dist_bufs{nullptr, dist_buf};
//
// auto int64_type = std::make_shared<arrow::Int64Type>();
// auto float_type = std::make_shared<arrow::FloatType>();
//
// auto id_array_data = arrow::ArrayData::Make(int64_type, elems, id_bufs);
// auto dist_array_data = arrow::ArrayData::Make(float_type, elems, dist_bufs);
//
// auto ids = std::make_shared<NumericArray<arrow::Int64Type>>(id_array_data);
// auto dists = std::make_shared<NumericArray<arrow::FloatType>>(dist_array_data);
// std::vector<ArrayPtr> array{ids, dists};
//
// return std::make_shared<Dataset>(array, nullptr);
return
std
::
make_shared
<
Dataset
>
((
void
*
)
res_ids
,
(
void
*
)
res_dis
);
}
IndexModelPtr
...
...
core/src/index/unittest/test_ivf.cpp
浏览文件 @
1e991d0f
...
...
@@ -181,11 +181,13 @@ TEST_P(IVFTest, clone_test) {
// PrintResult(result, nq, k);
auto
AssertEqual
=
[
&
](
knowhere
::
DatasetPtr
p1
,
knowhere
::
DatasetPtr
p2
)
{
auto
ids_p1
=
p1
->
array
()[
0
]
;
auto
ids_p2
=
p2
->
array
()[
0
]
;
auto
ids_p1
=
p1
->
ids
()
;
auto
ids_p2
=
p2
->
ids
()
;
for
(
int
i
=
0
;
i
<
nq
*
k
;
++
i
)
{
EXPECT_EQ
(
*
(
ids_p2
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
)),
*
(
ids_p1
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
)));
EXPECT_EQ
(
*
((
int64_t
*
)(
ids_p2
)
+
i
),
*
((
int64_t
*
)(
ids_p1
)
+
i
));
// EXPECT_EQ(*(ids_p2->data()->GetValues<int64_t>(1, i)), *(ids_p1->data()->GetValues<int64_t>(1,
// i)));
}
};
...
...
core/src/index/unittest/test_kdt.cpp
浏览文件 @
1e991d0f
...
...
@@ -66,15 +66,19 @@ TEST_F(KDTTest, kdt_basic) {
AssertAnns
(
result
,
nq
,
k
);
{
auto
ids
=
result
->
array
()[
0
];
auto
dists
=
result
->
array
()[
1
];
// auto ids = result->array()[0];
// auto dists = result->array()[1];
auto
ids
=
result
->
ids
();
auto
dists
=
result
->
dist
();
std
::
stringstream
ss_id
;
std
::
stringstream
ss_dist
;
for
(
auto
i
=
0
;
i
<
nq
;
i
++
)
{
for
(
auto
j
=
0
;
j
<
k
;
++
j
)
{
ss_id
<<
*
ids
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
*
k
+
j
)
<<
" "
;
ss_dist
<<
*
dists
->
data
()
->
GetValues
<
float
>
(
1
,
i
*
k
+
j
)
<<
" "
;
ss_id
<<
*
((
int64_t
*
)(
ids
)
+
i
*
k
+
j
)
<<
" "
;
ss_dist
<<
*
((
float
*
)(
dists
)
+
i
*
k
+
j
)
<<
" "
;
// ss_id << *ids->data()->GetValues<int64_t>(1, i * k + j) << " ";
// ss_dist << *dists->data()->GetValues<float>(1, i * k + j) << " ";
}
ss_id
<<
std
::
endl
;
ss_dist
<<
std
::
endl
;
...
...
core/src/index/unittest/utils.cpp
浏览文件 @
1e991d0f
...
...
@@ -151,9 +151,10 @@ generate_query_dataset(int64_t nb, int64_t dim, float* xb) {
void
AssertAnns
(
const
knowhere
::
DatasetPtr
&
result
,
const
int
&
nq
,
const
int
&
k
)
{
auto
ids
=
result
->
array
()[
0
]
;
auto
ids
=
result
->
ids
()
;
for
(
auto
i
=
0
;
i
<
nq
;
i
++
)
{
EXPECT_EQ
(
i
,
*
(
ids
->
data
()
->
GetValues
<
int64_t
>
(
1
,
i
*
k
)));
EXPECT_EQ
(
i
,
*
((
int64_t
*
)(
ids
)
+
i
*
k
));
// EXPECT_EQ(i, *(ids->data()->GetValues<int64_t>(1, i * k)));
}
}
...
...
core/src/metrics/Metrics.cpp
浏览文件 @
1e991d0f
...
...
@@ -16,8 +16,10 @@
// under the License.
#include "metrics/Metrics.h"
#include "PrometheusMetrics.h"
#include "server/Config.h"
#ifdef MILVUS_WITH_PROMETHEUS
#include "metrics/prometheus/PrometheusMetrics.h"
#endif
#include <string>
...
...
@@ -37,11 +39,15 @@ Metrics::CreateMetricsCollector() {
config
.
GetMetricConfigCollector
(
collector_type_str
);
#ifdef MILVUS_WITH_PROMETHEUS
if
(
collector_type_str
==
"prometheus"
)
{
return
PrometheusMetrics
::
GetInstance
();
}
else
{
return
MetricsBase
::
GetInstance
();
}
#else
return
MetricsBase
::
GetInstance
();
#endif
}
}
// namespace server
...
...
core/src/metrics/PrometheusMetrics.cpp
→
core/src/metrics/
prometheus/
PrometheusMetrics.cpp
浏览文件 @
1e991d0f
...
...
@@ -15,9 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "SystemInfo.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include "cache/GpuCacheMgr.h"
#include "metrics/SystemInfo.h"
#include "server/Config.h"
#include "utils/Log.h"
...
...
core/src/metrics/PrometheusMetrics.h
→
core/src/metrics/
prometheus/
PrometheusMetrics.h
浏览文件 @
1e991d0f
...
...
@@ -24,7 +24,7 @@
#include <string>
#include <vector>
#include "MetricBase.h"
#include "
metrics/
MetricBase.h"
#include "utils/Error.h"
#define METRICS_NOW_TIME std::chrono::system_clock::now()
...
...
core/src/wrapper/VecImpl.cpp
浏览文件 @
1e991d0f
...
...
@@ -84,8 +84,8 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
Config
search_cfg
=
cfg
;
auto
res
=
index_
->
Search
(
dataset
,
search_cfg
);
auto
ids_array
=
res
->
array
()[
0
];
auto
dis_array
=
res
->
array
()[
1
];
//
auto ids_array = res->array()[0];
//
auto dis_array = res->array()[1];
//{
// auto& ids = ids_array;
...
...
@@ -104,12 +104,14 @@ VecIndexImpl::Search(const int64_t& nq, const float* xq, float* dist, int64_t* i
// std::cout << "dist\n" << ss_dist.str() << std::endl;
//}
auto
p_ids
=
ids_array
->
data
()
->
GetValues
<
int64_t
>
(
1
,
0
);
auto
p_dist
=
dis_array
->
data
()
->
GetValues
<
float
>
(
1
,
0
);
//
auto p_ids = ids_array->data()->GetValues<int64_t>(1, 0);
//
auto p_dist = dis_array->data()->GetValues<float>(1, 0);
// TODO(linxj): avoid copy here.
memcpy
(
ids
,
p_ids
,
sizeof
(
int64_t
)
*
nq
*
k
);
memcpy
(
dist
,
p_dist
,
sizeof
(
float
)
*
nq
*
k
);
memcpy
(
ids
,
res
->
ids
(),
sizeof
(
int64_t
)
*
nq
*
k
);
memcpy
(
dist
,
res
->
dist
(),
sizeof
(
float
)
*
nq
*
k
);
free
(
res
->
ids
());
free
(
res
->
dist
());
}
catch
(
knowhere
::
KnowhereException
&
e
)
{
WRAPPER_LOG_ERROR
<<
e
.
what
();
return
Status
(
KNOWHERE_UNEXPECTED_ERROR
,
e
.
what
());
...
...
core/unittest/CMakeLists.txt
浏览文件 @
1e991d0f
...
...
@@ -110,12 +110,18 @@ set(unittest_libs
pthread
metrics
gfortran
prometheus-cpp-pull
prometheus-cpp-push
prometheus-cpp-core
dl
z
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
unittest_libs
${
unittest_libs
}
prometheus-cpp-push
prometheus-cpp-pull
prometheus-cpp-core
)
endif
()
set
(
unittest_libs
${
unittest_libs
}
dl
z
)
if
(
MILVUS_GPU_VERSION
)
include_directories
(
"
${
CUDA_INCLUDE_DIRS
}
"
)
...
...
@@ -135,4 +141,4 @@ add_subdirectory(db)
add_subdirectory
(
wrapper
)
add_subdirectory
(
metrics
)
add_subdirectory
(
scheduler
)
add_subdirectory
(
server
)
\ No newline at end of file
add_subdirectory
(
server
)
core/unittest/metrics/CMakeLists.txt
浏览文件 @
1e991d0f
...
...
@@ -18,10 +18,15 @@
#-------------------------------------------------------------------------------
set
(
test_files
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_metricbase.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_metrics.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/test_prometheus.cpp
${
CMAKE_CURRENT_SOURCE_DIR
}
/utils.cpp
)
test_metricbase.cpp
test_metrics.cpp
utils.cpp
)
if
(
MILVUS_WITH_PROMETHEUS
)
set
(
test_files
${
test_files
}
test_prometheus.cpp
)
endif
()
add_executable
(
test_metrics
${
common_files
}
...
...
core/unittest/metrics/test_prometheus.cpp
浏览文件 @
1e991d0f
...
...
@@ -15,8 +15,8 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "server/Config.h"
#include "metrics/prometheus/PrometheusMetrics.h"
#include <gtest/gtest.h>
#include <iostream>
...
...
core/unittest/wrapper/utils.h
浏览文件 @
1e991d0f
...
...
@@ -24,12 +24,13 @@
#include <cstdlib>
#include <cstdio>
#include <fstream>
#include <src/wrapper/ConfAdapter.h>
#include "wrapper/VecIndex.h"
#include "wrapper/utils.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "wrapper/ConfAdapterMgr.h"
#include "wrapper/ConfAdapter.h"
class
DataGenBase
;
...
...
tests/milvus-java-test/pom.xml
浏览文件 @
1e991d0f
...
...
@@ -99,7 +99,7 @@
<dependency>
<groupId>
io.milvus
</groupId>
<artifactId>
milvus-sdk-java
</artifactId>
<version>
0.
2.0-SNAPSHOT
</version>
<version>
0.
3.0
</version>
</dependency>
<!-- <dependency>-->
...
...
@@ -134,4 +134,4 @@
</dependencies>
</project>
\ No newline at end of file
</project>
tests/milvus_python_test/test_ping.py
浏览文件 @
1e991d0f
import
logging
import
pytest
__version__
=
'0.
5.1
'
__version__
=
'0.
6.0
'
class
TestPing
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录