Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
milvus
milvus
提交
3c0c4646
M
milvus
项目概览
milvus
/
milvus
12 个月 前同步成功
通知
261
Star
22476
Fork
2472
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3c0c4646
编写于
11月 07, 2019
作者:
Z
Zhiru Zhu
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
issue 229, 230 and 237
上级
b4189371
变更
15
隐藏空白更改
内联
并排
Showing
15 changed file
with
536 addition
and
166 deletion
+536
-166
core/CMakeLists.txt
core/CMakeLists.txt
+12
-12
core/build.sh
core/build.sh
+8
-8
core/cmake/DefineOptions.cmake
core/cmake/DefineOptions.cmake
+38
-38
core/conf/server_config.template
core/conf/server_config.template
+4
-5
core/src/CMakeLists.txt
core/src/CMakeLists.txt
+3
-0
core/src/index/CMakeLists.txt
core/src/index/CMakeLists.txt
+7
-7
core/src/index/cmake/DefineOptionsCore.cmake
core/src/index/cmake/DefineOptionsCore.cmake
+4
-15
core/src/main.cpp
core/src/main.cpp
+8
-2
core/src/server/Config.cpp
core/src/server/Config.cpp
+83
-60
core/src/server/Config.h
core/src/server/Config.h
+12
-0
core/src/utils/StringHelpFunctions.cpp
core/src/utils/StringHelpFunctions.cpp
+29
-15
core/src/utils/StringHelpFunctions.h
core/src/utils/StringHelpFunctions.h
+4
-1
core/unittest/CMakeLists.txt
core/unittest/CMakeLists.txt
+1
-0
core/unittest/server/test_config.cpp
core/unittest/server/test_config.cpp
+321
-0
core/unittest/server/test_util.cpp
core/unittest/server/test_util.cpp
+2
-3
未找到文件。
core/CMakeLists.txt
浏览文件 @
3c0c4646
...
...
@@ -31,16 +31,16 @@ GET_CURRENT_TIME(BUILD_TIME)
string
(
REGEX REPLACE
"
\n
"
""
BUILD_TIME
${
BUILD_TIME
}
)
message
(
STATUS
"Build time =
${
BUILD_TIME
}
"
)
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
execute_process
(
COMMAND
"git"
rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
if
(
GIT_BRANCH_NAME STREQUAL
""
)
if
(
GIT_BRANCH_NAME STREQUAL
""
)
execute_process
(
COMMAND
"git"
symbolic-ref --short -q HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
endif
()
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
endif
()
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
GET_GIT_BRANCH_NAME
(
GIT_BRANCH_NAME
)
message
(
STATUS
"GIT_BRANCH_NAME =
${
GIT_BRANCH_NAME
}
"
)
if
(
NOT GIT_BRANCH_NAME STREQUAL
""
)
if
(
NOT GIT_BRANCH_NAME STREQUAL
""
)
string
(
REGEX REPLACE
"
\n
"
""
GIT_BRANCH_NAME
${
GIT_BRANCH_NAME
}
)
endif
()
...
...
@@ -69,7 +69,7 @@ if (MILVUS_VERSION_MAJOR STREQUAL ""
OR MILVUS_VERSION_PATCH STREQUAL
""
)
message
(
WARNING
"Failed to determine Milvus version from git branch name"
)
set
(
MILVUS_VERSION
"0.6.0"
)
endif
()
endif
()
message
(
STATUS
"Build version =
${
MILVUS_VERSION
}
"
)
configure_file
(
${
CMAKE_CURRENT_SOURCE_DIR
}
/src/config.h.in
${
CMAKE_CURRENT_SOURCE_DIR
}
/src/config.h @ONLY
)
...
...
@@ -118,17 +118,17 @@ include(DefineOptions)
include
(
BuildUtils
)
include
(
ThirdPartyPackages
)
set
(
MILVUS_GPU_VERSION false
)
if
(
MILVUS_CPU_VERSION
)
message
(
STATUS
"Building Milvus CPU version"
)
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
else
()
set
(
MILVUS_CPU_VERSION false
)
if
(
MILVUS_GPU_VERSION
)
message
(
STATUS
"Building Milvus GPU version"
)
set
(
MILVUS_GPU_VERSION true
)
add_compile_definitions
(
"MILVUS_GPU_VERSION"
)
enable_language
(
CUDA
)
find_package
(
CUDA 10 REQUIRED
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda"
)
else
()
message
(
STATUS
"Building Milvus CPU version"
)
set
(
MILVUS_CPU_VERSION true
)
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
endif
()
if
(
CMAKE_BUILD_TYPE STREQUAL
"Release"
)
...
...
core/build.sh
浏览文件 @
3c0c4646
...
...
@@ -12,7 +12,7 @@ USE_JFROG_CACHE="OFF"
RUN_CPPLINT
=
"OFF"
CUSTOMIZATION
=
"OFF"
# default use ori faiss
CUDA_COMPILER
=
/usr/local/cuda/bin/nvcc
CPU_VERSION
=
"OFF"
GPU_VERSION
=
"OFF"
#defaults to CPU version
WITH_MKL
=
"OFF"
CUSTOMIZED_FAISS_URL
=
"
${
FAISS_URL
:-
NONE
}
"
...
...
@@ -51,7 +51,7 @@ do
c
)
BUILD_COVERAGE
=
"ON"
;;
g
)
z
)
PROFILING
=
"ON"
;;
j
)
...
...
@@ -60,8 +60,8 @@ do
x
)
CUSTOMIZATION
=
"OFF"
# force use ori faiss
;;
z
)
C
PU_VERSION
=
"ON"
g
)
G
PU_VERSION
=
"ON"
;;
m
)
WITH_MKL
=
"ON"
...
...
@@ -77,14 +77,14 @@ parameter:
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-
g
: profiling(default: OFF)
-
z
: profiling(default: OFF)
-j: use jfrog cache build directory(default: OFF)
-
z: build pure C
PU version(default: OFF)
-
g: build G
PU version(default: OFF)
-m: build with MKL(default: OFF)
-h: help
usage:
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} [-u] [-l] [-r] [-c] [-
g] [-j] [-z
] [-m] [-h]
./build.sh -p
\$
{INSTALL_PREFIX} -t
\$
{BUILD_TYPE} [-u] [-l] [-r] [-c] [-
z] [-j] [-g
] [-m] [-h]
"
exit
0
;;
...
...
@@ -116,7 +116,7 @@ CMAKE_CMD="cmake \
-DUSE_JFROG_CACHE=
${
USE_JFROG_CACHE
}
\
-DCUSTOMIZATION=
${
CUSTOMIZATION
}
\
-DFAISS_URL=
${
CUSTOMIZED_FAISS_URL
}
\
-DMILVUS_
CPU_VERSION=
${
C
PU_VERSION
}
\
-DMILVUS_
GPU_VERSION=
${
G
PU_VERSION
}
\
-DBUILD_FAISS_WITH_MKL=
${
WITH_MKL
}
\
../"
echo
${
CMAKE_CMD
}
...
...
core/cmake/DefineOptions.cmake
浏览文件 @
3c0c4646
...
...
@@ -13,16 +13,16 @@ macro(define_option name description default)
endmacro
()
function
(
list_join lst glue out
)
if
(
"
${${
lst
}}
"
STREQUAL
""
)
if
(
"
${${
lst
}}
"
STREQUAL
""
)
set
(
${
out
}
""
PARENT_SCOPE
)
return
()
endif
()
endif
()
list
(
GET
${
lst
}
0 joined
)
list
(
REMOVE_AT
${
lst
}
0
)
foreach
(
item
${${
lst
}}
)
foreach
(
item
${${
lst
}}
)
set
(
joined
"
${
joined
}${
glue
}${
item
}
"
)
endforeach
()
endforeach
()
set
(
${
out
}
${
joined
}
PARENT_SCOPE
)
endfunction
()
...
...
@@ -35,15 +35,15 @@ macro(define_option_string name description default)
set
(
"
${
name
}
_OPTION_ENUM"
${
ARGN
}
)
list_join
(
"
${
name
}
_OPTION_ENUM"
"|"
"
${
name
}
_OPTION_ENUM"
)
if
(
NOT
(
"
${${
name
}
_OPTION_ENUM
}
"
STREQUAL
""
))
if
(
NOT
(
"
${${
name
}
_OPTION_ENUM
}
"
STREQUAL
""
))
set_property
(
CACHE
${
name
}
PROPERTY STRINGS
${
ARGN
}
)
endif
()
endif
()
endmacro
()
#----------------------------------------------------------------------
set_option_category
(
"
C
PU version"
)
set_option_category
(
"
G
PU version"
)
define_option
(
MILVUS_
CPU_VERSION
"Build CPU version only
"
OFF
)
define_option
(
MILVUS_
GPU_VERSION
"Build GPU version
"
OFF
)
#----------------------------------------------------------------------
set_option_category
(
"Thirdparty"
)
...
...
@@ -51,11 +51,11 @@ set_option_category("Thirdparty")
set
(
MILVUS_DEPENDENCY_SOURCE_DEFAULT
"AUTO"
)
define_option_string
(
MILVUS_DEPENDENCY_SOURCE
"Method to use for acquiring MILVUS's build dependencies"
"
${
MILVUS_DEPENDENCY_SOURCE_DEFAULT
}
"
"AUTO"
"BUNDLED"
"SYSTEM"
)
"Method to use for acquiring MILVUS's build dependencies"
"
${
MILVUS_DEPENDENCY_SOURCE_DEFAULT
}
"
"AUTO"
"BUNDLED"
"SYSTEM"
)
define_option
(
MILVUS_VERBOSE_THIRDPARTY_BUILD
"Show output from ExternalProjects rather than just logging to files"
ON
)
...
...
@@ -75,14 +75,14 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
if
(
MILVUS_ENABLE_PROFILING STREQUAL
"ON"
)
define_option
(
MILVUS_WITH_LIBUNWIND
"Build with libunwind"
ON
)
define_option
(
MILVUS_WITH_GPERFTOOLS
"Build with gperftools"
ON
)
endif
()
endif
()
define_option
(
MILVUS_WITH_GRPC
"Build with GRPC"
ON
)
define_option
(
MILVUS_WITH_ZLIB
"Build with zlib compression"
ON
)
#----------------------------------------------------------------------
if
(
MSVC
)
if
(
MSVC
)
set_option_category
(
"MSVC"
)
define_option
(
MSVC_LINK_VERBOSE
...
...
@@ -90,7 +90,7 @@ if(MSVC)
OFF
)
define_option
(
MILVUS_USE_STATIC_CRT
"Build MILVUS with statically linked CRT"
OFF
)
endif
()
endif
()
#----------------------------------------------------------------------
...
...
@@ -99,9 +99,9 @@ set_option_category("Test and benchmark")
unset
(
MILVUS_BUILD_TESTS CACHE
)
if
(
BUILD_UNIT_TEST
)
define_option
(
MILVUS_BUILD_TESTS
"Build the MILVUS googletest unit tests"
ON
)
else
()
else
()
define_option
(
MILVUS_BUILD_TESTS
"Build the MILVUS googletest unit tests"
OFF
)
endif
(
BUILD_UNIT_TEST
)
endif
(
BUILD_UNIT_TEST
)
#----------------------------------------------------------------------
macro
(
config_summary
)
...
...
@@ -113,12 +113,12 @@ macro(config_summary)
message
(
STATUS
" Generator:
${
CMAKE_GENERATOR
}
"
)
message
(
STATUS
" Build type:
${
CMAKE_BUILD_TYPE
}
"
)
message
(
STATUS
" Source directory:
${
CMAKE_CURRENT_SOURCE_DIR
}
"
)
if
(
${
CMAKE_EXPORT_COMPILE_COMMANDS
}
)
if
(
${
CMAKE_EXPORT_COMPILE_COMMANDS
}
)
message
(
STATUS
" Compile commands:
${
CMAKE_CURRENT_BINARY_DIR
}
/compile_commands.json"
)
endif
()
endif
()
foreach
(
category
${
MILVUS_OPTION_CATEGORIES
}
)
foreach
(
category
${
MILVUS_OPTION_CATEGORIES
}
)
message
(
STATUS
)
message
(
STATUS
"
${
category
}
options:"
)
...
...
@@ -126,50 +126,50 @@ macro(config_summary)
set
(
option_names
${
MILVUS_
${
category
}
_OPTION_NAMES
}
)
set
(
max_value_length 0
)
foreach
(
name
${
option_names
}
)
foreach
(
name
${
option_names
}
)
string
(
LENGTH
"
\"
${${
name
}}
\"
"
value_length
)
if
(
${
max_value_length
}
LESS
${
value_length
}
)
if
(
${
max_value_length
}
LESS
${
value_length
}
)
set
(
max_value_length
${
value_length
}
)
endif
()
endforeach
()
endif
()
endforeach
()
foreach
(
name
${
option_names
}
)
if
(
"
${${
name
}
_OPTION_TYPE
}
"
STREQUAL
"string"
)
foreach
(
name
${
option_names
}
)
if
(
"
${${
name
}
_OPTION_TYPE
}
"
STREQUAL
"string"
)
set
(
value
"
\"
${${
name
}}
\"
"
)
else
()
else
()
set
(
value
"
${${
name
}}
"
)
endif
()
endif
()
set
(
default
${${
name
}
_OPTION_DEFAULT
}
)
set
(
description
${${
name
}
_OPTION_DESCRIPTION
}
)
string
(
LENGTH
${
description
}
description_length
)
if
(
${
description_length
}
LESS 70
)
if
(
${
description_length
}
LESS 70
)
string
(
SUBSTRING
" "
${
description_length
}
-1 description_padding
)
else
()
else
()
set
(
description_padding
"
"
)
endif
()
endif
()
set
(
comment
"[
${
name
}
]"
)
if
(
"
${
value
}
"
STREQUAL
"
${
default
}
"
)
if
(
"
${
value
}
"
STREQUAL
"
${
default
}
"
)
set
(
comment
"[default]
${
comment
}
"
)
endif
()
endif
()
if
(
NOT
(
"
${${
name
}
_OPTION_ENUM
}
"
STREQUAL
""
))
if
(
NOT
(
"
${${
name
}
_OPTION_ENUM
}
"
STREQUAL
""
))
set
(
comment
"
${
comment
}
[
${${
name
}
_OPTION_ENUM
}
]"
)
endif
()
endif
()
string
(
SUBSTRING
"
${
value
}
"
0
${
max_value_length
}
value
)
message
(
STATUS
"
${
description
}
${
description_padding
}
${
value
}
${
comment
}
"
)
endforeach
()
endforeach
()
endforeach
()
endforeach
()
endmacro
()
core/conf/server_config.template
浏览文件 @
3c0c4646
...
...
@@ -32,9 +32,9 @@ cache_config:
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
#
Skip
the following config if you are using GPU version
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
#
Uncomment
the following config if you are using GPU version
#
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
#
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
engine_config:
use_blas_threshold: 1100 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
...
...
@@ -44,5 +44,4 @@ engine_config:
resource_config:
search_resources: # define the devices used for search computation, must be in format: cpu or gpux
- cpu
- gpu0
index_build_device: gpu0 # CPU / GPU used for building index, must be in format: cpu / gpux
index_build_device: cpu # CPU / GPU used for building index, must be in format: cpu / gpux
core/src/CMakeLists.txt
浏览文件 @
3c0c4646
...
...
@@ -24,6 +24,9 @@ include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
include_directories
(
${
MILVUS_ENGINE_SRC
}
/grpc/gen-milvus
)
add_subdirectory
(
index
)
if
(
BUILD_FAISS_WITH_MKL
)
add_compile_definitions
(
"WITH_MKL"
)
endif
()
set
(
INDEX_INCLUDE_DIRS
${
INDEX_INCLUDE_DIRS
}
PARENT_SCOPE
)
foreach
(
dir
${
INDEX_INCLUDE_DIRS
}
)
...
...
core/src/index/CMakeLists.txt
浏览文件 @
3c0c4646
...
...
@@ -22,7 +22,7 @@ cmake_minimum_required(VERSION 3.14)
message
(
STATUS
"------------------------------KNOWHERE-----------------------------------"
)
message
(
STATUS
"Building using CMake version:
${
CMAKE_VERSION
}
"
)
set
(
KNOWHERE_VERSION
"0.
5
.0"
)
set
(
KNOWHERE_VERSION
"0.
6
.0"
)
string
(
REGEX MATCH
"^[0-9]+
\\
.[0-9]+
\\
.[0-9]+"
KNOWHERE_BASE_VERSION
"
${
KNOWHERE_VERSION
}
"
)
project
(
knowhere VERSION
"
${
KNOWHERE_BASE_VERSION
}
"
LANGUAGES C CXX
)
set
(
CMAKE_CXX_STANDARD 14
)
...
...
@@ -72,17 +72,17 @@ include(ExternalProject)
include
(
DefineOptionsCore
)
include
(
BuildUtilsCore
)
set
(
KNOWHERE_GPU_VERSION false
)
if
(
MILVUS_CPU_VERSION OR KNOWHERE_CPU_VERSION
)
message
(
STATUS
"Building Knowhere CPU version"
)
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
else
()
set
(
KNOWHERE_CPU_VERSION false
)
if
(
MILVUS_GPU_VERSION OR KNOWHERE_GPU_VERSION
)
message
(
STATUS
"Building Knowhere GPU version"
)
add_compile_definitions
(
"MILVUS_GPU_VERSION"
)
set
(
KNOWHERE_GPU_VERSION true
)
enable_language
(
CUDA
)
find_package
(
CUDA 10 REQUIRED
)
set
(
CUDA_NVCC_FLAGS
"
${
CUDA_NVCC_FLAGS
}
-Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda"
)
else
()
message
(
STATUS
"Building Knowhere CPU version"
)
set
(
KNOWHERE_CPU_VERSION true
)
add_compile_definitions
(
"MILVUS_CPU_VERSION"
)
endif
()
include
(
ThirdPartyPackagesCore
)
...
...
core/src/index/cmake/DefineOptionsCore.cmake
浏览文件 @
3c0c4646
...
...
@@ -41,12 +41,12 @@ macro(define_option_string name description default)
endmacro
()
#----------------------------------------------------------------------
set_option_category
(
"
C
PU version"
)
set_option_category
(
"
G
PU version"
)
if
(
MILVUS_
C
PU_VERSION
)
define_option
(
KNOWHERE_
CPU_VERSION
"Build CPU version only
"
ON
)
if
(
MILVUS_
G
PU_VERSION
)
define_option
(
KNOWHERE_
GPU_VERSION
"Build GPU version
"
ON
)
else
()
define_option
(
KNOWHERE_
CPU_VERSION
"Build CPU version only
"
OFF
)
define_option
(
KNOWHERE_
GPU_VERSION
"Build GPU version
"
OFF
)
endif
()
#----------------------------------------------------------------------
...
...
@@ -81,17 +81,6 @@ define_option(KNOWHERE_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option
(
BUILD_FAISS_WITH_MKL
"Build FAISS with MKL"
OFF
)
#----------------------------------------------------------------------
if
(
MSVC
)
set_option_category
(
"MSVC"
)
define_option
(
MSVC_LINK_VERBOSE
"Pass verbose linking options when linking libraries and executables"
OFF
)
define_option
(
KNOWHERE_USE_STATIC_CRT
"Build KNOWHERE with statically linked CRT"
OFF
)
endif
()
#----------------------------------------------------------------------
set_option_category
(
"Test and benchmark"
)
...
...
core/src/main.cpp
浏览文件 @
3c0c4646
...
...
@@ -51,7 +51,13 @@ print_banner() {
std
::
cout
<<
" /_/ /_/___/____/___/
\\
____/___/ "
<<
std
::
endl
;
std
::
cout
<<
std
::
endl
;
std
::
cout
<<
"Welcome to Milvus!"
<<
std
::
endl
;
std
::
cout
<<
"Milvus "
<<
BUILD_TYPE
<<
" version: v"
<<
MILVUS_VERSION
<<
", built at "
<<
BUILD_TIME
<<
std
::
endl
;
std
::
cout
<<
"Milvus "
<<
BUILD_TYPE
<<
" version: v"
<<
MILVUS_VERSION
<<
", built at "
<<
BUILD_TIME
<<
", with "
#ifdef WITH_MKL
<<
"MKL"
#else
<<
"OpenBLAS"
#endif
<<
" library."
<<
std
::
endl
;
#ifdef MILVUS_CPU_VERSION
std
::
cout
<<
"You are using Milvus CPU version"
<<
std
::
endl
;
#else
...
...
@@ -137,7 +143,7 @@ main(int argc, char* argv[]) {
s
=
server
.
Start
();
if
(
s
.
ok
())
{
std
::
cout
<<
"Milvus server start
successfully.
"
<<
std
::
endl
;
std
::
cout
<<
"Milvus server start
ed successfully!
"
<<
std
::
endl
;
}
else
{
goto
FAIL
;
}
...
...
core/src/server/Config.cpp
浏览文件 @
3c0c4646
...
...
@@ -25,6 +25,7 @@
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace
milvus
{
...
...
@@ -343,6 +344,11 @@ Config::ResetDefaultConfig() {
return
s
;
}
s
=
SetResourceConfigSearchResources
(
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
);
if
(
!
s
.
ok
())
{
return
s
;
}
s
=
SetResourceConfigIndexBuildDevice
(
CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT
);
if
(
!
s
.
ok
())
{
return
s
;
...
...
@@ -403,8 +409,7 @@ Status
Config
::
CheckServerConfigDeployMode
(
const
std
::
string
&
value
)
{
if
(
value
!=
"single"
&&
value
!=
"cluster_readonly"
&&
value
!=
"cluster_writable"
)
{
return
Status
(
SERVER_INVALID_ARGUMENT
,
"server_config.deploy_mode is not one of "
"single, cluster_readonly, and cluster_writable."
);
"server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."
);
}
return
Status
::
OK
();
}
...
...
@@ -592,15 +597,15 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
else
{
uint64_t
gpu_cache_capacity
=
std
::
stoi
(
value
)
*
GB
;
int
gpu_index
;
Status
s
=
GetResourceConfigIndexBuildDevice
(
gpu_index
);
int
device_id
;
Status
s
=
GetResourceConfigIndexBuildDevice
(
device_id
);
if
(
!
s
.
ok
())
{
return
s
;
}
size_t
gpu_memory
;
if
(
!
ValidationUtil
::
GetGpuMemory
(
gpu_index
,
gpu_memory
).
ok
())
{
std
::
string
msg
=
"Fail to get GPU memory for GPU device: "
+
std
::
to_string
(
gpu_index
);
if
(
!
ValidationUtil
::
GetGpuMemory
(
device_id
,
gpu_memory
).
ok
())
{
std
::
string
msg
=
"Fail to get GPU memory for GPU device: "
+
std
::
to_string
(
device_id
);
return
Status
(
SERVER_UNEXPECTED_ERROR
,
msg
);
}
else
if
(
gpu_cache_capacity
>=
gpu_memory
)
{
std
::
string
msg
=
"Invalid gpu cache capacity: "
+
value
+
...
...
@@ -689,29 +694,33 @@ Config::CheckResourceConfigMode(const std::string& value) {
}
Status
Check
GpuDevi
ce
(
const
std
::
string
&
value
)
{
Check
Resour
ce
(
const
std
::
string
&
value
)
{
std
::
string
s
=
value
;
std
::
transform
(
s
.
begin
(),
s
.
end
(),
s
.
begin
(),
::
tolower
);
#ifdef MILVUS_CPU_VERSION
if
(
s
!=
"cpu"
)
{
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Invalid CPU resource: "
+
s
);
}
#else
const
std
::
regex
pat
(
"gpu(
\\
d+)"
);
std
::
c
match
m
;
if
(
!
std
::
regex_match
(
value
.
c_str
()
,
m
,
pat
))
{
std
::
string
msg
=
"Invalid
gpu devi
ce: "
+
value
+
". Possible reason: resource_config.search_resources
does not match your hardware.
"
;
const
std
::
regex
pat
(
"
cpu|
gpu(
\\
d+)"
);
std
::
s
match
m
;
if
(
!
std
::
regex_match
(
s
,
m
,
pat
))
{
std
::
string
msg
=
"Invalid
search resour
ce: "
+
value
+
". Possible reason: resource_config.search_resources
is not in the format of cpux or gpux
"
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
int32_t
gpu_index
=
std
::
stoi
(
value
.
substr
(
3
));
if
(
!
ValidationUtil
::
ValidateGpuIndex
(
gpu_index
).
ok
())
{
std
::
string
msg
=
"Invalid gpu device: "
+
value
+
". Possible reason: resource_config.search_resources does not match your hardware."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
if
(
s
.
compare
(
0
,
3
,
"gpu"
)
==
0
)
{
int32_t
gpu_index
=
std
::
stoi
(
s
.
substr
(
3
));
if
(
!
ValidationUtil
::
ValidateGpuIndex
(
gpu_index
).
ok
())
{
std
::
string
msg
=
"Invalid search resource: "
+
value
+
". Possible reason: resource_config.search_resources does not match your hardware."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
}
#endif
return
Status
::
OK
();
}
...
...
@@ -724,38 +733,20 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
bool
cpu_found
=
false
,
gpu_found
=
false
;
for
(
auto
&
device
:
value
)
{
if
(
device
==
"cpu"
)
{
cpu_found
=
true
;
continue
;
for
(
auto
&
resource
:
value
)
{
auto
status
=
CheckResource
(
resource
);
if
(
!
status
.
ok
())
{
return
Status
(
SERVER_INVALID_ARGUMENT
,
status
.
message
());
}
if
(
CheckGpuDevice
(
device
).
ok
())
{
gpu_found
=
true
;
}
else
{
std
::
string
msg
=
"Invalid search resource: "
+
device
+
". Possible reason: resource_config.search_resources does not match your hardware."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
}
if
(
cpu_found
&&
!
gpu_found
)
{
std
::
string
msg
=
"Invalid search resource. Possible reason: resource_config.search_resources has only CPU resource."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
return
Status
::
OK
();
}
Status
Config
::
CheckResourceConfigIndexBuildDevice
(
const
std
::
string
&
value
)
{
// if (value == "cpu") {
// return Status::OK();
// }
if
(
!
CheckGpuDevice
(
value
).
ok
())
{
std
::
string
msg
=
"Invalid index build device: "
+
value
+
". Possible reason: resource_config.index_build_device does not match your hardware."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
auto
status
=
CheckResource
(
value
);
if
(
!
status
.
ok
())
{
return
Status
(
SERVER_INVALID_ARGUMENT
,
status
.
message
());
}
return
Status
::
OK
();
}
...
...
@@ -796,6 +787,22 @@ Config::GetConfigStr(const std::string& parent_key, const std::string& child_key
return
value
;
}
std
::
string
Config
::
GetConfigSequenceStr
(
const
std
::
string
&
parent_key
,
const
std
::
string
&
child_key
,
const
std
::
string
&
delim
,
const
std
::
string
&
default_value
)
{
std
::
string
value
;
if
(
!
GetConfigValueInMem
(
parent_key
,
child_key
,
value
).
ok
())
{
std
::
vector
<
std
::
string
>
sequence
=
GetConfigNode
(
parent_key
).
GetSequence
(
child_key
);
if
(
sequence
.
empty
())
{
value
=
default_value
;
}
else
{
server
::
StringHelpFunctions
::
MergeStringWithDelimeter
(
sequence
,
delim
,
value
);
}
SetConfigValueInMem
(
parent_key
,
child_key
,
value
);
}
return
value
;
}
Status
Config
::
GetServerConfigAddress
(
std
::
string
&
value
)
{
value
=
GetConfigStr
(
CONFIG_SERVER
,
CONFIG_SERVER_ADDRESS
,
CONFIG_SERVER_ADDRESS_DEFAULT
);
...
...
@@ -1019,8 +1026,10 @@ Config::GetResourceConfigMode(std::string& value) {
Status
Config
::
GetResourceConfigSearchResources
(
std
::
vector
<
std
::
string
>&
value
)
{
ConfigNode
resource_config
=
GetConfigNode
(
CONFIG_RESOURCE
);
value
=
resource_config
.
GetSequence
(
CONFIG_RESOURCE_SEARCH_RESOURCES
);
std
::
string
str
=
GetConfigSequenceStr
(
CONFIG_RESOURCE
,
CONFIG_RESOURCE_SEARCH_RESOURCES
,
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
,
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
);
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
str
,
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
,
value
);
return
CheckResourceConfigSearchResources
(
value
);
}
...
...
@@ -1033,10 +1042,10 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
return
s
;
}
if
(
str
!
=
"cpu"
)
{
value
=
std
::
stoi
(
str
.
substr
(
3
))
;
if
(
str
=
=
"cpu"
)
{
value
=
CPU_DEVICE_ID
;
}
else
{
value
=
-
1
;
value
=
std
::
stoi
(
str
.
substr
(
3
))
;
}
return
Status
::
OK
();
...
...
@@ -1163,7 +1172,7 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_METRIC_ENABLE_MONITOR
,
value
);
SetConfigValueInMem
(
CONFIG_
METRIC
,
CONFIG_METRIC_ENABLE_MONITOR
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1174,7 +1183,7 @@ Config::SetMetricConfigCollector(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_METRIC_COLLECTOR
,
value
);
SetConfigValueInMem
(
CONFIG_
METRIC
,
CONFIG_METRIC_COLLECTOR
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1185,7 +1194,7 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_METRIC_PROMETHEUS_PORT
,
value
);
SetConfigValueInMem
(
CONFIG_
METRIC
,
CONFIG_METRIC_PROMETHEUS_PORT
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1197,7 +1206,7 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_CACHE_CPU_CACHE_CAPACITY
,
value
);
SetConfigValueInMem
(
CONFIG_
CACHE
,
CONFIG_CACHE_CPU_CACHE_CAPACITY
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1208,7 +1217,7 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_CACHE_CPU_CACHE_THRESHOLD
,
value
);
SetConfigValueInMem
(
CONFIG_
CACHE
,
CONFIG_CACHE_CPU_CACHE_THRESHOLD
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1219,7 +1228,7 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_CACHE_GPU_CACHE_CAPACITY
,
value
);
SetConfigValueInMem
(
CONFIG_
CACHE
,
CONFIG_CACHE_GPU_CACHE_CAPACITY
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1230,7 +1239,7 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_CACHE_GPU_CACHE_THRESHOLD
,
value
);
SetConfigValueInMem
(
CONFIG_
CACHE
,
CONFIG_CACHE_GPU_CACHE_THRESHOLD
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1241,7 +1250,7 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_CACHE_CACHE_INSERT_DATA
,
value
);
SetConfigValueInMem
(
CONFIG_
CACHE
,
CONFIG_CACHE_CACHE_INSERT_DATA
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1253,7 +1262,7 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_ENGINE_USE_BLAS_THRESHOLD
,
value
);
SetConfigValueInMem
(
CONFIG_
ENGINE
,
CONFIG_ENGINE_USE_BLAS_THRESHOLD
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1264,7 +1273,7 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_ENGINE_OMP_THREAD_NUM
,
value
);
SetConfigValueInMem
(
CONFIG_
ENGINE
,
CONFIG_ENGINE_OMP_THREAD_NUM
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1275,7 +1284,7 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_ENGINE_GPU_SEARCH_THRESHOLD
,
value
);
SetConfigValueInMem
(
CONFIG_
ENGINE
,
CONFIG_ENGINE_GPU_SEARCH_THRESHOLD
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1287,7 +1296,21 @@ Config::SetResourceConfigMode(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_DB
,
CONFIG_RESOURCE_MODE
,
value
);
SetConfigValueInMem
(
CONFIG_RESOURCE
,
CONFIG_RESOURCE_MODE
,
value
);
return
Status
::
OK
();
}
Status
Config
::
SetResourceConfigSearchResources
(
const
std
::
string
&
value
)
{
std
::
vector
<
std
::
string
>
res_vec
;
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
value
,
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
,
res_vec
);
Status
s
=
CheckResourceConfigSearchResources
(
res_vec
);
if
(
!
s
.
ok
())
{
return
s
;
}
SetConfigValueInMem
(
CONFIG_RESOURCE
,
CONFIG_RESOURCE_SEARCH_RESOURCES
,
value
);
return
Status
::
OK
();
}
...
...
@@ -1298,7 +1321,7 @@ Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
return
s
;
}
SetConfigValueInMem
(
CONFIG_
DB
,
CONFIG_RESOURCE_INDEX_BUILD_DEVICE
,
value
);
SetConfigValueInMem
(
CONFIG_
RESOURCE
,
CONFIG_RESOURCE_INDEX_BUILD_DEVICE
,
value
);
return
Status
::
OK
();
}
...
...
core/src/server/Config.h
浏览文件 @
3c0c4646
...
...
@@ -92,12 +92,19 @@ static const char* CONFIG_RESOURCE = "resource_config";
static
const
char
*
CONFIG_RESOURCE_MODE
=
"mode"
;
static
const
char
*
CONFIG_RESOURCE_MODE_DEFAULT
=
"simple"
;
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES
=
"search_resources"
;
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
=
","
;
#ifdef MILVUS_CPU_VERSION
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
=
"cpu"
;
#else
static
const
char
*
CONFIG_RESOURCE_SEARCH_RESOURCES_DEFAULT
=
"cpu,gpu0"
;
#endif
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE
=
"index_build_device"
;
#ifdef MILVUS_CPU_VERSION
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT
=
"cpu"
;
#else
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT
=
"gpu0"
;
#endif
const
int32_t
CPU_DEVICE_ID
=
-
1
;
class
Config
{
public:
...
...
@@ -185,6 +192,9 @@ class Config {
std
::
string
GetConfigStr
(
const
std
::
string
&
parent_key
,
const
std
::
string
&
child_key
,
const
std
::
string
&
default_value
=
""
);
std
::
string
GetConfigSequenceStr
(
const
std
::
string
&
parent_key
,
const
std
::
string
&
child_key
,
const
std
::
string
&
delim
=
","
,
const
std
::
string
&
default_value
=
""
);
public:
/* server config */
...
...
@@ -306,6 +316,8 @@ class Config {
Status
SetResourceConfigMode
(
const
std
::
string
&
value
);
Status
SetResourceConfigSearchResources
(
const
std
::
string
&
value
);
Status
SetResourceConfigIndexBuildDevice
(
const
std
::
string
&
value
);
private:
...
...
core/src/utils/StringHelpFunctions.cpp
浏览文件 @
3c0c4646
...
...
@@ -39,39 +39,53 @@ StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qou
}
}
Status
void
StringHelpFunctions
::
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>&
result
)
{
if
(
str
.
empty
())
{
return
Status
::
OK
()
;
return
;
}
size_t
last
=
0
;
size_t
index
=
str
.
find_first_of
(
delimeter
,
last
);
while
(
index
!=
std
::
string
::
npos
)
{
result
.
emplace_back
(
str
.
substr
(
last
,
index
-
last
));
last
=
index
+
1
;
index
=
str
.
find_first_of
(
delimeter
,
last
);
size_t
prev
=
0
,
pos
=
0
;
while
(
true
)
{
pos
=
str
.
find_first_of
(
delimeter
,
prev
);
if
(
pos
==
std
::
string
::
npos
)
{
result
.
emplace_back
(
str
.
substr
(
prev
));
break
;
}
else
{
result
.
emplace_back
(
str
.
substr
(
prev
,
pos
-
prev
));
prev
=
pos
+
1
;
}
}
if
(
index
-
last
>
0
)
{
std
::
string
temp
=
str
.
substr
(
last
);
result
.
emplace_back
(
temp
);
}
void
StringHelpFunctions
::
MergeStringWithDelimeter
(
const
std
::
vector
<
std
::
string
>&
strs
,
const
std
::
string
&
delimeter
,
std
::
string
&
result
)
{
if
(
strs
.
empty
())
{
result
=
""
;
return
;
}
return
Status
::
OK
();
result
=
strs
[
0
];
for
(
size_t
i
=
1
;
i
<
strs
.
size
();
i
++
)
{
result
=
result
+
delimeter
+
strs
[
i
];
}
}
Status
StringHelpFunctions
::
SplitStringByQuote
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
const
std
::
string
&
quote
,
std
::
vector
<
std
::
string
>&
result
)
{
if
(
quote
.
empty
())
{
return
SplitStringByDelimeter
(
str
,
delimeter
,
result
);
SplitStringByDelimeter
(
str
,
delimeter
,
result
);
return
Status
::
OK
();
}
size_t
last
=
0
;
size_t
index
=
str
.
find_first_of
(
quote
,
last
);
if
(
index
==
std
::
string
::
npos
)
{
return
SplitStringByDelimeter
(
str
,
delimeter
,
result
);
SplitStringByDelimeter
(
str
,
delimeter
,
result
);
return
Status
::
OK
();
}
std
::
string
process_str
=
str
;
...
...
@@ -116,7 +130,7 @@ StringHelpFunctions::SplitStringByQuote(const std::string& str, const std::strin
}
if
(
!
process_str
.
empty
())
{
return
SplitStringByDelimeter
(
process_str
,
delimeter
,
result
);
SplitStringByDelimeter
(
process_str
,
delimeter
,
result
);
}
return
Status
::
OK
();
...
...
core/src/utils/StringHelpFunctions.h
浏览文件 @
3c0c4646
...
...
@@ -43,9 +43,12 @@ class StringHelpFunctions {
// ,b, | b |
// ,, | |
// a a
static
Status
static
void
SplitStringByDelimeter
(
const
std
::
string
&
str
,
const
std
::
string
&
delimeter
,
std
::
vector
<
std
::
string
>&
result
);
static
void
MergeStringWithDelimeter
(
const
std
::
vector
<
std
::
string
>&
strs
,
const
std
::
string
&
delimeter
,
std
::
string
&
result
);
// assume the table has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
...
...
core/unittest/CMakeLists.txt
浏览文件 @
3c0c4646
...
...
@@ -77,6 +77,7 @@ set(helper_files
${
MILVUS_ENGINE_SRC
}
/utils/CommonUtil.cpp
${
MILVUS_ENGINE_SRC
}
/utils/TimeRecorder.cpp
${
MILVUS_ENGINE_SRC
}
/utils/Status.cpp
${
MILVUS_ENGINE_SRC
}
/utils/StringHelpFunctions.cpp
${
MILVUS_ENGINE_SRC
}
/utils/ValidationUtil.cpp
${
MILVUS_ENGINE_SRC
}
/external/easyloggingpp/easylogging++.cc
)
...
...
core/unittest/server/test_config.cpp
浏览文件 @
3c0c4646
...
...
@@ -22,6 +22,7 @@
#include "server/Config.h"
#include "server/utils.h"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
namespace
{
...
...
@@ -98,6 +99,326 @@ TEST_F(ConfigTest, CONFIG_TEST) {
ASSERT_TRUE
(
seqs
.
empty
());
}
TEST_F
(
ConfigTest
,
SERVER_CONFIG_VALID_TEST
)
{
std
::
string
config_path
(
CONFIG_PATH
);
milvus
::
server
::
Config
&
config
=
milvus
::
server
::
Config
::
GetInstance
();
milvus
::
Status
s
;
std
::
string
str_val
;
int32_t
int32_val
;
int64_t
int64_val
;
float
float_val
;
bool
bool_val
;
/* server config */
std
::
string
server_addr
=
"192.168.1.155"
;
s
=
config
.
SetServerConfigAddress
(
server_addr
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetServerConfigAddress
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
server_addr
);
std
::
string
server_port
=
"12345"
;
s
=
config
.
SetServerConfigPort
(
server_port
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetServerConfigPort
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
server_port
);
std
::
string
server_mode
=
"cluster_readonly"
;
s
=
config
.
SetServerConfigDeployMode
(
server_mode
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetServerConfigDeployMode
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
server_mode
);
std
::
string
server_time_zone
=
"UTC+6"
;
s
=
config
.
SetServerConfigTimeZone
(
server_time_zone
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetServerConfigTimeZone
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
server_time_zone
);
/* db config */
std
::
string
db_primary_path
=
"/home/zilliz"
;
s
=
config
.
SetDBConfigPrimaryPath
(
db_primary_path
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigPrimaryPath
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
db_primary_path
);
std
::
string
db_secondary_path
=
"/home/zilliz"
;
s
=
config
.
SetDBConfigSecondaryPath
(
db_secondary_path
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigSecondaryPath
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
db_secondary_path
);
std
::
string
db_backend_url
=
"mysql://root:123456@127.0.0.1:19530/milvus"
;
s
=
config
.
SetDBConfigBackendUrl
(
db_backend_url
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigBackendUrl
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
db_backend_url
);
int32_t
db_archive_disk_threshold
=
100
;
s
=
config
.
SetDBConfigArchiveDiskThreshold
(
std
::
to_string
(
db_archive_disk_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigArchiveDiskThreshold
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
db_archive_disk_threshold
);
int32_t
db_archive_days_threshold
=
365
;
s
=
config
.
SetDBConfigArchiveDaysThreshold
(
std
::
to_string
(
db_archive_days_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigArchiveDaysThreshold
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
db_archive_days_threshold
);
int32_t
db_insert_buffer_size
=
2
;
s
=
config
.
SetDBConfigInsertBufferSize
(
std
::
to_string
(
db_insert_buffer_size
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigInsertBufferSize
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
db_insert_buffer_size
);
/* metric config */
bool
metric_enable_monitor
=
false
;
s
=
config
.
SetMetricConfigEnableMonitor
(
std
::
to_string
(
metric_enable_monitor
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetMetricConfigEnableMonitor
(
bool_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
bool_val
==
metric_enable_monitor
);
std
::
string
metric_collector
=
"prometheus"
;
s
=
config
.
SetMetricConfigCollector
(
metric_collector
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetMetricConfigCollector
(
str_val
);
ASSERT_TRUE
(
str_val
==
metric_collector
);
std
::
string
metric_prometheus_port
=
"2222"
;
s
=
config
.
SetMetricConfigPrometheusPort
(
metric_prometheus_port
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetMetricConfigPrometheusPort
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
metric_prometheus_port
);
/* cache config */
int64_t
cache_cpu_cache_capacity
=
5
;
s
=
config
.
SetCacheConfigCpuCacheCapacity
(
std
::
to_string
(
cache_cpu_cache_capacity
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetCacheConfigCpuCacheCapacity
(
int64_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int64_val
==
cache_cpu_cache_capacity
);
float
cache_cpu_cache_threshold
=
0.1
;
s
=
config
.
SetCacheConfigCpuCacheThreshold
(
std
::
to_string
(
cache_cpu_cache_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetCacheConfigCpuCacheThreshold
(
float_val
);
ASSERT_TRUE
(
float_val
==
cache_cpu_cache_threshold
);
int64_t
cache_gpu_cache_capacity
=
1
;
s
=
config
.
SetCacheConfigGpuCacheCapacity
(
std
::
to_string
(
cache_gpu_cache_capacity
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetCacheConfigGpuCacheCapacity
(
int64_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int64_val
==
cache_gpu_cache_capacity
);
float
cache_gpu_cache_threshold
=
0.2
;
s
=
config
.
SetCacheConfigGpuCacheThreshold
(
std
::
to_string
(
cache_gpu_cache_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetCacheConfigGpuCacheThreshold
(
float_val
);
ASSERT_TRUE
(
float_val
==
cache_gpu_cache_threshold
);
bool
cache_insert_data
=
true
;
s
=
config
.
SetCacheConfigCacheInsertData
(
std
::
to_string
(
cache_insert_data
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetCacheConfigCacheInsertData
(
bool_val
);
ASSERT_TRUE
(
bool_val
==
cache_insert_data
);
/* engine config */
int32_t
engine_use_blas_threshold
=
50
;
s
=
config
.
SetEngineConfigUseBlasThreshold
(
std
::
to_string
(
engine_use_blas_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigUseBlasThreshold
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
engine_use_blas_threshold
);
int32_t
engine_omp_thread_num
=
8
;
s
=
config
.
SetEngineConfigOmpThreadNum
(
std
::
to_string
(
engine_omp_thread_num
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigOmpThreadNum
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
engine_omp_thread_num
);
int32_t
engine_gpu_search_threshold
=
800
;
s
=
config
.
SetEngineConfigGpuSearchThreshold
(
std
::
to_string
(
engine_gpu_search_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigGpuSearchThreshold
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
engine_gpu_search_threshold
);
/* resource config */
std
::
string
resource_mode
=
"simple"
;
s
=
config
.
SetResourceConfigMode
(
resource_mode
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetResourceConfigMode
(
str_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
resource_mode
);
#ifdef MILVUS_CPU_VERSION
std
::
vector
<
std
::
string
>
search_resources
=
{
"cpu"
};
#else
std
::
vector
<
std
::
string
>
search_resources
=
{
"cpu"
,
"gpu0"
};
#endif
std
::
vector
<
std
::
string
>
res_vec
;
std
::
string
res_str
;
milvus
::
server
::
StringHelpFunctions
::
MergeStringWithDelimeter
(
search_resources
,
milvus
::
server
::
CONFIG_RESOURCE_SEARCH_RESOURCES_DELIMITER
,
res_str
);
s
=
config
.
SetResourceConfigSearchResources
(
res_str
);
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetResourceConfigSearchResources
(
res_vec
);
ASSERT_TRUE
(
s
.
ok
());
for
(
size_t
i
=
0
;
i
<
search_resources
.
size
();
i
++
)
{
ASSERT_TRUE
(
search_resources
[
i
]
==
res_vec
[
i
]);
}
#ifdef MILVUS_CPU_VERSION
int32_t
resource_index_build_device
=
milvus
::
server
::
CPU_DEVICE_ID
;
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"cpu"
);
#else
int32_t
resource_index_build_device
=
0
;
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"gpu"
+
std
::
to_string
(
resource_index_build_device
));
#endif
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetResourceConfigIndexBuildDevice
(
int32_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int32_val
==
resource_index_build_device
);
}
TEST_F
(
ConfigTest
,
SERVER_CONFIG_INVALID_TEST
)
{
std
::
string
config_path
(
CONFIG_PATH
);
milvus
::
server
::
Config
&
config
=
milvus
::
server
::
Config
::
GetInstance
();
milvus
::
Status
s
;
s
=
config
.
LoadConfigFile
(
""
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
LoadConfigFile
(
config_path
+
INVALID_CONFIG_FILE
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
LoadConfigFile
(
config_path
+
"dummy.yaml"
);
ASSERT_FALSE
(
s
.
ok
());
/* server config */
s
=
config
.
SetServerConfigAddress
(
"0.0.0"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigAddress
(
"0.0.0.256"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigPort
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigPort
(
"99999"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigDeployMode
(
"cluster"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigTimeZone
(
"GM"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigTimeZone
(
"GMT8"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetServerConfigTimeZone
(
"UTCA"
);
ASSERT_FALSE
(
s
.
ok
());
/* db config */
s
=
config
.
SetDBConfigPrimaryPath
(
""
);
ASSERT_FALSE
(
s
.
ok
());
// s = config.SetDBConfigSecondaryPath("");
// ASSERT_FALSE(s.ok());
s
=
config
.
SetDBConfigBackendUrl
(
"http://www.google.com"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigBackendUrl
(
"sqlite://:@:"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigBackendUrl
(
"mysql://root:123456@127.0.0.1/milvus"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigArchiveDiskThreshold
(
"0x10"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigArchiveDaysThreshold
(
"0x10"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigInsertBufferSize
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigInsertBufferSize
(
"0"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetDBConfigInsertBufferSize
(
"2048"
);
ASSERT_FALSE
(
s
.
ok
());
/* metric config */
s
=
config
.
SetMetricConfigEnableMonitor
(
"Y"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetMetricConfigCollector
(
"zilliz"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetMetricConfigPrometheusPort
(
"0xff"
);
ASSERT_FALSE
(
s
.
ok
());
/* cache config */
s
=
config
.
SetCacheConfigCpuCacheCapacity
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigCpuCacheCapacity
(
"0"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigCpuCacheCapacity
(
"2048"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigCpuCacheThreshold
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigCpuCacheThreshold
(
"1.0"
);
ASSERT_FALSE
(
s
.
ok
());
#ifdef MILVUS_GPU_VERSION
s
=
config
.
SetCacheConfigGpuCacheCapacity
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigGpuCacheCapacity
(
"128"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigGpuCacheThreshold
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetCacheConfigGpuCacheThreshold
(
"1.0"
);
ASSERT_FALSE
(
s
.
ok
());
#endif
s
=
config
.
SetCacheConfigCacheInsertData
(
"N"
);
ASSERT_FALSE
(
s
.
ok
());
/* engine config */
s
=
config
.
SetEngineConfigUseBlasThreshold
(
"0xff"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetEngineConfigOmpThreadNum
(
"a"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetEngineConfigOmpThreadNum
(
"10000"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetEngineConfigGpuSearchThreshold
(
"-1"
);
ASSERT_FALSE
(
s
.
ok
());
/* resource config */
s
=
config
.
SetResourceConfigMode
(
"default"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetResourceConfigSearchResources
(
"gpu10"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"gup2"
);
ASSERT_FALSE
(
s
.
ok
());
s
=
config
.
SetResourceConfigIndexBuildDevice
(
"gpu16"
);
ASSERT_FALSE
(
s
.
ok
());
}
TEST_F
(
ConfigTest
,
SERVER_CONFIG_TEST
)
{
std
::
string
config_path
(
CONFIG_PATH
);
milvus
::
server
::
Config
&
config
=
milvus
::
server
::
Config
::
GetInstance
();
...
...
core/unittest/server/test_util.cpp
浏览文件 @
3c0c4646
...
...
@@ -117,12 +117,11 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
str
=
"a,b,c"
;
std
::
vector
<
std
::
string
>
result
;
auto
status
=
milvus
::
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
str
,
","
,
result
);
ASSERT_TRUE
(
status
.
ok
());
milvus
::
server
::
StringHelpFunctions
::
SplitStringByDelimeter
(
str
,
","
,
result
);
ASSERT_EQ
(
result
.
size
(),
3UL
);
result
.
clear
();
status
=
milvus
::
server
::
StringHelpFunctions
::
SplitStringByQuote
(
str
,
","
,
"
\"
"
,
result
);
auto
status
=
milvus
::
server
::
StringHelpFunctions
::
SplitStringByQuote
(
str
,
","
,
"
\"
"
,
result
);
ASSERT_TRUE
(
status
.
ok
());
ASSERT_EQ
(
result
.
size
(),
3UL
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录