Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
22563d83
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
22563d83
编写于
9月 21, 2019
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-574 add GET config APIs
Former-commit-id: 6011f32d650d26367ab84f4bf8521889881f2629
上级
82e37e11
变更
13
显示空白变更内容
内联
并排
Showing
13 changed file
with
280 addition
and
112 deletion
+280
-112
cpp/conf/server_config.template
cpp/conf/server_config.template
+4
-4
cpp/src/cache/CpuCacheMgr.cpp
cpp/src/cache/CpuCacheMgr.cpp
+7
-10
cpp/src/cache/GpuCacheMgr.cpp
cpp/src/cache/GpuCacheMgr.cpp
+7
-10
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+1
-2
cpp/src/metrics/Metrics.cpp
cpp/src/metrics/Metrics.cpp
+3
-3
cpp/src/metrics/PrometheusMetrics.cpp
cpp/src/metrics/PrometheusMetrics.cpp
+3
-6
cpp/src/scheduler/SchedInst.cpp
cpp/src/scheduler/SchedInst.cpp
+3
-4
cpp/src/server/DBWrapper.cpp
cpp/src/server/DBWrapper.cpp
+13
-21
cpp/src/server/Server.cpp
cpp/src/server/Server.cpp
+4
-5
cpp/src/server/ServerConfig.cpp
cpp/src/server/ServerConfig.cpp
+191
-29
cpp/src/server/ServerConfig.h
cpp/src/server/ServerConfig.h
+38
-7
cpp/src/server/grpc_impl/GrpcServer.cpp
cpp/src/server/grpc_impl/GrpcServer.cpp
+3
-5
cpp/src/wrapper/KnowhereResource.cpp
cpp/src/wrapper/KnowhereResource.cpp
+3
-6
未找到文件。
cpp/conf/server_config.template
浏览文件 @
22563d83
...
...
@@ -5,13 +5,13 @@ server_config:
time_zone: UTC+8
db_config:
db_path: @MILVUS_DB_PATH@
# milvus database path
db_slave_path:
# secondary database path, split by semicolon
path: @MILVUS_DB_PATH@
# milvus database path
slave_path:
# secondary database path, split by semicolon
# URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite
db_
backend_url: sqlite://:@:/
backend_url: sqlite://:@:/
archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit
archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit
...
...
@@ -27,7 +27,7 @@ metric_config:
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory size used for cache
cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered
insert_immediately: false
# whether load data into cache when insert
cache_insert_data: false
# whether load data into cache when insert
engine_config:
blas_threshold: 20
...
...
cpp/src/cache/CpuCacheMgr.cpp
浏览文件 @
22563d83
...
...
@@ -29,19 +29,16 @@ namespace {
}
CpuCacheMgr
::
CpuCacheMgr
()
{
server
::
ConfigNode
&
config
=
server
::
ServerConfig
::
GetInstance
().
GetConfig
(
server
::
CONFIG_CACHE
);
int64_t
cap
=
config
.
GetInt64Value
(
server
::
CONFIG_CACHE_CPU_MEM_CAPACITY
,
std
::
stoi
(
server
::
CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT
));
cap
*=
unit
;
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
();
int64_t
cap
=
config
.
GetCacheConfigCpuMemCapacity
()
*
unit
;
cache_
=
std
::
make_shared
<
Cache
<
DataObjPtr
>>
(
cap
,
1UL
<<
32
);
double
free_percent
=
config
.
GetDoubleValue
(
server
::
CONFIG_CACHE_CPU_MEM_THRESHOLD
,
std
::
stof
(
server
::
CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT
));
if
(
free_percent
>
0.0
&&
free_percent
<=
1.0
)
{
cache_
->
set_freemem_percent
(
free_percent
);
float
cpu_mem_threshold
=
config
.
GetCacheConfigCpuMemThreshold
();
if
(
cpu_mem_threshold
>
0.0
&&
cpu_mem_threshold
<=
1.0
)
{
cache_
->
set_freemem_percent
(
cpu_mem_threshold
);
}
else
{
SERVER_LOG_ERROR
<<
"Invalid c
ache_free_percent: "
<<
free_percent
<<
", defaultly
set to "
<<
cache_
->
freemem_percent
();
SERVER_LOG_ERROR
<<
"Invalid c
pu_mem_threshold: "
<<
cpu_mem_threshold
<<
", by default
set to "
<<
cache_
->
freemem_percent
();
}
}
...
...
cpp/src/cache/GpuCacheMgr.cpp
浏览文件 @
22563d83
...
...
@@ -33,20 +33,17 @@ namespace {
}
GpuCacheMgr
::
GpuCacheMgr
()
{
server
::
ConfigNode
&
config
=
server
::
ServerConfig
::
GetInstance
().
GetConfig
(
server
::
CONFIG_CACHE
);
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
(
);
int64_t
cap
=
config
.
GetInt64Value
(
server
::
CONFIG_CACHE_GPU_MEM_CAPACITY
,
std
::
stoi
(
server
::
CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT
));
cap
*=
G_BYTE
;
int32_t
cap
=
config
.
GetCacheConfigGpuMemCapacity
()
*
G_BYTE
;
cache_
=
std
::
make_shared
<
Cache
<
DataObjPtr
>>
(
cap
,
1UL
<<
32
);
double
free_percent
=
config
.
GetDoubleValue
(
server
::
CONFIG_CACHE_GPU_MEM_THRESHOLD
,
std
::
stof
(
server
::
CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT
));
if
(
free_percent
>
0.0
&&
free_percent
<=
1.0
)
{
cache_
->
set_freemem_percent
(
free_percent
);
float
gpu_mem_threshold
=
config
.
GetCacheConfigGpuMemThreshold
();
if
(
gpu_mem_threshold
>
0.0
&&
gpu_mem_threshold
<=
1.0
)
{
cache_
->
set_freemem_percent
(
gpu_mem_threshold
);
}
else
{
SERVER_LOG_ERROR
<<
"Invalid gpu_
cache_free_percent: "
<<
free_percent
<<
", defaultly
set to "
<<
cache_
->
freemem_percent
();
SERVER_LOG_ERROR
<<
"Invalid gpu_
mem_threshold: "
<<
gpu_mem_threshold
<<
", by default
set to "
<<
cache_
->
freemem_percent
();
}
}
...
...
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
22563d83
...
...
@@ -335,8 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl
::
Init
()
{
using
namespace
zilliz
::
milvus
::
server
;
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
server_config
=
config
.
GetConfig
(
CONFIG_DB
);
gpu_num_
=
server_config
.
GetInt32Value
(
CONFIG_DB_BUILD_INDEX_GPU
,
std
::
stoi
(
CONFIG_DB_BUILD_INDEX_GPU_DEFAULT
));
gpu_num_
=
config
.
GetDBConfigBuildIndexGPU
();
return
Status
::
OK
();
}
...
...
cpp/src/metrics/Metrics.cpp
浏览文件 @
22563d83
...
...
@@ -16,6 +16,7 @@
// under the License.
#include "Metrics.h"
#include "server/ServerConfig.h"
#include "PrometheusMetrics.h"
...
...
@@ -31,9 +32,8 @@ Metrics::GetInstance() {
MetricsBase
&
Metrics
::
CreateMetricsCollector
()
{
ConfigNode
&
config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_METRIC
);
std
::
string
collector_type_str
=
config
.
GetValue
(
CONFIG_METRIC_COLLECTOR
,
CONFIG_METRIC_COLLECTOR_DEFAULT
);
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
std
::
string
collector_type_str
=
config
.
GetMetricConfigCollector
();
if
(
collector_type_str
==
"prometheus"
)
{
return
PrometheusMetrics
::
GetInstance
();
...
...
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @
22563d83
...
...
@@ -29,15 +29,12 @@ namespace server {
ErrorCode
PrometheusMetrics
::
Init
()
{
try
{
ConfigNode
&
metric_config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_METRIC
);
startup_
=
metric_config
.
GetBoolValue
(
CONFIG_METRIC_AUTO_BOOTUP
,
std
::
stoi
(
CONFIG_METRIC_AUTO_BOOTUP_DEFAULT
));
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
startup_
=
config
.
GetMetricConfigAutoBootup
();
if
(
!
startup_
)
return
SERVER_SUCCESS
;
// Following should be read from config file.
ConfigNode
&
prometheus_config
=
metric_config
.
GetChild
(
CONFIG_METRIC_PROMETHEUS
);
const
std
::
string
bind_address
=
prometheus_config
.
GetValue
(
CONFIG_METRIC_PROMETHEUS_PORT
,
CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT
);
const
std
::
string
bind_address
=
config
.
GetMetricConfigPrometheusPort
();
const
std
::
string
uri
=
std
::
string
(
"/tmp/metrics"
);
const
std
::
size_t
num_threads
=
2
;
...
...
cpp/src/scheduler/SchedInst.cpp
浏览文件 @
22563d83
...
...
@@ -38,10 +38,9 @@ std::mutex JobMgrInst::mutex_;
void
load_simple_config
()
{
server
::
ConfigNode
&
config
=
server
::
ServerConfig
::
GetInstance
().
GetConfig
(
server
::
CONFIG_RESOURCE
);
auto
mode
=
config
.
GetValue
(
server
::
CONFIG_RESOURCE_MODE
,
server
::
CONFIG_RESOURCE_MODE_DEFAULT
);
auto
pool
=
config
.
GetSequence
(
server
::
CONFIG_RESOURCE_POOL
);
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
();
auto
mode
=
config
.
GetResourceConfigMode
();
auto
pool
=
config
.
GetResourceConfigPool
();
bool
cpu
=
false
;
std
::
set
<
uint64_t
>
gpu_ids
;
for
(
auto
&
resource
:
pool
)
{
...
...
cpp/src/server/DBWrapper.cpp
浏览文件 @
22563d83
...
...
@@ -35,23 +35,20 @@ DBWrapper::DBWrapper() {
}
Status
DBWrapper
::
StartService
()
{
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
//db config
engine
::
DBOptions
opt
;
ConfigNode
&
db_config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_DB
);
opt
.
meta
.
backend_uri
=
db_config
.
GetValue
(
CONFIG_DB_BACKEND_URL
);
std
::
string
db_path
=
db_config
.
GetValue
(
CONFIG_DB_PATH
);
opt
.
meta
.
path
=
db_path
+
"/db"
;
opt
.
meta
.
backend_uri
=
config
.
GetDBConfigBackendUrl
();
opt
.
meta
.
path
=
config
.
GetDBConfigPath
()
+
"/db"
;
std
::
string
db_slave_path
=
db_config
.
GetValue
(
CONFIG_DB_SLAVE_PATH
);
std
::
string
db_slave_path
=
config
.
GetDBConfigSlavePath
(
);
StringHelpFunctions
::
SplitStringByDelimeter
(
db_slave_path
,
";"
,
opt
.
meta
.
slave_paths
);
// cache config
ConfigNode
&
cache_config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_CACHE
);
opt
.
insert_cache_immediately_
=
cache_config
.
GetBoolValue
(
CONFIG_CACHE_INSERT_IMMEDIATELY
,
std
::
stoi
(
CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT
));
opt
.
insert_cache_immediately_
=
config
.
GetCacheConfigCacheInsertData
();
ConfigNode
&
serverConfig
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_SERVER
);
std
::
string
mode
=
serverConfig
.
GetValue
(
CONFIG_SERVER_MODE
,
CONFIG_SERVER_MODE_DEFAULT
);
std
::
string
mode
=
config
.
GetServerConfigMode
();
if
(
mode
==
"single"
)
{
opt
.
mode
=
engine
::
DBOptions
::
MODE
::
SINGLE
;
}
...
...
@@ -67,9 +64,7 @@ Status DBWrapper::StartService() {
}
// engine config
ConfigNode
&
engine_config
=
ServerConfig
::
GetInstance
().
GetConfig
(
CONFIG_ENGINE
);
int32_t
omp_thread
=
engine_config
.
GetInt32Value
(
CONFIG_ENGINE_OMP_THREAD_NUM
,
std
::
stoi
(
CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT
));
int32_t
omp_thread
=
config
.
GetEngineConfigOmpThreadNum
();
if
(
omp_thread
>
0
)
{
omp_set_num_threads
(
omp_thread
);
SERVER_LOG_DEBUG
<<
"Specify openmp thread number: "
<<
omp_thread
;
...
...
@@ -82,19 +77,16 @@ Status DBWrapper::StartService() {
}
//init faiss global variable
faiss
::
distance_compute_blas_threshold
=
engine_config
.
GetInt32Value
(
CONFIG_ENGINE_BLAS_THRESHOLD
,
std
::
stoi
(
CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT
));
faiss
::
distance_compute_blas_threshold
=
config
.
GetEngineConfigBlasThreshold
();
//set archive config
engine
::
ArchiveConf
::
CriteriaT
criterial
;
int64_t
disk
=
db_config
.
GetInt64Value
(
CONFIG_DB_ARCHIVE_DISK_THRESHOLD
,
std
::
stoi
(
CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT
));
int64_t
days
=
db_config
.
GetInt64Value
(
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD
,
std
::
stoi
(
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT
));
if
(
disk
>
0
)
{
int32_t
disk
=
config
.
GetDBConfigArchiveDiskThreshold
();
int32_t
days
=
config
.
GetDBConfigArchiveDaysThreshold
();
if
(
disk
>
0
)
{
criterial
[
engine
::
ARCHIVE_CONF_DISK
]
=
disk
;
}
if
(
days
>
0
)
{
if
(
days
>
0
)
{
criterial
[
engine
::
ARCHIVE_CONF_DAYS
]
=
days
;
}
opt
.
meta
.
archive_conf
.
SetCriterias
(
criterial
);
...
...
cpp/src/server/Server.cpp
浏览文件 @
22563d83
...
...
@@ -162,9 +162,7 @@ Server::Start() {
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
server_config
=
config
.
GetConfig
(
CONFIG_SERVER
);
std
::
string
time_zone
=
server_config
.
GetValue
(
CONFIG_SERVER_TIME_ZONE
,
CONFIG_SERVER_TIME_ZONE_DEFAULT
);
std
::
string
time_zone
=
config
.
GetServerConfigTimeZone
();
if
(
time_zone
.
length
()
==
3
)
{
time_zone
=
"CUT"
;
}
else
{
...
...
@@ -232,8 +230,9 @@ Server::Stop() {
ErrorCode
Server
::
LoadConfig
()
{
ServerConfig
::
GetInstance
().
LoadConfigFile
(
config_filename_
);
auto
status
=
ServerConfig
::
GetInstance
().
ValidateConfig
();
ServerConfig
&
server_config
=
ServerConfig
::
GetInstance
();
server_config
.
LoadConfigFile
(
config_filename_
);
auto
status
=
server_config
.
ValidateConfig
();
if
(
!
status
.
ok
())
{
std
::
cerr
<<
"Failed to load config file: "
<<
config_filename_
<<
std
::
endl
;
exit
(
0
);
...
...
cpp/src/server/ServerConfig.cpp
浏览文件 @
22563d83
...
...
@@ -42,30 +42,29 @@ ServerConfig::GetInstance() {
}
Status
ServerConfig
::
LoadConfigFile
(
const
std
::
string
&
config_filename
)
{
std
::
string
filename
=
config_filename
;
ServerConfig
::
LoadConfigFile
(
const
std
::
string
&
filename
)
{
if
(
filename
.
empty
())
{
std
::
cerr
<<
"ERROR:
a config file is required
"
<<
std
::
endl
;
exit
(
1
);
//directly exit program if config file not specified
std
::
cerr
<<
"ERROR:
need specify config file
"
<<
std
::
endl
;
exit
(
1
);
}
struct
stat
dir
ectory
Stat
;
int
statOK
=
stat
(
filename
.
c_str
(),
&
dir
ectory
Stat
);
struct
stat
dirStat
;
int
statOK
=
stat
(
filename
.
c_str
(),
&
dirStat
);
if
(
statOK
!=
0
)
{
std
::
cerr
<<
"ERROR:
"
<<
filename
<<
" not found!"
<<
std
::
endl
;
exit
(
1
);
//directly exit program if config file not found
std
::
cerr
<<
"ERROR:
Config file not exist: "
<<
filename
<<
std
::
endl
;
exit
(
1
);
}
try
{
ConfigMgr
*
mgr
=
const_cast
<
ConfigMgr
*>
(
ConfigMgr
::
GetInstance
());
ErrorCode
err
=
mgr
->
LoadConfigFile
(
filename
);
if
(
err
!=
0
)
{
std
::
cerr
<<
"Server failed to load config file
"
<<
std
::
endl
;
exit
(
1
);
//directly exit program if the config file is illegal
std
::
cerr
<<
"Server failed to load config file
: "
<<
filename
<<
std
::
endl
;
exit
(
1
);
}
}
catch
(
YAML
::
Exception
&
e
)
{
std
::
cerr
<<
"Server failed to load config file: "
<<
std
::
endl
;
exit
(
1
);
//directly exit program if the config file is illegal
std
::
cerr
<<
"Server failed to load config file: "
<<
filename
<<
std
::
endl
;
exit
(
1
);
}
return
Status
::
OK
();
...
...
@@ -73,27 +72,25 @@ ServerConfig::LoadConfigFile(const std::string &config_filename) {
Status
ServerConfig
::
ValidateConfig
()
{
bool
okay
=
true
;
if
(
!
CheckServerConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Server config validation check fail"
)
;
}
if
(
!
CheckDBConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"DB config validation check fail"
)
;
}
if
(
!
CheckMetricConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Metric config validation check fail"
)
;
}
if
(
!
CheckCacheConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Cache config validation check fail"
)
;
}
if
(
!
CheckEngineConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Engine config validation check fail"
)
;
}
if
(
!
CheckResourceConfig
().
ok
())
{
okay
=
false
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
"Resource config validation check fail"
)
;
}
return
(
okay
?
Status
::
OK
()
:
Status
(
SERVER_INVALID_ARGUMENT
,
"Config validation not pass"
)
);
return
Status
::
OK
(
);
}
Status
...
...
@@ -201,18 +198,18 @@ ServerConfig::CheckDBConfig() {
okay
=
false
;
}
std
::
string
insert_
buffer_size_str
=
db_config
.
GetValue
(
CONFIG_DB_BUFFER_SIZE
,
CONFIG_DB_BUFFER_SIZE_DEFAULT
);
if
(
!
ValidationUtil
::
ValidateStringIsNumber
(
insert_
buffer_size_str
).
ok
())
{
std
::
cerr
<<
"ERROR:
insert_buffer_size "
<<
insert_
buffer_size_str
<<
" is not a number"
<<
std
::
endl
;
std
::
string
buffer_size_str
=
db_config
.
GetValue
(
CONFIG_DB_BUFFER_SIZE
,
CONFIG_DB_BUFFER_SIZE_DEFAULT
);
if
(
!
ValidationUtil
::
ValidateStringIsNumber
(
buffer_size_str
).
ok
())
{
std
::
cerr
<<
"ERROR:
buffer_size "
<<
buffer_size_str
<<
" is not a number"
<<
std
::
endl
;
okay
=
false
;
}
else
{
uint64_t
insert_buffer_size
=
(
uint64_t
)
std
::
stol
(
insert_
buffer_size_str
);
insert_
buffer_size
*=
GB
;
uint64_t
buffer_size
=
(
uint64_t
)
std
::
stol
(
buffer_size_str
);
buffer_size
*=
GB
;
unsigned
long
total_mem
=
0
,
free_mem
=
0
;
CommonUtil
::
GetSystemMemInfo
(
total_mem
,
free_mem
);
if
(
insert_
buffer_size
>=
total_mem
)
{
std
::
cerr
<<
"ERROR:
insert_
buffer_size exceed system memory"
<<
std
::
endl
;
if
(
buffer_size
>=
total_mem
)
{
std
::
cerr
<<
"ERROR: buffer_size exceed system memory"
<<
std
::
endl
;
okay
=
false
;
}
}
...
...
@@ -317,7 +314,7 @@ ServerConfig::CheckCacheConfig() {
}
std
::
string
insert_cache_immediately_str
=
cache_config
.
GetValue
(
CONFIG_CACHE_
INSERT_IMMEDIATELY
,
CONFIG_CACHE_INSERT_IMMEDIATELY
_DEFAULT
);
cache_config
.
GetValue
(
CONFIG_CACHE_
CACHE_INSERT_DATA
,
CONFIG_CACHE_CACHE_INSERT_DATA
_DEFAULT
);
if
(
!
ValidationUtil
::
ValidateStringIsBool
(
insert_cache_immediately_str
).
ok
())
{
std
::
cerr
<<
"ERROR: invalid insert_cache_immediately config: "
<<
insert_cache_immediately_str
<<
std
::
endl
;
okay
=
false
;
...
...
@@ -629,6 +626,171 @@ ServerConfig::GetConfig(const std::string &name) {
return
root_node
.
GetChild
(
name
);
}
/* server config */
std
::
string
ServerConfig
::
GetServerConfigAddress
()
{
ConfigNode
server_config
=
GetConfig
(
CONFIG_SERVER
);
return
server_config
.
GetValue
(
CONFIG_SERVER_ADDRESS
,
CONFIG_SERVER_ADDRESS_DEFAULT
);
}
std
::
string
ServerConfig
::
GetServerConfigPort
()
{
ConfigNode
server_config
=
GetConfig
(
CONFIG_SERVER
);
return
server_config
.
GetValue
(
CONFIG_SERVER_PORT
,
CONFIG_SERVER_PORT_DEFAULT
);
}
std
::
string
ServerConfig
::
GetServerConfigMode
()
{
ConfigNode
server_config
=
GetConfig
(
CONFIG_SERVER
);
return
server_config
.
GetValue
(
CONFIG_SERVER_MODE
,
CONFIG_SERVER_MODE_DEFAULT
);
}
std
::
string
ServerConfig
::
GetServerConfigTimeZone
()
{
ConfigNode
server_config
=
GetConfig
(
CONFIG_SERVER
);
return
server_config
.
GetValue
(
CONFIG_SERVER_TIME_ZONE
,
CONFIG_SERVER_TIME_ZONE_DEFAULT
);
}
/* db config */
std
::
string
ServerConfig
::
GetDBConfigPath
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetValue
(
CONFIG_DB_PATH
,
CONFIG_DB_PATH_DEFAULT
);
}
std
::
string
ServerConfig
::
GetDBConfigSlavePath
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetValue
(
CONFIG_DB_SLAVE_PATH
,
CONFIG_DB_SLAVE_PATH_DEFAULT
);
}
std
::
string
ServerConfig
::
GetDBConfigBackendUrl
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetValue
(
CONFIG_DB_BACKEND_URL
,
CONFIG_DB_BACKEND_URL_DEFAULT
);
}
int32_t
ServerConfig
::
GetDBConfigArchiveDiskThreshold
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetInt32Value
(
CONFIG_DB_ARCHIVE_DISK_THRESHOLD
,
std
::
stoi
(
CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT
));
}
int32_t
ServerConfig
::
GetDBConfigArchiveDaysThreshold
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetInt32Value
(
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD
,
std
::
stoi
(
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT
));
}
int32_t
ServerConfig
::
GetDBConfigBufferSize
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetInt32Value
(
CONFIG_DB_BUFFER_SIZE
,
std
::
stoi
(
CONFIG_DB_BUFFER_SIZE_DEFAULT
));
}
int32_t
ServerConfig
::
GetDBConfigBuildIndexGPU
()
{
ConfigNode
db_config
=
GetConfig
(
CONFIG_DB
);
return
db_config
.
GetInt32Value
(
CONFIG_DB_BUILD_INDEX_GPU
,
std
::
stoi
(
CONFIG_DB_BUILD_INDEX_GPU_DEFAULT
));
}
/* metric config */
bool
ServerConfig
::
GetMetricConfigAutoBootup
()
{
ConfigNode
metric_config
=
GetConfig
(
CONFIG_METRIC
);
return
metric_config
.
GetBoolValue
(
CONFIG_METRIC_AUTO_BOOTUP
,
std
::
stoi
(
CONFIG_METRIC_AUTO_BOOTUP_DEFAULT
));
}
std
::
string
ServerConfig
::
GetMetricConfigCollector
()
{
ConfigNode
metric_config
=
GetConfig
(
CONFIG_METRIC
);
return
metric_config
.
GetValue
(
CONFIG_METRIC_COLLECTOR
,
CONFIG_METRIC_COLLECTOR_DEFAULT
);
}
std
::
string
ServerConfig
::
GetMetricConfigPrometheusPort
()
{
ConfigNode
metric_config
=
GetConfig
(
CONFIG_METRIC
);
return
metric_config
.
GetValue
(
CONFIG_METRIC_PROMETHEUS_PORT
,
CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT
);
}
/* cache config */
int32_t
ServerConfig
::
GetCacheConfigCpuMemCapacity
()
{
ConfigNode
cache_config
=
GetConfig
(
CONFIG_CACHE
);
return
cache_config
.
GetInt32Value
(
CONFIG_CACHE_CPU_MEM_CAPACITY
,
std
::
stoi
(
CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT
));
}
float
ServerConfig
::
GetCacheConfigCpuMemThreshold
()
{
ConfigNode
cache_config
=
GetConfig
(
CONFIG_CACHE
);
return
cache_config
.
GetFloatValue
(
CONFIG_CACHE_CPU_MEM_THRESHOLD
,
std
::
stof
(
CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT
));
}
int32_t
ServerConfig
::
GetCacheConfigGpuMemCapacity
()
{
ConfigNode
cache_config
=
GetConfig
(
CONFIG_CACHE
);
return
cache_config
.
GetInt32Value
(
CONFIG_CACHE_GPU_MEM_CAPACITY
,
std
::
stoi
(
CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT
));
}
float
ServerConfig
::
GetCacheConfigGpuMemThreshold
()
{
ConfigNode
cache_config
=
GetConfig
(
CONFIG_CACHE
);
return
cache_config
.
GetFloatValue
(
CONFIG_CACHE_GPU_MEM_THRESHOLD
,
std
::
stof
(
CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT
));
}
bool
ServerConfig
::
GetCacheConfigCacheInsertData
()
{
ConfigNode
cache_config
=
GetConfig
(
CONFIG_CACHE
);
return
cache_config
.
GetBoolValue
(
CONFIG_CACHE_CACHE_INSERT_DATA
,
std
::
stoi
(
CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT
));
}
/* engine config */
int32_t
ServerConfig
::
GetEngineConfigBlasThreshold
()
{
ConfigNode
engine_config
=
GetConfig
(
CONFIG_ENGINE
);
return
engine_config
.
GetInt32Value
(
CONFIG_ENGINE_BLAS_THRESHOLD
,
std
::
stoi
(
CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT
));
}
int32_t
ServerConfig
::
GetEngineConfigOmpThreadNum
()
{
ConfigNode
engine_config
=
GetConfig
(
CONFIG_ENGINE
);
return
engine_config
.
GetInt32Value
(
CONFIG_ENGINE_OMP_THREAD_NUM
,
std
::
stoi
(
CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT
));
}
/* resource config */
std
::
string
ServerConfig
::
GetResourceConfigMode
()
{
ConfigNode
resource_config
=
GetConfig
(
CONFIG_RESOURCE
);
return
resource_config
.
GetValue
(
CONFIG_RESOURCE_MODE
,
CONFIG_RESOURCE_MODE_DEFAULT
);
}
std
::
vector
<
std
::
string
>
ServerConfig
::
GetResourceConfigPool
()
{
ConfigNode
resource_config
=
GetConfig
(
CONFIG_RESOURCE
);
return
resource_config
.
GetSequence
(
CONFIG_RESOURCE_POOL
);
}
}
}
...
...
cpp/src/server/ServerConfig.h
浏览文件 @
22563d83
...
...
@@ -39,11 +39,11 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
/* db config */
static
const
char
*
CONFIG_DB
=
"db_config"
;
static
const
char
*
CONFIG_DB_PATH
=
"
db_
path"
;
static
const
char
*
CONFIG_DB_PATH
=
"path"
;
static
const
char
*
CONFIG_DB_PATH_DEFAULT
=
"/tmp/milvus"
;
static
const
char
*
CONFIG_DB_SLAVE_PATH
=
"
db_
slave_path"
;
static
const
char
*
CONFIG_DB_SLAVE_PATH
=
"slave_path"
;
static
const
char
*
CONFIG_DB_SLAVE_PATH_DEFAULT
=
""
;
static
const
char
*
CONFIG_DB_BACKEND_URL
=
"
db_
backend_url"
;
static
const
char
*
CONFIG_DB_BACKEND_URL
=
"backend_url"
;
static
const
char
*
CONFIG_DB_BACKEND_URL_DEFAULT
=
"sqlite://:@:/"
;
static
const
char
*
CONFIG_DB_ARCHIVE_DISK_THRESHOLD
=
"archive_disk_threshold"
;
static
const
char
*
CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT
=
"0"
;
...
...
@@ -64,8 +64,8 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold";
static
const
char
*
CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT
=
"0.85"
;
static
const
char
*
CONFIG_CACHE_GPU_MEM_THRESHOLD
=
"gpu_mem_threshold"
;
static
const
char
*
CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT
=
"0.85"
;
static
const
char
*
CONFIG_CACHE_
INSERT_IMMEDIATELY
=
"insert_immediately
"
;
static
const
char
*
CONFIG_CACHE_
INSERT_IMMEDIATELY
_DEFAULT
=
"0"
;
static
const
char
*
CONFIG_CACHE_
CACHE_INSERT_DATA
=
"cache_insert_data
"
;
static
const
char
*
CONFIG_CACHE_
CACHE_INSERT_DATA
_DEFAULT
=
"0"
;
/* metric config */
static
const
char
*
CONFIG_METRIC
=
"metric_config"
;
...
...
@@ -95,20 +95,51 @@ class ServerConfig {
public:
static
ServerConfig
&
GetInstance
();
Status
LoadConfigFile
(
const
std
::
string
&
config_
filename
);
Status
LoadConfigFile
(
const
std
::
string
&
filename
);
Status
ValidateConfig
();
void
PrintAll
()
const
;
private:
ConfigNode
GetConfig
(
const
std
::
string
&
name
)
const
;
ConfigNode
&
GetConfig
(
const
std
::
string
&
name
);
private:
Status
CheckServerConfig
();
Status
CheckDBConfig
();
Status
CheckMetricConfig
();
Status
CheckCacheConfig
();
Status
CheckEngineConfig
();
Status
CheckResourceConfig
();
public:
std
::
string
GetServerConfigAddress
();
std
::
string
GetServerConfigPort
();
std
::
string
GetServerConfigMode
();
std
::
string
GetServerConfigTimeZone
();
std
::
string
GetDBConfigPath
();
std
::
string
GetDBConfigSlavePath
();
std
::
string
GetDBConfigBackendUrl
();
int32_t
GetDBConfigArchiveDiskThreshold
();
int32_t
GetDBConfigArchiveDaysThreshold
();
int32_t
GetDBConfigBufferSize
();
int32_t
GetDBConfigBuildIndexGPU
();
bool
GetMetricConfigAutoBootup
();
std
::
string
GetMetricConfigCollector
();
std
::
string
GetMetricConfigPrometheusPort
();
int32_t
GetCacheConfigCpuMemCapacity
();
float
GetCacheConfigCpuMemThreshold
();
int32_t
GetCacheConfigGpuMemCapacity
();
float
GetCacheConfigGpuMemThreshold
();
bool
GetCacheConfigCacheInsertData
();
int32_t
GetEngineConfigBlasThreshold
();
int32_t
GetEngineConfigOmpThreadNum
();
std
::
string
GetResourceConfigMode
();
std
::
vector
<
std
::
string
>
GetResourceConfigPool
();
};
}
...
...
cpp/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @
22563d83
...
...
@@ -74,12 +74,10 @@ GrpcServer::Stop() {
Status
GrpcServer
::
StartService
()
{
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
server_config
=
config
.
GetConfig
(
CONFIG_SERVER
);
ConfigNode
engine_config
=
config
.
GetConfig
(
CONFIG_ENGINE
);
std
::
string
address
=
server_config
.
GetValue
(
CONFIG_SERVER_ADDRESS
,
CONFIG_SERVER_ADDRESS_DEFAULT
);
int32_t
port
=
server_config
.
GetInt32Value
(
CONFIG_SERVER_PORT
,
std
::
stoi
(
CONFIG_SERVER_PORT_DEFAULT
));
std
::
string
address
=
config
.
GetServerConfigAddress
();
std
::
string
port
=
config
.
GetServerConfigPort
();
std
::
string
server_address
(
address
+
":"
+
std
::
to_string
(
port
)
);
std
::
string
server_address
(
address
+
":"
+
port
);
::
grpc
::
ServerBuilder
builder
;
builder
.
SetOption
(
std
::
unique_ptr
<::
grpc
::
ServerBuilderOption
>
(
new
NoReusePortOption
));
...
...
cpp/src/wrapper/KnowhereResource.cpp
浏览文件 @
22563d83
...
...
@@ -38,16 +38,13 @@ ErrorCode KnowhereResource::Initialize() {
GpuResourcesArray
gpu_resources
;
//get build index gpu resource
server
::
ServerConfig
&
root_config
=
server
::
ServerConfig
::
GetInstance
();
server
::
ConfigNode
&
db_config
=
root_config
.
GetConfig
(
server
::
CONFIG_DB
);
server
::
ServerConfig
&
config
=
server
::
ServerConfig
::
GetInstance
();
int32_t
build_index_gpu
=
db_config
.
GetInt32Value
(
server
::
CONFIG_DB_BUILD_INDEX_GPU
,
std
::
stoi
(
server
::
CONFIG_DB_BUILD_INDEX_GPU_DEFAULT
));
int32_t
build_index_gpu
=
config
.
GetDBConfigBuildIndexGPU
();
gpu_resources
.
insert
(
std
::
make_pair
(
build_index_gpu
,
GpuResourceSetting
()));
//get search gpu resource
server
::
ConfigNode
&
res_config
=
root_config
.
GetConfig
(
server
::
CONFIG_RESOURCE
);
auto
pool
=
res_config
.
GetSequence
(
server
::
CONFIG_RESOURCE_POOL
);
auto
pool
=
config
.
GetResourceConfigPool
();
std
::
set
<
uint64_t
>
gpu_ids
;
for
(
auto
&
resource
:
pool
)
{
if
(
resource
.
length
()
<
4
||
resource
.
substr
(
0
,
3
)
!=
"gpu"
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录