Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
0d0d08c2
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0d0d08c2
编写于
9月 25, 2019
作者:
Y
yudong.cai
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
MS-574 let all config APIs return Status
Former-commit-id: 7a843fa57779637761a80def4705a37f9805ceaf
上级
b0fe84cd
变更
13
展开全部
隐藏空白更改
内联
并排
Showing
13 changed file
with
543 addition
and
277 deletion
+543
-277
cpp/src/cache/CpuCacheMgr.cpp
cpp/src/cache/CpuCacheMgr.cpp
+13
-2
cpp/src/cache/GpuCacheMgr.cpp
cpp/src/cache/GpuCacheMgr.cpp
+12
-2
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+2
-1
cpp/src/db/engine/ExecutionEngineImpl.h
cpp/src/db/engine/ExecutionEngineImpl.h
+1
-1
cpp/src/metrics/Metrics.cpp
cpp/src/metrics/Metrics.cpp
+3
-1
cpp/src/metrics/PrometheusMetrics.cpp
cpp/src/metrics/PrometheusMetrics.cpp
+5
-2
cpp/src/scheduler/SchedInst.cpp
cpp/src/scheduler/SchedInst.cpp
+5
-2
cpp/src/server/Config.cpp
cpp/src/server/Config.cpp
+370
-218
cpp/src/server/Config.h
cpp/src/server/Config.h
+71
-32
cpp/src/server/DBWrapper.cpp
cpp/src/server/DBWrapper.cpp
+34
-11
cpp/src/server/Server.cpp
cpp/src/server/Server.cpp
+7
-1
cpp/src/server/grpc_impl/GrpcServer.cpp
cpp/src/server/grpc_impl/GrpcServer.cpp
+11
-2
cpp/src/wrapper/KnowhereResource.cpp
cpp/src/wrapper/KnowhereResource.cpp
+9
-2
未找到文件。
cpp/src/cache/CpuCacheMgr.cpp
浏览文件 @
0d0d08c2
...
...
@@ -30,10 +30,21 @@ namespace {
CpuCacheMgr
::
CpuCacheMgr
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
int64_t
cap
=
config
.
GetCacheConfigCpuMemCapacity
()
*
unit
;
Status
s
;
int32_t
cpu_mem_cap
;
s
=
config
.
GetCacheConfigCpuMemCapacity
(
cpu_mem_cap
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
s
.
message
();
}
int64_t
cap
=
cpu_mem_cap
*
unit
;
cache_
=
std
::
make_shared
<
Cache
<
DataObjPtr
>>
(
cap
,
1UL
<<
32
);
float
cpu_mem_threshold
=
config
.
GetCacheConfigCpuMemThreshold
();
float
cpu_mem_threshold
;
s
=
config
.
GetCacheConfigCpuMemThreshold
(
cpu_mem_threshold
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
s
.
message
();
}
if
(
cpu_mem_threshold
>
0.0
&&
cpu_mem_threshold
<=
1.0
)
{
cache_
->
set_freemem_percent
(
cpu_mem_threshold
);
}
else
{
...
...
cpp/src/cache/GpuCacheMgr.cpp
浏览文件 @
0d0d08c2
...
...
@@ -34,11 +34,21 @@ namespace {
GpuCacheMgr
::
GpuCacheMgr
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
Status
s
;
int32_t
cap
=
config
.
GetCacheConfigGpuMemCapacity
()
*
G_BYTE
;
int32_t
gpu_mem_cap
;
s
=
config
.
GetCacheConfigGpuMemCapacity
(
gpu_mem_cap
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
s
.
message
();
}
int32_t
cap
=
gpu_mem_cap
*
G_BYTE
;
cache_
=
std
::
make_shared
<
Cache
<
DataObjPtr
>>
(
cap
,
1UL
<<
32
);
float
gpu_mem_threshold
=
config
.
GetCacheConfigGpuMemThreshold
();
float
gpu_mem_threshold
;
s
=
config
.
GetCacheConfigGpuMemThreshold
(
gpu_mem_threshold
);
if
(
!
s
.
ok
())
{
SERVER_LOG_ERROR
<<
s
.
message
();
}
if
(
gpu_mem_threshold
>
0.0
&&
gpu_mem_threshold
<=
1.0
)
{
cache_
->
set_freemem_percent
(
gpu_mem_threshold
);
}
else
{
...
...
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
0d0d08c2
...
...
@@ -336,7 +336,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl
::
Init
()
{
using
namespace
zilliz
::
milvus
::
server
;
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
gpu_num_
=
config
.
GetDBConfigBuildIndexGPU
();
Status
s
=
config
.
GetDBConfigBuildIndexGPU
(
gpu_num_
);
if
(
!
s
.
ok
())
return
s
;
return
Status
::
OK
();
}
...
...
cpp/src/db/engine/ExecutionEngineImpl.h
浏览文件 @
0d0d08c2
...
...
@@ -101,7 +101,7 @@ protected:
std
::
string
location_
;
int32_t
nlist_
=
0
;
int
64
_t
gpu_num_
=
0
;
int
32
_t
gpu_num_
=
0
;
};
...
...
cpp/src/metrics/Metrics.cpp
浏览文件 @
0d0d08c2
...
...
@@ -33,7 +33,9 @@ Metrics::GetInstance() {
MetricsBase
&
Metrics
::
CreateMetricsCollector
()
{
Config
&
config
=
Config
::
GetInstance
();
std
::
string
collector_type_str
=
config
.
GetMetricConfigCollector
();
std
::
string
collector_type_str
;
config
.
GetMetricConfigCollector
(
collector_type_str
);
if
(
collector_type_str
==
"prometheus"
)
{
return
PrometheusMetrics
::
GetInstance
();
...
...
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @
0d0d08c2
...
...
@@ -31,11 +31,14 @@ ErrorCode
PrometheusMetrics
::
Init
()
{
try
{
Config
&
config
=
Config
::
GetInstance
();
startup_
=
config
.
GetMetricConfigAutoBootup
();
Status
s
=
config
.
GetMetricConfigAutoBootup
(
startup_
);
if
(
!
s
.
ok
())
return
s
.
code
();
if
(
!
startup_
)
return
SERVER_SUCCESS
;
// Following should be read from config file.
const
std
::
string
bind_address
=
config
.
GetMetricConfigPrometheusPort
();
std
::
string
bind_address
;
s
=
config
.
GetMetricConfigPrometheusPort
(
bind_address
);
if
(
!
s
.
ok
())
return
s
.
code
();
const
std
::
string
uri
=
std
::
string
(
"/tmp/metrics"
);
const
std
::
size_t
num_threads
=
2
;
...
...
cpp/src/scheduler/SchedInst.cpp
浏览文件 @
0d0d08c2
...
...
@@ -39,8 +39,11 @@ std::mutex JobMgrInst::mutex_;
void
load_simple_config
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
auto
mode
=
config
.
GetResourceConfigMode
();
auto
pool
=
config
.
GetResourceConfigPool
();
std
::
string
mode
;
config
.
GetResourceConfigMode
(
mode
);
std
::
vector
<
std
::
string
>
pool
;
config
.
GetResourceConfigPool
(
pool
);
bool
cpu
=
false
;
std
::
set
<
uint64_t
>
gpu_ids
;
for
(
auto
&
resource
:
pool
)
{
...
...
cpp/src/server/Config.cpp
浏览文件 @
0d0d08c2
此差异已折叠。
点击以展开。
cpp/src/server/Config.h
浏览文件 @
0d0d08c2
...
...
@@ -102,8 +102,7 @@ class Config {
void
PrintAll
()
const
;
private:
ConfigNode
GetConfig
(
const
std
::
string
&
name
)
const
;
ConfigNode
&
GetConfig
(
const
std
::
string
&
name
);
ConfigNode
&
GetConfigNode
(
const
std
::
string
&
name
);
Status
CheckServerConfig
();
Status
CheckDBConfig
();
...
...
@@ -116,40 +115,80 @@ class Config {
const
std
::
string
&
child_key
,
std
::
string
&
value
);
void
SetConfigValueInMem
(
const
std
::
string
&
parent_key
,
Status
SetConfigValueInMem
(
const
std
::
string
&
parent_key
,
const
std
::
string
&
child_key
,
std
::
string
&
value
);
private:
/* server config */
Status
GetServerConfigStrAddress
(
std
::
string
&
value
);
Status
GetServerConfigStrPort
(
std
::
string
&
value
);
Status
GetServerConfigStrMode
(
std
::
string
&
value
);
Status
GetServerConfigStrTimeZone
(
std
::
string
&
value
);
/* db config */
Status
GetDBConfigStrPath
(
std
::
string
&
value
);
Status
GetDBConfigStrSlavePath
(
std
::
string
&
value
);
Status
GetDBConfigStrBackendUrl
(
std
::
string
&
value
);
Status
GetDBConfigStrArchiveDiskThreshold
(
std
::
string
&
value
);
Status
GetDBConfigStrArchiveDaysThreshold
(
std
::
string
&
value
);
Status
GetDBConfigStrBufferSize
(
std
::
string
&
value
);
Status
GetDBConfigStrBuildIndexGPU
(
std
::
string
&
value
);
/* metric config */
Status
GetMetricConfigStrAutoBootup
(
std
::
string
&
value
);
Status
GetMetricConfigStrCollector
(
std
::
string
&
value
);
Status
GetMetricConfigStrPrometheusPort
(
std
::
string
&
value
);
/* cache config */
Status
GetCacheConfigStrCpuMemCapacity
(
std
::
string
&
value
);
Status
GetCacheConfigStrCpuMemThreshold
(
std
::
string
&
value
);
Status
GetCacheConfigStrGpuMemCapacity
(
std
::
string
&
value
);
Status
GetCacheConfigStrGpuMemThreshold
(
std
::
string
&
value
);
Status
GetCacheConfigStrCacheInsertData
(
std
::
string
&
value
);
/* engine config */
Status
GetEngineConfigStrBlasThreshold
(
std
::
string
&
value
);
Status
GetEngineConfigStrOmpThreadNum
(
std
::
string
&
value
);
/* resource config */
Status
GetResourceConfigStrMode
(
std
::
string
&
value
);
public:
std
::
string
GetServerConfigAddress
();
std
::
string
GetServerConfigPort
();
std
::
string
GetServerConfigMode
();
std
::
string
GetServerConfigTimeZone
();
std
::
string
GetDBConfigPath
();
std
::
string
GetDBConfigSlavePath
();
std
::
string
GetDBConfigBackendUrl
();
int32_t
GetDBConfigArchiveDiskThreshold
();
int32_t
GetDBConfigArchiveDaysThreshold
();
int32_t
GetDBConfigBufferSize
();
int32_t
GetDBConfigBuildIndexGPU
();
bool
GetMetricConfigAutoBootup
();
std
::
string
GetMetricConfigCollector
();
std
::
string
GetMetricConfigPrometheusPort
();
int32_t
GetCacheConfigCpuMemCapacity
();
float
GetCacheConfigCpuMemThreshold
();
int32_t
GetCacheConfigGpuMemCapacity
();
float
GetCacheConfigGpuMemThreshold
();
bool
GetCacheConfigCacheInsertData
();
int32_t
GetEngineConfigBlasThreshold
();
int32_t
GetEngineConfigOmpThreadNum
();
std
::
string
GetResourceConfigMode
();
std
::
vector
<
std
::
string
>
GetResourceConfigPool
();
/* server config */
Status
GetServerConfigAddress
(
std
::
string
&
value
);
Status
GetServerConfigPort
(
std
::
string
&
value
);
Status
GetServerConfigMode
(
std
::
string
&
value
);
Status
GetServerConfigTimeZone
(
std
::
string
&
value
);
/* db config */
Status
GetDBConfigPath
(
std
::
string
&
value
);
Status
GetDBConfigSlavePath
(
std
::
string
&
value
);
Status
GetDBConfigBackendUrl
(
std
::
string
&
value
);
Status
GetDBConfigArchiveDiskThreshold
(
int32_t
&
value
);
Status
GetDBConfigArchiveDaysThreshold
(
int32_t
&
value
);
Status
GetDBConfigBufferSize
(
int32_t
&
value
);
Status
GetDBConfigBuildIndexGPU
(
int32_t
&
value
);
/* metric config */
Status
GetMetricConfigAutoBootup
(
bool
&
value
);
Status
GetMetricConfigCollector
(
std
::
string
&
value
);
Status
GetMetricConfigPrometheusPort
(
std
::
string
&
value
);
/* cache config */
Status
GetCacheConfigCpuMemCapacity
(
int32_t
&
value
);
Status
GetCacheConfigCpuMemThreshold
(
float
&
value
);
Status
GetCacheConfigGpuMemCapacity
(
int32_t
&
value
);
Status
GetCacheConfigGpuMemThreshold
(
float
&
value
);
Status
GetCacheConfigCacheInsertData
(
bool
&
value
);
/* engine config */
Status
GetEngineConfigBlasThreshold
(
int32_t
&
value
);
Status
GetEngineConfigOmpThreadNum
(
int32_t
&
value
);
/* resource config */
Status
GetResourceConfigMode
(
std
::
string
&
value
);
Status
GetResourceConfigPool
(
std
::
vector
<
std
::
string
>&
value
);
private:
std
::
unordered_map
<
std
::
string
,
std
::
unordered_map
<
std
::
string
,
std
::
string
>>
config_map_
;
...
...
cpp/src/server/DBWrapper.cpp
浏览文件 @
0d0d08c2
...
...
@@ -36,19 +36,33 @@ DBWrapper::DBWrapper() {
Status
DBWrapper
::
StartService
()
{
Config
&
config
=
Config
::
GetInstance
();
Status
s
;
//db config
engine
::
DBOptions
opt
;
opt
.
meta
.
backend_uri
=
config
.
GetDBConfigBackendUrl
();
opt
.
meta
.
path
=
config
.
GetDBConfigPath
()
+
"/db"
;
std
::
string
db_slave_path
=
config
.
GetDBConfigSlavePath
();
s
=
config
.
GetDBConfigBackendUrl
(
opt
.
meta
.
backend_uri
);
if
(
!
s
.
ok
())
return
s
;
std
::
string
path
;
s
=
config
.
GetDBConfigPath
(
path
);
if
(
!
s
.
ok
())
return
s
;
opt
.
meta
.
path
=
path
+
"/db"
;
std
::
string
db_slave_path
;
s
=
config
.
GetDBConfigSlavePath
(
db_slave_path
);
if
(
!
s
.
ok
())
return
s
;
StringHelpFunctions
::
SplitStringByDelimeter
(
db_slave_path
,
";"
,
opt
.
meta
.
slave_paths
);
// cache config
opt
.
insert_cache_immediately_
=
config
.
GetCacheConfigCacheInsertData
();
s
=
config
.
GetCacheConfigCacheInsertData
(
opt
.
insert_cache_immediately_
);
if
(
!
s
.
ok
())
return
s
;
std
::
string
mode
;
s
=
config
.
GetServerConfigMode
(
mode
);
if
(
!
s
.
ok
())
return
s
;
std
::
string
mode
=
config
.
GetServerConfigMode
();
if
(
mode
==
"single"
)
{
opt
.
mode
=
engine
::
DBOptions
::
MODE
::
SINGLE
;
}
...
...
@@ -64,8 +78,10 @@ Status DBWrapper::StartService() {
}
// engine config
int32_t
omp_thread
=
config
.
GetEngineConfigOmpThreadNum
();
if
(
omp_thread
>
0
)
{
int32_t
omp_thread
;
s
=
config
.
GetEngineConfigOmpThreadNum
(
omp_thread
);
if
(
!
s
.
ok
())
return
s
;
if
(
omp_thread
>
0
)
{
omp_set_num_threads
(
omp_thread
);
SERVER_LOG_DEBUG
<<
"Specify openmp thread number: "
<<
omp_thread
;
}
else
{
...
...
@@ -77,15 +93,22 @@ Status DBWrapper::StartService() {
}
//init faiss global variable
faiss
::
distance_compute_blas_threshold
=
config
.
GetEngineConfigBlasThreshold
();
int32_t
blas_threshold
;
s
=
config
.
GetEngineConfigBlasThreshold
(
blas_threshold
);
if
(
!
s
.
ok
())
return
s
;
faiss
::
distance_compute_blas_threshold
=
blas_threshold
;
//set archive config
engine
::
ArchiveConf
::
CriteriaT
criterial
;
int32_t
disk
=
config
.
GetDBConfigArchiveDiskThreshold
();
int32_t
days
=
config
.
GetDBConfigArchiveDaysThreshold
();
int32_t
disk
,
days
;
s
=
config
.
GetDBConfigArchiveDiskThreshold
(
disk
);
if
(
!
s
.
ok
())
return
s
;
if
(
disk
>
0
)
{
criterial
[
engine
::
ARCHIVE_CONF_DISK
]
=
disk
;
}
s
=
config
.
GetDBConfigArchiveDaysThreshold
(
days
);
if
(
!
s
.
ok
())
return
s
;
if
(
days
>
0
)
{
criterial
[
engine
::
ARCHIVE_CONF_DAYS
]
=
days
;
}
...
...
cpp/src/server/Server.cpp
浏览文件 @
0d0d08c2
...
...
@@ -161,7 +161,13 @@ Server::Start() {
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
Config
&
config
=
Config
::
GetInstance
();
std
::
string
time_zone
=
config
.
GetServerConfigTimeZone
();
std
::
string
time_zone
;
Status
s
=
config
.
GetServerConfigTimeZone
(
time_zone
);
if
(
!
s
.
ok
())
{
std
::
cerr
<<
"Fail to get server config timezone"
<<
std
::
endl
;
return
;
}
if
(
time_zone
.
length
()
==
3
)
{
time_zone
=
"CUT"
;
}
else
{
...
...
cpp/src/server/grpc_impl/GrpcServer.cpp
浏览文件 @
0d0d08c2
...
...
@@ -74,8 +74,17 @@ GrpcServer::Stop() {
Status
GrpcServer
::
StartService
()
{
Config
&
config
=
Config
::
GetInstance
();
std
::
string
address
=
config
.
GetServerConfigAddress
();
std
::
string
port
=
config
.
GetServerConfigPort
();
std
::
string
address
,
port
;
Status
s
;
s
=
config
.
GetServerConfigAddress
(
address
);
if
(
!
s
.
ok
())
{
return
s
;
}
s
=
config
.
GetServerConfigPort
(
port
);
if
(
!
s
.
ok
())
{
return
s
;
}
std
::
string
server_address
(
address
+
":"
+
port
);
...
...
cpp/src/wrapper/KnowhereResource.cpp
浏览文件 @
0d0d08c2
...
...
@@ -36,15 +36,22 @@ ErrorCode KnowhereResource::Initialize() {
};
using
GpuResourcesArray
=
std
::
map
<
int64_t
,
GpuResourceSetting
>
;
GpuResourcesArray
gpu_resources
;
Status
s
;
//get build index gpu resource
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
int32_t
build_index_gpu
=
config
.
GetDBConfigBuildIndexGPU
();
int32_t
build_index_gpu
;
s
=
config
.
GetDBConfigBuildIndexGPU
(
build_index_gpu
);
if
(
!
s
.
ok
())
return
s
.
code
();
gpu_resources
.
insert
(
std
::
make_pair
(
build_index_gpu
,
GpuResourceSetting
()));
//get search gpu resource
auto
pool
=
config
.
GetResourceConfigPool
();
std
::
vector
<
std
::
string
>
pool
;
s
=
config
.
GetResourceConfigPool
(
pool
);
if
(
!
s
.
ok
())
return
s
.
code
();
std
::
set
<
uint64_t
>
gpu_ids
;
for
(
auto
&
resource
:
pool
)
{
if
(
resource
.
length
()
<
4
||
resource
.
substr
(
0
,
3
)
!=
"gpu"
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录