Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
68457e33
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
68457e33
编写于
11月 20, 2019
作者:
J
Jin Hai
提交者:
GitHub
11月 20, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #430 from cydrain/caiyd_use_all_int64
#306 use int64 for config integer
上级
8225ac00
82e6b273
变更
19
隐藏空白更改
内联
并排
Showing
19 changed file
with
96 addition
and
96 deletion
+96
-96
CHANGELOG.md
CHANGELOG.md
+5
-4
core/src/db/Options.h
core/src/db/Options.h
+1
-1
core/src/db/engine/ExecutionEngineImpl.cpp
core/src/db/engine/ExecutionEngineImpl.cpp
+2
-2
core/src/db/engine/ExecutionEngineImpl.h
core/src/db/engine/ExecutionEngineImpl.h
+2
-2
core/src/scheduler/SchedInst.cpp
core/src/scheduler/SchedInst.cpp
+2
-2
core/src/scheduler/optimizer/BuildIndexPass.cpp
core/src/scheduler/optimizer/BuildIndexPass.cpp
+1
-1
core/src/scheduler/optimizer/FaissFlatPass.h
core/src/scheduler/optimizer/FaissFlatPass.h
+2
-2
core/src/scheduler/optimizer/FaissIVFFlatPass.h
core/src/scheduler/optimizer/FaissIVFFlatPass.h
+2
-2
core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp
core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp
+1
-1
core/src/scheduler/optimizer/FaissIVFSQ8HPass.h
core/src/scheduler/optimizer/FaissIVFSQ8HPass.h
+2
-2
core/src/scheduler/optimizer/FaissIVFSQ8Pass.h
core/src/scheduler/optimizer/FaissIVFSQ8Pass.h
+2
-2
core/src/server/Config.cpp
core/src/server/Config.cpp
+35
-35
core/src/server/Config.h
core/src/server/Config.h
+8
-8
core/src/server/DBWrapper.cpp
core/src/server/DBWrapper.cpp
+4
-4
core/src/utils/CommonUtil.cpp
core/src/utils/CommonUtil.cpp
+1
-1
core/src/utils/CommonUtil.h
core/src/utils/CommonUtil.h
+1
-1
core/src/wrapper/KnowhereResource.cpp
core/src/wrapper/KnowhereResource.cpp
+2
-2
core/unittest/server/test_config.cpp
core/unittest/server/test_config.cpp
+22
-23
core/unittest/server/test_util.cpp
core/unittest/server/test_util.cpp
+1
-1
未找到文件。
CHANGELOG.md
浏览文件 @
68457e33
...
...
@@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
346 - Support build index with multiple gpu
## Improvement
-
\#
275 - Rename C++ SDK IndexType
-
\#
284 - Change C++ SDK to shared library
-
\#
255 - Add ivfsq8 test report detailed version
-
\#
260 - C++ SDK README
-
\#
266 - Rpc request source code refactor
-
\#
314 - add Find FAISS in CMake
-
\#
275 - Rename C++ SDK IndexType
-
\#
284 - Change C++ SDK to shared library
-
\#
306 - Use int64 for all config integer
-
\#
310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue
-
\#
314 - add Find FAISS in CMake
-
\#
322 - Add option to enable / disable prometheus
-
\#
358 - Add more information in build.sh and install.md
-
\#
255 - Add ivfsq8 test report detailed version
-
\#
404 - Add virtual method Init() in Pass abstract class
-
\#
409 - Add a Fallback pass in optimizer
...
...
core/src/db/Options.h
浏览文件 @
68457e33
...
...
@@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk";
static
const
char
*
ARCHIVE_CONF_DAYS
=
"days"
;
struct
ArchiveConf
{
using
CriteriaT
=
std
::
map
<
std
::
string
,
int
>
;
using
CriteriaT
=
std
::
map
<
std
::
string
,
int
64_t
>
;
explicit
ArchiveConf
(
const
std
::
string
&
type
,
const
std
::
string
&
criterias
=
std
::
string
());
...
...
core/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
68457e33
...
...
@@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const {
const
std
::
string
key
=
location_
+
".quantizer"
;
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
int
32
_t
>
gpus
;
std
::
vector
<
int
64
_t
>
gpus
;
Status
s
=
config
.
GetGpuResourceConfigSearchResources
(
gpus
);
if
(
!
s
.
ok
())
{
ENGINE_LOG_ERROR
<<
s
.
message
();
...
...
@@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl
::
Init
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
int
32
_t
>
gpu_ids
;
std
::
vector
<
int
64
_t
>
gpu_ids
;
Status
s
=
config
.
GetGpuResourceConfigBuildIndexResources
(
gpu_ids
);
for
(
auto
id
:
gpu_ids
)
{
if
(
gpu_num_
==
id
)
{
...
...
core/src/db/engine/ExecutionEngineImpl.h
浏览文件 @
68457e33
...
...
@@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine {
int64_t
dim_
;
std
::
string
location_
;
int
32
_t
nlist_
=
0
;
int
32
_t
gpu_num_
=
0
;
int
64
_t
nlist_
=
0
;
int
64
_t
gpu_num_
=
0
;
};
}
// namespace engine
...
...
core/src/scheduler/SchedInst.cpp
浏览文件 @
68457e33
...
...
@@ -55,9 +55,9 @@ load_simple_config() {
// get resources
#ifdef MILVUS_GPU_VERSION
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
int
32
_t
>
gpu_ids
;
std
::
vector
<
int
64
_t
>
gpu_ids
;
config
.
GetGpuResourceConfigSearchResources
(
gpu_ids
);
std
::
vector
<
int
32
_t
>
build_gpu_ids
;
std
::
vector
<
int
64
_t
>
build_gpu_ids
;
config
.
GetGpuResourceConfigBuildIndexResources
(
build_gpu_ids
);
auto
pcie
=
Connection
(
"pcie"
,
12000
);
...
...
core/src/scheduler/optimizer/BuildIndexPass.cpp
浏览文件 @
68457e33
...
...
@@ -26,7 +26,7 @@ namespace scheduler {
void
BuildIndexPass
::
Init
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
int
32
_t
>
build_resources
;
std
::
vector
<
int
64
_t
>
build_resources
;
Status
s
=
config
.
GetGpuResourceConfigBuildIndexResources
(
build_resources
);
if
(
!
s
.
ok
())
{
throw
;
...
...
core/src/scheduler/optimizer/FaissFlatPass.h
浏览文件 @
68457e33
...
...
@@ -45,9 +45,9 @@ class FaissFlatPass : public Pass {
Run
(
const
TaskPtr
&
task
)
override
;
private:
int
32_t
threshold_
=
std
::
numeric_limits
<
int32
_t
>::
max
();
int
64_t
threshold_
=
std
::
numeric_limits
<
int64
_t
>::
max
();
int64_t
count_
=
0
;
std
::
vector
<
int
32
_t
>
gpus
;
std
::
vector
<
int
64
_t
>
gpus
;
};
using
FaissFlatPassPtr
=
std
::
shared_ptr
<
FaissFlatPass
>
;
...
...
core/src/scheduler/optimizer/FaissIVFFlatPass.h
浏览文件 @
68457e33
...
...
@@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass {
Run
(
const
TaskPtr
&
task
)
override
;
private:
int
32_t
threshold_
=
std
::
numeric_limits
<
int32
_t
>::
max
();
int
64_t
threshold_
=
std
::
numeric_limits
<
int64
_t
>::
max
();
int64_t
count_
=
0
;
std
::
vector
<
int
32
_t
>
gpus
;
std
::
vector
<
int
64
_t
>
gpus
;
};
using
FaissIVFFlatPassPtr
=
std
::
shared_ptr
<
FaissIVFFlatPass
>
;
...
...
core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp
浏览文件 @
68457e33
...
...
@@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() {
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
Status
s
=
config
.
GetEngineConfigGpuSearchThreshold
(
threshold_
);
if
(
!
s
.
ok
())
{
threshold_
=
std
::
numeric_limits
<
int
32
_t
>::
max
();
threshold_
=
std
::
numeric_limits
<
int
64
_t
>::
max
();
}
s
=
config
.
GetGpuResourceConfigSearchResources
(
gpus
);
}
...
...
core/src/scheduler/optimizer/FaissIVFSQ8HPass.h
浏览文件 @
68457e33
...
...
@@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass {
Run
(
const
TaskPtr
&
task
)
override
;
private:
int
32_t
threshold_
=
std
::
numeric_limits
<
int32
_t
>::
max
();
int
64_t
threshold_
=
std
::
numeric_limits
<
int64
_t
>::
max
();
int64_t
count_
=
0
;
std
::
vector
<
int
32
_t
>
gpus
;
std
::
vector
<
int
64
_t
>
gpus
;
};
using
FaissIVFSQ8HPassPtr
=
std
::
shared_ptr
<
FaissIVFSQ8HPass
>
;
...
...
core/src/scheduler/optimizer/FaissIVFSQ8Pass.h
浏览文件 @
68457e33
...
...
@@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass {
Run
(
const
TaskPtr
&
task
)
override
;
private:
int
32_t
threshold_
=
std
::
numeric_limits
<
int32
_t
>::
max
();
int
64_t
threshold_
=
std
::
numeric_limits
<
int64
_t
>::
max
();
int64_t
count_
=
0
;
std
::
vector
<
int
32
_t
>
gpus
;
std
::
vector
<
int
64
_t
>
gpus
;
};
using
FaissIVFSQ8PassPtr
=
std
::
shared_ptr
<
FaissIVFSQ8Pass
>
;
...
...
core/src/server/Config.cpp
浏览文件 @
68457e33
...
...
@@ -113,19 +113,19 @@ Config::ValidateConfig() {
return
s
;
}
int
32
_t
db_archive_disk_threshold
;
int
64
_t
db_archive_disk_threshold
;
s
=
GetDBConfigArchiveDiskThreshold
(
db_archive_disk_threshold
);
if
(
!
s
.
ok
())
{
return
s
;
}
int
32
_t
db_archive_days_threshold
;
int
64
_t
db_archive_days_threshold
;
s
=
GetDBConfigArchiveDaysThreshold
(
db_archive_days_threshold
);
if
(
!
s
.
ok
())
{
return
s
;
}
int
32
_t
db_insert_buffer_size
;
int
64
_t
db_insert_buffer_size
;
s
=
GetDBConfigInsertBufferSize
(
db_insert_buffer_size
);
if
(
!
s
.
ok
())
{
return
s
;
...
...
@@ -170,19 +170,19 @@ Config::ValidateConfig() {
}
/* engine config */
int
32
_t
engine_use_blas_threshold
;
int
64
_t
engine_use_blas_threshold
;
s
=
GetEngineConfigUseBlasThreshold
(
engine_use_blas_threshold
);
if
(
!
s
.
ok
())
{
return
s
;
}
int
32
_t
engine_omp_thread_num
;
int
64
_t
engine_omp_thread_num
;
s
=
GetEngineConfigOmpThreadNum
(
engine_omp_thread_num
);
if
(
!
s
.
ok
())
{
return
s
;
}
int
32
_t
engine_gpu_search_threshold
;
int
64
_t
engine_gpu_search_threshold
;
s
=
GetEngineConfigGpuSearchThreshold
(
engine_gpu_search_threshold
);
if
(
!
s
.
ok
())
{
return
s
;
...
...
@@ -208,13 +208,13 @@ Config::ValidateConfig() {
return
s
;
}
std
::
vector
<
int
32
_t
>
search_resources
;
std
::
vector
<
int
64
_t
>
search_resources
;
s
=
GetGpuResourceConfigSearchResources
(
search_resources
);
if
(
!
s
.
ok
())
{
return
s
;
}
std
::
vector
<
int
32
_t
>
index_build_resources
;
std
::
vector
<
int
64
_t
>
index_build_resources
;
s
=
GetGpuResourceConfigBuildIndexResources
(
index_build_resources
);
if
(
!
s
.
ok
())
{
return
s
;
...
...
@@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
". Possible reason: db_config.insert_buffer_size is not a positive integer."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
else
{
int64_t
buffer_size
=
std
::
sto
i
(
value
)
*
GB
;
int64_t
buffer_size
=
std
::
sto
ll
(
value
)
*
GB
;
if
(
buffer_size
<=
0
)
{
std
::
string
msg
=
"Invalid insert buffer size: "
+
value
+
". Possible reason: db_config.insert_buffer_size is not a positive integer."
;
...
...
@@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
else
{
int64_t
cpu_cache_capacity
=
std
::
sto
i
(
value
)
*
GB
;
int64_t
cpu_cache_capacity
=
std
::
sto
ll
(
value
)
*
GB
;
if
(
cpu_cache_capacity
<=
0
)
{
std
::
string
msg
=
"Invalid cpu cache capacity: "
+
value
+
". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."
;
...
...
@@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
std
::
cerr
<<
"WARNING: cpu cache capacity value is too big"
<<
std
::
endl
;
}
int
32
_t
buffer_value
;
int
64
_t
buffer_value
;
Status
s
=
GetDBConfigInsertBufferSize
(
buffer_value
);
if
(
!
s
.
ok
())
{
return
s
;
...
...
@@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
int
32_t
omp_thread
=
std
::
stoi
(
value
);
uint32
_t
sys_thread_cnt
=
8
;
int
64_t
omp_thread
=
std
::
stoll
(
value
);
int64
_t
sys_thread_cnt
=
8
;
CommonUtil
::
GetSystemAvailableThreads
(
sys_thread_cnt
);
if
(
omp_thread
>
s
tatic_cast
<
int32_t
>
(
sys_thread_cnt
)
)
{
if
(
omp_thread
>
s
ys_thread_cnt
)
{
std
::
string
msg
=
"Invalid omp thread num: "
+
value
+
". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
...
...
@@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) {
". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
else
{
uint64_t
gpu_cache_capacity
=
std
::
stoi
(
value
)
*
GB
;
std
::
vector
<
int
32
_t
>
gpu_ids
;
int64_t
gpu_cache_capacity
=
std
::
stoll
(
value
)
*
GB
;
std
::
vector
<
int
64
_t
>
gpu_ids
;
Status
s
=
GetGpuResourceConfigBuildIndexResources
(
gpu_ids
);
if
(
!
s
.
ok
())
{
return
s
;
}
for
(
int
32
_t
gpu_id
:
gpu_ids
)
{
for
(
int
64
_t
gpu_id
:
gpu_ids
)
{
size_t
gpu_memory
;
if
(
!
ValidationUtil
::
GetGpuMemory
(
gpu_id
,
gpu_memory
).
ok
())
{
std
::
string
msg
=
"Fail to get GPU memory for GPU device: "
+
std
::
to_string
(
gpu_id
);
...
...
@@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) {
}
Status
Config
::
GetDBConfigArchiveDiskThreshold
(
int
32
_t
&
value
)
{
Config
::
GetDBConfigArchiveDiskThreshold
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_DB
,
CONFIG_DB_ARCHIVE_DISK_THRESHOLD
,
CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT
);
Status
s
=
CheckDBConfigArchiveDiskThreshold
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
Status
Config
::
GetDBConfigArchiveDaysThreshold
(
int
32
_t
&
value
)
{
Config
::
GetDBConfigArchiveDaysThreshold
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_DB
,
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD
,
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT
);
Status
s
=
CheckDBConfigArchiveDaysThreshold
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
Status
Config
::
GetDBConfigInsertBufferSize
(
int
32
_t
&
value
)
{
Config
::
GetDBConfigInsertBufferSize
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_DB
,
CONFIG_DB_INSERT_BUFFER_SIZE
,
CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT
);
Status
s
=
CheckDBConfigInsertBufferSize
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
...
...
@@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) {
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
...
...
@@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) {
}
Status
Config
::
GetEngineConfigUseBlasThreshold
(
int
32
_t
&
value
)
{
Config
::
GetEngineConfigUseBlasThreshold
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_ENGINE
,
CONFIG_ENGINE_USE_BLAS_THRESHOLD
,
CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT
);
Status
s
=
CheckEngineConfigUseBlasThreshold
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
Status
Config
::
GetEngineConfigOmpThreadNum
(
int
32
_t
&
value
)
{
Config
::
GetEngineConfigOmpThreadNum
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_ENGINE
,
CONFIG_ENGINE_OMP_THREAD_NUM
,
CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT
);
Status
s
=
CheckEngineConfigOmpThreadNum
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
Status
Config
::
GetEngineConfigGpuSearchThreshold
(
int
32
_t
&
value
)
{
Config
::
GetEngineConfigGpuSearchThreshold
(
int
64
_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_ENGINE
,
CONFIG_ENGINE_GPU_SEARCH_THRESHOLD
,
CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT
);
Status
s
=
CheckEngineConfigGpuSearchThreshold
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
...
...
@@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) {
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
sto
i
(
str
);
value
=
std
::
sto
ll
(
str
);
return
Status
::
OK
();
}
...
...
@@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) {
}
Status
Config
::
GetGpuResourceConfigSearchResources
(
std
::
vector
<
int
32
_t
>&
value
)
{
Config
::
GetGpuResourceConfigSearchResources
(
std
::
vector
<
int
64
_t
>&
value
)
{
bool
gpu_resource_enable
=
false
;
Status
s
=
GetGpuResourceConfigEnable
(
gpu_resource_enable
);
if
(
!
s
.
ok
())
{
...
...
@@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector<int32_t>& value) {
return
s
;
}
for
(
std
::
string
&
res
:
res_vec
)
{
value
.
push_back
(
std
::
sto
i
(
res
.
substr
(
3
)));
value
.
push_back
(
std
::
sto
ll
(
res
.
substr
(
3
)));
}
return
Status
::
OK
();
}
Status
Config
::
GetGpuResourceConfigBuildIndexResources
(
std
::
vector
<
int
32
_t
>&
value
)
{
Config
::
GetGpuResourceConfigBuildIndexResources
(
std
::
vector
<
int
64
_t
>&
value
)
{
bool
gpu_resource_enable
=
false
;
Status
s
=
GetGpuResourceConfigEnable
(
gpu_resource_enable
);
if
(
!
s
.
ok
())
{
...
...
@@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector<int32_t>& value) {
return
s
;
}
for
(
std
::
string
&
res
:
res_vec
)
{
value
.
push_back
(
std
::
sto
i
(
res
.
substr
(
3
)));
value
.
push_back
(
std
::
sto
ll
(
res
.
substr
(
3
)));
}
return
Status
::
OK
();
}
...
...
core/src/server/Config.h
浏览文件 @
68457e33
...
...
@@ -210,11 +210,11 @@ class Config {
Status
GetDBConfigBackendUrl
(
std
::
string
&
value
);
Status
GetDBConfigArchiveDiskThreshold
(
int
32
_t
&
value
);
GetDBConfigArchiveDiskThreshold
(
int
64
_t
&
value
);
Status
GetDBConfigArchiveDaysThreshold
(
int
32
_t
&
value
);
GetDBConfigArchiveDaysThreshold
(
int
64
_t
&
value
);
Status
GetDBConfigInsertBufferSize
(
int
32
_t
&
value
);
GetDBConfigInsertBufferSize
(
int
64
_t
&
value
);
Status
GetDBConfigPreloadTable
(
std
::
string
&
value
);
...
...
@@ -236,11 +236,11 @@ class Config {
/* engine config */
Status
GetEngineConfigUseBlasThreshold
(
int
32
_t
&
value
);
GetEngineConfigUseBlasThreshold
(
int
64
_t
&
value
);
Status
GetEngineConfigOmpThreadNum
(
int
32
_t
&
value
);
GetEngineConfigOmpThreadNum
(
int
64
_t
&
value
);
Status
GetEngineConfigGpuSearchThreshold
(
int
32
_t
&
value
);
GetEngineConfigGpuSearchThreshold
(
int
64
_t
&
value
);
/* gpu resource config */
Status
...
...
@@ -250,9 +250,9 @@ class Config {
Status
GetGpuResourceConfigCacheThreshold
(
float
&
value
);
Status
GetGpuResourceConfigSearchResources
(
std
::
vector
<
int
32
_t
>&
value
);
GetGpuResourceConfigSearchResources
(
std
::
vector
<
int
64
_t
>&
value
);
Status
GetGpuResourceConfigBuildIndexResources
(
std
::
vector
<
int
32
_t
>&
value
);
GetGpuResourceConfigBuildIndexResources
(
std
::
vector
<
int
64
_t
>&
value
);
public:
/* server config */
...
...
core/src/server/DBWrapper.cpp
浏览文件 @
68457e33
...
...
@@ -89,7 +89,7 @@ DBWrapper::StartService() {
}
// engine config
int
32
_t
omp_thread
;
int
64
_t
omp_thread
;
s
=
config
.
GetEngineConfigOmpThreadNum
(
omp_thread
);
if
(
!
s
.
ok
())
{
std
::
cerr
<<
s
.
ToString
()
<<
std
::
endl
;
...
...
@@ -100,7 +100,7 @@ DBWrapper::StartService() {
omp_set_num_threads
(
omp_thread
);
SERVER_LOG_DEBUG
<<
"Specify openmp thread number: "
<<
omp_thread
;
}
else
{
uint32
_t
sys_thread_cnt
=
8
;
int64
_t
sys_thread_cnt
=
8
;
if
(
CommonUtil
::
GetSystemAvailableThreads
(
sys_thread_cnt
))
{
omp_thread
=
static_cast
<
int32_t
>
(
ceil
(
sys_thread_cnt
*
0.5
));
omp_set_num_threads
(
omp_thread
);
...
...
@@ -108,7 +108,7 @@ DBWrapper::StartService() {
}
// init faiss global variable
int
32
_t
use_blas_threshold
;
int
64
_t
use_blas_threshold
;
s
=
config
.
GetEngineConfigUseBlasThreshold
(
use_blas_threshold
);
if
(
!
s
.
ok
())
{
std
::
cerr
<<
s
.
ToString
()
<<
std
::
endl
;
...
...
@@ -119,7 +119,7 @@ DBWrapper::StartService() {
// set archive config
engine
::
ArchiveConf
::
CriteriaT
criterial
;
int
32
_t
disk
,
days
;
int
64
_t
disk
,
days
;
s
=
config
.
GetDBConfigArchiveDiskThreshold
(
disk
);
if
(
!
s
.
ok
())
{
std
::
cerr
<<
s
.
ToString
()
<<
std
::
endl
;
...
...
core/src/utils/CommonUtil.cpp
浏览文件 @
68457e33
...
...
@@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) {
}
bool
CommonUtil
::
GetSystemAvailableThreads
(
uint32
_t
&
thread_count
)
{
CommonUtil
::
GetSystemAvailableThreads
(
int64
_t
&
thread_count
)
{
// threadCnt = std::thread::hardware_concurrency();
thread_count
=
sysconf
(
_SC_NPROCESSORS_CONF
);
thread_count
*=
THREAD_MULTIPLY_CPU
;
...
...
core/src/utils/CommonUtil.h
浏览文件 @
68457e33
...
...
@@ -30,7 +30,7 @@ class CommonUtil {
static
bool
GetSystemMemInfo
(
uint64_t
&
total_mem
,
uint64_t
&
free_mem
);
static
bool
GetSystemAvailableThreads
(
uint32
_t
&
thread_count
);
GetSystemAvailableThreads
(
int64
_t
&
thread_count
);
static
bool
IsFileExist
(
const
std
::
string
&
path
);
...
...
core/src/wrapper/KnowhereResource.cpp
浏览文件 @
68457e33
...
...
@@ -48,7 +48,7 @@ KnowhereResource::Initialize() {
// get build index gpu resource
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
std
::
vector
<
int
32
_t
>
build_index_gpus
;
std
::
vector
<
int
64
_t
>
build_index_gpus
;
s
=
config
.
GetGpuResourceConfigBuildIndexResources
(
build_index_gpus
);
if
(
!
s
.
ok
())
return
s
;
...
...
@@ -58,7 +58,7 @@ KnowhereResource::Initialize() {
}
// get search gpu resource
std
::
vector
<
int
32
_t
>
search_gpus
;
std
::
vector
<
int
64
_t
>
search_gpus
;
s
=
config
.
GetGpuResourceConfigSearchResources
(
search_gpus
);
if
(
!
s
.
ok
())
return
s
;
...
...
core/unittest/server/test_config.cpp
浏览文件 @
68457e33
...
...
@@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
milvus
::
server
::
Config
&
config
=
milvus
::
server
::
Config
::
GetInstance
();
milvus
::
Status
s
;
std
::
string
str_val
;
int32_t
int32_val
;
int64_t
int64_val
;
float
float_val
;
bool
bool_val
;
...
...
@@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
str_val
==
db_backend_url
);
int
32
_t
db_archive_disk_threshold
=
100
;
int
64
_t
db_archive_disk_threshold
=
100
;
s
=
config
.
SetDBConfigArchiveDiskThreshold
(
std
::
to_string
(
db_archive_disk_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigArchiveDiskThreshold
(
int
32
_val
);
s
=
config
.
GetDBConfigArchiveDiskThreshold
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
db_archive_disk_threshold
);
ASSERT_TRUE
(
int
64
_val
==
db_archive_disk_threshold
);
int
32
_t
db_archive_days_threshold
=
365
;
int
64
_t
db_archive_days_threshold
=
365
;
s
=
config
.
SetDBConfigArchiveDaysThreshold
(
std
::
to_string
(
db_archive_days_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigArchiveDaysThreshold
(
int
32
_val
);
s
=
config
.
GetDBConfigArchiveDaysThreshold
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
db_archive_days_threshold
);
ASSERT_TRUE
(
int
64
_val
==
db_archive_days_threshold
);
int
32
_t
db_insert_buffer_size
=
2
;
int
64
_t
db_insert_buffer_size
=
2
;
s
=
config
.
SetDBConfigInsertBufferSize
(
std
::
to_string
(
db_insert_buffer_size
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetDBConfigInsertBufferSize
(
int
32
_val
);
s
=
config
.
GetDBConfigInsertBufferSize
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
db_insert_buffer_size
);
ASSERT_TRUE
(
int
64
_val
==
db_insert_buffer_size
);
/* metric config */
bool
metric_enable_monitor
=
false
;
...
...
@@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE
(
bool_val
==
cache_insert_data
);
/* engine config */
int
32
_t
engine_use_blas_threshold
=
50
;
int
64
_t
engine_use_blas_threshold
=
50
;
s
=
config
.
SetEngineConfigUseBlasThreshold
(
std
::
to_string
(
engine_use_blas_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigUseBlasThreshold
(
int
32
_val
);
s
=
config
.
GetEngineConfigUseBlasThreshold
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
engine_use_blas_threshold
);
ASSERT_TRUE
(
int
64
_val
==
engine_use_blas_threshold
);
int
32
_t
engine_omp_thread_num
=
8
;
int
64
_t
engine_omp_thread_num
=
8
;
s
=
config
.
SetEngineConfigOmpThreadNum
(
std
::
to_string
(
engine_omp_thread_num
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigOmpThreadNum
(
int
32
_val
);
s
=
config
.
GetEngineConfigOmpThreadNum
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
engine_omp_thread_num
);
ASSERT_TRUE
(
int
64
_val
==
engine_omp_thread_num
);
int
32
_t
engine_gpu_search_threshold
=
800
;
int
64
_t
engine_gpu_search_threshold
=
800
;
s
=
config
.
SetEngineConfigGpuSearchThreshold
(
std
::
to_string
(
engine_gpu_search_threshold
));
ASSERT_TRUE
(
s
.
ok
());
s
=
config
.
GetEngineConfigGpuSearchThreshold
(
int
32
_val
);
s
=
config
.
GetEngineConfigGpuSearchThreshold
(
int
64
_val
);
ASSERT_TRUE
(
s
.
ok
());
ASSERT_TRUE
(
int
32
_val
==
engine_gpu_search_threshold
);
ASSERT_TRUE
(
int
64
_val
==
engine_gpu_search_threshold
);
/* gpu resource config */
bool
resource_enable_gpu
=
true
;
...
...
@@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE
(
float_val
==
gpu_cache_threshold
);
std
::
vector
<
std
::
string
>
search_resources
=
{
"gpu0"
};
std
::
vector
<
int
32
_t
>
search_res_vec
;
std
::
vector
<
int
64
_t
>
search_res_vec
;
std
::
string
search_res_str
;
milvus
::
server
::
StringHelpFunctions
::
MergeStringWithDelimeter
(
search_resources
,
milvus
::
server
::
CONFIG_GPU_RESOURCE_DELIMITER
,
search_res_str
);
...
...
@@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
s
=
config
.
GetGpuResourceConfigSearchResources
(
search_res_vec
);
ASSERT_TRUE
(
s
.
ok
());
for
(
size_t
i
=
0
;
i
<
search_resources
.
size
();
i
++
)
{
ASSERT_TRUE
(
std
::
sto
i
(
search_resources
[
i
].
substr
(
3
))
==
search_res_vec
[
i
]);
ASSERT_TRUE
(
std
::
sto
ll
(
search_resources
[
i
].
substr
(
3
))
==
search_res_vec
[
i
]);
}
std
::
vector
<
std
::
string
>
build_index_resources
=
{
"gpu0"
};
std
::
vector
<
int
32
_t
>
build_index_res_vec
;
std
::
vector
<
int
64
_t
>
build_index_res_vec
;
std
::
string
build_index_res_str
;
milvus
::
server
::
StringHelpFunctions
::
MergeStringWithDelimeter
(
build_index_resources
,
milvus
::
server
::
CONFIG_GPU_RESOURCE_DELIMITER
,
build_index_res_str
);
...
...
@@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
s
=
config
.
GetGpuResourceConfigBuildIndexResources
(
build_index_res_vec
);
ASSERT_TRUE
(
s
.
ok
());
for
(
size_t
i
=
0
;
i
<
build_index_resources
.
size
();
i
++
)
{
ASSERT_TRUE
(
std
::
sto
i
(
build_index_resources
[
i
].
substr
(
3
))
==
build_index_res_vec
[
i
]);
ASSERT_TRUE
(
std
::
sto
ll
(
build_index_resources
[
i
].
substr
(
3
))
==
build_index_res_vec
[
i
]);
}
#endif
}
...
...
core/unittest/server/test_util.cpp
浏览文件 @
68457e33
...
...
@@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) {
ASSERT_GT
(
total_mem
,
0
);
ASSERT_GT
(
free_mem
,
0
);
uint32
_t
thread_cnt
=
0
;
int64
_t
thread_cnt
=
0
;
milvus
::
server
::
CommonUtil
::
GetSystemAvailableThreads
(
thread_cnt
);
ASSERT_GT
(
thread_cnt
,
0
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录