Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
39e5374f
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
39e5374f
编写于
10月 31, 2019
作者:
H
Heisenberg
浏览文件
操作
浏览文件
下载
差异文件
Merge branch '0.5.1' into 0.5.1-yzb
Former-commit-id: 116e0cb95fde8473b3ec02764c08b05ccd8c695e
上级
620633fb
5c25d561
变更
9
展开全部
隐藏空白更改
内联
并排
Showing
9 changed file
with
272 addition
and
186 deletion
+272
-186
CHANGELOG.md
CHANGELOG.md
+2
-0
core/CMakeLists.txt
core/CMakeLists.txt
+5
-1
core/cmake/ThirdPartyPackages.cmake
core/cmake/ThirdPartyPackages.cmake
+193
-182
core/conf/server_config.template
core/conf/server_config.template
+1
-0
core/src/index/cmake/ThirdPartyPackagesCore.cmake
core/src/index/cmake/ThirdPartyPackagesCore.cmake
+2
-1
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
+11
-1
core/src/scheduler/optimizer/LargeSQ8HPass.h
core/src/scheduler/optimizer/LargeSQ8HPass.h
+5
-1
core/src/server/Config.cpp
core/src/server/Config.cpp
+45
-0
core/src/server/Config.h
core/src/server/Config.h
+8
-0
未找到文件。
CHANGELOG.md
浏览文件 @
39e5374f
...
...
@@ -5,11 +5,13 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.5.1 (TODO)
## Bug
-
\#
134 - JFrog cache error
## Feature
-
\#
90 - The server start error messages could be improved to enhance user experience
-
\#
104 - test_scheduler core dump
-
\#
115 - Using new structure for tasktable
-
\#
139 - New config opion use_gpu_threshold
## Improvement
-
\#
64 - Improvement dump function in scheduler
...
...
core/CMakeLists.txt
浏览文件 @
39e5374f
...
...
@@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message
(
STATUS
"Build time =
${
BUILD_TIME
}
"
)
MACRO
(
GET_GIT_BRANCH_NAME GIT_BRANCH_NAME
)
execute_process
(
COMMAND
"git"
symbolic-ref --short HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
execute_process
(
COMMAND
"git"
rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
if
(
GIT_BRANCH_NAME STREQUAL
""
)
execute_process
(
COMMAND
"git"
symbolic-ref --short -q HEAD OUTPUT_VARIABLE
${
GIT_BRANCH_NAME
}
)
endif
()
ENDMACRO
(
GET_GIT_BRANCH_NAME
)
GET_GIT_BRANCH_NAME
(
GIT_BRANCH_NAME
)
message
(
STATUS
"GIT_BRANCH_NAME =
${
GIT_BRANCH_NAME
}
"
)
if
(
NOT GIT_BRANCH_NAME STREQUAL
""
)
string
(
REGEX REPLACE
"
\n
"
""
GIT_BRANCH_NAME
${
GIT_BRANCH_NAME
}
)
endif
()
...
...
core/cmake/ThirdPartyPackages.cmake
浏览文件 @
39e5374f
此差异已折叠。
点击以展开。
core/conf/server_config.template
浏览文件 @
39e5374f
...
...
@@ -36,6 +36,7 @@ cache_config:
engine_config:
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
use_gpu_threshold: 1000
resource_config:
search_resources: # define the GPUs used for search computation, must be in format: gpux
...
...
core/src/index/cmake/ThirdPartyPackagesCore.cmake
浏览文件 @
39e5374f
...
...
@@ -245,7 +245,8 @@ if(CUSTOMIZATION)
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
set
(
FAISS_MD5
"f3b2ce3364c3fa7febd3aa7fdd0fe380"
)
# commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
set
(
FAISS_MD5
"bb30722c22390ce5f6759ccb216c1b2a"
)
# commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
endif
()
else
()
set
(
FAISS_SOURCE_URL
"https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz"
)
...
...
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
浏览文件 @
39e5374f
...
...
@@ -21,11 +21,20 @@
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "server/Config.h"
#include "utils/Log.h"
namespace
milvus
{
namespace
scheduler
{
LargeSQ8HPass
::
LargeSQ8HPass
()
{
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
Status
s
=
config
.
GetEngineConfigUseGpuThreshold
(
threshold_
);
if
(
!
s
.
ok
())
{
threshold_
=
std
::
numeric_limits
<
int32_t
>::
max
();
}
}
bool
LargeSQ8HPass
::
Run
(
const
TaskPtr
&
task
)
{
if
(
task
->
Type
()
!=
TaskType
::
SearchTask
)
{
...
...
@@ -40,7 +49,8 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
auto
search_job
=
std
::
static_pointer_cast
<
SearchJob
>
(
search_task
->
job_
.
lock
());
// TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu
if
(
search_job
->
nq
()
<
100
)
{
if
(
search_job
->
nq
()
<
threshold_
)
{
return
false
;
}
...
...
core/src/scheduler/optimizer/LargeSQ8HPass.h
浏览文件 @
39e5374f
...
...
@@ -18,6 +18,7 @@
#include <condition_variable>
#include <deque>
#include <limits>
#include <list>
#include <memory>
#include <mutex>
...
...
@@ -34,11 +35,14 @@ namespace scheduler {
class
LargeSQ8HPass
:
public
Pass
{
public:
LargeSQ8HPass
()
=
default
;
LargeSQ8HPass
();
public:
bool
Run
(
const
TaskPtr
&
task
)
override
;
private:
int32_t
threshold_
=
std
::
numeric_limits
<
int32_t
>::
max
();
};
using
LargeSQ8HPassPtr
=
std
::
shared_ptr
<
LargeSQ8HPass
>
;
...
...
core/src/server/Config.cpp
浏览文件 @
39e5374f
...
...
@@ -193,6 +193,12 @@ Config::ValidateConfig() {
return
s
;
}
int32_t
engine_use_gpu_threshold
;
s
=
GetEngineConfigUseGpuThreshold
(
engine_use_gpu_threshold
);
if
(
!
s
.
ok
())
{
return
s
;
}
/* resource config */
std
::
string
resource_mode
;
s
=
GetResourceConfigMode
(
resource_mode
);
...
...
@@ -324,6 +330,11 @@ Config::ResetDefaultConfig() {
return
s
;
}
s
=
SetEngineConfigUseGpuThreshold
(
CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT
);
if
(
!
s
.
ok
())
{
return
s
;
}
/* resource config */
s
=
SetResourceConfigMode
(
CONFIG_RESOURCE_MODE_DEFAULT
);
if
(
!
s
.
ok
())
{
...
...
@@ -656,6 +667,16 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
return
Status
::
OK
();
}
Status
Config
::
CheckEngineConfigUseGpuThreshold
(
const
std
::
string
&
value
)
{
if
(
!
ValidationUtil
::
ValidateStringIsNumber
(
value
).
ok
())
{
std
::
string
msg
=
"Invalid gpu threshold: "
+
value
+
". Possible reason: engine_config.use_gpu_threshold is not a positive integer."
;
return
Status
(
SERVER_INVALID_ARGUMENT
,
msg
);
}
return
Status
::
OK
();
}
Status
Config
::
CheckResourceConfigMode
(
const
std
::
string
&
value
)
{
if
(
value
!=
"simple"
)
{
...
...
@@ -951,6 +972,19 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) {
return
Status
::
OK
();
}
Status
Config
::
GetEngineConfigUseGpuThreshold
(
int32_t
&
value
)
{
std
::
string
str
=
GetConfigStr
(
CONFIG_ENGINE
,
CONFIG_ENGINE_USE_GPU_THRESHOLD
,
CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT
);
Status
s
=
CheckEngineConfigUseGpuThreshold
(
str
);
if
(
!
s
.
ok
())
{
return
s
;
}
value
=
std
::
stoi
(
str
);
return
Status
::
OK
();
}
Status
Config
::
GetResourceConfigMode
(
std
::
string
&
value
)
{
value
=
GetConfigStr
(
CONFIG_RESOURCE
,
CONFIG_RESOURCE_MODE
,
CONFIG_RESOURCE_MODE_DEFAULT
);
...
...
@@ -1203,6 +1237,17 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return
Status
::
OK
();
}
Status
Config
::
SetEngineConfigUseGpuThreshold
(
const
std
::
string
&
value
)
{
Status
s
=
CheckEngineConfigUseGpuThreshold
(
value
);
if
(
!
s
.
ok
())
{
return
s
;
}
SetConfigValueInMem
(
CONFIG_DB
,
CONFIG_ENGINE_USE_GPU_THRESHOLD
,
value
);
return
Status
::
OK
();
}
/* resource config */
Status
Config
::
SetResourceConfigMode
(
const
std
::
string
&
value
)
{
...
...
core/src/server/Config.h
浏览文件 @
39e5374f
...
...
@@ -84,6 +84,8 @@ static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold";
static
const
char
*
CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT
=
"20"
;
static
const
char
*
CONFIG_ENGINE_OMP_THREAD_NUM
=
"omp_thread_num"
;
static
const
char
*
CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT
=
"0"
;
static
const
char
*
CONFIG_ENGINE_USE_GPU_THRESHOLD
=
"use_gpu_threshold"
;
static
const
char
*
CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT
=
"1000"
;
/* resource config */
static
const
char
*
CONFIG_RESOURCE
=
"resource_config"
;
...
...
@@ -166,6 +168,8 @@ class Config {
CheckEngineConfigUseBlasThreshold
(
const
std
::
string
&
value
);
Status
CheckEngineConfigOmpThreadNum
(
const
std
::
string
&
value
);
Status
CheckEngineConfigUseGpuThreshold
(
const
std
::
string
&
value
);
/* resource config */
Status
...
...
@@ -230,6 +234,8 @@ class Config {
GetEngineConfigUseBlasThreshold
(
int32_t
&
value
);
Status
GetEngineConfigOmpThreadNum
(
int32_t
&
value
);
Status
GetEngineConfigUseGpuThreshold
(
int32_t
&
value
);
/* resource config */
Status
...
...
@@ -289,6 +295,8 @@ class Config {
SetEngineConfigUseBlasThreshold
(
const
std
::
string
&
value
);
Status
SetEngineConfigOmpThreadNum
(
const
std
::
string
&
value
);
Status
SetEngineConfigUseGpuThreshold
(
const
std
::
string
&
value
);
/* resource config */
Status
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录