Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
4b73bda0
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4b73bda0
编写于
8月 24, 2019
作者:
S
starlord
浏览文件
操作
浏览文件
下载
差异文件
MS-413 Remove thrift dependency
Former-commit-id: 62ed2b780dc79f8415b3d63b902fb3757e523ef7
上级
86ca848f
cc5d6a71
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
62 addition
and
11 deletion
+62
-11
cpp/CHANGELOG.md
cpp/CHANGELOG.md
+3
-0
cpp/conf/server_config.template
cpp/conf/server_config.template
+14
-0
cpp/src/db/engine/ExecutionEngineImpl.cpp
cpp/src/db/engine/ExecutionEngineImpl.cpp
+12
-8
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
+23
-3
cpp/unittest/metrics/CMakeLists.txt
cpp/unittest/metrics/CMakeLists.txt
+10
-0
未找到文件。
cpp/CHANGELOG.md
浏览文件 @
4b73bda0
...
...
@@ -5,6 +5,8 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.4.0 (2019-07-28)
## Bug
-
MS-411 - Fix metric unittest linking error
-
MS-412 - Fix gpu cache logical error
## Improvement
-
MS-327 - Clean code for milvus
...
...
@@ -48,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
MS-408 - Add device_id in resource construct function
-
MS-409 - Using new scheduler
-
MS-413 - Remove thrift dependency
-
MS-410 - Add resource config comment
## New Feature
-
MS-343 - Implement ResourceMgr
...
...
cpp/conf/server_config.template
浏览文件 @
4b73bda0
...
...
@@ -40,6 +40,17 @@ engine_config:
omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute
resource_config:
# resource list, length: 0~N
# please set a DISK resource and a CPU resource least, or system will not return query result.
#
# example:
# resource_name: # resource name, just using in connections below
# type: DISK # resource type, optional: DISK/CPU/GPU
# memory: 256 # memory size, unit: GB
# device_id: 0
# enable_loader: true # if is enable loader, optional: true, false
# enable_executor: false # if is enable executor, optional: true, false
resources:
ssda:
type: DISK
...
...
@@ -69,7 +80,10 @@ resource_config:
enable_loader: true
enable_executor: true
# connection list, length: 0~N
# format: -${resource_name}===${resource_name}
connections:
- ssda===cpu
- cpu===gtx1060
- cpu===gtx1660
cpp/src/db/engine/ExecutionEngineImpl.cpp
浏览文件 @
4b73bda0
...
...
@@ -139,9 +139,11 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
}
Status
ExecutionEngineImpl
::
CopyToGpu
(
uint64_t
device_id
)
{
index_
=
zilliz
::
milvus
::
cache
::
GpuCacheMgr
::
GetInstance
(
device_id
)
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index_
!=
nullptr
);
if
(
!
index_
)
{
auto
index
=
zilliz
::
milvus
::
cache
::
GpuCacheMgr
::
GetInstance
(
device_id
)
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index
!=
nullptr
);
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
try
{
index_
=
index_
->
CopyToGpu
(
device_id
);
ENGINE_LOG_DEBUG
<<
"CPU to GPU"
<<
device_id
;
...
...
@@ -161,9 +163,11 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
}
Status
ExecutionEngineImpl
::
CopyToCpu
()
{
index_
=
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index_
!=
nullptr
);
if
(
!
index_
)
{
auto
index
=
zilliz
::
milvus
::
cache
::
CpuCacheMgr
::
GetInstance
()
->
GetIndex
(
location_
);
bool
already_in_cache
=
(
index
!=
nullptr
);
if
(
already_in_cache
)
{
index_
=
index
;
}
else
{
try
{
index_
=
index_
->
CopyToCpu
();
ENGINE_LOG_DEBUG
<<
"GPU to CPU"
;
...
...
@@ -175,7 +179,7 @@ Status ExecutionEngineImpl::CopyToCpu() {
}
}
if
(
!
already_in_cache
)
{
if
(
!
already_in_cache
)
{
Cache
();
}
return
Status
::
OK
();
...
...
@@ -276,7 +280,7 @@ Status ExecutionEngineImpl::Init() {
using
namespace
zilliz
::
milvus
::
server
;
ServerConfig
&
config
=
ServerConfig
::
GetInstance
();
ConfigNode
server_config
=
config
.
GetConfig
(
CONFIG_SERVER
);
gpu_num_
=
server_config
.
GetInt32Value
(
"gpu_index"
,
0
);
gpu_num_
=
server_config
.
GetInt32Value
(
"gpu_index"
,
0
);
return
Status
::
OK
();
}
...
...
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
浏览文件 @
4b73bda0
...
...
@@ -5,6 +5,7 @@
******************************************************************************/
#include <list>
#include <random>
#include "Action.h"
...
...
@@ -38,6 +39,22 @@ push_task_round_robin(TaskTable &self_task_table, std::list<ResourcePtr> &neighb
}
}
void
push_task_randomly
(
TaskTable
&
self_task_table
,
std
::
vector
<
ResourcePtr
>
&
neighbours
)
{
std
::
random_device
rd
;
std
::
mt19937
mt
(
rd
());
std
::
uniform_int_distribution
<
uint64_t
>
dist
(
0
,
neighbours
.
size
()
-
1
);
CacheMgr
cache
;
auto
indexes
=
PickToMove
(
self_task_table
,
cache
,
self_task_table
.
Size
());
for
(
auto
index
:
indexes
)
{
if
(
self_task_table
.
Move
(
index
))
{
auto
task
=
self_task_table
.
Get
(
index
)
->
task
;
neighbours
[
dist
(
mt
)]
->
task_table
().
Put
(
task
);
}
}
}
void
Action
::
PushTaskToNeighbour
(
const
ResourceWPtr
&
res
)
{
auto
self
=
res
.
lock
();
...
...
@@ -60,18 +77,21 @@ Action::PushTaskToNeighbourHasExecutor(const ResourceWPtr &res) {
auto
self
=
res
.
lock
();
if
(
not
self
)
return
;
std
::
list
<
ResourcePtr
>
neighbours
;
std
::
list
<
ResourcePtr
>
l_neighbours
;
std
::
vector
<
ResourcePtr
>
v_neighbours
;
for
(
auto
&
neighbour_node
:
self
->
GetNeighbours
())
{
auto
node
=
neighbour_node
.
neighbour_node
.
lock
();
if
(
not
node
)
continue
;
auto
resource
=
std
::
static_pointer_cast
<
Resource
>
(
node
);
if
(
resource
->
HasExecutor
())
{
neighbours
.
emplace_back
(
resource
);
l_neighbours
.
push_back
(
resource
);
v_neighbours
.
push_back
(
resource
);
}
}
push_task_round_robin
(
self
->
task_table
(),
neighbours
);
// push_task_round_robin(self->task_table(), l_neighbours);
push_task_randomly
(
self
->
task_table
(),
v_neighbours
);
}
...
...
cpp/unittest/metrics/CMakeLists.txt
浏览文件 @
4b73bda0
...
...
@@ -19,6 +19,11 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/cache cache_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/wrapper wrapper_src
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/wrapper/knowhere knowhere_src
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/action scheduler_action_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/event scheduler_event_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/resource scheduler_resource_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler/task scheduler_task_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/scheduler scheduler_srcs
)
aux_source_directory
(
${
MILVUS_ENGINE_SRC
}
/src/metrics metrics_src
)
aux_source_directory
(
./ test_srcs
)
...
...
@@ -52,6 +57,11 @@ set(count_test_src
${
db_meta_files
}
${
db_scheduler_srcs
}
${
wrapper_src
}
${
scheduler_action_srcs
}
${
scheduler_event_srcs
}
${
scheduler_resource_srcs
}
${
scheduler_task_srcs
}
${
scheduler_srcs
}
${
knowhere_src
}
${
metrics_src
}
${
test_srcs
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录