Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
milvus
提交
96116d2f
milvus
项目概览
BaiXuePrincess
/
milvus
与 Fork 源项目一致
从无法访问的项目Fork
通知
7
Star
4
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
milvus
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
96116d2f
编写于
11月 01, 2019
作者:
F
fishpenguin
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
#164 - Add CPU version for building index
Former-commit-id: fc19b6ab41af57528797cfc88c7f9d6b54d64963
上级
490269e8
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
76 addition
and
85 deletion
+76
-85
CHANGELOG.md
CHANGELOG.md
+1
-0
core/src/scheduler/JobMgr.cpp
core/src/scheduler/JobMgr.cpp
+17
-12
core/src/scheduler/SchedInst.cpp
core/src/scheduler/SchedInst.cpp
+1
-1
core/src/scheduler/SchedInst.h
core/src/scheduler/SchedInst.h
+0
-1
core/src/scheduler/TaskCreator.cpp
core/src/scheduler/TaskCreator.cpp
+9
-2
core/src/scheduler/action/PushTaskToNeighbour.cpp
core/src/scheduler/action/PushTaskToNeighbour.cpp
+35
-67
core/src/scheduler/optimizer/OnlyGPUPass.cpp
core/src/scheduler/optimizer/OnlyGPUPass.cpp
+2
-1
core/src/server/Config.cpp
core/src/server/Config.cpp
+9
-1
core/src/server/Config.h
core/src/server/Config.h
+2
-0
未找到文件。
CHANGELOG.md
浏览文件 @
96116d2f
...
...
@@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA.
-
\#
115 - Using new structure for tasktable
-
\#
139 - New config option use_gpu_threshold
-
\#
146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT
-
\#
164 - Add CPU version for building index
## Improvement
-
\#
64 - Improvement dump function in scheduler
...
...
core/src/scheduler/JobMgr.cpp
浏览文件 @
96116d2f
...
...
@@ -104,20 +104,25 @@ JobMgr::build_task(const JobPtr& job) {
void
JobMgr
::
calculate_path
(
const
TaskPtr
&
task
)
{
if
(
task
->
type_
!=
TaskType
::
SearchTask
)
{
return
;
}
if
(
task
->
type_
==
TaskType
::
SearchTask
)
{
if
(
task
->
label
()
->
Type
()
!=
TaskLabelType
::
SPECIFIED_RESOURCE
)
{
return
;
}
if
(
task
->
label
()
->
Type
()
!=
TaskLabelType
::
SPECIFIED_RESOURCE
)
{
return
;
std
::
vector
<
std
::
string
>
path
;
auto
spec_label
=
std
::
static_pointer_cast
<
SpecResLabel
>
(
task
->
label
());
auto
src
=
res_mgr_
->
GetDiskResources
()[
0
];
auto
dest
=
spec_label
->
resource
();
ShortestPath
(
src
.
lock
(),
dest
.
lock
(),
res_mgr_
,
path
);
task
->
path
()
=
Path
(
path
,
path
.
size
()
-
1
);
}
else
if
(
task
->
type_
==
TaskType
::
BuildIndexTask
)
{
auto
spec_label
=
std
::
static_pointer_cast
<
SpecResLabel
>
(
task
->
label
());
auto
src
=
res_mgr_
->
GetDiskResources
()[
0
];
auto
dest
=
spec_label
->
resource
();
std
::
vector
<
std
::
string
>
path
;
ShortestPath
(
src
.
lock
(),
dest
.
lock
(),
res_mgr_
,
path
);
task
->
path
()
=
Path
(
path
,
path
.
size
()
-
1
);
}
std
::
vector
<
std
::
string
>
path
;
auto
spec_label
=
std
::
static_pointer_cast
<
SpecResLabel
>
(
task
->
label
());
auto
src
=
res_mgr_
->
GetDiskResources
()[
0
];
auto
dest
=
spec_label
->
resource
();
ShortestPath
(
src
.
lock
(),
dest
.
lock
(),
res_mgr_
,
path
);
task
->
path
()
=
Path
(
path
,
path
.
size
()
-
1
);
}
}
// namespace scheduler
...
...
core/src/scheduler/SchedInst.cpp
浏览文件 @
96116d2f
...
...
@@ -75,7 +75,7 @@ load_simple_config() {
}
}
if
(
not
find_build_gpu_id
)
{
if
(
not
find_build_gpu_id
&&
build_gpu_id
!=
server
::
CPU_DEVICE_ID
)
{
ResMgrInst
::
GetInstance
()
->
Add
(
ResourceFactory
::
Create
(
std
::
to_string
(
build_gpu_id
),
"GPU"
,
build_gpu_id
,
true
,
true
));
ResMgrInst
::
GetInstance
()
->
Connect
(
"cpu"
,
std
::
to_string
(
build_gpu_id
),
pcie
);
...
...
core/src/scheduler/SchedInst.h
浏览文件 @
96116d2f
...
...
@@ -106,7 +106,6 @@ class OptimizerInst {
has_cpu
=
true
;
}
}
std
::
vector
<
PassPtr
>
pass_list
;
pass_list
.
push_back
(
std
::
make_shared
<
LargeSQ8HPass
>
());
pass_list
.
push_back
(
std
::
make_shared
<
HybridPass
>
());
...
...
core/src/scheduler/TaskCreator.cpp
浏览文件 @
96116d2f
...
...
@@ -70,8 +70,15 @@ TaskCreator::Create(const DeleteJobPtr& job) {
std
::
vector
<
TaskPtr
>
TaskCreator
::
Create
(
const
BuildIndexJobPtr
&
job
)
{
std
::
vector
<
TaskPtr
>
tasks
;
// TODO(yukun): remove "disk" hardcode here
ResourcePtr
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"disk"
);
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
int32_t
build_index_id
;
Status
stat
=
config
.
GetResourceConfigIndexBuildDevice
(
build_index_id
);
ResourcePtr
res_ptr
;
if
(
build_index_id
==
server
::
CPU_DEVICE_ID
)
{
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
"cpu"
);
}
else
{
res_ptr
=
ResMgrInst
::
GetInstance
()
->
GetResource
(
ResourceType
::
GPU
,
build_index_id
);
}
for
(
auto
&
to_index_file
:
job
->
to_index_files
())
{
auto
label
=
std
::
make_shared
<
SpecResLabel
>
(
std
::
weak_ptr
<
Resource
>
(
res_ptr
));
...
...
core/src/scheduler/action/PushTaskToNeighbour.cpp
浏览文件 @
96116d2f
...
...
@@ -138,73 +138,41 @@ Action::SpecifiedResourceLabelTaskScheduler(const ResourceMgrPtr& res_mgr, Resou
std
::
shared_ptr
<
LoadCompletedEvent
>
event
)
{
auto
task_item
=
event
->
task_table_item_
;
auto
task
=
event
->
task_table_item_
->
task
;
if
(
resource
->
type
()
==
ResourceType
::
DISK
)
{
// step 1: calculate shortest path per resource, from disk to compute resource
auto
compute_resources
=
res_mgr
->
GetComputeResources
();
std
::
vector
<
std
::
vector
<
std
::
string
>>
paths
;
std
::
vector
<
uint64_t
>
transport_costs
;
for
(
auto
&
res
:
compute_resources
)
{
std
::
vector
<
std
::
string
>
path
;
uint64_t
transport_cost
=
ShortestPath
(
resource
,
res
,
res_mgr
,
path
);
transport_costs
.
push_back
(
transport_cost
);
paths
.
emplace_back
(
path
);
}
// if (task->job_.lock()->type() == JobType::SEARCH) {
// auto label = task->label();
// auto spec_label = std::static_pointer_cast<SpecResLabel>(label);
// if (spec_label->resource().lock()->type() == ResourceType::CPU) {
// std::vector<std::string> spec_path;
// spec_path.push_back(spec_label->resource().lock()->name());
// spec_path.push_back(resource->name());
// task->path() = Path(spec_path, spec_path.size() - 1);
// } else {
// // step 2: select min cost, cost(resource) = avg_cost * task_to_do + transport_cost
// uint64_t min_cost = std::numeric_limits<uint64_t>::max();
// uint64_t min_cost_idx = 0;
// for (uint64_t i = 0; i < compute_resources.size(); ++i) {
// if (compute_resources[i]->TotalTasks() == 0) {
// min_cost_idx = i;
// break;
// }
// uint64_t cost = compute_resources[i]->TaskAvgCost() *
// compute_resources[i]->NumOfTaskToExec() +
// transport_costs[i];
// if (min_cost > cost) {
// min_cost = cost;
// min_cost_idx = i;
// }
// }
//
// // step 3: set path in task
// Path task_path(paths[min_cost_idx], paths[min_cost_idx].size() - 1);
// task->path() = task_path;
// }
//
// } else
if
(
task
->
job_
.
lock
()
->
type
()
==
JobType
::
BUILD
)
{
// step2: Read device id in config
// get build index gpu resource
server
::
Config
&
config
=
server
::
Config
::
GetInstance
();
int32_t
build_index_gpu
;
Status
stat
=
config
.
GetResourceConfigIndexBuildDevice
(
build_index_gpu
);
bool
find_gpu_res
=
false
;
if
(
res_mgr
->
GetResource
(
ResourceType
::
GPU
,
build_index_gpu
)
!=
nullptr
)
{
for
(
uint64_t
i
=
0
;
i
<
compute_resources
.
size
();
++
i
)
{
if
(
compute_resources
[
i
]
->
name
()
==
res_mgr
->
GetResource
(
ResourceType
::
GPU
,
build_index_gpu
)
->
name
())
{
find_gpu_res
=
true
;
Path
task_path
(
paths
[
i
],
paths
[
i
].
size
()
-
1
);
task
->
path
()
=
task_path
;
break
;
}
}
}
if
(
not
find_gpu_res
)
{
task
->
path
()
=
Path
(
paths
[
0
],
paths
[
0
].
size
()
-
1
);
}
}
}
// if (resource->type() == ResourceType::DISK) {
// // step 1: calculate shortest path per resource, from disk to compute resource
// auto compute_resources = res_mgr->GetComputeResources();
// std::vector<std::vector<std::string>> paths;
// std::vector<uint64_t> transport_costs;
// for (auto& res : compute_resources) {
// std::vector<std::string> path;
// uint64_t transport_cost = ShortestPath(resource, res, res_mgr, path);
// transport_costs.push_back(transport_cost);
// paths.emplace_back(path);
// }
// if (task->job_.lock()->type() == JobType::BUILD) {
// // step2: Read device id in config
// // get build index gpu resource
// server::Config& config = server::Config::GetInstance();
// int32_t build_index_gpu;
// Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
//
// bool find_gpu_res = false;
// if (res_mgr->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) {
// for (uint64_t i = 0; i < compute_resources.size(); ++i) {
// if (compute_resources[i]->name() ==
// res_mgr->GetResource(ResourceType::GPU, build_index_gpu)->name()) {
// find_gpu_res = true;
// Path task_path(paths[i], paths[i].size() - 1);
// task->path() = task_path;
// break;
// }
// }
// }
// if (not find_gpu_res) {
// task->path() = Path(paths[0], paths[0].size() - 1);
// }
// }
// }
if
(
resource
->
name
()
==
task
->
path
().
Last
())
{
resource
->
WakeupExecutor
();
...
...
core/src/scheduler/optimizer/OnlyGPUPass.cpp
浏览文件 @
96116d2f
...
...
@@ -46,7 +46,8 @@ OnlyGPUPass::Run(const TaskPtr& task) {
auto
label
=
std
::
make_shared
<
SpecResLabel
>
(
std
::
weak_ptr
<
Resource
>
(
res_ptr
));
task
->
label
()
=
label
;
specified_gpu_id_
=
specified_gpu_id_
++
%
gpu_id
.
size
();
++
specified_gpu_id_
;
specified_gpu_id_
=
specified_gpu_id_
%
gpu_id
.
size
();
return
true
;
}
...
...
core/src/server/Config.cpp
浏览文件 @
96116d2f
...
...
@@ -596,6 +596,9 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
return
s
;
}
if
(
gpu_index
==
server
::
CPU_DEVICE_ID
)
return
Status
::
OK
();
size_t
gpu_memory
;
if
(
!
ValidationUtil
::
GetGpuMemory
(
gpu_index
,
gpu_memory
).
ok
())
{
std
::
string
msg
=
"Fail to get GPU memory for GPU device: "
+
std
::
to_string
(
gpu_index
);
...
...
@@ -1013,7 +1016,12 @@ Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
return
s
;
}
value
=
std
::
stoi
(
str
.
substr
(
3
));
if
(
str
==
"cpu"
)
{
value
=
CPU_DEVICE_ID
;
}
else
{
value
=
std
::
stoi
(
str
.
substr
(
3
));
}
return
Status
::
OK
();
}
...
...
core/src/server/Config.h
浏览文件 @
96116d2f
...
...
@@ -95,6 +95,8 @@ static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE
=
"index_build_device"
;
static
const
char
*
CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT
=
"gpu0"
;
const
int32_t
CPU_DEVICE_ID
=
-
1
;
class
Config
{
public:
static
Config
&
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录