Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Oneflow-Inc
oneflow
提交
ba8cfc3a
O
oneflow
项目概览
Oneflow-Inc
/
oneflow
上一次同步 2 年多
通知
13
Star
2733
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
O
oneflow
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ba8cfc3a
编写于
1月 08, 2020
作者:
L
lixinqi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refine signature of InterJobMemSharingUtil::MergeMemReusedChunkBetweenUserJobs
上级
eb320970
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
18 addition
and
14 deletion
+18
-14
oneflow/core/job/inter_job_mem_sharing_util.cpp
oneflow/core/job/inter_job_mem_sharing_util.cpp
+2
-3
oneflow/core/job/inter_job_mem_sharing_util.h
oneflow/core/job/inter_job_mem_sharing_util.h
+2
-2
oneflow/core/job/job_build_and_infer_ctx.cpp
oneflow/core/job/job_build_and_infer_ctx.cpp
+1
-3
oneflow/core/job/oneflow.cpp
oneflow/core/job/oneflow.cpp
+13
-6
未找到文件。
oneflow/core/job/inter_job_mem_sharing_util.cpp
浏览文件 @
ba8cfc3a
...
...
@@ -337,9 +337,8 @@ void InterJobMemSharingUtil::MergeMemSharedInterfaceMemBlockBetweenJobs(
}
void
InterJobMemSharingUtil
::
MergeMemReusedChunkBetweenUserJobs
(
const
std
::
vector
<
std
::
shared_ptr
<
Job
>>&
jobs
,
Plan
*
plan
,
int64_t
user_job_size
)
{
if
(
user_job_size
==
1
)
{
return
;
}
std
::
vector
<
std
::
shared_ptr
<
Job
>>
user_jobs
(
jobs
.
begin
(),
jobs
.
begin
()
+
user_job_size
);
const
std
::
vector
<
std
::
shared_ptr
<
Job
>>&
user_jobs
,
Plan
*
plan
)
{
if
(
user_jobs
.
size
()
==
1
)
{
return
;
}
std
::
vector
<
HashSet
<
int64_t
>>
reuse_mem_job_groups
=
GetMutualExclusionJobGroups
(
user_jobs
);
HashMap
<
int64_t
,
ChunkProto
>
chunk_id2chunk
;
...
...
oneflow/core/job/inter_job_mem_sharing_util.h
浏览文件 @
ba8cfc3a
...
...
@@ -10,8 +10,8 @@ struct InterJobMemSharingUtil {
static
void
MergeMemSharedInterfaceMemBlockBetweenJobs
(
const
std
::
vector
<
std
::
shared_ptr
<
Job
>>&
jobs
,
Plan
*
plan
);
static
void
MergeMemReusedChunkBetweenUserJobs
(
const
std
::
vector
<
std
::
shared_ptr
<
Job
>>&
jobs
,
Plan
*
plan
,
int64_t
user_job_size
);
static
void
MergeMemReusedChunkBetweenUserJobs
(
const
std
::
vector
<
std
::
shared_ptr
<
Job
>>&
user_
jobs
,
Plan
*
plan
);
};
}
// namespace oneflow
...
...
oneflow/core/job/job_build_and_infer_ctx.cpp
浏览文件 @
ba8cfc3a
...
...
@@ -41,14 +41,12 @@ Maybe<void> JobBuildAndInferCtx::SetJobConf(const JobConfigProto& job_conf) {
return
Maybe
<
void
>::
Ok
();
}
REGISTER_FUNCTION_CONFIG_DEF
().
Bool
(
"is_user_function"
,
true
,
"is user defined function"
);
Maybe
<
void
>
JobBuildAndInferCtx
::
Complete
()
{
CHECK_NOTNULL
(
Global
<
JobDesc
>::
Get
());
Global
<
JobDesc
>::
Delete
();
auto
scope
=
std
::
make_unique
<
GlobalJobDescScope
>
(
job_
->
job_conf
(),
job_id_
);
auto
DoPass
=
[
&
](
const
std
::
string
&
pass_name
)
{
FunctionPass
(
pass_name
)(
job_
);
};
if
(
GlobalJobDesc
().
Bool
(
"
is_user_function
"
))
{
if
(
GlobalJobDesc
().
Bool
(
"
__is_user_function__
"
))
{
DoPass
(
"CompleteOfrecordDecoder"
);
DoPass
(
"SetDefaultVariableConf"
);
DoPass
(
"AutoMixedPrecision"
);
...
...
oneflow/core/job/oneflow.cpp
浏览文件 @
ba8cfc3a
...
...
@@ -674,19 +674,26 @@ void MakePushJob(const std::string& job_name, const std::string& op_name,
job_conf
->
set_default_data_type
(
data_type
);
}
REGISTER_FUNCTION_CONFIG_DEF
().
Bool
(
"__is_user_function__"
,
true
,
"is user defined function"
);
void
CompileAndMergePlanOnMaster
(
const
PbRpf
<
Job
>&
conf_jobs
,
Plan
*
plan
)
{
size_t
user_job_size
=
conf_jobs
.
size
();
std
::
vector
<
std
::
shared_ptr
<
Job
>>
jobs
(
conf_jobs
.
size
());
FOR_RANGE
(
int
,
i
,
0
,
jobs
.
size
())
{
jobs
.
at
(
i
).
reset
(
new
Job
(
conf_jobs
.
Get
(
i
)));
}
std
::
vector
<
std
::
shared_ptr
<
Job
>>
function_jobs
;
function_jobs
.
reserve
(
jobs
.
size
());
FOR_RANGE
(
int
,
i
,
0
,
jobs
.
size
())
{
JobDesc
job_desc
(
jobs
.
at
(
i
)
->
job_conf
(),
i
);
if
(
job_desc
.
Bool
(
"__is_user_function__"
))
{
function_jobs
.
push_back
(
jobs
.
at
(
i
));
}
}
if
(
Global
<
MachineCtx
>::
Get
()
->
IsThisMachineMaster
())
{
HashMap
<
std
::
string
,
ParallelBlobConf
>
push_op_name2parallel_blob_conf
;
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kInputConf
},
jobs
,
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kInputConf
},
function_
jobs
,
&
push_op_name2parallel_blob_conf
);
HashMap
<
std
::
string
,
ParallelBlobConf
>
pull_op_name2parallel_blob_conf
;
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kReturnConf
},
jobs
,
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kReturnConf
},
function_
jobs
,
&
pull_op_name2parallel_blob_conf
);
HashMap
<
std
::
string
,
ParallelBlobConf
>
var_op_name2parallel_blob_conf
;
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kVariableConf
},
jobs
,
FilterOpName2ParallelBlobConf
({
OperatorConf
::
kVariableConf
},
function_
jobs
,
&
var_op_name2parallel_blob_conf
);
for
(
const
auto
&
pair
:
push_op_name2parallel_blob_conf
)
{
auto
push_job
=
std
::
make_shared
<
Job
>
();
...
...
@@ -700,7 +707,7 @@ void CompileAndMergePlanOnMaster(const PbRpf<Job>& conf_jobs, Plan* plan) {
pull_job
.
get
());
jobs
.
emplace_back
(
pull_job
);
}
MakeModelIoJobs
(
jobs
,
var_op_name2parallel_blob_conf
,
MakeModelIoJobs
(
function_
jobs
,
var_op_name2parallel_blob_conf
,
[
&
](
Job
*
job
)
{
jobs
.
emplace_back
(
new
Job
(
*
job
));
});
}
std
::
vector
<
Plan
>
sub_plans
(
jobs
.
size
());
...
...
@@ -711,7 +718,7 @@ void CompileAndMergePlanOnMaster(const PbRpf<Job>& conf_jobs, Plan* plan) {
}
if
(
Global
<
MachineCtx
>::
Get
()
->
IsThisMachineMaster
())
{
MergeSubPlanWithoutGenNetTopo
(
plan
,
sub_plans
);
InterJobMemSharingUtil
::
MergeMemReusedChunkBetweenUserJobs
(
jobs
,
plan
,
user_job_size
);
InterJobMemSharingUtil
::
MergeMemReusedChunkBetweenUserJobs
(
function_jobs
,
plan
);
InterJobMemSharingUtil
::
MergeMemSharedInterfaceMemBlockBetweenJobs
(
jobs
,
plan
);
FinishGlobalCriticalSectionDesc
(
*
plan
,
jobs
.
size
());
Plan
main_plan
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录