Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
一只大灰猫
PaddleRec
提交
eabfd85d
P
PaddleRec
项目概览
一只大灰猫
/
PaddleRec
与 Fork 源项目一致
Fork自
PaddlePaddle / PaddleRec
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleRec
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
eabfd85d
编写于
5月 28, 2020
作者:
X
xjqbest
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix
上级
cf7f8499
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
52 addition
and
21 deletion
+52
-21
core/model.py
core/model.py
+6
-2
core/trainers/single_trainer.py
core/trainers/single_trainer.py
+46
-19
未找到文件。
core/model.py
浏览文件 @
eabfd85d
...
...
@@ -87,9 +87,13 @@ class Model(object):
if
dataset_class
==
"DataLoader"
:
self
.
_init_dataloader
()
def
_init_dataloader
(
self
):
def
_init_dataloader
(
self
,
is_infer
=
False
):
if
is_infer
:
data
=
self
.
_infer_data_var
else
:
data
=
self
.
_data_var
self
.
_data_loader
=
fluid
.
io
.
DataLoader
.
from_generator
(
feed_list
=
self
.
_data_var
,
feed_list
=
data
,
capacity
=
64
,
use_double_buffer
=
False
,
iterable
=
False
)
...
...
core/trainers/single_trainer.py
浏览文件 @
eabfd85d
...
...
@@ -93,7 +93,10 @@ class SingleTrainer(TranspileTrainer):
for
model_dict
in
self
.
_env
[
"executor"
]:
if
model_dict
[
"dataset_name"
]
==
dataset_name
:
model
=
self
.
_model
[
model_dict
[
"name"
]][
3
]
inputs
=
model
.
get_inputs
()
if
model_dict
[
"is_infer"
]:
inputs
=
model
.
_infer_data_var
else
:
inputs
=
model
.
_data_var
dataset
.
set_use_var
(
inputs
)
break
return
dataset
...
...
@@ -158,23 +161,32 @@ class SingleTrainer(TranspileTrainer):
envs
.
path_adapter
(
self
.
_env
[
"workspace"
]))
model
=
envs
.
lazy_instance_by_fliename
(
model_path
,
"Model"
)(
self
.
_env
)
model
.
_data_var
=
model
.
input_data
(
dataset_name
=
model_dict
[
"dataset_name"
])
is_infer
=
model_dict
.
get
(
"is_infer"
,
False
)
if
is_infer
:
model
.
_infer_data_var
=
model
.
input_data
(
dataset_name
=
model_dict
[
"dataset_name"
])
else
:
model
.
_data_var
=
model
.
input_data
(
dataset_name
=
model_dict
[
"dataset_name"
])
if
envs
.
get_global_env
(
"dataset."
+
dataset_name
+
".type"
)
==
"DataLoader"
:
model
.
_init_dataloader
()
model
.
_init_dataloader
(
is_infer
=
is_infer
)
self
.
_get_dataloader
(
dataset_name
,
model
.
_data_loader
)
model
.
net
(
model
.
_data_var
,
is_infer
=
model_dict
.
get
(
"is_infer"
,
False
))
optimizer
=
model
.
_build_optimizer
(
opt_name
,
opt_lr
,
opt_strategy
)
optimizer
.
minimize
(
model
.
_cost
)
if
is_infer
:
model
.
net
(
model
.
_infer_data_var
,
True
)
else
:
model
.
net
(
model
.
_data_var
,
False
)
optimizer
=
model
.
_build_optimizer
(
opt_name
,
opt_lr
,
opt_strategy
)
optimizer
.
minimize
(
model
.
_cost
)
model_dict
[
"is_infer"
]
=
is_infer
self
.
_model
[
model_dict
[
"name"
]][
0
]
=
train_program
self
.
_model
[
model_dict
[
"name"
]][
1
]
=
startup_program
self
.
_model
[
model_dict
[
"name"
]][
2
]
=
scope
self
.
_model
[
model_dict
[
"name"
]][
3
]
=
model
self
.
_model
[
model_dict
[
"name"
]][
4
]
=
train_program
.
clone
()
for
dataset
in
self
.
_env
[
"dataset"
]:
if
dataset
[
"type"
]
!=
"DataLoader"
:
...
...
@@ -223,7 +235,10 @@ class SingleTrainer(TranspileTrainer):
fetch_vars
=
[]
fetch_alias
=
[]
fetch_period
=
20
metrics
=
model_class
.
get_metrics
()
if
model_dict
[
"is_infer"
]:
metrics
=
model_class
.
get_infer_results
()
else
:
metrics
=
model_class
.
get_metrics
()
if
metrics
:
fetch_vars
=
metrics
.
values
()
fetch_alias
=
metrics
.
keys
()
...
...
@@ -231,24 +246,36 @@ class SingleTrainer(TranspileTrainer):
program
=
self
.
_model
[
model_name
][
0
]
reader
=
self
.
_dataset
[
reader_name
]
with
fluid
.
scope_guard
(
scope
):
self
.
_exe
.
train_from_dataset
(
program
=
program
,
dataset
=
reader
,
fetch_list
=
fetch_vars
,
fetch_info
=
fetch_alias
,
print_period
=
fetch_period
)
if
model_dict
[
"is_infer"
]:
self
.
_exe
.
infer_from_dataset
(
program
=
program
,
dataset
=
reader
,
fetch_list
=
fetch_vars
,
fetch_info
=
fetch_alias
,
print_period
=
fetch_period
)
else
:
self
.
_exe
.
train_from_dataset
(
program
=
program
,
dataset
=
reader
,
fetch_list
=
fetch_vars
,
fetch_info
=
fetch_alias
,
print_period
=
fetch_period
)
def
_executor_dataloader_train
(
self
,
model_dict
):
reader_name
=
model_dict
[
"dataset_name"
]
model_name
=
model_dict
[
"name"
]
model_class
=
self
.
_model
[
model_name
][
3
]
program
=
self
.
_model
[
model_name
][
0
].
clone
()
program
=
fluid
.
compiler
.
CompiledProgram
(
program
).
with_data_parallel
(
loss_name
=
model_class
.
get_avg_cost
().
name
)
if
not
model_dict
[
"is_infer"
]:
program
=
fluid
.
compiler
.
CompiledProgram
(
program
).
with_data_parallel
(
loss_name
=
model_class
.
get_avg_cost
().
name
)
fetch_vars
=
[]
fetch_alias
=
[]
fetch_period
=
20
metrics
=
model_class
.
get_metrics
()
if
model_dict
[
"is_infer"
]:
metrics
=
model_class
.
get_infer_results
()
else
:
metrics
=
model_class
.
get_metrics
()
if
metrics
:
fetch_vars
=
metrics
.
values
()
fetch_alias
=
metrics
.
keys
()
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录