Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
PaddleRec
提交
4ece6cc5
P
PaddleRec
项目概览
BaiXuePrincess
/
PaddleRec
与 Fork 源项目一致
Fork自
PaddlePaddle / PaddleRec
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleRec
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4ece6cc5
编写于
4月 01, 2020
作者:
T
tangwei
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add cluster training
上级
1e953617
变更
5
显示空白变更内容
内联
并排
Showing
5 changed file
with
220 addition
and
6 deletion
+220
-6
examples/ctr-dnn_train.yaml
examples/ctr-dnn_train.yaml
+3
-0
models/ctr_dnn/model.py
models/ctr_dnn/model.py
+0
-4
trainer/cluster_train.py
trainer/cluster_train.py
+213
-0
trainer/cluster_train_local.py
trainer/cluster_train_local.py
+0
-0
trainer/single_train.py
trainer/single_train.py
+4
-2
未找到文件。
examples/ctr-dnn_train.yaml
浏览文件 @
4ece6cc5
...
...
@@ -29,6 +29,9 @@ train:
threads
:
12
epochs
:
10
trainer
:
"
SingleTraining"
role_maler
:
"
PaddleCloudRoleMaker"
strategy
:
mode
:
"
async"
reader
:
mode
:
"
dataset"
...
...
models/ctr_dnn/model.py
浏览文件 @
4ece6cc5
...
...
@@ -125,10 +125,6 @@ class Train(object):
optimizer
=
fluid
.
optimizer
.
Adam
(
learning_rate
,
lazy_mode
=
True
)
return
optimizer
def
optimize
(
self
):
optimizer
=
self
.
optimizer
()
optimizer
.
minimize
(
self
.
loss
)
class
Evaluate
(
object
):
def
input
(
self
):
...
...
trainer/cluster_train.py
0 → 100644
浏览文件 @
4ece6cc5
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Training use fluid with one node only.
"""
from
__future__
import
print_function
import
os
import
time
import
numpy
as
np
import
logging
import
paddle.fluid
as
fluid
from
.trainer
import
Trainer
from
..utils
import
envs
from
..reader
import
dataset
from
paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler
import
fleet
from
paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy
import
StrategyFactory
from
paddle.fluid.incubate.fleet.base.role_maker
import
PaddleCloudRoleMaker
logging
.
basicConfig
(
format
=
"%(asctime)s - %(levelname)s - %(message)s"
)
logger
=
logging
.
getLogger
(
"fluid"
)
logger
.
setLevel
(
logging
.
INFO
)
def
need_save
(
epoch_id
,
epoch_interval
,
is_last
=
False
):
if
is_last
:
return
True
return
epoch_id
%
epoch_interval
==
0
class
ClusterTrainer
(
Trainer
):
def
__init__
(
self
,
config
=
None
,
yaml_file
=
None
):
Trainer
.
__init__
(
self
,
config
,
yaml_file
)
self
.
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
self
.
regist_context_processor
(
'uninit'
,
self
.
instance
)
self
.
regist_context_processor
(
'init_pass'
,
self
.
init
)
self
.
regist_context_processor
(
'server_pass'
,
self
.
server
)
self
.
regist_context_processor
(
'train_pass'
,
self
.
train
)
self
.
regist_context_processor
(
'terminal_pass'
,
self
.
terminal
)
def
build_role_maker
(
self
):
role_maker
=
envs
.
get_global_env
(
"train.role_maker"
)
if
role_maker
==
"PaddleCloudRoleMaker"
:
role
=
PaddleCloudRoleMaker
()
return
role
else
:
raise
ValueError
(
"only support PaddleCloudRoleMaker now"
)
def
build_strategy
(
self
):
mode
=
envs
.
get_global_env
(
"train.strategy.mode"
)
strategy
=
None
if
mode
==
"async"
:
strategy
=
StrategyFactory
.
create_async_strategy
()
elif
mode
==
"geo"
:
push_num
=
envs
.
get_global_env
(
"train.strategy.mode.push_num"
,
100
)
strategy
=
StrategyFactory
.
create_geo_strategy
(
push_num
)
elif
mode
==
"sync"
:
strategy
=
StrategyFactory
.
create_sync_strategy
()
elif
mode
==
"half_async"
:
strategy
=
StrategyFactory
.
create_half_async_strategy
()
return
strategy
def
instance
(
self
,
context
):
model_package
=
__import__
(
envs
.
get_global_env
(
"train.model.models"
))
train_model
=
getattr
(
model_package
,
'Train'
)
self
.
model
=
train_model
()
context
[
'status'
]
=
'init_pass'
def
init
(
self
,
context
):
fleet
.
init
(
self
.
build_role_maker
())
self
.
model
.
input
()
self
.
model
.
net
()
self
.
model
.
loss
()
self
.
metrics
=
self
.
model
.
metrics
()
self
.
loss
=
self
.
model
.
avg_loss
()
optimizer
=
self
.
model
.
get_optimizer
()
strategy
=
self
.
build_strategy
()
optimizer
=
fleet
.
distributed_optimizer
(
optimizer
,
strategy
)
optimizer
.
minimize
(
self
.
loss
)
if
fleet
.
is_server
():
context
[
'status'
]
=
'server_pass'
else
:
context
[
'status'
]
=
'train_pass'
def
server
(
self
,
context
):
fleet
.
init_server
()
fleet
.
run_server
()
context
[
'status'
]
=
'wait'
def
terminal
(
self
,
context
):
fleet
.
stop_worker
()
context
[
'is_exit'
]
=
True
def
train
(
self
,
context
):
print
(
"Need to be implement"
)
context
[
'is_exit'
]
=
True
class
ClusterTrainerWithDataloader
(
ClusterTrainer
):
pass
class
ClusterTrainerWithDataset
(
ClusterTrainer
):
def
_get_dataset
(
self
,
inputs
,
threads
,
batch_size
,
pipe_command
,
train_files_path
):
dataset
=
fluid
.
DatasetFactory
().
create_dataset
()
dataset
.
set_use_var
(
inputs
)
dataset
.
set_pipe_command
(
pipe_command
)
dataset
.
set_batch_size
(
batch_size
)
dataset
.
set_thread
(
threads
)
file_list
=
[
os
.
path
.
join
(
train_files_path
,
x
)
for
x
in
os
.
listdir
(
train_files_path
)
]
dataset
.
set_filelist
(
file_list
)
return
dataset
def
save
(
self
,
epoch_id
):
def
save_inference_model
():
is_save_inference
=
envs
.
get_global_env
(
"save.inference"
,
False
)
if
not
is_save_inference
:
return
save_interval
=
envs
.
get_global_env
(
"save.inference.epoch_interval"
,
1
)
if
not
need_save
(
epoch_id
,
save_interval
,
False
):
return
feed_varnames
=
envs
.
get_global_env
(
"save.inference.feed_varnames"
,
None
)
fetch_varnames
=
envs
.
get_global_env
(
"save.inference.fetch_varnames"
,
None
)
fetch_vars
=
[
fluid
.
global_scope
().
vars
[
varname
]
for
varname
in
fetch_varnames
]
dirname
=
envs
.
get_global_env
(
"save.inference.dirname"
,
None
)
assert
dirname
is
not
None
dirname
=
os
.
path
.
join
(
dirname
,
str
(
epoch_id
))
fluid
.
io
.
save_inference_model
(
dirname
,
feed_varnames
,
fetch_vars
,
self
.
exe
)
def
save_persistables
():
is_save_increment
=
envs
.
get_global_env
(
"save.increment"
,
False
)
if
not
is_save_increment
:
return
save_interval
=
envs
.
get_global_env
(
"save.increment.epoch_interval"
,
1
)
if
not
need_save
(
epoch_id
,
save_interval
,
False
):
return
dirname
=
envs
.
get_global_env
(
"save.inference.dirname"
,
None
)
assert
dirname
is
not
None
dirname
=
os
.
path
.
join
(
dirname
,
str
(
epoch_id
))
fluid
.
io
.
save_persistables
(
self
.
exe
,
dirname
)
is_save
=
envs
.
get_global_env
(
"save"
,
False
)
if
not
is_save
:
return
save_persistables
()
save_inference_model
()
def
train
(
self
,
context
):
inputs
=
self
.
model
.
input_vars
()
threads
=
envs
.
get_global_env
(
"threads"
)
batch_size
=
envs
.
get_global_env
(
"batch_size"
)
pipe_command
=
envs
.
get_global_env
(
"pipe_command"
)
train_data_path
=
envs
.
get_global_env
(
"train_data_path"
)
dataset
=
self
.
_get_dataset
(
inputs
,
threads
,
batch_size
,
pipe_command
,
train_data_path
)
fleet
.
init_worker
()
self
.
exe
.
run
(
fleet
.
startup_program
)
epochs
=
envs
.
get_global_env
(
"epochs"
)
for
i
in
range
(
epochs
):
self
.
exe
.
train_from_dataset
(
program
=
fluid
.
default_main_program
(),
dataset
=
dataset
,
fetch_list
=
[
self
.
metrics
],
fetch_info
=
[
"epoch {} auc "
.
format
(
i
)],
print_period
=
100
)
self
.
save
(
i
)
context
[
'status'
]
=
'infer_pass'
def
infer
(
self
,
context
):
context
[
'status'
]
=
'terminal_pass'
trainer/cluster_train_local.py
已删除
100644 → 0
浏览文件 @
1e953617
trainer/single_train.py
浏览文件 @
4ece6cc5
...
...
@@ -62,9 +62,11 @@ class SingleTrainer(Trainer):
def
init
(
self
,
context
):
self
.
model
.
input
()
self
.
model
.
net
()
self
.
model
.
loss
()
self
.
metrics
=
self
.
model
.
metrics
()
self
.
model
.
optimize
()
loss
=
self
.
model
.
avg_loss
()
optimizer
=
self
.
model
.
get_optimizer
()
optimizer
.
minimize
(
loss
)
# run startup program at once
self
.
exe
.
run
(
fluid
.
default_startup_program
())
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录