Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
models
提交
62683816
M
models
项目概览
PaddlePaddle
/
models
大约 1 年 前同步成功
通知
222
Star
6828
Fork
2962
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
602
列表
看板
标记
里程碑
合并请求
255
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
602
Issue
602
列表
看板
标记
里程碑
合并请求
255
合并请求
255
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
62683816
编写于
3月 26, 2019
作者:
Z
zhang wenhui
提交者:
GitHub
3月 26, 2019
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #1920 from frankwhzhang/fix_bug
add ssr distribute_train
上级
8a6f5942
5be55aba
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
267 addition
and
1 deletion
+267
-1
fluid/PaddleRec/ssr/README.md
fluid/PaddleRec/ssr/README.md
+4
-1
fluid/PaddleRec/ssr/cluster_train.py
fluid/PaddleRec/ssr/cluster_train.py
+205
-0
fluid/PaddleRec/ssr/cluster_train.sh
fluid/PaddleRec/ssr/cluster_train.sh
+58
-0
未找到文件。
fluid/PaddleRec/ssr/README.md
浏览文件 @
62683816
...
...
@@ -39,7 +39,10 @@ cpu 单机多卡训练
CPU_NUM
=
10 python train.py
--train_dir
train_data
--use_cuda
0
--parallel
1
--batch_size
50
--model_dir
model_output
--num_devices
10
```
多机训练 参考fluid/PaddleRec/gru4rec下的配置
本地模拟多机训练
```
bash
sh cluster_train.sh
```
## Inference
...
...
fluid/PaddleRec/ssr/cluster_train.py
0 → 100644
浏览文件 @
62683816
#Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
os
import
sys
import
time
import
argparse
import
logging
import
paddle.fluid
as
fluid
import
paddle
import
utils
import
numpy
as
np
from
nets
import
SequenceSemanticRetrieval
logging
.
basicConfig
(
format
=
"%(asctime)s - %(levelname)s - %(message)s"
)
logger
=
logging
.
getLogger
(
"fluid"
)
logger
.
setLevel
(
logging
.
INFO
)
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
"sequence semantic retrieval"
)
parser
.
add_argument
(
"--train_dir"
,
type
=
str
,
default
=
'train_data'
,
help
=
"Training file"
)
parser
.
add_argument
(
"--base_lr"
,
type
=
float
,
default
=
0.01
,
help
=
"learning rate"
)
parser
.
add_argument
(
'--vocab_path'
,
type
=
str
,
default
=
'vocab.txt'
,
help
=
'vocab file'
)
parser
.
add_argument
(
"--epochs"
,
type
=
int
,
default
=
10
,
help
=
"Number of epochs"
)
parser
.
add_argument
(
'--parallel'
,
type
=
int
,
default
=
0
,
help
=
'whether parallel'
)
parser
.
add_argument
(
'--use_cuda'
,
type
=
int
,
default
=
0
,
help
=
'whether use gpu'
)
parser
.
add_argument
(
'--print_batch'
,
type
=
int
,
default
=
10
,
help
=
'num of print batch'
)
parser
.
add_argument
(
'--model_dir'
,
type
=
str
,
default
=
'model_output'
,
help
=
'model dir'
)
parser
.
add_argument
(
"--hidden_size"
,
type
=
int
,
default
=
128
,
help
=
"hidden size"
)
parser
.
add_argument
(
"--batch_size"
,
type
=
int
,
default
=
50
,
help
=
"number of batch"
)
parser
.
add_argument
(
"--embedding_dim"
,
type
=
int
,
default
=
128
,
help
=
"embedding dim"
)
parser
.
add_argument
(
'--num_devices'
,
type
=
int
,
default
=
1
,
help
=
'Number of GPU devices'
)
parser
.
add_argument
(
'--step_num'
,
type
=
int
,
default
=
1000
,
help
=
'Number of steps'
)
parser
.
add_argument
(
'--enable_ce'
,
action
=
'store_true'
,
help
=
'If set, run the task with continuous evaluation logs.'
)
parser
.
add_argument
(
'--role'
,
type
=
str
,
default
=
'pserver'
,
help
=
'trainer or pserver'
)
parser
.
add_argument
(
'--endpoints'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The pserver endpoints, like: 127.0.0.1:6000, 127.0.0.1:6001'
)
parser
.
add_argument
(
'--current_endpoint'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The current_endpoint'
)
parser
.
add_argument
(
'--trainer_id'
,
type
=
int
,
default
=
0
,
help
=
'trainer id ,only trainer_id=0 save model'
)
parser
.
add_argument
(
'--trainers'
,
type
=
int
,
default
=
1
,
help
=
'The num of trianers, (default: 1)'
)
return
parser
.
parse_args
()
def
get_cards
(
args
):
return
args
.
num_devices
def
train_loop
(
main_program
,
avg_cost
,
acc
,
train_input_data
,
place
,
args
,
train_reader
):
data_list
=
[
var
.
name
for
var
in
train_input_data
]
feeder
=
fluid
.
DataFeeder
(
feed_list
=
data_list
,
place
=
place
)
exe
=
fluid
.
Executor
(
place
)
exe
.
run
(
fluid
.
default_startup_program
())
train_exe
=
exe
total_time
=
0.0
ce_info
=
[]
for
pass_id
in
range
(
args
.
epochs
):
epoch_idx
=
pass_id
+
1
print
(
"epoch_%d start"
%
epoch_idx
)
t0
=
time
.
time
()
i
=
0
for
batch_id
,
data
in
enumerate
(
train_reader
()):
i
+=
1
loss_val
,
correct_val
=
train_exe
.
run
(
feed
=
feeder
.
feed
(
data
),
fetch_list
=
[
avg_cost
.
name
,
acc
.
name
])
ce_info
.
append
(
float
(
np
.
mean
(
correct_val
))
/
args
.
batch_size
)
if
i
%
args
.
print_batch
==
0
:
logger
.
info
(
"Train --> pass: {} batch_id: {} avg_cost: {}, acc: {}"
.
format
(
pass_id
,
batch_id
,
np
.
mean
(
loss_val
),
float
(
np
.
mean
(
correct_val
))
/
args
.
batch_size
))
if
args
.
enable_ce
and
i
>
args
.
step_num
:
break
t1
=
time
.
time
()
total_time
+=
t1
-
t0
print
(
"epoch:%d num_steps:%d time_cost(s):%f"
%
(
epoch_idx
,
i
,
total_time
/
epoch_idx
))
save_dir
=
"%s/epoch_%d"
%
(
args
.
model_dir
,
epoch_idx
)
fluid
.
io
.
save_params
(
executor
=
exe
,
dirname
=
save_dir
)
print
(
"model saved in %s"
%
save_dir
)
# only for ce
if
args
.
enable_ce
:
ce_acc
=
0
try
:
ce_acc
=
ce_info
[
-
2
]
except
:
print
(
"ce info error"
)
epoch_idx
=
args
.
epochs
device
=
get_device
(
args
)
if
args
.
use_cuda
:
gpu_num
=
device
[
1
]
print
(
"kpis
\t
each_pass_duration_gpu%s
\t
%s"
%
(
gpu_num
,
total_time
/
epoch_idx
))
print
(
"kpis
\t
train_acc_gpu%s
\t
%s"
%
(
gpu_num
,
ce_acc
))
else
:
cpu_num
=
device
[
1
]
threads_num
=
device
[
2
]
print
(
"kpis
\t
each_pass_duration_cpu%s_thread%s
\t
%s"
%
(
cpu_num
,
threads_num
,
total_time
/
epoch_idx
))
print
(
"kpis
\t
train_acc_cpu%s_thread%s
\t
%s"
%
(
cpu_num
,
threads_num
,
ce_acc
))
def
train
(
args
):
if
args
.
enable_ce
:
SEED
=
102
fluid
.
default_startup_program
().
random_seed
=
SEED
fluid
.
default_main_program
().
random_seed
=
SEED
use_cuda
=
True
if
args
.
use_cuda
else
False
parallel
=
True
if
args
.
parallel
else
False
print
(
"use_cuda:"
,
use_cuda
,
"parallel:"
,
parallel
)
train_reader
,
vocab_size
=
utils
.
construct_train_data
(
args
.
train_dir
,
args
.
vocab_path
,
args
.
batch_size
*
get_cards
(
args
))
place
=
fluid
.
CUDAPlace
(
0
)
if
use_cuda
else
fluid
.
CPUPlace
()
ssr
=
SequenceSemanticRetrieval
(
vocab_size
,
args
.
embedding_dim
,
args
.
hidden_size
)
# Train program
train_input_data
,
cos_pos
,
avg_cost
,
acc
=
ssr
.
train
()
# Optimization to minimize lost
optimizer
=
fluid
.
optimizer
.
Adagrad
(
learning_rate
=
args
.
base_lr
)
optimizer
.
minimize
(
avg_cost
)
print
(
"run distribute training"
)
t
=
fluid
.
DistributeTranspiler
()
t
.
transpile
(
args
.
trainer_id
,
pservers
=
args
.
endpoints
,
trainers
=
args
.
trainers
)
if
args
.
role
==
"pserver"
:
print
(
"run psever"
)
pserver_prog
=
t
.
get_pserver_program
(
args
.
current_endpoint
)
pserver_startup
=
t
.
get_startup_program
(
args
.
current_endpoint
,
pserver_prog
)
exe
=
fluid
.
Executor
(
fluid
.
CPUPlace
())
exe
.
run
(
pserver_startup
)
exe
.
run
(
pserver_prog
)
elif
args
.
role
==
"trainer"
:
print
(
"run trainer"
)
train_loop
(
t
.
get_trainer_program
(),
avg_cost
,
acc
,
train_input_data
,
place
,
args
,
train_reader
)
def
get_device
(
args
):
if
args
.
use_cuda
:
gpus
=
os
.
environ
.
get
(
"CUDA_VISIBLE_DEVICES"
,
1
)
gpu_num
=
len
(
gpus
.
split
(
','
))
return
"gpu"
,
gpu_num
else
:
threads_num
=
os
.
environ
.
get
(
'NUM_THREADS'
,
1
)
cpu_num
=
os
.
environ
.
get
(
'CPU_NUM'
,
1
)
return
"cpu"
,
int
(
cpu_num
),
int
(
threads_num
)
def
main
():
args
=
parse_args
()
train
(
args
)
if
__name__
==
"__main__"
:
main
()
fluid/PaddleRec/ssr/cluster_train.sh
0 → 100644
浏览文件 @
62683816
#!/bin/bash
#export GLOG_v=30
#export GLOG_logtostderr=1
# start pserver0
python cluster_train.py
\
--train_dir
train_data
\
--model_dir
cluster_model
\
--vocab_path
vocab.txt
\
--batch_size
5
\
--role
pserver
\
--endpoints
127.0.0.1:6000,127.0.0.1:6001
\
--current_endpoint
127.0.0.1:6000
\
--trainers
2
\
>
pserver0.log 2>&1 &
# start pserver1
python cluster_train.py
\
--train_dir
train_data
\
--model_dir
cluster_model
\
--vocab_path
vocab.txt
\
--batch_size
5
\
--role
pserver
\
--endpoints
127.0.0.1:6000,127.0.0.1:6001
\
--current_endpoint
127.0.0.1:6001
\
--trainers
2
\
>
pserver1.log 2>&1 &
# start trainer0
#CUDA_VISIBLE_DEVICES=1 python cluster_train.py \
python cluster_train.py
\
--train_dir
train_data
\
--model_dir
cluster_model
\
--vocab_path
vocab.txt
\
--batch_size
5
\
--print_batch
10
\
--use_cuda
0
\
--role
trainer
\
--endpoints
127.0.0.1:6000,127.0.0.1:6001
\
--trainers
2
\
--trainer_id
0
\
>
trainer0.log 2>&1 &
# start trainer1
#CUDA_VISIBLE_DEVICES=2 python cluster_train.py \
python cluster_train.py
\
--train_dir
train_data
\
--model_dir
cluster_model
\
--vocab_path
vocab.txt
\
--batch_size
5
\
--print_batch
10
\
--use_cuda
0
\
--role
trainer
\
--endpoints
127.0.0.1:6000,127.0.0.1:6001
\
--trainers
2
\
--trainer_id
1
\
>
trainer1.log 2>&1 &
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录