Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
weixin_51232023
models
提交
024743c3
M
models
项目概览
weixin_51232023
/
models
与 Fork 源项目一致
Fork自
PaddlePaddle / models
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
024743c3
编写于
2月 27, 2019
作者:
Z
zhangwenhui03
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix bug
上级
f503908d
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
63 addition
and
30 deletion
+63
-30
fluid/PaddleRec/gru4rec/cluster_train.py
fluid/PaddleRec/gru4rec/cluster_train.py
+46
-22
fluid/PaddleRec/gru4rec/infer.py
fluid/PaddleRec/gru4rec/infer.py
+17
-8
未找到文件。
fluid/PaddleRec/gru4rec/cluster_train.py
浏览文件 @
024743c3
...
...
@@ -13,22 +13,26 @@ import net
SEED
=
102
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
"gru4rec benchmark."
)
parser
.
add_argument
(
'--train_dir'
,
type
=
str
,
default
=
'train_data'
,
help
=
'train file address'
)
parser
.
add_argument
(
'--vocab_path'
,
type
=
str
,
default
=
'vocab.txt'
,
help
=
'vocab file address'
)
parser
.
add_argument
(
'--is_local'
,
type
=
int
,
default
=
1
,
help
=
'whether local'
)
'--train_dir'
,
type
=
str
,
default
=
'train_data'
,
help
=
'train file address'
)
parser
.
add_argument
(
'--hid_size'
,
type
=
int
,
default
=
100
,
help
=
'hid size'
)
'--vocab_path'
,
type
=
str
,
default
=
'vocab.txt'
,
help
=
'vocab file address'
)
parser
.
add_argument
(
'--is_local'
,
type
=
int
,
default
=
1
,
help
=
'whether local'
)
parser
.
add_argument
(
'--hid_size'
,
type
=
int
,
default
=
100
,
help
=
'hid size'
)
parser
.
add_argument
(
'--model_dir'
,
type
=
str
,
default
=
'model_recall20'
,
help
=
'model dir'
)
parser
.
add_argument
(
'--batch_size'
,
type
=
int
,
default
=
5
,
help
=
'num of batch size'
)
parser
.
add_argument
(
'--pass_num'
,
type
=
int
,
default
=
10
,
help
=
'num of epoch'
)
parser
.
add_argument
(
'--pass_num'
,
type
=
int
,
default
=
10
,
help
=
'num of epoch'
)
parser
.
add_argument
(
'--print_batch'
,
type
=
int
,
default
=
10
,
help
=
'num of print batch'
)
parser
.
add_argument
(
...
...
@@ -40,19 +44,33 @@ def parse_args():
parser
.
add_argument
(
'--role'
,
type
=
str
,
default
=
'pserver'
,
help
=
'trainer or pserver'
)
parser
.
add_argument
(
'--endpoints'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The pserver endpoints, like: 127.0.0.1:6000, 127.0.0.1:6001'
)
parser
.
add_argument
(
'--current_endpoint'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The current_endpoint'
)
parser
.
add_argument
(
'--trainer_id'
,
type
=
int
,
default
=
0
,
help
=
'trainer id ,only trainer_id=0 save model'
)
parser
.
add_argument
(
'--trainers'
,
type
=
int
,
default
=
1
,
help
=
'The num of trianers, (default: 1)'
)
'--endpoints'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The pserver endpoints, like: 127.0.0.1:6000, 127.0.0.1:6001'
)
parser
.
add_argument
(
'--current_endpoint'
,
type
=
str
,
default
=
'127.0.0.1:6000'
,
help
=
'The current_endpoint'
)
parser
.
add_argument
(
'--trainer_id'
,
type
=
int
,
default
=
0
,
help
=
'trainer id ,only trainer_id=0 save model'
)
parser
.
add_argument
(
'--trainers'
,
type
=
int
,
default
=
1
,
help
=
'The num of trianers, (default: 1)'
)
args
=
parser
.
parse_args
()
return
args
def
get_cards
(
args
):
return
args
.
num_devices
def
train
():
""" do training """
args
=
parse_args
()
...
...
@@ -67,12 +85,13 @@ def train():
buffer_size
=
1000
,
word_freq_threshold
=
0
,
is_train
=
True
)
# Train program
src_wordseq
,
dst_wordseq
,
avg_cost
,
acc
=
net
.
network
(
vocab_size
=
vocab_size
,
hid_size
=
hid_size
)
src_wordseq
,
dst_wordseq
,
avg_cost
,
acc
=
net
.
all_vocab_network
(
vocab_size
=
vocab_size
,
hid_size
=
hid_size
)
# Optimization to minimize lost
sgd_optimizer
=
fluid
.
optimizer
.
SGD
(
learning_rate
=
args
.
base_lr
)
sgd_optimizer
.
minimize
(
avg_cost
)
def
train_loop
(
main_program
):
""" train network """
pass_num
=
args
.
pass_num
...
...
@@ -97,9 +116,11 @@ def train():
lod_dst_wordseq
=
utils
.
to_lodtensor
([
dat
[
1
]
for
dat
in
data
],
place
)
ret_avg_cost
=
exe
.
run
(
main_program
,
feed
=
{
"src_wordseq"
:
lod_src_wordseq
,
"dst_wordseq"
:
lod_dst_wordseq
},
fetch_list
=
fetch_list
)
feed
=
{
"src_wordseq"
:
lod_src_wordseq
,
"dst_wordseq"
:
lod_dst_wordseq
},
fetch_list
=
fetch_list
)
avg_ppl
=
np
.
exp
(
ret_avg_cost
[
0
])
newest_ppl
=
np
.
mean
(
avg_ppl
)
if
i
%
args
.
print_batch
==
0
:
...
...
@@ -113,7 +134,8 @@ def train():
feed_var_names
=
[
"src_wordseq"
,
"dst_wordseq"
]
fetch_vars
=
[
avg_cost
,
acc
]
if
args
.
trainer_id
==
0
:
fluid
.
io
.
save_inference_model
(
save_dir
,
feed_var_names
,
fetch_vars
,
exe
)
fluid
.
io
.
save_inference_model
(
save_dir
,
feed_var_names
,
fetch_vars
,
exe
)
print
(
"model saved in %s"
%
save_dir
)
print
(
"finish training"
)
...
...
@@ -123,7 +145,8 @@ def train():
else
:
print
(
"run distribute training"
)
t
=
fluid
.
DistributeTranspiler
()
t
.
transpile
(
args
.
trainer_id
,
pservers
=
args
.
endpoints
,
trainers
=
args
.
trainers
)
t
.
transpile
(
args
.
trainer_id
,
pservers
=
args
.
endpoints
,
trainers
=
args
.
trainers
)
if
args
.
role
==
"pserver"
:
print
(
"run psever"
)
pserver_prog
=
t
.
get_pserver_program
(
args
.
current_endpoint
)
...
...
@@ -136,5 +159,6 @@ def train():
print
(
"run trainer"
)
train_loop
(
t
.
get_trainer_program
())
if
__name__
==
"__main__"
:
train
()
fluid/PaddleRec/gru4rec/infer.py
浏览文件 @
024743c3
...
...
@@ -11,23 +11,27 @@ import paddle
import
utils
def
parse_args
():
parser
=
argparse
.
ArgumentParser
(
"gru4rec benchmark."
)
parser
.
add_argument
(
'--test_dir'
,
type
=
str
,
default
=
'test_data'
,
help
=
'test file address'
)
parser
.
add_argument
(
'--start_index'
,
type
=
int
,
default
=
'1'
,
help
=
'start index'
)
'--start_index'
,
type
=
int
,
default
=
'1'
,
help
=
'start index'
)
parser
.
add_argument
(
'--last_index'
,
type
=
int
,
default
=
'10'
,
help
=
'end index'
)
parser
.
add_argument
(
'--
last_index'
,
type
=
int
,
default
=
'10'
,
help
=
'end index'
)
'--
model_dir'
,
type
=
str
,
default
=
'model_recall20'
,
help
=
'model dir'
)
parser
.
add_argument
(
'--
model_dir'
,
type
=
str
,
default
=
'model_recall20'
,
help
=
'model dir'
)
'--
use_cuda'
,
type
=
int
,
default
=
'0'
,
help
=
'whether use cuda'
)
parser
.
add_argument
(
'--
use_cuda'
,
type
=
int
,
default
=
'1'
,
help
=
'whether use cuda'
)
'--
batch_size'
,
type
=
int
,
default
=
'5'
,
help
=
'batch_size'
)
parser
.
add_argument
(
'--
batch_size'
,
type
=
int
,
default
=
'5'
,
help
=
'batch_size'
)
'--
vocab_path'
,
type
=
str
,
default
=
'vocab.txt'
,
help
=
'vocab file'
)
args
=
parser
.
parse_args
()
return
args
def
infer
(
test_reader
,
use_cuda
,
model_path
):
""" inference function """
place
=
fluid
.
CUDAPlace
(
0
)
if
use_cuda
else
fluid
.
CPUPlace
()
...
...
@@ -72,11 +76,16 @@ if __name__ == "__main__":
test_dir
=
args
.
test_dir
model_dir
=
args
.
model_dir
batch_size
=
args
.
batch_size
vocab_path
=
args
.
vocab_path
use_cuda
=
True
if
args
.
use_cuda
else
False
print
(
"start index: "
,
start_index
,
" last_index:"
,
last_index
)
print
(
"start index: "
,
start_index
,
" last_index:"
,
last_index
)
vocab_size
,
test_reader
=
utils
.
prepare_data
(
test_dir
,
""
,
batch_size
=
batch_size
,
buffer_size
=
1000
,
word_freq_threshold
=
0
,
is_train
=
False
)
test_dir
,
vocab_path
,
batch_size
=
batch_size
,
buffer_size
=
1000
,
word_freq_threshold
=
0
,
is_train
=
False
)
for
epoch
in
range
(
start_index
,
last_index
+
1
):
epoch_path
=
model_dir
+
"/epoch_"
+
str
(
epoch
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录