Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
models
提交
d52ff5a0
M
models
项目概览
PaddlePaddle
/
models
1 年多 前同步成功
通知
222
Star
6828
Fork
2962
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
602
列表
看板
标记
里程碑
合并请求
255
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
models
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
602
Issue
602
列表
看板
标记
里程碑
合并请求
255
合并请求
255
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
d52ff5a0
编写于
5月 19, 2020
作者:
L
liu zhengxi
提交者:
GitHub
5月 19, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
update apis according to cards, test=develop (#4638)
上级
aa86b479
变更
2
显示空白变更内容
内联
并排
Showing
2 changed file
with
70 addition
and
87 deletion
+70
-87
PaddleNLP/machine_translation/transformer/utils/input_field.py
...eNLP/machine_translation/transformer/utils/input_field.py
+0
-52
PaddleNLP/similarity_net/utils.py
PaddleNLP/similarity_net/utils.py
+70
-35
未找到文件。
PaddleNLP/machine_translation/transformer/utils/input_field.py
浏览文件 @
d52ff5a0
...
@@ -25,58 +25,6 @@ import numpy as np
...
@@ -25,58 +25,6 @@ import numpy as np
import
paddle.fluid
as
fluid
import
paddle.fluid
as
fluid
class
Placeholder
(
object
):
def
__init__
(
self
):
self
.
shapes
=
[]
self
.
dtypes
=
[]
self
.
lod_levels
=
[]
self
.
names
=
[]
def
__init__
(
self
,
input_shapes
):
self
.
shapes
=
[]
self
.
dtypes
=
[]
self
.
lod_levels
=
[]
self
.
names
=
[]
for
new_holder
in
input_shapes
:
shape
=
new_holder
[
0
]
dtype
=
new_holder
[
1
]
lod_level
=
new_holder
[
2
]
if
len
(
new_holder
)
>=
3
else
0
name
=
new_holder
[
3
]
if
len
(
new_holder
)
>=
4
else
""
self
.
append_placeholder
(
shape
,
dtype
,
lod_level
=
lod_level
,
name
=
name
)
def
append_placeholder
(
self
,
shape
,
dtype
,
lod_level
=
0
,
name
=
""
):
self
.
shapes
.
append
(
shape
)
self
.
dtypes
.
append
(
dtype
)
self
.
lod_levels
.
append
(
lod_level
)
self
.
names
.
append
(
name
)
def
build
(
self
,
capacity
,
reader_name
,
use_double_buffer
=
False
):
pyreader
=
fluid
.
layers
.
py_reader
(
capacity
=
capacity
,
shapes
=
self
.
shapes
,
dtypes
=
self
.
dtypes
,
lod_levels
=
self
.
lod_levels
,
name
=
reader_name
,
use_double_buffer
=
use_double_buffer
)
return
[
pyreader
,
fluid
.
layers
.
read_file
(
pyreader
)]
def
__add__
(
self
,
new_holder
):
assert
isinstance
(
new_holder
,
tuple
)
or
isinstance
(
new_holder
,
list
)
assert
len
(
new_holder
)
>=
2
shape
=
new_holder
[
0
]
dtype
=
new_holder
[
1
]
lod_level
=
new_holder
[
2
]
if
len
(
new_holder
)
>=
3
else
0
name
=
new_holder
[
3
]
if
len
(
new_holder
)
>=
4
else
""
self
.
append_placeholder
(
shape
,
dtype
,
lod_level
=
lod_level
,
name
=
name
)
class
InputField
(
object
):
class
InputField
(
object
):
"""
"""
A high-level API for handling inputs in PaddlePaddle.
A high-level API for handling inputs in PaddlePaddle.
...
...
PaddleNLP/similarity_net/utils.py
浏览文件 @
d52ff5a0
...
@@ -30,6 +30,7 @@ import io
...
@@ -30,6 +30,7 @@ import io
******functions for file processing******
******functions for file processing******
"""
"""
def
load_vocab
(
file_path
):
def
load_vocab
(
file_path
):
"""
"""
load the given vocabulary
load the given vocabulary
...
@@ -56,8 +57,11 @@ def get_result_file(args):
...
@@ -56,8 +57,11 @@ def get_result_file(args):
"""
"""
with
io
.
open
(
args
.
test_data_dir
,
"r"
,
encoding
=
"utf8"
)
as
test_file
:
with
io
.
open
(
args
.
test_data_dir
,
"r"
,
encoding
=
"utf8"
)
as
test_file
:
with
io
.
open
(
"predictions.txt"
,
"r"
,
encoding
=
"utf8"
)
as
predictions_file
:
with
io
.
open
(
with
io
.
open
(
args
.
test_result_path
,
"w"
,
encoding
=
"utf8"
)
as
test_result_file
:
"predictions.txt"
,
"r"
,
encoding
=
"utf8"
)
as
predictions_file
:
with
io
.
open
(
args
.
test_result_path
,
"w"
,
encoding
=
"utf8"
)
as
test_result_file
:
test_datas
=
[
line
.
strip
(
"
\n
"
)
for
line
in
test_file
]
test_datas
=
[
line
.
strip
(
"
\n
"
)
for
line
in
test_file
]
predictions
=
[
line
.
strip
(
"
\n
"
)
for
line
in
predictions_file
]
predictions
=
[
line
.
strip
(
"
\n
"
)
for
line
in
predictions_file
]
for
test_data
,
prediction
in
zip
(
test_datas
,
predictions
):
for
test_data
,
prediction
in
zip
(
test_datas
,
predictions
):
...
@@ -165,49 +169,81 @@ class ArgumentGroup(object):
...
@@ -165,49 +169,81 @@ class ArgumentGroup(object):
help
=
help
+
' Default: %(default)s.'
,
help
=
help
+
' Default: %(default)s.'
,
**
kwargs
)
**
kwargs
)
class
ArgConfig
(
object
):
class
ArgConfig
(
object
):
def
__init__
(
self
):
def
__init__
(
self
):
parser
=
argparse
.
ArgumentParser
()
parser
=
argparse
.
ArgumentParser
()
model_g
=
ArgumentGroup
(
parser
,
"model"
,
"model configuration and paths."
)
model_g
=
ArgumentGroup
(
parser
,
"model"
,
model_g
.
add_arg
(
"config_path"
,
str
,
None
,
"Path to the json file for EmoTect model config."
)
"model configuration and paths."
)
model_g
.
add_arg
(
"init_checkpoint"
,
str
,
None
,
"Init checkpoint to resume training from."
)
model_g
.
add_arg
(
"config_path"
,
str
,
None
,
model_g
.
add_arg
(
"output_dir"
,
str
,
None
,
"Directory path to save checkpoints"
)
"Path to the json file for EmoTect model config."
)
model_g
.
add_arg
(
"task_mode"
,
str
,
None
,
"task mode: pairwise or pointwise"
)
model_g
.
add_arg
(
"init_checkpoint"
,
str
,
None
,
"Init checkpoint to resume training from."
)
model_g
.
add_arg
(
"output_dir"
,
str
,
None
,
"Directory path to save checkpoints"
)
model_g
.
add_arg
(
"task_mode"
,
str
,
None
,
"task mode: pairwise or pointwise"
)
train_g
=
ArgumentGroup
(
parser
,
"training"
,
"training options."
)
train_g
=
ArgumentGroup
(
parser
,
"training"
,
"training options."
)
train_g
.
add_arg
(
"epoch"
,
int
,
10
,
"Number of epoches for training."
)
train_g
.
add_arg
(
"epoch"
,
int
,
10
,
"Number of epoches for training."
)
train_g
.
add_arg
(
"save_steps"
,
int
,
200
,
"The steps interval to save checkpoints."
)
train_g
.
add_arg
(
"save_steps"
,
int
,
200
,
train_g
.
add_arg
(
"validation_steps"
,
int
,
100
,
"The steps interval to evaluate model performance."
)
"The steps interval to save checkpoints."
)
train_g
.
add_arg
(
"validation_steps"
,
int
,
100
,
"The steps interval to evaluate model performance."
)
log_g
=
ArgumentGroup
(
parser
,
"logging"
,
"logging related"
)
log_g
=
ArgumentGroup
(
parser
,
"logging"
,
"logging related"
)
log_g
.
add_arg
(
"skip_steps"
,
int
,
10
,
"The steps interval to print loss."
)
log_g
.
add_arg
(
"skip_steps"
,
int
,
10
,
log_g
.
add_arg
(
"verbose_result"
,
bool
,
True
,
"Whether to output verbose result."
)
"The steps interval to print loss."
)
log_g
.
add_arg
(
"test_result_path"
,
str
,
"test_result"
,
"Directory path to test result."
)
log_g
.
add_arg
(
"verbose_result"
,
bool
,
True
,
log_g
.
add_arg
(
"infer_result_path"
,
str
,
"infer_result"
,
"Directory path to infer result."
)
"Whether to output verbose result."
)
log_g
.
add_arg
(
"test_result_path"
,
str
,
"test_result"
,
data_g
=
ArgumentGroup
(
parser
,
"data"
,
"Data paths, vocab paths and data processing options"
)
"Directory path to test result."
)
data_g
.
add_arg
(
"train_data_dir"
,
str
,
None
,
"Directory path to training data."
)
log_g
.
add_arg
(
"infer_result_path"
,
str
,
"infer_result"
,
data_g
.
add_arg
(
"valid_data_dir"
,
str
,
None
,
"Directory path to valid data."
)
"Directory path to infer result."
)
data_g
.
add_arg
(
"test_data_dir"
,
str
,
None
,
"Directory path to testing data."
)
data_g
.
add_arg
(
"infer_data_dir"
,
str
,
None
,
"Directory path to infer data."
)
data_g
=
ArgumentGroup
(
parser
,
"data"
,
"Data paths, vocab paths and data processing options"
)
data_g
.
add_arg
(
"train_data_dir"
,
str
,
None
,
"Directory path to training data."
)
data_g
.
add_arg
(
"valid_data_dir"
,
str
,
None
,
"Directory path to valid data."
)
data_g
.
add_arg
(
"test_data_dir"
,
str
,
None
,
"Directory path to testing data."
)
data_g
.
add_arg
(
"infer_data_dir"
,
str
,
None
,
"Directory path to infer data."
)
data_g
.
add_arg
(
"vocab_path"
,
str
,
None
,
"Vocabulary path."
)
data_g
.
add_arg
(
"vocab_path"
,
str
,
None
,
"Vocabulary path."
)
data_g
.
add_arg
(
"batch_size"
,
int
,
32
,
"Total examples' number in batch for training."
)
data_g
.
add_arg
(
"batch_size"
,
int
,
32
,
"Total examples' number in batch for training."
)
run_type_g
=
ArgumentGroup
(
parser
,
"run_type"
,
"running type options."
)
run_type_g
=
ArgumentGroup
(
parser
,
"run_type"
,
"running type options."
)
run_type_g
.
add_arg
(
"use_cuda"
,
bool
,
False
,
"If set, use GPU for training."
)
run_type_g
.
add_arg
(
"use_cuda"
,
bool
,
False
,
run_type_g
.
add_arg
(
"task_name"
,
str
,
None
,
"The name of task to perform sentiment classification."
)
"If set, use GPU for training."
)
run_type_g
.
add_arg
(
"do_train"
,
bool
,
False
,
"Whether to perform training."
)
run_type_g
.
add_arg
(
"task_name"
,
str
,
None
,
"The name of task to perform sentiment classification."
)
run_type_g
.
add_arg
(
"do_train"
,
bool
,
False
,
"Whether to perform training."
)
run_type_g
.
add_arg
(
"do_valid"
,
bool
,
False
,
"Whether to perform dev."
)
run_type_g
.
add_arg
(
"do_valid"
,
bool
,
False
,
"Whether to perform dev."
)
run_type_g
.
add_arg
(
"do_test"
,
bool
,
False
,
"Whether to perform testing."
)
run_type_g
.
add_arg
(
"do_test"
,
bool
,
False
,
run_type_g
.
add_arg
(
"do_infer"
,
bool
,
False
,
"Whether to perform inference."
)
"Whether to perform testing."
)
run_type_g
.
add_arg
(
"compute_accuracy"
,
bool
,
False
,
"Whether to compute accuracy."
)
run_type_g
.
add_arg
(
"do_infer"
,
bool
,
False
,
run_type_g
.
add_arg
(
"lamda"
,
float
,
0.91
,
"When task_mode is pairwise, lamda is the threshold for calculating the accuracy."
)
"Whether to perform inference."
)
run_type_g
.
add_arg
(
"compute_accuracy"
,
bool
,
False
,
"Whether to compute accuracy."
)
run_type_g
.
add_arg
(
"lamda"
,
float
,
0.91
,
"When task_mode is pairwise, lamda is the threshold for calculating the accuracy."
)
custom_g
=
ArgumentGroup
(
parser
,
"customize"
,
"customized options."
)
custom_g
=
ArgumentGroup
(
parser
,
"customize"
,
"customized options."
)
self
.
custom_g
=
custom_g
self
.
custom_g
=
custom_g
parser
.
add_argument
(
'--enable_ce'
,
action
=
'store_true'
,
help
=
'If set, run the task with continuous evaluation logs.'
)
parser
.
add_argument
(
'--enable_ce'
,
action
=
'store_true'
,
help
=
'If set, run the task with continuous evaluation logs.'
)
self
.
parser
=
parser
self
.
parser
=
parser
...
@@ -355,10 +391,9 @@ def init_checkpoint(exe, init_checkpoint_path, main_program):
...
@@ -355,10 +391,9 @@ def init_checkpoint(exe, init_checkpoint_path, main_program):
return
False
return
False
return
os
.
path
.
exists
(
os
.
path
.
join
(
init_checkpoint_path
,
var
.
name
))
return
os
.
path
.
exists
(
os
.
path
.
join
(
init_checkpoint_path
,
var
.
name
))
fluid
.
io
.
load_vars
(
var_list
=
[]
exe
,
for
var
in
main_program
.
list_vars
():
i
nit_checkpoint_path
,
i
f
fluid
.
io
.
is_persistable
(
var
)
and
existed_persitables
(
var
):
main_program
=
main_program
,
var_list
.
append
(
var
)
predicate
=
existed_persitables
)
fluid
.
load
(
main_program
,
init_checkpoint_path
,
exe
,
var_list
=
var_list
)
print
(
"Load model from {}"
.
format
(
init_checkpoint_path
))
print
(
"Load model from {}"
.
format
(
init_checkpoint_path
))
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录