未验证 提交 872f494e 编写于 作者: X Xing Wu 提交者: GitHub

update 1.7_api for PaddleTextGEN lexical_analysis sentiment_classification (#4304)

* update 1.7_api for PaddleTextGEN lexical_analysis sentiment_classification
上级 6f04c0da
运行本目录下的范例模型需要安装PaddlePaddle Fluid 1.6版。如果您的 PaddlePaddle 安装版本低于此要求,请按照[安装文档](https://www.paddlepaddle.org.cn/#quick-start)中的说明更新 PaddlePaddle 安装版本。 运行本目录下的范例模型需要安装PaddlePaddle Fluid 1.7版本。如果您的 PaddlePaddle 安装版本低于此要求,请按照[安装文档](https://www.paddlepaddle.org.cn/#quick-start)中的说明更新 PaddlePaddle 安装版本。
# Sequence to Sequence (Seq2Seq) # Sequence to Sequence (Seq2Seq)
......
...@@ -93,7 +93,7 @@ def infer(): ...@@ -93,7 +93,7 @@ def infer():
# clone from default main program and use it as the validation program # clone from default main program and use it as the validation program
main_program = fluid.default_main_program() main_program = fluid.default_main_program()
main_program = main_program.clone(for_test=True) main_program = main_program.clone(for_test=True)
print([param.name for param in main_program.blocks[0].all_parameters()]) print([param.name for param in main_program.all_parameters()])
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = Executor(place) exe = Executor(place)
......
...@@ -16,7 +16,7 @@ Lexical Analysis of Chinese,简称 LAC,是一个联合的词法分析模型 ...@@ -16,7 +16,7 @@ Lexical Analysis of Chinese,简称 LAC,是一个联合的词法分析模型
#### 1.PaddlePaddle 安装 #### 1.PaddlePaddle 安装
本项目依赖 PaddlePaddle 1.6.0 及以上版本和PaddleHub 1.0.0及以上版本 ,PaddlePaddle安装请参考官网 [快速安装](http://www.paddlepaddle.org/paddle#quick-start),PaddleHub安装参考 [PaddleHub](https://github.com/PaddlePaddle/PaddleHub) 本项目依赖 PaddlePaddle 1.7 及以上版本和PaddleHub 1.0.0及以上版本 ,PaddlePaddle安装请参考官网 [快速安装](http://www.paddlepaddle.org/paddle#quick-start),PaddleHub安装参考 [PaddleHub](https://github.com/PaddlePaddle/PaddleHub)
> Warning: GPU 和 CPU 版本的 PaddlePaddle 分别是 paddlepaddle-gpu 和 paddlepaddle,请安装时注意区别。 > Warning: GPU 和 CPU 版本的 PaddlePaddle 分别是 paddlepaddle-gpu 和 paddlepaddle,请安装时注意区别。
......
...@@ -35,9 +35,9 @@ def create_model(args, vocab_size, num_labels, mode='train'): ...@@ -35,9 +35,9 @@ def create_model(args, vocab_size, num_labels, mode='train'):
"""create lac model""" """create lac model"""
# model's input data # model's input data
words = fluid.data(name='words', shape=[-1, 1], dtype='int64', lod_level=1) words = fluid.data(name='words', shape=[None, 1], dtype='int64', lod_level=1)
targets = fluid.data( targets = fluid.data(
name='targets', shape=[-1, 1], dtype='int64', lod_level=1) name='targets', shape=[None, 1], dtype='int64', lod_level=1)
# for inference process # for inference process
if mode == 'infer': if mode == 'infer':
...@@ -88,9 +88,10 @@ def create_pyreader(args, ...@@ -88,9 +88,10 @@ def create_pyreader(args,
return_reader=False, return_reader=False,
mode='train'): mode='train'):
# init reader # init reader
device_count = len(fluid.cuda_places()) if args.use_cuda else len(fluid.cpu_places())
if model == 'lac': if model == 'lac':
pyreader = fluid.io.PyReader( pyreader = fluid.io.DataLoader.from_generator(
feed_list=feed_list, feed_list=feed_list,
capacity=50, capacity=50,
use_double_buffer=True, use_double_buffer=True,
...@@ -101,19 +102,19 @@ def create_pyreader(args, ...@@ -101,19 +102,19 @@ def create_pyreader(args,
# create lac pyreader # create lac pyreader
if mode == 'train': if mode == 'train':
pyreader.decorate_sample_list_generator( pyreader.set_sample_list_generator(
fluid.io.batch( fluid.io.batch(
fluid.io.shuffle( fluid.io.shuffle(
reader.file_reader(file_name), reader.file_reader(file_name),
buf_size=args.traindata_shuffle_buffer), buf_size=args.traindata_shuffle_buffer),
batch_size=args.batch_size), batch_size=args.batch_size/device_count),
places=place) places=place)
else: else:
pyreader.decorate_sample_list_generator( pyreader.set_sample_list_generator(
fluid.io.batch( fluid.io.batch(
reader.file_reader( reader.file_reader(
file_name, mode=mode), file_name, mode=mode),
batch_size=args.batch_size), batch_size=args.batch_size/device_count),
places=place) places=place)
elif model == 'ernie': elif model == 'ernie':
...@@ -162,19 +163,19 @@ def create_ernie_model(args, ernie_config): ...@@ -162,19 +163,19 @@ def create_ernie_model(args, ernie_config):
# ERNIE's input data # ERNIE's input data
src_ids = fluid.data( src_ids = fluid.data(
name='src_ids', shape=[-1, args.max_seq_len, 1], dtype='int64') name='src_ids', shape=[None, args.max_seq_len, 1], dtype='int64')
sent_ids = fluid.data( sent_ids = fluid.data(
name='sent_ids', shape=[-1, args.max_seq_len, 1], dtype='int64') name='sent_ids', shape=[None, args.max_seq_len, 1], dtype='int64')
pos_ids = fluid.data( pos_ids = fluid.data(
name='pos_ids', shape=[-1, args.max_seq_len, 1], dtype='int64') name='pos_ids', shape=[None, args.max_seq_len, 1], dtype='int64')
input_mask = fluid.data( input_mask = fluid.data(
name='input_mask', shape=[-1, args.max_seq_len, 1], dtype='float32') name='input_mask', shape=[None, args.max_seq_len, 1], dtype='float32')
padded_labels = fluid.data( padded_labels = fluid.data(
name='padded_labels', shape=[-1, args.max_seq_len, 1], dtype='int64') name='padded_labels', shape=[None, args.max_seq_len, 1], dtype='int64')
seq_lens = fluid.data( seq_lens = fluid.data(
name='seq_lens', shape=[-1], dtype='int64', lod_level=0) name='seq_lens', shape=[None], dtype='int64', lod_level=0)
squeeze_labels = fluid.layers.squeeze(padded_labels, axes=[-1]) squeeze_labels = fluid.layers.squeeze(padded_labels, axes=[-1])
......
...@@ -29,7 +29,7 @@ ...@@ -29,7 +29,7 @@
1. PaddlePaddle 安装 1. PaddlePaddle 安装
本项目依赖于 PaddlePaddle Fluid 1.6 及以上版本,请参考 [安装指南](http://www.paddlepaddle.org/#quick-start) 进行安装 本项目依赖于 PaddlePaddle Fluid 1.7 及以上版本,请参考 [安装指南](http://www.paddlepaddle.org/#quick-start) 进行安装
2. 代码安装 2. 代码安装
......
...@@ -107,12 +107,12 @@ def test_inference_model(args): ...@@ -107,12 +107,12 @@ def test_inference_model(args):
params_filename="params.pdparams") params_filename="params.pdparams")
infer_data_generator = processor.data_generator( infer_data_generator = processor.data_generator(
batch_size=args.batch_size, batch_size=args.batch_size/dev_count,
phase="infer", phase="infer",
epoch=1, epoch=1,
shuffle=False) shuffle=False)
infer_pyreader.decorate_sample_list_generator(infer_data_generator) infer_pyreader.set_sample_list_generator(infer_data_generator)
inference(exe, test_prog, infer_pyreader, inference(exe, test_prog, infer_pyreader,
[probs.name], "infer") [probs.name], "infer")
......
...@@ -129,7 +129,7 @@ def test_inference_model(args): ...@@ -129,7 +129,7 @@ def test_inference_model(args):
assert (args.inference_model_dir) assert (args.inference_model_dir)
infer_data_generator = reader.data_generator( infer_data_generator = reader.data_generator(
input_file=args.test_set, input_file=args.test_set,
batch_size=args.batch_size, batch_size=args.batch_size/dev_count,
phase="infer", phase="infer",
epoch=1, epoch=1,
shuffle=False) shuffle=False)
...@@ -140,7 +140,7 @@ def test_inference_model(args): ...@@ -140,7 +140,7 @@ def test_inference_model(args):
model_filename="model.pdmodel", model_filename="model.pdmodel",
params_filename="params.pdparams") params_filename="params.pdparams")
infer_pyreader.decorate_batch_generator(infer_data_generator) infer_pyreader.set_batch_generator(infer_data_generator)
inference(exe, test_prog, infer_pyreader, inference(exe, test_prog, infer_pyreader,
[probs.name], "infer") [probs.name], "infer")
......
...@@ -39,14 +39,14 @@ def create_model(args, ...@@ -39,14 +39,14 @@ def create_model(args,
Create Model for sentiment classification Create Model for sentiment classification
""" """
data = fluid.layers.data( data = fluid.data(
name="src_ids", shape=[-1, args.max_seq_len], dtype='int64') name="src_ids", shape=[None, args.max_seq_len], dtype='int64')
label = fluid.layers.data( label = fluid.data(
name="label", shape=[-1, 1], dtype="int64") name="label", shape=[None, 1], dtype="int64")
seq_len = fluid.layers.data( seq_len = fluid.data(
name="seq_len", shape=[-1], dtype="int64") name="seq_len", shape=[None], dtype="int64")
data_reader = fluid.io.PyReader(feed_list=[data, label, seq_len], data_reader = fluid.io.DataLoader.from_generator(feed_list=[data, label, seq_len],
capacity=4, iterable=False) capacity=4, iterable=False)
if args.model_type == "bilstm_net": if args.model_type == "bilstm_net":
...@@ -151,7 +151,7 @@ def main(args): ...@@ -151,7 +151,7 @@ def main(args):
if args.do_train: if args.do_train:
train_data_generator = processor.data_generator( train_data_generator = processor.data_generator(
batch_size=args.batch_size, batch_size=args.batch_size/dev_count,
phase='train', phase='train',
epoch=args.epoch, epoch=args.epoch,
shuffle=True) shuffle=True)
...@@ -187,7 +187,7 @@ def main(args): ...@@ -187,7 +187,7 @@ def main(args):
if args.do_val: if args.do_val:
test_data_generator = processor.data_generator( test_data_generator = processor.data_generator(
batch_size=args.batch_size, batch_size=args.batch_size/dev_count,
phase='dev', phase='dev',
epoch=1, epoch=1,
shuffle=False) shuffle=False)
...@@ -204,7 +204,7 @@ def main(args): ...@@ -204,7 +204,7 @@ def main(args):
if args.do_infer: if args.do_infer:
infer_data_generator = processor.data_generator( infer_data_generator = processor.data_generator(
batch_size=args.batch_size, batch_size=args.batch_size/dev_count,
phase='infer', phase='infer',
epoch=1, epoch=1,
shuffle=False) shuffle=False)
...@@ -238,15 +238,15 @@ def main(args): ...@@ -238,15 +238,15 @@ def main(args):
if args.do_train: if args.do_train:
train_exe = exe train_exe = exe
train_reader.decorate_sample_list_generator(train_data_generator) train_reader.set_sample_list_generator(train_data_generator)
else: else:
train_exe = None train_exe = None
if args.do_val: if args.do_val:
test_exe = exe test_exe = exe
test_reader.decorate_sample_list_generator(test_data_generator) test_reader.set_sample_list_generator(test_data_generator)
if args.do_infer: if args.do_infer:
test_exe = exe test_exe = exe
infer_reader.decorate_sample_list_generator(infer_data_generator) infer_reader.set_sample_list_generator(infer_data_generator)
if args.do_train: if args.do_train:
train_reader.start() train_reader.start()
......
...@@ -37,18 +37,18 @@ from config import PDConfig ...@@ -37,18 +37,18 @@ from config import PDConfig
from utils import init_checkpoint from utils import init_checkpoint
def ernie_pyreader(args, pyreader_name): def ernie_pyreader(args, pyreader_name):
src_ids = fluid.layers.data( src_ids = fluid.data(
name="src_ids", shape=[-1, args.max_seq_len, 1], dtype="int64") name="src_ids", shape=[None, args.max_seq_len, 1], dtype="int64")
sent_ids = fluid.layers.data( sent_ids = fluid.data(
name="sent_ids", shape=[-1, args.max_seq_len, 1], dtype="int64") name="sent_ids", shape=[None, args.max_seq_len, 1], dtype="int64")
pos_ids = fluid.layers.data( pos_ids = fluid.data(
name="pos_ids", shape=[-1, args.max_seq_len, 1], dtype="int64") name="pos_ids", shape=[None, args.max_seq_len, 1], dtype="int64")
input_mask = fluid.layers.data( input_mask = fluid.data(
name="input_mask", shape=[-1, args.max_seq_len, 1], dtype="float32") name="input_mask", shape=[None, args.max_seq_len, 1], dtype="float32")
labels = fluid.layers.data( labels = fluid.data(
name="labels", shape=[-1, 1], dtype="int64") name="labels", shape=[None, 1], dtype="int64")
seq_lens = fluid.layers.data( seq_lens = fluid.data(
name="seq_lens", shape=[-1], dtype="int64") name="seq_lens", shape=[None], dtype="int64")
pyreader = fluid.io.DataLoader.from_generator( pyreader = fluid.io.DataLoader.from_generator(
feed_list=[src_ids, sent_ids, pos_ids, input_mask, labels, seq_lens], feed_list=[src_ids, sent_ids, pos_ids, input_mask, labels, seq_lens],
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册