未验证 提交 c27a0a2e 编写于 作者: Y Yibing Liu 提交者: GitHub

Upgrade APIs for word2vec & label_semantic_roles (#2115)

* Upgrade APIs for word2vec & label_semantic_roles

* Some more fixes
上级 f8af27a8
...@@ -388,8 +388,8 @@ The data introduction section mentions the payment of the CoNLL 2005 training se ...@@ -388,8 +388,8 @@ The data introduction section mentions the payment of the CoNLL 2005 training se
crf_decode = fluid.layers.crf_decoding( crf_decode = fluid.layers.crf_decoding(
input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))
train_data = paddle.batch( train_data = fluid.io.batch(
paddle.reader.shuffle( fluid.io.shuffle(
paddle.dataset.conll05.test(), buf_size=8192), paddle.dataset.conll05.test(), buf_size=8192),
batch_size=BATCH_SIZE) batch_size=BATCH_SIZE)
......
...@@ -143,8 +143,8 @@ def train(use_cuda, save_dirname=None, is_local=True): ...@@ -143,8 +143,8 @@ def train(use_cuda, save_dirname=None, is_local=True):
# define network topology # define network topology
feature_out = db_lstm(**locals()) feature_out = db_lstm(**locals())
target = fluid.layers.data( target = fluid.data(
name='target', shape=[1], dtype='int64', lod_level=1) name='target', shape=[None, 1], dtype='int64', lod_level=1)
crf_cost = fluid.layers.linear_chain_crf( crf_cost = fluid.layers.linear_chain_crf(
input=feature_out, input=feature_out,
label=target, label=target,
...@@ -165,11 +165,11 @@ def train(use_cuda, save_dirname=None, is_local=True): ...@@ -165,11 +165,11 @@ def train(use_cuda, save_dirname=None, is_local=True):
input=feature_out, param_attr=fluid.ParamAttr(name='crfw')) input=feature_out, param_attr=fluid.ParamAttr(name='crfw'))
if args.enable_ce: if args.enable_ce:
train_data = paddle.batch( train_data = fluid.io.batch(
paddle.dataset.conll05.test(), batch_size=BATCH_SIZE) paddle.dataset.conll05.test(), batch_size=BATCH_SIZE)
else: else:
train_data = paddle.batch( train_data = fluid.io.batch(
paddle.reader.shuffle( fluid.io.shuffle(
paddle.dataset.conll05.test(), buf_size=8192), paddle.dataset.conll05.test(), buf_size=8192),
batch_size=BATCH_SIZE) batch_size=BATCH_SIZE)
......
...@@ -289,15 +289,15 @@ def optimizer_func(): ...@@ -289,15 +289,15 @@ def optimizer_func():
- Now we can start training. This version is much simpler than before. We have ready-made training and test sets: `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()`. Both will return a reader. In PaddlePaddle, the reader is a Python function that reads the next piece of data when called each time . It is a Python generator. - Now we can start training. This version is much simpler than before. We have ready-made training and test sets: `paddle.dataset.imikolov.train()` and `paddle.dataset.imikolov.test()`. Both will return a reader. In PaddlePaddle, the reader is a Python function that reads the next piece of data when called each time . It is a Python generator.
`paddle.batch` will read in a reader and output a batched reader. We can also output the training of each step and batch during the training process. `fluid.io.batch` will read in a reader and output a batched reader. We can also output the training of each step and batch during the training process.
```python ```python
def train(if_use_cuda, params_dirname, is_sparse=True): def train(if_use_cuda, params_dirname, is_sparse=True):
place = fluid.CUDAPlace(0) if if_use_cuda else fluid.CPUPlace() place = fluid.CUDAPlace(0) if if_use_cuda else fluid.CPUPlace()
train_reader = paddle.batch( train_reader = fluid.io.batch(
paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE) paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
test_reader = paddle.batch( test_reader = fluid.io.batch(
paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE) paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE)
first_word = fluid.data(name='firstw', shape=[None, 1], dtype='int64') first_word = fluid.data(name='firstw', shape=[None, 1], dtype='int64')
......
...@@ -98,9 +98,9 @@ def optimizer_func(): ...@@ -98,9 +98,9 @@ def optimizer_func():
def train(if_use_cuda, params_dirname, is_sparse=True): def train(if_use_cuda, params_dirname, is_sparse=True):
place = fluid.CUDAPlace(0) if if_use_cuda else fluid.CPUPlace() place = fluid.CUDAPlace(0) if if_use_cuda else fluid.CPUPlace()
train_reader = paddle.batch( train_reader = fluid.io.batch(
paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE) paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
test_reader = paddle.batch( test_reader = fluid.io.batch(
paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE) paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE)
first_word = fluid.data(name='firstw', shape=[None, 1], dtype='int64') first_word = fluid.data(name='firstw', shape=[None, 1], dtype='int64')
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册