Merge pull request #54 from 0YuanZhang0/seq_tag

seq_tag

examples/sequence_tagging/README.md

@@ -6,7 +6,7 @@ Sequence Tagging，是一个序列标注模型，模型可用于实现，分词
 
 |模型|Precision|Recall|F1-score|
 |:-:|:-:|:-:|:-:|
-|Lexical Analysis|88.26%|89.20%|88.73%|
+|Lexical Analysis|89.57%|89.96%|89.76%|
 
 ## 2. 快速开始
 
@@ -22,7 +22,7 @@ Sequence Tagging，是一个序列标注模型，模型可用于实现，分词
 克隆工具集代码库到本地
 ```bash
  git clone https://github.com/PaddlePaddle/hapi.git
- cd hapi/sequence_tagging
+ cd hapi/examples/sequence_tagging
 ```
 
 #### 3. 环境依赖
@@ -70,7 +70,7 @@ python -u train.py \
           --dynamic False
 
 # --device: 使用gpu设备还是cpu设备
-# --dynamic： 是否使用动态图模式进行训练，如果使用静态图训练，设置为True, 动态图设置为False
+# --dynamic： 是否使用动态图模式进行训练，如果使用静态图训练，设置为False, 动态图设置为True
 ```
 
 GPU上多卡训练
@@ -84,7 +84,7 @@ python -m paddle.distributed.launch --selected_gpus=0,1,2,3  train.py \
           --dynamic False
 
 # --device: 使用gpu设备还是cpu设备
-# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为True, 动态图设置为False
+# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为False, 动态图设置为True
 ```
 
 CPU上训练
@@ -95,7 +95,7 @@ python -u train.py \
           --dynamic False
 
 # --device: 使用gpu设备还是cpu设备
-# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为True, 动态图设置为False
+# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为False, 动态图设置为True
 ```
 
 ### 模型预测
@@ -105,15 +105,13 @@ python -u train.py \
 python predict.py \
       --init_from_checkpoint  model_baseline/params \
       --output_file predict.result  \
-      --mode predict \
       --device cpu  \
       --dynamic False
 
 # --init_from_checkpoint: 初始化模型
 # --output_file: 预测结果文件
 # --device: 使用gpu还是cpu设备
-# --mode: 开启模式, 设置为train时，进行训练，设置为predict时进行预测
-# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为True, 动态图设置为False
+# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为False, 动态图设置为True
 ```
 
 ### 模型评估
@@ -123,14 +121,12 @@ python predict.py \
 # baseline model
 python eval.py \
         --init_from_checkpoint  ./model_baseline/params \
-        --mode predict \
         --device cpu  \
         --dynamic False
 
 # --init_from_checkpoint: 初始化模型
 # --device: 使用gpu还是cpu设备
-# --mode: 开启模式, 设置为train时，进行训练，设置为predict时进行预测
-# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为True, 动态图设置为False
+# --dynamic: 是否使用动态图模式进行训练，如果使用静态图训练，设置为False, 动态图设置为True
 ```
 
 
@@ -168,7 +164,7 @@ Overall Architecture of GRU-CRF-MODEL
 训练使用的数据可以由用户根据实际的应用场景，自己组织数据。除了第一行是 `text_a\tlabel` 固定的开头，后面的每行数据都是由两列组成，以制表符分隔，第一列是 utf-8 编码的中文文本，以 `\002` 分割，第二列是对应每个字的标注，以 `\002` 分隔。我们采用 IOB2 标注体系，即以 X-B 作为类型为 X 的词的开始，以 X-I 作为类型为 X 的词的持续，以 O 表示不关注的字（实际上，在词性、专名联合标注中，不存在 O ）。示例如下：
 
 ```text
-除\002了\002他\002续\002任\002十\002二\002届\002政\002协\002委\002员\002,\002马\002化\002腾\002,\002雷\002军\002,\002李\002彦\002宏\002也\002被\002推\002选\002为\002新\002一\002届\002全\002国\002人\002大\002代\002表\002或\002全\002国\002政\002协\002委\002员	p-B\002p-I\002r-B\002v-B\002v-I\002m-B\002m-I\002m-I\002ORG-B\002ORG-I\002n-B\002n-I\002w-B\002PER-B\002PER-I\002PER-I\002w-B\002PER-B\002PER-I\002w-B\002PER-B\002PER-I\002PER-I\002d-B\002p-B\002v-B\002v-I\002v-B\002a-B\002m-B\002m-I\002ORG-B\002ORG-I\002ORG-I\002ORG-I\002n-B\002n-I\002c-B\002n-B\002n-I\002ORG-B\002ORG-I\002n-B\002n-I
+除\002了\002他\002续\002任\002十\002二\002届\002政\002协\002委\002员\002,\002马\002化\002腾\002,\002雷\002军\002,\002李\002彦\002宏\002也\002被\002推\002选\002为\002新\002一\002届\002全\002国\002人\002大\002代\002表\002或\002全\002国\002政\002协\002委\002员    p-B\002p-I\002r-B\002v-B\002v-I\002m-B\002m-I\002m-I\002ORG-B\002ORG-I\002n-B\002n-I\002w-B\002PER-B\002PER-I\002PER-I\002w-B\002PER-B\002PER-I\002w-B\002PER-B\002PER-I\002PER-I\002d-B\002p-B\002v-B\002v-I\002v-B\002a-B\002m-B\002m-I\002ORG-B\002ORG-I\002ORG-I\002ORG-I\002n-B\002n-I\002c-B\002n-B\002n-I\002ORG-B\002ORG-I\002n-B\002n-I
 ```
 
 + 我们随同代码一并发布了完全版的模型和相关的依赖数据。但是，由于模型的训练数据过于庞大，我们没有发布训练数据，仅在`data`目录下放置少数样本用以示例输入数据格式。
@@ -196,6 +192,7 @@ Overall Architecture of GRU-CRF-MODEL
 ├── eval.py                               # 词法分析评估的脚本
 ├── downloads.py                      # 用于下载数据和模型的脚本
 ├── downloads.sh                      # 用于下载数据和模型的脚本
+├── sequence_tagging.yaml             # 模型训练、预测、评估相关配置参数
 └──reader.py                           # 文件读取相关函数
 ```
 
@@ -207,11 +204,11 @@ Overall Architecture of GRU-CRF-MODEL
 
 ```text
 @article{jiao2018LAC,
-	title={Chinese Lexical Analysis with Deep Bi-GRU-CRF Network},
-	author={Jiao, Zhenyu and Sun, Shuqi and Sun, Ke},
-	journal={arXiv preprint arXiv:1807.01882},
-	year={2018},
-	url={https://arxiv.org/abs/1807.01882}
+    title={Chinese Lexical Analysis with Deep Bi-GRU-CRF Network},
+    author={Jiao, Zhenyu and Sun, Shuqi and Sun, Ke},
+    journal={arXiv preprint arXiv:1807.01882},
+    year={2018},
+    url={https://arxiv.org/abs/1807.01882}
 }
 ```
 ### 如何贡献代码

examples/sequence_tagging/downloads.py

@@ -35,7 +35,7 @@ FILE_INFO = {
     },
     'MODEL': {
         'name': 'sequence_tagging_dy.tar.gz',
-        'md5': "1125d374c03c8218b6e47325dcf607e3"
+        'md5': "6ba37ceea8f1f764ba1fe227295a6a3b"
     },
 }
 

examples/sequence_tagging/eval.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-SequenceTagging network structure
+SequenceTagging eval structure
 """
 
 from __future__ import division
@@ -25,18 +25,16 @@ import math
 import argparse
 import numpy as np
 
-from train import SeqTagging
+from train import SeqTagging, ChunkEval, LacLoss
 from utils.configure import PDConfig
 from utils.check import check_gpu, check_version
-from utils.metrics import chunk_count
-from reader import LacDataset, create_lexnet_data_generator, create_dataloader
+from reader import LacDataset, LacDataLoader
 
 work_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.join(work_dir, "../"))
 from hapi.model import set_device, Input
 
 import paddle.fluid as fluid
-from paddle.fluid.optimizer import AdamOptimizer
 from paddle.fluid.layers.utils import flatten
 
 
@@ -44,51 +42,33 @@ def main(args):
     place = set_device(args.device)
     fluid.enable_dygraph(place) if args.dynamic else None
 
-    inputs = [Input([None, None], 'int64', name='words'), 
-              Input([None], 'int64', name='length')] 
+    inputs = [
+        Input(
+            [None, None], 'int64', name='words'), Input(
+                [None], 'int64', name='length'), Input(
+                    [None, None], 'int64', name='target')
+    ]
+    labels = [Input([None, None], 'int64', name='labels')]
 
-    feed_list = None if args.dynamic else [x.forward() for x in inputs]
     dataset = LacDataset(args)
-    eval_path = args.test_file
-
-    chunk_evaluator = fluid.metrics.ChunkEvaluator()
-    chunk_evaluator.reset()
-
-    eval_generator = create_lexnet_data_generator(
-        args, reader=dataset, file_name=eval_path, place=place, mode="test")
-
-    eval_dataset = create_dataloader(
-        eval_generator, place, feed_list=feed_list)
+    eval_dataset = LacDataLoader(args, place, phase="test")
 
     vocab_size = dataset.vocab_size
     num_labels = dataset.num_labels
-    model = SeqTagging(args, vocab_size, num_labels)
-
-    optim = AdamOptimizer(
-        learning_rate=args.base_learning_rate,
-        parameter_list=model.parameters())
+    model = SeqTagging(args, vocab_size, num_labels, mode="test")
 
     model.mode = "test"
-    model.prepare(inputs=inputs)
+    model.prepare(
+        metrics=ChunkEval(num_labels),
+        inputs=inputs,
+        labels=labels,
+        device=place)
     model.load(args.init_from_checkpoint, skip_mismatch=True)
 
-    for data in eval_dataset():
-        if len(data) == 1: 
-            batch_data = data[0]
-            targets = np.array(batch_data[2])
-        else: 
-            batch_data = data
-            targets = batch_data[2].numpy()
-        inputs_data = [batch_data[0], batch_data[1]]
-        crf_decode, length = model.test(inputs=inputs_data)
-        num_infer_chunks, num_label_chunks, num_correct_chunks = chunk_count(crf_decode, targets, length, dataset.id2label_dict)
-        chunk_evaluator.update(num_infer_chunks, num_label_chunks, num_correct_chunks)
-    
-    precision, recall, f1 = chunk_evaluator.eval()
-    print("[test] P: %.5f, R: %.5f, F1: %.5f" % (precision, recall, f1))
+    model.evaluate(eval_dataset.dataloader, batch_size=args.batch_size)
 
 
-if __name__ == '__main__': 
+if __name__ == '__main__':
     args = PDConfig(yaml_file="sequence_tagging.yaml")
     args.build()
     args.Print()

examples/sequence_tagging/predict.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """
-SequenceTagging network structure
+SequenceTagging predict structure
 """
 
 from __future__ import division
@@ -28,14 +28,13 @@ import numpy as np
 from train import SeqTagging
 from utils.check import check_gpu, check_version
 from utils.configure import PDConfig
-from reader import LacDataset, create_lexnet_data_generator, create_dataloader
+from reader import LacDataset, LacDataLoader
 
 work_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.join(work_dir, "../"))
 from hapi.model import set_device, Input
 
 import paddle.fluid as fluid
-from paddle.fluid.optimizer import AdamOptimizer
 from paddle.fluid.layers.utils import flatten
 
 
@@ -43,26 +42,18 @@ def main(args):
     place = set_device(args.device)
     fluid.enable_dygraph(place) if args.dynamic else None
 
-    inputs = [Input([None, None], 'int64', name='words'), 
-              Input([None], 'int64', name='length')]
+    inputs = [
+        Input(
+            [None, None], 'int64', name='words'), Input(
+                [None], 'int64', name='length')
+    ]
 
-    feed_list = None if args.dynamic else [x.forward() for x in inputs]
     dataset = LacDataset(args)
-    predict_path = args.predict_file
-
-    predict_generator = create_lexnet_data_generator(
-        args, reader=dataset, file_name=predict_path, place=place, mode="predict")
-
-    predict_dataset = create_dataloader(
-        predict_generator, place, feed_list=feed_list)
+    predict_dataset = LacDataLoader(args, place, phase="predict")
 
     vocab_size = dataset.vocab_size
     num_labels = dataset.num_labels
-    model = SeqTagging(args, vocab_size, num_labels)
-
-    optim = AdamOptimizer(
-        learning_rate=args.base_learning_rate,
-        parameter_list=model.parameters())
+    model = SeqTagging(args, vocab_size, num_labels, mode="predict")
 
     model.mode = "test"
     model.prepare(inputs=inputs)
@@ -70,20 +61,20 @@ def main(args):
     model.load(args.init_from_checkpoint, skip_mismatch=True)
 
     f = open(args.output_file, "wb")
-    for data in predict_dataset(): 
-        if len(data) == 1: 
+    for data in predict_dataset.dataloader:
+        if len(data) == 1:
             input_data = data[0]
-        else: 
+        else:
             input_data = data
-        results, length = model.test(inputs=flatten(input_data))
-        for i in range(len(results)): 
+        results, length = model.test_batch(inputs=flatten(input_data))
+        for i in range(len(results)):
             word_len = length[i]
-            word_ids = results[i][: word_len]
+            word_ids = results[i][:word_len]
             tags = [dataset.id2label_dict[str(id)] for id in word_ids]
             f.write("\002".join(tags) + "\n")
-        
 
-if __name__ == '__main__': 
+
+if __name__ == '__main__':
     args = PDConfig(yaml_file="sequence_tagging.yaml")
     args.build()
     args.Print()

examples/sequence_tagging/reader.py

@@ -19,12 +19,19 @@ from __future__ import division
 from __future__ import print_function
 
 import io
+import os
+import leveldb
 import numpy as np
+import shutil
+from functools import partial
 
 import paddle
+from paddle.io import BatchSampler, DataLoader, Dataset
+from paddle.fluid.dygraph.parallel import ParallelEnv
+from hapi.distributed import DistributedBatchSampler
 
 
-class LacDataset(object):
+class LacDataset(Dataset):
     """
     Load lexical analysis dataset
     """
@@ -34,6 +41,7 @@ class LacDataset(object):
         self.label_dict_path = args.label_dict_path
         self.word_rep_dict_path = args.word_rep_dict_path
         self._load_dict()
+        self.examples = []
 
     def _load_dict(self):
         self.word2id_dict = self.load_kv_dict(
@@ -108,152 +116,135 @@ class LacDataset(object):
             label_ids.append(label_id)
         return label_ids
 
-    def file_reader(self,
-                    filename,
-                    mode="train",
-                    batch_size=32,
-                    max_seq_len=126):
+    def file_reader(self, filename, phase="train"):
         """
         yield (word_idx, target_idx) one by one from file,
             or yield (word_idx, ) in `infer` mode
         """
-
-        def wrapper():
-            fread = io.open(filename, "r", encoding="utf-8")
-            if mode == "train": 
-                headline = next(fread)
-                headline = headline.strip().split('\t')
-                assert len(headline) == 2 and headline[0] == "text_a" and headline[
-                    1] == "label"
-                buf = []
-                for line in fread:
-                    words, labels = line.strip("\n").split("\t")
-                    if len(words) < 1:
-                        continue
-                    word_ids = self.word_to_ids(words.split("\002"))
-                    label_ids = self.label_to_ids(labels.split("\002"))
-                    assert len(word_ids) == len(label_ids)
-                    words_len = np.int64(len(word_ids))
-                        
-                    word_ids = word_ids[0:max_seq_len]
-                    words_len = np.int64(len(word_ids))
-                    word_ids += [0 for _ in range(max_seq_len - words_len)]
-                    label_ids = label_ids[0:max_seq_len]
-                    label_ids += [0 for _ in range(max_seq_len - words_len)]
-                    assert len(word_ids) == len(label_ids)
-                    yield word_ids, label_ids, words_len
-            elif mode == "test": 
-                headline = next(fread)
+        self.phase = phase
+        with io.open(filename, "r", encoding="utf8") as fr:
+            if phase in ["train", "test"]:
+                headline = next(fr)
                 headline = headline.strip().split('\t')
-                assert len(headline) == 2 and headline[0] == "text_a" and headline[
-                           1] == "label"
-                buf = []
-                for line in fread:
-                    words, labels = line.strip("\n").split("\t")
-                    if len(words) < 1:
-                        continue
-                    word_ids = self.word_to_ids(words.split("\002"))
-                    label_ids = self.label_to_ids(labels.split("\002"))
-                    assert len(word_ids) == len(label_ids)
-                    words_len = np.int64(len(word_ids))
-                    yield word_ids, label_ids, words_len
-            else: 
-                for line in fread: 
-                    words = line.strip("\n").split('\t')[0]
-                    if words == u"text_a": 
+                assert len(headline) == 2 and headline[
+                    0] == "text_a" and headline[1] == "label"
+
+                for line in fr:
+                    line_str = line.strip("\n")
+                    if len(line_str) < 1 and len(line_str.split('\t')) < 2:
                         continue
-                    if "\002" not in words: 
-                        word_ids = self.word_to_ids(words)
-                    else: 
-                        word_ids = self.word_to_ids(words.split("\002"))
-                    words_len = np.int64(len(word_ids))
-                    yield word_ids, words_len
 
-            fread.close()
+                    self.examples.append(line_str)
+            else:
+                for idx, line in enumerate(fr):
+                    words = line.strip("\n").split("\t")[0]
+                    self.examples.append(words)
+
+    def __getitem__(self, idx):
+        line_str = self.examples[idx]
+        if self.phase in ["train", "test"]:
+            words, labels = line_str.split('\t')
+            word_ids = self.word_to_ids(words.split("\002"))
+            label_ids = self.label_to_ids(labels.split("\002"))
+            assert len(word_ids) == len(label_ids)
+            return word_ids, label_ids
+        else:
+            words = [w for w in line_str]
+            word_ids = self.word_to_ids(words)
+            return word_ids
+
+    def __len__(self):
 
-        return wrapper
+        return len(self.examples)
 
 
-def create_lexnet_data_generator(args, reader, file_name, place, mode="train"): 
-    def padding_data(max_len, batch_data): 
+def create_lexnet_data_generator(args, insts, phase="train"):
+    def padding_data(max_len, batch_data, if_len=False):
         padding_batch_data = []
-        for data in batch_data: 
+        padding_lens = []
+        for data in batch_data:
+            data = data[:max_len]
+            if if_len:
+                seq_len = np.int64(len(data))
+                padding_lens.append(seq_len)
             data += [0 for _ in range(max_len - len(data))]
             padding_batch_data.append(data)
-        return padding_batch_data
-
-    def wrapper(): 
-        if mode == "train": 
-            batch_words, batch_labels, seq_lens = [], [], []
-            for epoch in xrange(args.epoch):
-                for instance in reader.file_reader(
-                        file_name, mode, max_seq_len=args.max_seq_len)():
-                    words, labels, words_len = instance
-                    if len(seq_lens) < args.batch_size:
-                        batch_words.append(words)
-                        batch_labels.append(labels)
-                        seq_lens.append(words_len)
-                    if len(seq_lens) == args.batch_size: 
-                        yield batch_words, seq_lens, batch_labels, batch_labels
-                        batch_words, batch_labels, seq_lens = [], [], []
-
-            if len(seq_lens) > 0:
-                yield batch_words, seq_lens, batch_labels, batch_labels
-        elif mode == "test": 
-            batch_words, batch_labels, seq_lens, max_len = [], [], [], 0
-            for instance in reader.file_reader(
-                file_name, mode, max_seq_len=args.max_seq_len)():
-                words, labels, words_len = instance
-                max_len = words_len if words_len > max_len else max_len
-                if len(seq_lens) < args.batch_size:
-                    batch_words.append(words)
-                    seq_lens.append(words_len)
-                    batch_labels.append(labels)
-                if len(seq_lens) == args.batch_size: 
-                    padding_batch_words = padding_data(max_len, batch_words)
-                    padding_batch_labels = padding_data(max_len, batch_labels)
-                    yield padding_batch_words, seq_lens, padding_batch_labels, padding_batch_labels
-                    batch_words, batch_labels, seq_lens, max_len = [], [], [], 0
-            if len(seq_lens) > 0: 
-                padding_batch_words = padding_data(max_len, batch_words)
-                padding_batch_labels = padding_data(max_len, batch_labels)
-                yield padding_batch_words, seq_lens, padding_batch_labels, padding_batch_labels
-
-        else: 
-            batch_words, seq_lens, max_len = [], [], 0
-            for instance in reader.file_reader(
-                   file_name, mode, max_seq_len=args.max_seq_len)():
-                words, words_len = instance
-                if len(seq_lens) < args.batch_size:
-                    batch_words.append(words)
-                    seq_lens.append(words_len)
-                    max_len = words_len if words_len > max_len else max_len
-                if len(seq_lens) == args.batch_size: 
-                    padding_batch_words = padding_data(max_len, batch_words)
-                    yield padding_batch_words, seq_lens
-                    batch_words, seq_lens, max_len = [], [], 0
-            if len(seq_lens) > 0: 
-                padding_batch_words = padding_data(max_len, batch_words)
-                yield padding_batch_words, seq_lens
-
-    return wrapper
-
-
-def create_dataloader(generator, place, feed_list=None):
-    if not feed_list:
-        data_loader = paddle.io.DataLoader.from_generator(
-            capacity=50,
-            use_double_buffer=True,
-            iterable=True,
-            return_list=True)
+        if if_len:
+            return np.array(padding_batch_data), np.array(padding_lens)
+        else:
+            return np.array(padding_batch_data)
+
+    if phase == "train":
+        batch_words = [inst[0] for inst in insts]
+        batch_labels = [inst[1] for inst in insts]
+        padding_batch_words, padding_lens = padding_data(
+            args.max_seq_len, batch_words, if_len=True)
+        padding_batch_labels = padding_data(args.max_seq_len, batch_labels)
+        return [
+            padding_batch_words, padding_lens, padding_batch_labels,
+            padding_batch_labels
+        ]
+    elif phase == "test":
+        batch_words = [inst[0] for inst in insts]
+        seq_len = [len(inst[0]) for inst in insts]
+        max_seq_len = max(seq_len)
+        batch_labels = [inst[1] for inst in insts]
+        padding_batch_words, padding_lens = padding_data(
+            max_seq_len, batch_words, if_len=True)
+        padding_batch_labels = padding_data(max_seq_len, batch_labels)
+        return [
+            padding_batch_words, padding_lens, padding_batch_labels,
+            padding_batch_labels
+        ]
     else:
-        data_loader = paddle.io.DataLoader.from_generator(
-            feed_list=feed_list,
-            capacity=50,
-            use_double_buffer=True,
-            iterable=True,
+        batch_words = insts
+        seq_len = [len(inst) for inst in insts]
+        max_seq_len = max(seq_len)
+        padding_batch_words, padding_lens = padding_data(
+            max_seq_len, batch_words, if_len=True)
+        return [padding_batch_words, padding_lens]
+
+
+class LacDataLoader(object):
+    def __init__(self,
+                 args,
+                 place,
+                 phase="train",
+                 shuffle=False,
+                 num_workers=0,
+                 drop_last=False):
+        assert phase in [
+            "train", "test", "predict"
+        ], "phase should be in [train, test, predict], but get %s" % phase
+
+        if phase == "train":
+            file_name = args.train_file
+        elif phase == "test":
+            file_name = args.test_file
+        elif phase == "predict":
+            file_name = args.predict_file
+
+        self.dataset = LacDataset(args)
+        self.dataset.file_reader(file_name, phase=phase)
+
+        if phase == "train":
+            self.sampler = DistributedBatchSampler(
+                dataset=self.dataset,
+                batch_size=args.batch_size,
+                shuffle=shuffle,
+                drop_last=drop_last)
+        else:
+            self.sampler = BatchSampler(
+                dataset=self.dataset,
+                batch_size=args.batch_size,
+                shuffle=shuffle,
+                drop_last=drop_last)
+
+        self.dataloader = DataLoader(
+            dataset=self.dataset,
+            batch_sampler=self.sampler,
+            places=place,
+            collate_fn=partial(
+                create_lexnet_data_generator, args, phase=phase),
+            num_workers=num_workers,
             return_list=True)
-    data_loader.set_batch_generator(generator, places=place)
-    return data_loader
-
-

examples/sequence_tagging/sequence_tagging.yaml

 word_dict_path: "./conf/word.dic"
 label_dict_path: "./conf/tag.dic"
 word_rep_dict_path: "./conf/q2b.dic"
-device: "cpu"
+device: "gpu"
 dynamic: True
 epoch: 10
 base_learning_rate: 0.001
@@ -14,7 +14,7 @@ batch_size: 300
 max_seq_len: 126
 num_devices: 1
 save_dir: "model"
-init_from_checkpoint: "model_baseline/params"
+init_from_checkpoint: ""
 init_from_pretrain_model: ""
 save_freq: 1
 eval_freq: 1
@@ -22,4 +22,3 @@ output_file: "predict.result"
 test_file: "./data/test.tsv"
 train_file: "./data/train.tsv"
 predict_file: "./data/infer.tsv"
-mode: "train"

examples/sequence_tagging/train.py

@@ -28,21 +28,23 @@ import numpy as np
 work_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(os.path.join(work_dir, "../"))
 
-
 from hapi.metrics import Metric
 from hapi.model import Model, Input, Loss, set_device
 from hapi.text.text import SequenceTagging
 
 from utils.check import check_gpu, check_version
 from utils.configure import PDConfig
-from reader import LacDataset, create_lexnet_data_generator, create_dataloader 
+from reader import LacDataset, LacDataLoader
 
 import paddle.fluid as fluid
 from paddle.fluid.optimizer import AdamOptimizer
 
+__all__ = ["SeqTagging", "LacLoss", "ChunkEval"]
+
 
 class SeqTagging(Model):
-    def __init__(self, args, vocab_size, num_labels, length=None):
+    def __init__(self, args, vocab_size, num_labels, length=None,
+                 mode="train"):
         super(SeqTagging, self).__init__()
         """
         define the lexical analysis network structure
@@ -53,7 +55,7 @@ class SeqTagging(Model):
             for infer: return the prediction
             otherwise: return the prediction
         """
-        self.mode_type = args.mode
+        self.mode_type = mode
         self.word_emb_dim = args.word_emb_dim
         self.vocab_size = vocab_size
         self.num_labels = num_labels
@@ -65,19 +67,19 @@ class SeqTagging(Model):
         self.bigru_num = args.bigru_num
         self.batch_size = args.batch_size
         self.init_bound = 0.1
-        self.length=length
+        self.length = length
 
         self.sequence_tagging = SequenceTagging(
-                        vocab_size=self.vocab_size,
-                        num_labels=self.num_labels,
-                        batch_size=self.batch_size,
-                        word_emb_dim=self.word_emb_dim,
-                        grnn_hidden_dim=self.grnn_hidden_dim,
-                        emb_learning_rate=self.emb_lr,
-                        crf_learning_rate=self.crf_lr,
-                        bigru_num=self.bigru_num,
-                        init_bound=self.init_bound,
-                        length=self.length)
+            vocab_size=self.vocab_size,
+            num_labels=self.num_labels,
+            batch_size=self.batch_size,
+            word_emb_dim=self.word_emb_dim,
+            grnn_hidden_dim=self.grnn_hidden_dim,
+            emb_learning_rate=self.emb_lr,
+            crf_learning_rate=self.crf_lr,
+            bigru_num=self.bigru_num,
+            init_bound=self.init_bound,
+            length=self.length)
 
     def forward(self, *inputs):
         """
@@ -85,10 +87,10 @@ class SeqTagging(Model):
         """
         word = inputs[0]
         lengths = inputs[1]
-        if self.mode_type == "train" or self.mode_type == "test": 
+        if self.mode_type == "train" or self.mode_type == "test":
             target = inputs[2]
             outputs = self.sequence_tagging(word, lengths, target)
-        else: 
+        else:
             outputs = self.sequence_tagging(word, lengths)
         return outputs
 
@@ -156,7 +158,7 @@ class ChunkEval(Metric):
             int(math.ceil((num_labels - 1) / 2.0)), "IOB")
         self.reset()
 
-    def add_metric_op(self, *args): 
+    def add_metric_op(self, *args):
         crf_decode = args[0]
         lengths = args[2]
         label = args[3]
@@ -207,30 +209,25 @@ def main(args):
     place = set_device(args.device)
     fluid.enable_dygraph(place) if args.dynamic else None
 
-    inputs = [Input([None, None], 'int64', name='words'),
-              Input([None], 'int64', name='length'), 
-              Input([None, None], 'int64', name='target')]
+    inputs = [
+        Input(
+            [None, None], 'int64', name='words'), Input(
+                [None], 'int64', name='length'), Input(
+                    [None, None], 'int64', name='target')
+    ]
 
     labels = [Input([None, None], 'int64', name='labels')]
 
-    feed_list = None if args.dynamic else [x.forward() for x in inputs + labels]
-    dataset = LacDataset(args)
-    train_path = args.train_file
-    test_path = args.test_file
-
-    train_generator = create_lexnet_data_generator(
-        args, reader=dataset, file_name=train_path, place=place, mode="train")
-    test_generator = create_lexnet_data_generator(
-        args, reader=dataset, file_name=test_path, place=place, mode="test")
+    feed_list = None if args.dynamic else [
+        x.forward() for x in inputs + labels
+    ]
 
-    train_dataset = create_dataloader(
-        train_generator, place, feed_list=feed_list)
-    test_dataset = create_dataloader(
-        test_generator, place, feed_list=feed_list)
+    dataset = LacDataset(args)
+    train_dataset = LacDataLoader(args, place, phase="train")
 
     vocab_size = dataset.vocab_size
     num_labels = dataset.num_labels
-    model = SeqTagging(args, vocab_size, num_labels)
+    model = SeqTagging(args, vocab_size, num_labels, mode="train")
 
     optim = AdamOptimizer(
         learning_rate=args.base_learning_rate,
@@ -250,8 +247,7 @@ def main(args):
     if args.init_from_pretrain_model:
         model.load(args.init_from_pretrain_model, reset_optimizer=True)
 
-    model.fit(train_dataset,
-              test_dataset,
+    model.fit(train_dataset.dataloader,
               epochs=args.epoch,
               batch_size=args.batch_size,
               eval_freq=args.eval_freq,
@@ -263,7 +259,7 @@ if __name__ == '__main__':
     args = PDConfig(yaml_file="sequence_tagging.yaml")
     args.build()
     args.Print()
-    
+
     use_gpu = True if args.device == "gpu" else False
     check_gpu(use_gpu)
     check_version()

examples/sequence_tagging/utils/configure.py

@@ -195,13 +195,19 @@ class PDConfig(object):
                                "Whether to perform predicting.")
         self.default_g.add_arg("do_eval", bool, False,
                                "Whether to perform evaluating.")
-        self.default_g.add_arg("do_save_inference_model", bool, False,
-                               "Whether to perform model saving for inference.")
+        self.default_g.add_arg(
+            "do_save_inference_model", bool, False,
+            "Whether to perform model saving for inference.")
 
         # NOTE: args for profiler
-        self.default_g.add_arg("is_profiler", int, 0, "the switch of profiler tools. (used for benchmark)")
-        self.default_g.add_arg("profiler_path", str, './', "the profiler output file path. (used for benchmark)")
-        self.default_g.add_arg("max_iter", int, 0, "the max train batch num.(used for benchmark)")
+        self.default_g.add_arg(
+            "is_profiler", int, 0,
+            "the switch of profiler tools. (used for benchmark)")
+        self.default_g.add_arg(
+            "profiler_path", str, './',
+            "the profiler output file path. (used for benchmark)")
+        self.default_g.add_arg("max_iter", int, 0,
+                               "the max train batch num.(used for benchmark)")
 
         self.parser = parser
 

examples/sequence_tagging/utils/metrics.py

@@ -23,7 +23,7 @@ import paddle.fluid as fluid
 __all__ = ['chunk_count', "build_chunk"]
 
 
-def build_chunk(data_list, id2label_dict): 
+def build_chunk(data_list, id2label_dict):
     """
     Assembly entity
     """
@@ -31,29 +31,29 @@ def build_chunk(data_list, id2label_dict):
     ner_dict = {}
     ner_str = ""
     ner_start = 0
-    for i in range(len(tag_list)): 
+    for i in range(len(tag_list)):
         tag = tag_list[i]
-        if tag == u"O": 
-            if i != 0: 
+        if tag == u"O":
+            if i != 0:
                 key = "%d_%d" % (ner_start, i - 1)
                 ner_dict[key] = ner_str
             ner_start = i
-            ner_str = tag 
-        elif tag.endswith(u"B"): 
-            if i != 0: 
+            ner_str = tag
+        elif tag.endswith(u"B"):
+            if i != 0:
                 key = "%d_%d" % (ner_start, i - 1)
                 ner_dict[key] = ner_str
             ner_start = i
             ner_str = tag.split('-')[0]
-        elif tag.endswith(u"I"): 
-            if tag.split('-')[0] != ner_str: 
-                if i != 0: 
+        elif tag.endswith(u"I"):
+            if tag.split('-')[0] != ner_str:
+                if i != 0:
                     key = "%d_%d" % (ner_start, i - 1)
                     ner_dict[key] = ner_str
                 ner_start = i
                 ner_str = tag.split('-')[0]
     return ner_dict
-                    
+
 
 def chunk_count(infer_numpy, label_numpy, seq_len, id2label_dict):
     """
@@ -62,15 +62,14 @@ def chunk_count(infer_numpy, label_numpy, seq_len, id2label_dict):
     num_infer_chunks, num_label_chunks, num_correct_chunks = 0, 0, 0
     assert infer_numpy.shape[0] == label_numpy.shape[0]
 
-    for i in range(infer_numpy.shape[0]): 
-        infer_list = infer_numpy[i][: seq_len[i]]
-        label_list = label_numpy[i][: seq_len[i]]
+    for i in range(infer_numpy.shape[0]):
+        infer_list = infer_numpy[i][:seq_len[i]]
+        label_list = label_numpy[i][:seq_len[i]]
         infer_dict = build_chunk(infer_list, id2label_dict)
         num_infer_chunks += len(infer_dict)
         label_dict = build_chunk(label_list, id2label_dict)
         num_label_chunks += len(label_dict)
-        for key in infer_dict: 
-            if key in label_dict and label_dict[key] == infer_dict[key]: 
+        for key in infer_dict:
+            if key in label_dict and label_dict[key] == infer_dict[key]:
                 num_correct_chunks += 1
     return num_infer_chunks, num_label_chunks, num_correct_chunks
-

hapi/text/text.py

@@ -19,6 +19,7 @@ from __future__ import print_function
 import os
 import six
 import sys
+
 if six.PY2:
     reload(sys)
     sys.setdefaultencoding('utf8')
@@ -37,7 +38,7 @@ import paddle
 import paddle.fluid as fluid
 import paddle.fluid.layers.utils as utils
 from paddle.fluid.layers.utils import map_structure, flatten, pack_sequence_as
-from paddle.fluid.dygraph import to_variable, Embedding, Linear, LayerNorm, GRUUnit
+from paddle.fluid.dygraph import to_variable, Embedding, Linear, LayerNorm, GRUUnit, Conv2D
 from paddle.fluid.data_feeder import convert_dtype
 
 from paddle.fluid import layers
@@ -49,7 +50,8 @@ __all__ = [
     'BeamSearchDecoder', 'MultiHeadAttention', 'FFN',
     'TransformerEncoderLayer', 'TransformerEncoder', 'TransformerDecoderLayer',
     'TransformerDecoder', 'TransformerBeamSearchDecoder', 'Linear_chain_crf',
-    'Crf_decoding', 'SequenceTagging'
+    'Crf_decoding', 'SequenceTagging', 'GRUEncoderLayer', 'CNNEncoder',
+    'BOWEncoder', 'SimpleConvPoolLayer', 'GRUEncoder', 'DynamicGRU', 'LSTMEncoder'
 ]
 
 
@@ -87,12 +89,12 @@ class RNNCell(Layer):
         batch_ref = flatten(batch_ref)[0]
 
         def _is_shape_sequence(seq):
-            if sys.version_info < (3, ):
+            if sys.version_info < (3,):
                 integer_types = (
                     int,
-                    long, )
+                    long,)
             else:
-                integer_types = (int, )
+                integer_types = (int,)
             """For shape, list/tuple of integer is the finest-grained objection"""
             if (isinstance(seq, list) or isinstance(seq, tuple)):
                 if reduce(
@@ -247,8 +249,8 @@ class BasicLSTMCell(RNNCell):
 
         self.use_customized_weight = False
         for _weights in [
-                forget_gate_weights, input_gate_weights, output_gate_weights,
-                cell_weights
+            forget_gate_weights, input_gate_weights, output_gate_weights,
+            cell_weights
         ]:
             for _key in _weights:
                 if _weights[_key] is not None:
@@ -273,7 +275,7 @@ class BasicLSTMCell(RNNCell):
                 is_bias=True)
         else:
             if "w" in forget_gate_weights and forget_gate_weights[
-                    "w"] is not None:
+                "w"] is not None:
                 self.fg_w = forget_gate_weights["w"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -287,7 +289,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "h" in forget_gate_weights and forget_gate_weights[
-                    "h"] is not None:
+                "h"] is not None:
                 self.fg_h = forget_gate_weights["h"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -301,7 +303,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "b" in forget_gate_weights and forget_gate_weights[
-                    "b"] is not None:
+                "b"] is not None:
                 self.fg_b = forget_gate_weights["b"]
             else:
                 if self._bias_attr is not None and self._bias_attr.name is not None:
@@ -316,7 +318,7 @@ class BasicLSTMCell(RNNCell):
                     is_bias=True)
 
             if "w" in input_gate_weights and input_gate_weights[
-                    "w"] is not None:
+                "w"] is not None:
                 self.ig_w = input_gate_weights["w"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -331,7 +333,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "h" in input_gate_weights and input_gate_weights[
-                    "h"] is not None:
+                "h"] is not None:
                 self.ig_h = input_gate_weights["h"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -346,7 +348,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "b" in input_gate_weights and input_gate_weights[
-                    "b"] is not None:
+                "b"] is not None:
                 self.ig_b = input_gate_weights["b"]
             else:
                 if self._bias_attr is not None and self._bias_attr.name is not None:
@@ -361,7 +363,7 @@ class BasicLSTMCell(RNNCell):
                     is_bias=True)
 
             if "w" in output_gate_weights and output_gate_weights[
-                    "w"] is not None:
+                "w"] is not None:
                 self.og_w = output_gate_weights["w"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -375,7 +377,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "h" in output_gate_weights and output_gate_weights[
-                    "h"] is not None:
+                "h"] is not None:
                 self.og_h = output_gate_weights["h"]
             else:
                 if self._param_attr is not None and self._param_attr.name is not None:
@@ -390,7 +392,7 @@ class BasicLSTMCell(RNNCell):
                     dtype=self._dtype)
 
             if "b" in output_gate_weights and output_gate_weights[
-                    "b"] is not None:
+                "b"] is not None:
                 self.og_b = output_gate_weights["b"]
             else:
                 if self._bias_attr is not None and self._bias_attr.name is not None:
@@ -545,7 +547,7 @@ class BasicGRUCell(RNNCell):
 
         self.use_customized_weight = False
         for _weights in [
-                update_gate_weights, reset_gate_weights, cell_weights
+            update_gate_weights, reset_gate_weights, cell_weights
         ]:
             for _key in _weights:
                 if _weights[_key] is not None:
@@ -601,7 +603,7 @@ class BasicGRUCell(RNNCell):
 
             # create the parameters of gates in gru
             if "w" in update_gate_weights and update_gate_weights[
-                    "w"] is not None:
+                "w"] is not None:
                 self.ug_w = update_gate_weights["w"]
             else:
                 if gate_param_attr is not None and gate_param_attr.name is not None:
@@ -615,7 +617,7 @@ class BasicGRUCell(RNNCell):
                     dtype=self._dtype)
 
             if "h" in update_gate_weights and update_gate_weights[
-                    "h"] is not None:
+                "h"] is not None:
                 self.ug_h = update_gate_weights["h"]
             else:
                 if gate_param_attr is not None and gate_param_attr.name is not None:
@@ -629,7 +631,7 @@ class BasicGRUCell(RNNCell):
                     dtype=self._dtype)
 
             if "b" in update_gate_weights and update_gate_weights[
-                    "b"] is not None:
+                "b"] is not None:
                 self.ug_b = update_gate_weights["b"]
             else:
                 if gate_bias_attr is not None and gate_bias_attr.name is not None:
@@ -645,7 +647,7 @@ class BasicGRUCell(RNNCell):
 
             # reset gate parameters
             if "w" in reset_gate_weights and reset_gate_weights[
-                    "w"] is not None:
+                "w"] is not None:
                 self.rg_w = reset_gate_weights["w"]
             else:
                 if gate_param_attr is not None and gate_param_attr.name is not None:
@@ -659,7 +661,7 @@ class BasicGRUCell(RNNCell):
                     dtype=self._dtype)
 
             if "h" in reset_gate_weights and reset_gate_weights[
-                    "h"] is not None:
+                "h"] is not None:
                 self.rg_h = reset_gate_weights["h"]
             else:
                 if gate_param_attr is not None and gate_param_attr.name is not None:
@@ -673,7 +675,7 @@ class BasicGRUCell(RNNCell):
                     dtype=self._dtype)
 
             if "b" in reset_gate_weights and reset_gate_weights[
-                    "b"] is not None:
+                "b"] is not None:
                 self.rg_b = reused_params["b"]
             else:
                 if gate_bias_attr is not None and gate_bias_attr.name is not None:
@@ -763,7 +765,7 @@ class BasicGRUCell(RNNCell):
         c = self._activation(candidate)
         new_hidden = u * pre_hidden + (1 - u) * c
 
-        return new_hidden
+        return new_hidden, new_hidden
 
     @property
     def state_shape(self):
@@ -801,7 +803,7 @@ class RNN(fluid.dygraph.Layer):
                 new_state = fluid.layers.elementwise_mul(
                     new_state, step_mask,
                     axis=0) - fluid.layers.elementwise_mul(
-                        state, (step_mask - 1), axis=0)
+                    state, (step_mask - 1), axis=0)
                 return new_state
 
             flat_inputs = flatten(inputs)
@@ -847,8 +849,8 @@ class RNN(fluid.dygraph.Layer):
                 outputs = map_structure(
                     lambda x: ArrayWrapper(x),
                     step_outputs) if i == 0 else map_structure(
-                        lambda x, x_array: x_array.append(x), step_outputs,
-                        outputs)
+                    lambda x, x_array: x_array.append(x), step_outputs,
+                    outputs)
 
             final_outputs = map_structure(
                 lambda x: fluid.layers.stack(x.array,
@@ -917,7 +919,7 @@ class DynamicDecode(Layer):
                     step_mask.stop_gradient = True
                 new_state = layers.elementwise_mul(
                     state, step_mask, axis=0) - layers.elementwise_mul(
-                        new_state, (step_mask - 1), axis=0)
+                    new_state, (step_mask - 1), axis=0)
                 if convert_dtype(state_dtype) in ["bool"]:
                     new_state = layers.cast(new_state, dtype=state_dtype)
                 return new_state
@@ -959,8 +961,8 @@ class DynamicDecode(Layer):
                 outputs = map_structure(
                     lambda x: ArrayWrapper(x),
                     step_outputs) if step_idx == 0 else map_structure(
-                        lambda x, x_array: x_array.append(x), step_outputs,
-                        outputs)
+                    lambda x, x_array: x_array.append(x), step_outputs,
+                    outputs)
                 inputs, states, finished, sequence_lengths = (
                     next_inputs, next_states, next_finished,
                     next_sequence_lengths)
@@ -989,7 +991,7 @@ class DynamicDecode(Layer):
 
             return (final_outputs, final_states,
                     sequence_lengths) if self.return_length else (
-                        final_outputs, final_states)
+                final_outputs, final_states)
         else:
             return fluid.layers.dynamic_decode(
                 self.decoder,
@@ -1040,7 +1042,7 @@ class TransformerBeamSearchDecoder(layers.BeamSearchDecoder):
         x = layers.reshape(
             x, [0] * (len(x.shape) - var_dim_in_state
                       ) + [self.batch_size * self.beam_size] +
-            [int(size) for size in x.shape[-var_dim_in_state + 2:]])
+               [int(size) for size in x.shape[-var_dim_in_state + 2:]])
         x = layers.transpose(
             x,
             list(range((len(x.shape) + 1 - var_dim_in_state), len(x.shape))) +
@@ -1051,9 +1053,9 @@ class TransformerBeamSearchDecoder(layers.BeamSearchDecoder):
         var_dim_size = layers.shape(x)[self.var_dim_in_state]
         x = layers.reshape(
             x, [-1, self.beam_size] +
-            [int(size)
-             for size in x.shape[1:self.var_dim_in_state]] + [var_dim_size] +
-            [int(size) for size in x.shape[self.var_dim_in_state + 1:]])
+               [int(size)
+                for size in x.shape[1:self.var_dim_in_state]] + [var_dim_size] +
+               [int(size) for size in x.shape[self.var_dim_in_state + 1:]])
         return x
 
     def step(self, time, inputs, states, **kwargs):
@@ -1116,7 +1118,7 @@ class PrePostProcessLayer(Layer):
             elif cmd == "d":  # add dropout
                 self.functors.append(lambda x: layers.dropout(
                     x, dropout_prob=dropout_rate, is_test=False)
-                                     if dropout_rate else x)
+                if dropout_rate else x)
 
     def forward(self, x, residual=None):
         for i, cmd in enumerate(self.process_cmd):
@@ -1217,7 +1219,7 @@ class MultiHeadAttention(Layer):
 
         # scale dot product attention
         product = layers.matmul(
-            x=q, y=k, transpose_y=True, alpha=self.d_model**-0.5)
+            x=q, y=k, transpose_y=True, alpha=self.d_model ** -0.5)
         if attn_bias:
             product += attn_bias
         weights = layers.softmax(product)
@@ -1307,7 +1309,6 @@ class TransformerEncoderLayer(Layer):
                  reused_ffn_weights={"reused_fc1": None,
                                      "reused_fc2": None},
                  reused_post_ffn_layernorm=None):
-
         super(TransformerEncoderLayer, self).__init__()
 
         self.preprocesser1 = PrePostProcessLayer(preprocess_cmd, d_model,
@@ -1555,7 +1556,7 @@ class TransformerDecoder(Layer):
         ]
 
 
-#TODO: we should merge GRUCell with BasicGRUCell
+# TODO: we should merge GRUCell with BasicGRUCell
 class GRUCell(RNNCell):
     def __init__(self,
                  input_size,
@@ -1589,7 +1590,7 @@ class GRUCell(RNNCell):
         return [self.hidden_size]
 
 
-#TODO: we should merge GRUCell with BasicGRUCell
+# TODO: we should merge GRUCell with BasicGRUCell
 class GRUEncoderCell(RNNCell):
     def __init__(self,
                  num_layers,
@@ -1605,7 +1606,7 @@ class GRUEncoderCell(RNNCell):
             self.gru_cells.append(
                 self.add_sublayer(
                     "gru_%d" % i,
-                    #BasicGRUCell(
+                    # BasicGRUCell(
                     GRUCell(
                         input_size=input_size if i == 0 else hidden_size,
                         hidden_size=hidden_size,
@@ -1672,7 +1673,6 @@ class Linear_chain_crf(fluid.dygraph.Layer):
         self._transition = value
 
     def forward(self, input, label, length=None):
-
         alpha = self._helper.create_variable_for_type_inference(
             dtype=self._dtype)
         emission_exps = self._helper.create_variable_for_type_inference(
@@ -1723,7 +1723,6 @@ class Crf_decoding(fluid.dygraph.Layer):
         self._transition = value
 
     def forward(self, input, label=None, length=None):
-
         viterbi_path = self._helper.create_variable_for_type_inference(
             dtype=self._dtype)
         this_inputs = {
@@ -1741,6 +1740,64 @@ class Crf_decoding(fluid.dygraph.Layer):
         return viterbi_path
 
 
+class GRUEncoderLayer(Layer):
+    def __init__(self,
+                 input_dim,
+                 grnn_hidden_dim,
+                 init_bound,
+                 num_layers=1,
+                 h_0=None,
+                 is_bidirection=False):
+        super(GRUEncoderLayer, self).__init__()
+        self.h_0 = h_0
+        self.num_layers = num_layers
+        self.is_bidirection = is_bidirection
+        self.gru_list = []
+        self.gru_r_list = []
+        for i in range(num_layers):
+            self.basic_gru_cell = BasicGRUCell(
+                input_size=input_dim if i == 0 else input_dim * 2,
+                hidden_size=grnn_hidden_dim,
+                param_attr=fluid.ParamAttr(
+                    initializer=fluid.initializer.UniformInitializer(
+                        low=-init_bound, high=init_bound),
+                    regularizer=fluid.regularizer.L2DecayRegularizer(
+                        regularization_coeff=1e-4)))
+            self.gru_list.append(
+                self.add_sublayer(
+                    "gru_%d" % i,
+                    RNN(self.basic_gru_cell,
+                        is_reverse=False,
+                        time_major=False)))
+        if self.is_bidirection:
+            for i in range(num_layers):
+                self.basic_gru_cell_r = BasicGRUCell(
+                    input_size=input_dim if i == 0 else input_dim * 2,
+                    hidden_size=grnn_hidden_dim,
+                    param_attr=fluid.ParamAttr(
+                        initializer=fluid.initializer.UniformInitializer(
+                            low=-init_bound, high=init_bound),
+                        regularizer=fluid.regularizer.L2DecayRegularizer(
+                            regularization_coeff=1e-4)))
+                self.gru_r_list.append(
+                    self.add_sublayer(
+                        "gru_r_%d" % i,
+                        RNN(self.basic_gru_cell_r,
+                            is_reverse=True,
+                            time_major=False)))
+
+    def forward(self, input_feature):
+        for i in range(self.num_layers):
+            pre_gru, pre_state = self.gru_list[i](input_feature)
+            if self.is_bidirection:
+                gru_r, r_state = self.gru_r_list[i](input_feature)
+                out = fluid.layers.concat(input=[pre_gru, gru_r], axis=-1)
+            else:
+                out = pre_gru
+            input_feature = out
+        return out
+
+
 class SequenceTagging(fluid.dygraph.Layer):
     def __init__(self,
                  vocab_size,
@@ -1790,26 +1847,13 @@ class SequenceTagging(fluid.dygraph.Layer):
             force_cpu=True,
             name='h_0')
 
-        self.bigru_units = []
-        for i in range(self.bigru_num):
-            if i == 0:
-                self.bigru_units.append(
-                    self.add_sublayer(
-                        "bigru_units%d" % i,
-                        BiGRU(
-                            self.grnn_hidden_dim,
-                            self.grnn_hidden_dim,
-                            self.init_bound,
-                            h_0=h_0)))
-            else:
-                self.bigru_units.append(
-                    self.add_sublayer(
-                        "bigru_units%d" % i,
-                        BiGRU(
-                            self.grnn_hidden_dim * 2,
-                            self.grnn_hidden_dim,
-                            self.init_bound,
-                            h_0=h_0)))
+        self.gru_encoder = GRUEncoderLayer(
+            input_dim=self.grnn_hidden_dim,
+            grnn_hidden_dim=self.grnn_hidden_dim,
+            init_bound=self.init_bound,
+            num_layers=self.bigru_num,
+            h_0=h_0,
+            is_bidirection=True)
 
         self.fc = Linear(
             input_dim=self.grnn_hidden_dim * 2,
@@ -1837,10 +1881,7 @@ class SequenceTagging(fluid.dygraph.Layer):
         word_embed = self.word_embedding(word)
         input_feature = word_embed
 
-        for i in range(self.bigru_num):
-            bigru_output = self.bigru_units[i](input_feature)
-            input_feature = bigru_output
-
+        bigru_output = self.gru_encoder(input_feature)
         emission = self.fc(bigru_output)
 
         if target is not None:
@@ -1854,3 +1895,227 @@ class SequenceTagging(fluid.dygraph.Layer):
             self.linear_chain_crf.weight = self.crf_decoding.weight
             crf_decode = self.crf_decoding(input=emission, length=lengths)
             return crf_decode, lengths
+
+class SimpleConvPoolLayer(Layer):
+    def __init__(self,
+                 num_channels,
+                 num_filters,
+                 filter_size,
+                 use_cudnn=False,
+                 act=None
+                 ):
+        super(SimpleConvPoolLayer, self).__init__()
+        self._conv2d = Conv2D(num_channels=num_channels,
+                              num_filters=num_filters,
+                              filter_size=filter_size,
+                              padding=[1, 1],
+                              use_cudnn=use_cudnn,
+                              act=act)
+
+    def forward(self, input):
+        x = self._conv2d(input)
+        x = fluid.layers.reduce_max(x, dim=-1)
+        x = fluid.layers.reshape(x, shape=[x.shape[0], -1])
+        return x
+
+
+class CNNEncoder(Layer):
+    """
+    simple CNNEncoder for simnet
+    """
+    def __init__(self,
+                 dict_size,
+                 emb_dim,
+                 filter_size,
+                 num_filters,
+                 hidden_dim,
+                 seq_len,
+                 padding_idx,
+                 act
+                 ):
+        super(CNNEncoder, self).__init__()
+        self.dict_size = dict_size
+        self.emb_dim = emb_dim
+        self.filter_size = filter_size
+        self.num_filters = num_filters
+        self.hidden_dim = hidden_dim
+        self.seq_len = seq_len
+        self.padding_idx = padding_idx
+        self.act = act
+        self.channels = 1
+        self.emb_layer = Embedding(size=[self.dict_size, self.emb_dim],
+                                   is_sparse=True,
+                                   padding_idx=self.padding_idx,
+                                   param_attr=fluid.ParamAttr(name='emb', initializer=fluid.initializer.Xavier()))
+        self.cnn_layer = SimpleConvPoolLayer(
+            self.channels,
+            self.num_filters,
+            self.filter_size,
+            use_cudnn=False,
+            act=self.act
+        )
+
+    def forward(self, input):
+        emb = self.emb_layer(input)
+        emb_reshape = fluid.layers.reshape(
+            emb, shape=[-1, self.channels, self.seq_len, self.hidden_dim])
+        emb_out=self.cnn_layer(emb_reshape)
+        return emb_out
+
+class BOWEncoder(Layer):
+    """
+    simple BOWEncoder for simnet
+    """
+    def __init__(self,
+                 dict_size,
+                 emb_dim,
+                 bow_dim,
+                 seq_len,
+                 padding_idx
+                 ):
+        super(BOWEncoder, self).__init__()
+        self.dict_size = dict_size
+        self.bow_dim = bow_dim
+        self.seq_len = seq_len
+        self.emb_dim = emb_dim
+        self.padding_idx=padding_idx
+        self.emb_layer = Embedding(size=[self.dict_size, self.emb_dim],
+                                   is_sparse=True,
+                                   padding_idx=self.padding_idx,
+                                   param_attr=fluid.ParamAttr(name='emb', initializer=fluid.initializer.Xavier()))
+
+    def forward(self, input):
+        emb = self.emb_layer(input)
+        emb_reshape = fluid.layers.reshape(
+            emb, shape=[-1, self.seq_len, self.bow_dim])
+        bow_emb = fluid.layers.reduce_sum(emb_reshape, dim=1)
+        return bow_emb
+
+class DynamicGRU(fluid.dygraph.Layer):
+    def __init__(self,
+                 size,
+                 h_0=None,
+                 param_attr=None,
+                 bias_attr=None,
+                 is_reverse=False,
+                 gate_activation='sigmoid',
+                 candidate_activation='tanh',
+                 origin_mode=False,
+                 init_size=None):
+        super(DynamicGRU, self).__init__()
+
+        self.gru_unit = GRUUnit(
+            size * 3,
+            param_attr=param_attr,
+            bias_attr=bias_attr,
+            activation=candidate_activation,
+            gate_activation=gate_activation,
+            origin_mode=origin_mode)
+
+        self.size = size
+        self.h_0 = h_0
+        self.is_reverse = is_reverse
+
+    def forward(self, inputs):
+        hidden = self.h_0
+        res = []
+        for i in range(inputs.shape[1]):
+            if self.is_reverse:
+                i = inputs.shape[1] - 1 - i
+            input_ = inputs[:, i:i + 1, :]
+            input_ = fluid.layers.reshape(
+                input_, [-1, input_.shape[2]], inplace=False)
+            hidden, reset, gate = self.gru_unit(input_, hidden)
+            hidden_ = fluid.layers.reshape(
+                hidden, [-1, 1, hidden.shape[1]], inplace=False)
+            res.append(hidden_)
+        if self.is_reverse:
+            res = res[::-1]
+        res = fluid.layers.concat(res, axis=1)
+        return res
+
+class GRUEncoder(Layer):
+    """
+    simple GRUEncoder for simnet
+    """
+    def __init__(self,
+                 dict_size,
+                 emb_dim,
+                 gru_dim,
+                 hidden_dim,
+                 padding_idx,
+                 seq_len
+                 ):
+        super(GRUEncoder, self).__init__()
+        self.dict_size = dict_size
+        self.emb_dim = emb_dim
+        self.gru_dim = gru_dim
+        self.seq_len=seq_len
+        self.hidden_dim = hidden_dim
+        self.padding_idx=self.padding_idx
+        self.emb_layer = Embedding(size=[self.dict_size, self.emb_dim],
+                                                    is_sparse=True,
+                                                    padding_idx=self.padding_idx,
+                                                    param_attr=fluid.ParamAttr(name='emb',
+                                                                               initializer=fluid.initializer.Xavier()))
+        self.gru_layer = DynamicGRU(self.gru_dim)
+        self.proj_layer = Linear(input_dim=self.hidden_dim, output_dim=self.gru_dim * 3)
+
+    def forward(self, input):
+        emb = self.emb_layer(input)
+        emb_proj = self.proj_layer(emb)
+        h_0 = np.zeros((emb_proj.shape[0], self.hidden_dim), dtype="float32")
+        h_0 = to_variable(h_0)
+        gru = self.gru_layer(emb_proj, h_0=h_0)
+        gru = fluid.layers.reduce_max(gru, dim=1)
+        gru = fluid.layers.tanh(gru)
+        return gru
+
+class LSTMEncoder(Layer):
+    """
+    simple LSTMEncoder for simnet
+    """
+    def __init__(self,
+                 dict_size,
+                 emb_dim,
+                 lstm_dim,
+                 hidden_dim,
+                 seq_len,
+                 padding_idx,
+                 is_reverse
+                 ):
+        """
+        initialize
+        """
+        super(LSTMEncoder, self).__init__()
+        self.dict_size = dict_size
+        self.emb_dim = emb_dim
+        self.lstm_dim = lstm_dim
+        self.hidden_dim = hidden_dim
+        self.seq_len = seq_len
+        self.is_reverse = False
+        self.padding_idx=padding_idx
+
+        self.emb_layer = Embedding(size=[self.dict_size, self.emb_dim],
+                                   is_sparse=True,
+                                   padding_idx=self.padding_idx,
+                                   param_attr=fluid.ParamAttr(name='emb', initializer=fluid.initializer.Xavier()))
+
+        self.lstm_cell = BasicLSTMCell(
+            hidden_size=self.lstm_dim, input_size=self.lstm_dim * 4
+        )
+        self.lstm_layer = RNN(
+            cell=self.lstm_cell, time_major=True, is_reverse=self.is_reverse
+        )
+        self.proj_layer = Linear(input_dim=self.hidden_dim, output_dim=self.lstm_dim * 4)
+
+    def forward(self, input):
+        emb = self.emb_layer(input)
+        emb_proj = self.proj_layer(emb)
+        emb_lstm, _ = self.lstm_layer(emb_proj)
+        emb_reduce = fluid.layers.reduce_max(emb_lstm, dim=1)
+        emb_reshape = fluid.layers.reshape(
+            emb_reduce, shape=[-1, self.seq_len, self.hidden_dim])
+        emb_lstm = fluid.layers.reduce_sum(emb_reshape, dim=1)
+        emb_last = fluid.layers.tanh(emb_lstm)
+        return emb_last