Paddingernie16 (#3598)

* Update README.md

* padding ernie

* modify ernie

* modify code style

* remove squeeze_labels

* fix lod/ernie input_mask dtype

* padding LAC/ERNIE revert squeeze

* padding LAC/ERNIE revert label shape

* padding LAC/ERNIE add padded_token_embeddings

PaddleNLP/lexical_analysis/README.md

@@ -16,7 +16,7 @@ Lexical Analysis of Chinese，简称 LAC，是一个联合的词法分析模型
 
 #### 1.PaddlePaddle 安装
 
-本项目依赖 PaddlePaddle 1.4.0 及以上版本和PaddleHub 1.0.0及以上版本 ，PaddlePaddle安装请参考官网 [快速安装](http://www.paddlepaddle.org/paddle#quick-start)，PaddleHub安装参考 [PaddleHub](https://github.com/PaddlePaddle/PaddleHub)。
+本项目依赖 PaddlePaddle 1.6.0 及以上版本和PaddleHub 1.0.0及以上版本 ，PaddlePaddle安装请参考官网 [快速安装](http://www.paddlepaddle.org/paddle#quick-start)，PaddleHub安装参考 [PaddleHub](https://github.com/PaddlePaddle/PaddleHub)。
 
 > Warning: GPU 和 CPU 版本的 PaddlePaddle 分别是 paddlepaddle-gpu 和 paddlepaddle，请安装时注意区别。
 

PaddleNLP/lexical_analysis/creator.py

@@ -59,7 +59,7 @@ def create_model(args,  vocab_size, num_labels, mode = 'train'):
         "targets": targets,
         "avg_cost":avg_cost,
         "crf_decode": crf_decode,
-        "precision" : precision,
+        "precision": precision,
         "recall": recall,
         "f1_score": f1_score,
         "chunk_evaluator": chunk_evaluator,
@@ -143,9 +143,10 @@ def create_ernie_model(args, ernie_config):
     src_ids = fluid.layers.data(name='src_ids', shape=[args.max_seq_len, 1], dtype='int64',lod_level=0)
     sent_ids = fluid.layers.data(name='sent_ids', shape=[args.max_seq_len, 1], dtype='int64',lod_level=0)
     pos_ids = fluid.layers.data(name='pos_ids', shape=[args.max_seq_len, 1], dtype='int64',lod_level=0)
-    input_mask = fluid.layers.data(name='input_mask', shape=[args.max_seq_len, 1], dtype='int64',lod_level=0)
+    input_mask = fluid.layers.data(name='input_mask', shape=[args.max_seq_len, 1], dtype='float32',lod_level=0)
     padded_labels =fluid.layers.data(name='padded_labels', shape=[args.max_seq_len, 1], dtype='int64',lod_level=0)
-    seq_lens = fluid.layers.data(name='seq_lens', shape=[1], dtype='int64',lod_level=0)
+    seq_lens = fluid.layers.data(name='seq_lens', shape=[-1], dtype='int64',lod_level=0)
+    squeeze_labels = fluid.layers.squeeze(padded_labels, axes=[-1])
 
     ernie_inputs = {
         "src_ids": src_ids,
@@ -156,44 +157,43 @@ def create_ernie_model(args, ernie_config):
     }
     embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config)
 
-    words = fluid.layers.sequence_unpad(src_ids, seq_lens)
-    labels = fluid.layers.sequence_unpad(padded_labels, seq_lens)
-
-    token_embeddings = embeddings["token_embeddings"]
+    padded_token_embeddings = embeddings["padded_token_embeddings"]
 
     emission = fluid.layers.fc(
         size=args.num_labels,
-        input=token_embeddings,
+        input=padded_token_embeddings,
         param_attr=fluid.ParamAttr(
             initializer=fluid.initializer.Uniform(
                 low=-args.init_bound, high=args.init_bound),
             regularizer=fluid.regularizer.L2DecayRegularizer(
-                regularization_coeff=1e-4)))
+                regularization_coeff=1e-4)),
+        num_flatten_dims=2)
 
     crf_cost = fluid.layers.linear_chain_crf(
         input=emission,
-        label=labels,
+        label=padded_labels,
         param_attr=fluid.ParamAttr(
             name='crfw',
-            learning_rate=args.crf_learning_rate))
+            learning_rate=args.crf_learning_rate),
+        length=seq_lens)
     avg_cost = fluid.layers.mean(x=crf_cost)
     crf_decode = fluid.layers.crf_decoding(
-        input=emission, param_attr=fluid.ParamAttr(name='crfw'))
-
+            input=emission, param_attr=fluid.ParamAttr(name='crfw'),length=seq_lens)
 
     (precision, recall, f1_score, num_infer_chunks, num_label_chunks,
      num_correct_chunks) = fluid.layers.chunk_eval(
-         input=crf_decode,
-         label=labels,
-         chunk_scheme="IOB",
-         num_chunk_types=int(math.ceil((args.num_labels - 1) / 2.0)))
+        input=crf_decode,
+        label=squeeze_labels,
+        chunk_scheme="IOB",
+        num_chunk_types=int(math.ceil((args.num_labels - 1) / 2.0)),
+        seq_length=seq_lens)
     chunk_evaluator = fluid.metrics.ChunkEvaluator()
     chunk_evaluator.reset()
 
     ret = {
         "feed_list": [src_ids, sent_ids, pos_ids, input_mask, padded_labels, seq_lens],
-        "words":words,
-        "labels":labels,
+        "words":src_ids,
+        "labels":padded_labels,
         "avg_cost":avg_cost,
         "crf_decode":crf_decode,
         "precision" : precision,

PaddleNLP/models/representation/ernie.py

@@ -80,6 +80,7 @@ def ernie_encoder_with_paddle_hub(ernie_inputs, max_seq_len):
     embeddings = {
         "sentence_embeddings": cls_feats,
         "token_embeddings": unpad_enc_out,
+        "padded_token_embeddings": enc_out
     }
 
     for k, v in embeddings.items():
@@ -106,6 +107,7 @@ def ernie_encoder(ernie_inputs, ernie_config):
     embeddings = {
         "sentence_embeddings": cls_feats,
         "token_embeddings": unpad_enc_out,
+        "padded_token_embeddings": enc_out
     }
 
     for k, v in embeddings.items():

PaddleNLP/preprocess/padding.py

@@ -69,7 +69,7 @@ def pad_batch_data(insts,
 
     if return_seq_lens:
         seq_lens = np.array([len(inst) for inst in insts])
-        return_list += [seq_lens.astype("int64").reshape([-1, 1])]
+        return_list += [seq_lens.astype("int64").reshape([-1])]
 
     return return_list if len(return_list) > 1 else return_list[0]