add predefine network usage demo

demo/text_classification/predict.py

@@ -60,7 +60,7 @@ if __name__ == '__main__':
     # Construct transfer learning network
     # Use "pooled_output" for classification tasks on an entire sentence.
     # Use "sequence_output" for token-level output.
-    token_feature = outputs["sequence_output"]
+    pooled_output = outputs["pooled_output"]
 
     # Setup feed list for data feeder
     # Must feed all the tensor of module need
@@ -80,15 +80,10 @@ if __name__ == '__main__':
         strategy=hub.AdamWeightDecayStrategy())
 
     # Define a classfication finetune task by PaddleHub's API
-    # network choice: bilstm, bow, cnn, dpcnn, gru, lstm (PaddleHub pre-defined network)
-    # If you wanna add network after ERNIE/BERT/RoBERTa/ELECTRA module,
-    # you must use the outputs["sequence_output"] as the token_feature of TextClassifierTask,
-    # rather than outputs["pooled_output"], and feature is None
     cls_task = hub.TextClassifierTask(
         data_reader=reader,
-        token_feature=token_feature,
+        feature=pooled_output,
         feed_list=feed_list,
-        network=args.network,
         num_classes=dataset.num_labels,
         config=config)
 

demo/text_classification/predict_predefine_net.py

+#coding:utf-8
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on classification task """
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import ast
+import numpy as np
+import os
+import time
+import paddle
+import paddle.fluid as fluid
+import paddlehub as hub
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+parser.add_argument("--checkpoint_dir", type=str, default=None, help="Directory to model checkpoint")
+parser.add_argument("--batch_size",     type=int,   default=1, help="Total examples' number in batch for training.")
+parser.add_argument("--max_seq_len", type=int, default=512, help="Number of words of the longest seqence.")
+parser.add_argument("--use_gpu", type=ast.literal_eval, default=False, help="Whether use GPU for finetuning, input should be True or False")
+parser.add_argument("--use_data_parallel", type=ast.literal_eval, default=False, help="Whether use data parallel.")
+parser.add_argument("--network", type=str, default='bilstm', help="Pre-defined network which was connected after Transformer model, such as ERNIE, BERT ,RoBERTa and ELECTRA.")
+args = parser.parse_args()
+# yapf: enable.
+
+if __name__ == '__main__':
+    # Load Paddlehub ERNIE Tiny pretrained model
+    module = hub.Module(name="ernie_tiny")
+    inputs, outputs, program = module.context(
+        trainable=True, max_seq_len=args.max_seq_len)
+
+    # Download dataset and use accuracy as metrics
+    # Choose dataset: GLUE/XNLI/ChinesesGLUE/NLPCC-DBQA/LCQMC
+    dataset = hub.dataset.ChnSentiCorp()
+
+    # For ernie_tiny, it use sub-word to tokenize chinese sentence
+    # If not ernie tiny, sp_model_path and word_dict_path should be set None
+    reader = hub.reader.ClassifyReader(
+        dataset=dataset,
+        vocab_path=module.get_vocab_path(),
+        max_seq_len=args.max_seq_len,
+        sp_model_path=module.get_spm_path(),
+        word_dict_path=module.get_word_dict_path())
+
+    # Construct transfer learning network
+    # Use "pooled_output" for classification tasks on an entire sentence.
+    # Use "sequence_output" for token-level output.
+    token_feature = outputs["sequence_output"]
+
+    # Setup feed list for data feeder
+    # Must feed all the tensor of module need
+    feed_list = [
+        inputs["input_ids"].name,
+        inputs["position_ids"].name,
+        inputs["segment_ids"].name,
+        inputs["input_mask"].name,
+    ]
+
+    # Setup runing config for PaddleHub Finetune API
+    config = hub.RunConfig(
+        use_data_parallel=args.use_data_parallel,
+        use_cuda=args.use_gpu,
+        batch_size=args.batch_size,
+        checkpoint_dir=args.checkpoint_dir,
+        strategy=hub.AdamWeightDecayStrategy())
+
+    # Define a classfication finetune task by PaddleHub's API
+    # network choice: bilstm, bow, cnn, dpcnn, gru, lstm (PaddleHub pre-defined network)
+    # If you wanna add network after ERNIE/BERT/RoBERTa/ELECTRA module,
+    # you must use the outputs["sequence_output"] as the token_feature of TextClassifierTask,
+    # rather than outputs["pooled_output"], and feature is None
+    cls_task = hub.TextClassifierTask(
+        data_reader=reader,
+        token_feature=token_feature,
+        feed_list=feed_list,
+        network=args.network,
+        num_classes=dataset.num_labels,
+        config=config)
+
+    # Data to be prdicted
+    data = [["这个宾馆比较陈旧了，特价的房间也很一般。总体来说一般"], ["交通方便；环境很好；服务态度很好 房间较小"],
+            ["19天硬盘就罢工了~~~算上运来的一周都没用上15天~~~可就是不能换了~~~唉~~~~你说这算什么事呀~~~"]]
+
+    print(cls_task.predict(data=data, return_result=True))

demo/text_classification/run_cls.sh

+export FLAGS_eager_delete_tensor_gb=0.0
+export CUDA_VISIBLE_DEVICES=0
+
+CKPT_DIR="./ckpt_chnsenticorp"
+
+python -u text_cls.py \
+            --batch_size=24 \
+            --use_gpu=True \
+            --checkpoint_dir=${CKPT_DIR} \
+            --learning_rate=5e-5 \
+            --weight_decay=0.01 \
+            --max_seq_len=128 \
+            --warmup_proportion=0.1 \
+            --num_epoch=3 \
+            --use_data_parallel=True
+
+# The sugguested hyper parameters for difference task
+# for ChineseGLUE:
+# TNews: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# LCQMC: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# XNLI_zh: batch_size=32, weight_decay=0, num_epoch=2, max_seq_len=128, lr=5e-5
+# INEWS: batch_size=4, weight_decay=0, num_epoch=3, max_seq_len=512, lr=5e-5
+# DRCD: see demo: reading-comprehension
+# CMRC2018: see demo: reading-comprehension
+# BQ: batch_size=32, weight_decay=0, num_epoch=2, max_seq_len=100, lr=1e-5
+# MSRANER: see demo: sequence-labeling
+# THUCNEWS: batch_size=8, weight_decay=0, num_epoch=2, max_seq_len=512, lr=5e-5
+# IFLYTEKDATA: batch_size=16, weight_decay=0, num_epoch=5, max_seq_len=256, lr=1e-5
+
+# for other tasks:
+# ChnSentiCorp: batch_size=24, weight_decay=0.01, num_epoch=3, max_seq_len=128, lr=5e-5
+# NLPCC_DBQA: batch_size=8, weight_decay=0.01, num_epoch=3, max_seq_len=512, lr=2e-5
+# LCQMC: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=2e-5
+# QQP: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# QNLI: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# SST-2: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# CoLA: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+# MRPC: batch_size=32, weight_decay=0.01, num_epoch=3, max_seq_len=128, lr=5e-5
+# RTE: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=3e-5
+# MNLI: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+#       Specify the matched/mismatched dev and test dataset  with an underscore.
+#       mnli_m or mnli: dev and test in matched dataset.
+#       mnli_mm: dev and test in mismatched dataset.
+#      The difference can be seen in https://www.nyu.edu/projects/bowman/multinli/paper.pdf.
+#       If you are not sure which one to pick, just use mnli or mnli_m.
+# XNLI: batch_size=32, weight_decay=0, num_epoch=3, max_seq_len=128, lr=5e-5
+#       Specify the language with an underscore like xnli_zh.
+#       ar- Arabic      bg- Bulgarian      de- German
+#       el- Greek       en- English        es- Spanish
+#       fr- French      hi- Hindi          ru- Russian
+#       sw- Swahili     th- Thai           tr- Turkish
+#       ur- Urdu        vi- Vietnamese     zh- Chinese (Simplified)

demo/text_classification/run_classifier.sh → demo/text_classification/run_cls_predefine_net.sh

 export FLAGS_eager_delete_tensor_gb=0.0
 export CUDA_VISIBLE_DEVICES=0
 
-CKPT_DIR="./ckpt_chnsenticorp"
+CKPT_DIR="./ckpt_chnsenticorp_predefine_net"
 
-python -u text_classifier.py \
+python -u text_cls_predefine_net.py \
                    --batch_size=24 \
                    --use_gpu=True \
                    --checkpoint_dir=${CKPT_DIR} \

demo/text_classification/run_predict.sh

@@ -3,8 +3,8 @@ export CUDA_VISIBLE_DEVICES=0
 
 CKPT_DIR="./ckpt_chnsenticorp"
 
-python -u predict.py --checkpoint_dir=$CKPT_DIR \
+python -u predict.py \
+            --checkpoint_dir=$CKPT_DIR \
             --max_seq_len=128 \
             --use_gpu=True \
-                            --batch_size=24 \
-                            --network=bilstm
+            --batch_size=24

demo/text_classification/run_predict_predefine_net.sh

+export FLAGS_eager_delete_tensor_gb=0.0
+export CUDA_VISIBLE_DEVICES=0
+
+CKPT_DIR="./ckpt_chnsenticorp_predefine_net"
+
+python -u predict_predefine_net.py \
+                    --checkpoint_dir=$CKPT_DIR \
+                    --max_seq_len=128 \
+                    --use_gpu=True \
+                    --batch_size=24 \
+                    --network=bilstm

demo/text_classification/text_cls.py

+#coding:utf-8
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on classification task """
+
+import argparse
+import ast
+import paddlehub as hub
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+parser.add_argument("--num_epoch", type=int, default=3, help="Number of epoches for fine-tuning.")
+parser.add_argument("--use_gpu", type=ast.literal_eval, default=True, help="Whether use GPU for finetuning, input should be True or False")
+parser.add_argument("--learning_rate", type=float, default=5e-5, help="Learning rate used to train with warmup.")
+parser.add_argument("--weight_decay", type=float, default=0.01, help="Weight decay rate for L2 regularizer.")
+parser.add_argument("--warmup_proportion", type=float, default=0.1, help="Warmup proportion params for warmup strategy")
+parser.add_argument("--checkpoint_dir", type=str, default=None, help="Directory to model checkpoint")
+parser.add_argument("--max_seq_len", type=int, default=512, help="Number of words of the longest seqence.")
+parser.add_argument("--batch_size", type=int, default=32, help="Total examples' number in batch for training.")
+parser.add_argument("--use_data_parallel", type=ast.literal_eval, default=False, help="Whether use data parallel.")
+args = parser.parse_args()
+# yapf: enable.
+
+if __name__ == '__main__':
+
+    # Load Paddlehub ERNIE Tiny pretrained model
+    module = hub.Module(name="ernie_tiny")
+    inputs, outputs, program = module.context(
+        trainable=True, max_seq_len=args.max_seq_len)
+
+    # Download dataset and use accuracy as metrics
+    # Choose dataset: GLUE/XNLI/ChinesesGLUE/NLPCC-DBQA/LCQMC
+    # metric should be acc, f1 or matthews
+    dataset = hub.dataset.ChnSentiCorp()
+    metrics_choices = ["acc"]
+
+    # For ernie_tiny, it use sub-word to tokenize chinese sentence
+    # If not ernie tiny, sp_model_path and word_dict_path should be set None
+    reader = hub.reader.ClassifyReader(
+        dataset=dataset,
+        vocab_path=module.get_vocab_path(),
+        max_seq_len=args.max_seq_len,
+        sp_model_path=module.get_spm_path(),
+        word_dict_path=module.get_word_dict_path())
+
+    # Construct transfer learning network
+    # Use "pooled_output" for classification tasks on an entire sentence.
+    # Use "sequence_output" for token-level output.
+    pooled_output = outputs["pooled_output"]
+
+    # Setup feed list for data feeder
+    # Must feed all the tensor of module need
+    feed_list = [
+        inputs["input_ids"].name,
+        inputs["position_ids"].name,
+        inputs["segment_ids"].name,
+        inputs["input_mask"].name,
+    ]
+
+    # Select finetune strategy, setup config and finetune
+    strategy = hub.AdamWeightDecayStrategy(
+        warmup_proportion=args.warmup_proportion,
+        weight_decay=args.weight_decay,
+        learning_rate=args.learning_rate)
+
+    # Setup runing config for PaddleHub Finetune API
+    config = hub.RunConfig(
+        use_data_parallel=args.use_data_parallel,
+        use_cuda=args.use_gpu,
+        num_epoch=args.num_epoch,
+        batch_size=args.batch_size,
+        checkpoint_dir=args.checkpoint_dir,
+        strategy=strategy)
+
+    # Define a classfication finetune task by PaddleHub's API
+    cls_task = hub.TextClassifierTask(
+        data_reader=reader,
+        feature=pooled_output,
+        feed_list=feed_list,
+        num_classes=dataset.num_labels,
+        config=config,
+        metrics_choices=metrics_choices)
+
+    # Finetune and evaluate by PaddleHub's API
+    # will finish training, evaluation, testing, save model automatically
+    cls_task.finetune_and_eval()