Ce dgu (#2316)

* add ce for dialogue_general_understanding

* add ce for dialogue_general_understanding

PaddleNLP/dialogue_model_toolkit/dialogue_general_understanding/.run_ce.sh

+#!/bin/bash
+
+train_atis_slot(){
+  python -u train.py \
+  --task_name atis_slot \
+  --use_cuda true \
+  --do_train true \
+  --do_val true \
+  --do_test true \
+  --epoch 2 \
+  --batch_size 32 \
+  --data_dir ./data/atis/atis_slot \
+  --bert_config_path ./uncased_L-12_H-768_A-12/bert_config.json \
+  --vocab_path ./uncased_L-12_H-768_A-12/vocab.txt \
+  --init_pretraining_params ./uncased_L-12_H-768_A-12/params \
+  --checkpoints ./output/atis_slot \
+  --save_steps 100 \
+  --learning_rate 2e-5 \
+  --weight_decay  0.01 \
+  --max_seq_len 128 \
+  --skip_steps 10 \
+  --validation_steps 1000000 \
+  --num_iteration_per_drop_scope 10 \
+  --use_fp16 false \
+  --enable_ce
+}
+
+train_mrda(){
+  python -u train.py \
+  --task_name mrda \
+  --use_cuda true \
+  --do_train true \
+  --do_val true \
+  --do_test true \
+  --epoch 2 \
+  --batch_size 4096 \
+  --data_dir ./data/mrda \
+  --bert_config_path ./uncased_L-12_H-768_A-12/bert_config.json \
+  --vocab_path ./uncased_L-12_H-768_A-12/vocab.txt \
+  --init_pretraining_params ./uncased_L-12_H-768_A-12/params \
+  --checkpoints ./output/mrda \
+  --save_steps 500 \
+  --learning_rate 2e-5 \
+  --weight_decay  0.01 \
+  --max_seq_len 128 \
+  --skip_steps 200 \
+  --validation_steps 1000000 \
+  --num_iteration_per_drop_scope 10 \
+  --use_fp16 false \
+  --enable_ce
+}
+
+
+cudaid=${multi:=0,1,2,3}
+export CUDA_VISIBLE_DEVICES=$cudaid
+train_atis_slot | python _ce.py
+sleep 20
+
+cudaid=${single:=0}
+export CUDA_VISIBLE_DEVICES=$cudaid
+train_atis_slot | python _ce.py
+

PaddleNLP/dialogue_model_toolkit/dialogue_general_understanding/_ce.py

+# this file is only used for continuous evaluation test!
+
+import os
+import sys
+sys.path.append(os.environ['ceroot'])
+from kpi import CostKpi
+from kpi import DurationKpi
+from kpi import AccKpi
+
+each_step_duration_atis_slot_card1 = DurationKpi('each_step_duration_atis_slot_card1', 0.01, 0, actived=True)
+train_loss_atis_slot_card1 = CostKpi('train_loss_atis_slot_card1', 0.08, 0, actived=True)
+train_acc_atis_slot_card1 = CostKpi('train_acc_atis_slot_card1', 0.01, 0, actived=True)
+each_step_duration_atis_slot_card4 = DurationKpi('each_step_duration_atis_slot_card4', 0.06, 0, actived=True)
+train_loss_atis_slot_card4 = CostKpi('train_loss_atis_slot_card4', 0.03, 0, actived=True)
+train_acc_atis_slot_card4 = CostKpi('train_acc_atis_slot_card4', 0.01, 0, actived=True)
+
+tracking_kpis = [
+        each_step_duration_atis_slot_card1,
+        train_loss_atis_slot_card1,
+        train_acc_atis_slot_card1,
+        each_step_duration_atis_slot_card4,
+        train_loss_atis_slot_card4,
+        train_acc_atis_slot_card4,
+]
+
+
+def parse_log(log):
+    '''
+    This method should be implemented by model developers.
+
+    The suggestion:
+
+    each line in the log should be key, value, for example:
+
+    "
+    train_cost\t1.0
+    test_cost\t1.0
+    train_cost\t1.0
+    train_cost\t1.0
+    train_acc\t1.2
+    "
+    '''
+    for line in log.split('\n'):
+        fs = line.strip().split('\t')
+        print(fs)
+        if len(fs) == 3 and fs[0] == 'kpis':
+            kpi_name = fs[1]
+            kpi_value = float(fs[2])
+            yield kpi_name, kpi_value
+
+
+def log_to_ce(log):
+    kpi_tracker = {}
+    for kpi in tracking_kpis:
+        kpi_tracker[kpi.name] = kpi
+
+    for (kpi_name, kpi_value) in parse_log(log):
+        print(kpi_name, kpi_value)
+        kpi_tracker[kpi_name].add_record(kpi_value)
+        kpi_tracker[kpi_name].persist()
+
+
+if __name__ == '__main__':
+    log = sys.stdin.read()
+    log_to_ce(log)

PaddleNLP/dialogue_model_toolkit/dialogue_general_understanding/finetune_args.py

@@ -70,5 +70,6 @@ run_type_g.add_arg("task_name", str, None,
 run_type_g.add_arg("do_train", bool, True, "Whether to perform training.")
 run_type_g.add_arg("do_val", bool, True, "Whether to perform evaluation on dev data set.")
 run_type_g.add_arg("do_test", bool, True, "Whether to perform evaluation on test data set.")
+parser.add_argument('--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.')
 
 

PaddleNLP/dialogue_model_toolkit/dialogue_general_understanding/train.py

@@ -141,6 +141,8 @@ def main(args):
         print("Num warmup steps: %d" % warmup_steps)
 
         train_program = fluid.Program()
+        if args.random_seed is not None:
+            train_program.random_seed = args.random_seed
         with fluid.program_guard(train_program, startup_prog):
             with fluid.unique_name.guard():
                 results = create_model(
@@ -257,6 +259,7 @@ def main(args):
         steps = 0
         total_cost, total_acc, total_num_seqs = [], [], []
         time_begin = time.time()
+        ce_info = []
         while True:
             try: 
                 steps += 1
@@ -318,12 +321,14 @@ def main(args):
                                steps, np.sum(total_cost) / np.sum(total_num_seqs),
                                np.sum(total_acc) / np.sum(total_num_seqs),
                                args.skip_steps / used_time))
+                        ce_info.append([np.sum(total_cost) / np.sum(total_num_seqs), np.sum(total_acc) / np.sum(total_num_seqs), args.skip_steps / used_time])
                     else: 
                         print("%s epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
                             "speed: %f steps/s" %
                             (current_time, current_epoch, current_example, num_train_examples,
                             steps, np.sum(total_cost) / np.sum(total_num_seqs),
                             args.skip_steps / used_time))
+                        ce_info.append([np.sum(total_cost) / np.sum(total_num_seqs), args.skip_steps / used_time])
                     total_cost, total_acc, total_num_seqs = [], [], []
                     time_begin = time.time()
 
@@ -354,6 +359,25 @@ def main(args):
                 fluid.io.save_persistables(exe, save_path, train_program)
                 train_pyreader.reset()
                 break
+    if args.do_train and args.enable_ce:
+        card_num = get_cards()
+        print("zytest_card_num", card_num)
+        ce_loss = 0
+        ce_acc = 0
+        ce_time = 0
+        try:
+            ce_loss = ce_info[-2][0]
+            ce_acc = ce_info[-2][1]
+            ce_time = ce_info[-2][2]
+        except:
+            print("ce info error")
+        print("kpis\teach_step_duration_%s_card%s\t%s" %
+                (task_name, card_num, ce_time))
+        print("kpis\ttrain_loss_%s_card%s\t%f" %
+            (task_name, card_num, ce_loss))
+        print("kpis\ttrain_acc_%s_card%s\t%f" %
+            (task_name, card_num, ce_acc))
+
     #final eval on dev set
     if args.do_val: 
         test_pyreader.decorate_tensor_provider( 
@@ -375,6 +399,15 @@ def main(args):
         evaluate(test_exe, test_prog, test_pyreader, fetch_test_list, "test")
 
 
+def get_cards():
+    num = 0
+    cards = os.environ.get('CUDA_VISIBLE_DEVICES', '')
+    print("zytest_cards", cards)
+    if cards != '':
+        num = len(cards.split(","))
+    return num
+
+
 if __name__ == '__main__': 
     args = parser.parse_args()
     print_arguments(args)