release scripts

ERNIE/config/ernie_config.json

+{
+  "attention_probs_dropout_prob": 0.1, 
+  "hidden_act": "relu", 
+  "hidden_dropout_prob": 0.1, 
+  "hidden_size": 768, 
+  "initializer_range": 0.02, 
+  "max_position_embeddings": 513, 
+  "num_attention_heads": 12, 
+  "num_hidden_layers": 12, 
+  "type_vocab_size": 2, 
+  "vocab_size": 18000
+}

ERNIE/data/train_filelist

+./data/demo_train_set.gz	1.1

ERNIE/data/valid_filelist

+./data/demo_valid_set.gz	1.1

ERNIE/pretrain_args.py

@@ -38,7 +38,7 @@ train_g.add_arg("lr_scheduler",      str,    "linear_warmup_decay",
                 "scheduler of learning rate.", choices=['linear_warmup_decay', 'noam_decay'])
 train_g.add_arg("weight_decay",      float,  0.01,    "Weight decay rate for L2 regularizer.")
 train_g.add_arg("num_train_steps",   int,    1000000, "Total steps to perform pretraining.")
-train_g.add_arg("warmup_steps",      int,    4000,    "Total steps to perform warmup when pretraining.")
+train_g.add_arg("warmup_steps",      int,    5000,    "Total steps to perform warmup when pretraining.")
 train_g.add_arg("save_steps",        int,    10000,   "The steps interval to save checkpoints.")
 train_g.add_arg("validation_steps",  int,    1000,    "The steps interval to evaluate model performance.")
 train_g.add_arg("use_fp16",          bool,   False,   "Whether to use fp16 mixed precision training.")

ERNIE/script/pretrain.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+python -u ./train.py --use_cuda True \
+                --is_distributed False\
+                --use_fast_executor True \
+                --weight_sharing True \
+                --in_tokens true \
+                --batch_size 8192 \
+                --vocab_path ./config/vocab.txt \
+                --train_filelist ./data/train_filelist \
+                --valid_filelist ./data/valid_filelist \
+                --validation_steps 100 \
+                --num_train_steps 1000000 \
+                --checkpoints ./checkpoints \
+                --save_steps 10000 \
+                --ernie_config_path ./config/ernie_config.json \
+                --learning_rate 1e-4 \
+                --weight_decay 0.01 \
+                --max_seq_len 512 \
+                --skip_steps 10

ERNIE/script/run_ChnSentiCorp.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0
+
+python -u run_classifier.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train true \
+                   --do_val true \
+                   --do_test true \
+                   --batch_size 24 \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TASK_DATA_PATH}/chnsenticorp/train.tsv \
+                   --dev_set ${TASK_DATA_PATH}/chnsenticorp/dev.tsv \
+                   --test_set ${TASK_DATA_PATH}/chnsenticorp/test.tsv \
+                   --vocab_path config/vocab.txt \
+                   --checkpoints ./checkpoints \
+                   --save_steps 1000 \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --validation_steps 100 \
+                   --epoch 10 \
+                   --max_seq_len 256 \
+                   --ernie_config_path config/ernie_config.json \
+                   --learning_rate 5e-5 \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 2 \
+                   --random_seed 1

ERNIE/script/run_dbqa.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+python -u run_classifier.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train true \
+                   --do_val true \
+                   --do_test true \
+                   --batch_size 8 \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TASK_DATA_PATH}/nlpcc-dbqa/train.tsv \
+                   --dev_set ${TASK_DATA_PATH}/nlpcc-dbqa/dev.tsv \
+                   --test_set ${TASK_DATA_PATH}/nlpcc-dbqa/test.tsv \
+                   --vocab_path config/vocab.txt \
+                   --ernie_config_path config/ernie_config.json \
+                   --checkpoints "./checkpoints" \
+                   --save_steps 1000 \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --validation_steps 1000 \
+                   --epoch 3 \
+                   --max_seq_len 512 \
+                   --learning_rate 2e-5 \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 2 \
+                   --random_seed 1

ERNIE/script/run_lcqmc.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0
+
+python -u run_classifier.py \
+                   --use_cuda true \
+                   --verbose true \
+                   --do_train true \
+                   --do_val true \
+                   --do_test true \
+                   --batch_size 32 \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TASK_DATA_PATH}/lcqmc/train.tsv \
+                   --dev_set ${TASK_DATA_PATH}/lcqmc/dev.tsv \
+                   --test_set ${TASK_DATA_PATH}/lcqmc/test.tsv \
+                   --vocab_path config/vocab.txt \
+                   --checkpoints ./checkpoints \
+                   --save_steps 1000 \
+                   --weight_decay  0.0 \
+                   --warmup_proportion 0.0 \
+                   --validation_steps 100 \
+                   --epoch 3 \
+                   --max_seq_len 128 \
+                   --ernie_config_path config/ernie_config.json \
+                   --learning_rate 2e-5 \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 2 \
+                   --random_seed 1

ERNIE/script/run_msra_ner.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0
+
+python -u run_sequence_labeling.py \
+                   --use_cuda true \
+                   --do_train true \
+                   --do_val true \
+                   --do_test true \
+                   --batch_size 16 \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --num_labels 7 \
+                   --label_map_config ${TASK_DATA_PATH}/label_map.json \
+                   --train_set ${TASK_DATA_PATH}/msra_ner/train.tsv \
+                   --dev_set ${TASK_DATA_PATH}/msra_ner/dev.tsv \
+                   --test_set ${TASK_DATA_PATH}/msra_ner/test.tsv \
+                   --vocab_path config/vocab.txt \
+                   --ernie_config_path config/ernie_config.json \
+                   --checkpoints ./checkpoints \
+                   --save_steps 100000 \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --validation_steps 100 \
+                   --epoch 3 \
+                   --max_seq_len 256 \
+                   --learning_rate 5e-5 \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --random_seed 1

ERNIE/script/run_xnli.sh

+set -eux
+
+export FLAGS_sync_nccl_allreduce=1
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+
+python -u run_classifier.py \
+                   --use_cuda true \
+                   --do_train true \
+                   --do_val true \
+                   --do_test true \
+                   --verbose true \
+                   --batch_size 8192 \
+                   --in_tokens true \
+                   --init_pretraining_params ${MODEL_PATH} \
+                   --train_set ${TASK_DATA_PATH}/xnli/train.tsv \
+                   --dev_set ${TASK_DATA_PATH}/xnli/dev.tsv \
+                   --test_set ${TASK_DATA_PATH}/xnli/test.tsv \
+                   --vocab_path config/vocab.txt \
+                   --label_map ${TASK_DATA_PATH}/xnli/label_map.json \
+                   --ernie_config_path config/ernie_config.json \
+                   --checkpoints ./checkpoints \
+                   --save_steps 1000 \
+                   --weight_decay  0.01 \
+                   --warmup_proportion 0.0 \
+                   --validation_steps 25 \
+                   --epoch 3 \
+                   --max_seq_len 512 \
+                   --learning_rate 1e-4 \
+                   --skip_steps 10 \
+                   --num_iteration_per_drop_scope 1 \
+                   --num_labels 3 \
+                   --random_seed 1