update run scripts

examples/ted_en_zh/st1/conf/transformer_mtl_noam.yaml

@@ -11,9 +11,9 @@ data:
   max_output_input_ratio: 20.0
 
 collator:
-  vocab_filepath: data/lang_char/vocab.txt
+  vocab_filepath: data/lang_char/ted_en_zh_bpe8000.txt
   unit_type: 'spm'
-  spm_model_prefix: data/train_sp.en-zh-nlpr.zh-nlpr_bpe8000_tc
+  spm_model_prefix: data/lang_char/ted_en_zh_bpe8000
   mean_std_filepath: ""
   # augmentation_config: conf/augmentation.json
   batch_size: 10

examples/ted_en_zh/st1/local/data.sh

@@ -166,7 +166,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
         ${prep_dir}/test.en-zh.zh/feats.scp ${prep_dir}/train_sp.en-zh.zh/cmvn.ark ${prep_dir}/dump_feats/test.en-zh.zh ${feat_trans_dir}
 fi
 
-dict=${dict_dir}/ted_en_zh_${bpemode}${nbpe}_joint.txt
+dict=${dict_dir}/ted_en_zh_${bpemode}${nbpe}.txt
 nlsyms=${dict_dir}/ted_en_zh_non_lang_syms.txt
 bpemodel=${dict_dir}/ted_en_zh_${bpemode}${nbpe}
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then

examples/ted_en_zh/st1/local/train_finetune.sh

-#!/bin/bash
-
-if [ $# != 3 ];then
-    echo "usage: CUDA_VISIBLE_DEVICES=0 ${0} config_path ckpt_name ckpt_path"
-    exit -1
-fi
-
-ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
-echo "using $ngpu gpus..."
-
-config_path=$1
-ckpt_name=$2
-ckpt_path=$3
-
-mkdir -p exp
-
-# seed may break model convergence
-seed=0
-if [ ${seed} != 0 ]; then
-    export FLAGS_cudnn_deterministic=True
-fi
-
-python3 -u ${BIN_DIR}/train.py \
---ngpu ${ngpu} \
---config ${config_path} \
---output exp/${ckpt_name} \
---checkpoint_path ${ckpt_path} \
---seed ${seed}
-
-if [ ${seed} != 0 ]; then
-    unset FLAGS_cudnn_deterministic
-fi
-
-if [ $? -ne 0 ]; then
-    echo "Failed in training!"
-    exit 1
-fi
-
-exit 0
\ No newline at end of file

examples/ted_en_zh/st1/run.sh

@@ -6,7 +6,7 @@ gpus=0,1,2,3
 stage=1
 stop_stage=4
 conf_path=conf/transformer_mtl_noam.yaml
-ckpt_path=paddle.98
+ckpt= # paddle.98 # (finetune from FAT-ST pretrained model)
 avg_num=5
 data_path=./TED_EnZh # path to unzipped data
 source ${MAIN_ROOT}/utils/parse_options.sh || exit 1;
@@ -22,21 +22,20 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
 fi
 
 if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
-    # download pretrained
-    bash ./local/download_pretrain.sh || exit -1
-fi
-
-if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # train model, all `ckpt` under `exp` dir
-    CUDA_VISIBLE_DEVICES=${gpus} ./local/train_finetune.sh ${conf_path}  ${ckpt} ${ckpt_path}
+    if [ -n "${ckpt_path}" ]; then
+        echo "Finetune from Pretrained Model" ${ckpt_path}
+        ./local/download_pretrain.sh || exit -1
+    fi 
+    CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${ckpt} "${ckpt_path}"
 fi
 
-if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # avg n best model
     avg.sh best exp/${ckpt}/checkpoints ${avg_num}
 fi
 
-if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then
+if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
     # test ckpt avg_n
     CUDA_VISIBLE_DEVICES=0 ./local/test.sh ${conf_path} exp/${ckpt}/checkpoints/${avg_ckpt} || exit -1
 fi
\ No newline at end of file