Merge pull request #884 from PaddlePaddle/test

using simple test for multi decode type, and using gpu

examples/librispeech/s1/README.md

@@ -31,6 +31,8 @@
 
 
 ## Transformer
+
+### Test w/ 20s length filter
 | Model | Params | Config | Augmentation| Test set | Decode method | Loss | WER |  
 | --- | --- | --- | --- | --- | --- | --- | --- |
 | transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean | attention | 6.98 | 0.036 |  
@@ -38,7 +40,7 @@
 ### Test w/o length filter
 | Model | Params | Config | Augmentation| Test set | Decode method | Loss | WER |  
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | attention | 7.63 | 0.056832 |  
-| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | ctc_greedy_search | 7.63 | 0.059742 |  
-| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | ctc_prefix_beam_search | 7.63 | 0.059057 |  
-| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | attention_rescoring | 7.63 | 0.047417 |  
+| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | attention | 7.404532432556152 | 0.056204 |  
+| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | ctc_greedy_search | 7.404532432556152 | 0.058658 |  
+| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | ctc_prefix_beam_search | 7.404532432556152 | 0.058278 |  
+| transformer | 32.52 M | conf/transformer.yaml | spec_aug + shift | test-clean-all | attention_rescoring | 7.404532432556152 | 0.045591 |  

examples/librispeech/s1/local/test.sh

@@ -11,14 +11,12 @@ lmtag=
 recog_set="test-clean test-other dev-clean dev-other"
 recog_set="test-clean"
 
-# bpemode (unigram or bpe)
-nbpe=5000
-bpemode=unigram
-bpeprefix="data/bpe_${bpemode}_${nbpe}"
-bpemodel=${bpeprefix}.model
-
-if [ $# != 3 ];then
-    echo "usage: ${0} config_path dict_path ckpt_path_prefix"
+
+stage=0
+stop_stage=100
+
+if [ $# != 2 ];then
+    echo "usage: ${0} config_path ckpt_path_prefix"
     exit -1
 fi
 
@@ -26,8 +24,7 @@ ngpu=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')
 echo "using $ngpu gpus..."
 
 config_path=$1
-dict=$2
-ckpt_prefix=$3
+ckpt_prefix=$2
 
 chunk_mode=false
 if [[ ${config_path} =~ ^.*chunk_.*yaml$ ]];then
@@ -42,46 +39,75 @@ echo "chunk mode ${chunk_mode}"
 #    exit 1
 #fi
 
-pids=() # initialize pids
-
-for dmethd in attention ctc_greedy_search ctc_prefix_beam_search attention_rescoring; do
-(
-    for rtask in ${recog_set}; do
-    (
-        decode_dir=decode_${rtask}_${dmethd}_$(basename ${config_path%.*})_${lmtag}
-        feat_recog_dir=${datadir}
-        mkdir -p ${expdir}/${decode_dir}
-        mkdir -p ${feat_recog_dir}
 
-        # split data
-        split_json.sh ${feat_recog_dir}/manifest.${rtask} ${nj}
-
-        #### use CPU for decoding
-        ngpu=0
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+    for type in attention; do
+        echo "decoding ${type}"
+        if [ ${chunk_mode} == true ];then
+            # stream decoding only support batchsize=1
+            batch_size=1
+        else
+            batch_size=64
+        fi
+        python3 -u ${BIN_DIR}/test.py \
+            --nproc ${ngpu} \
+            --config ${config_path} \
+            --result_file ${ckpt_prefix}.${type}.rsl \
+            --checkpoint_path ${ckpt_prefix} \
+            --opts decoding.decoding_method ${type} \
+            --opts decoding.batch_size ${batch_size}
+
+        if [ $? -ne 0 ]; then
+            echo "Failed in evaluation!"
+            exit 1
+        fi
+        echo "decoding ${type} done."
+    done
+fi
 
-        # set batchsize 0 to disable batch decoding
+for type in ctc_greedy_search; do
+    echo "decoding ${type}"
+    if [ ${chunk_mode} == true ];then
+        # stream decoding only support batchsize=1
         batch_size=1
-        ${decode_cmd} JOB=1:${nj} ${expdir}/${decode_dir}/log/decode.JOB.log \
+    else
+        batch_size=64
+    fi
     python3 -u ${BIN_DIR}/test.py \
         --nproc ${ngpu} \
         --config ${config_path} \
-            --result_file ${expdir}/${decode_dir}/data.JOB.json \
+        --result_file ${ckpt_prefix}.${type}.rsl \
         --checkpoint_path ${ckpt_prefix} \
-            --opts decoding.decoding_method ${dmethd} \
-            --opts decoding.batch_size ${batch_size} \
-            --opts data.test_manifest ${feat_recog_dir}/split${nj}/JOB/manifest.${rtask}
+        --opts decoding.decoding_method ${type} \
+        --opts decoding.batch_size ${batch_size}
+
+    if [ $? -ne 0 ]; then
+        echo "Failed in evaluation!"
+        exit 1
+    fi
+    echo "decoding ${type} done."
+done
 
-        score_sclite.sh --bpe ${nbpe} --bpemodel ${bpemodel}.model --wer true ${expdir}/${decode_dir} ${dict}
 
-    ) &
-    pids+=($!) # store background pids
-    done
-) &
-pids+=($!) # store background pids
+
+for type in ctc_prefix_beam_search attention_rescoring; do
+    echo "decoding ${type}"
+    batch_size=1
+    python3 -u ${BIN_DIR}/test.py \
+        --nproc ${ngpu} \
+        --config ${config_path} \
+        --result_file ${ckpt_prefix}.${type}.rsl \
+        --checkpoint_path ${ckpt_prefix} \
+        --opts decoding.decoding_method ${type} \
+        --opts decoding.batch_size ${batch_size}
+
+    if [ $? -ne 0 ]; then
+        echo "Failed in evaluation!"
+        exit 1
+    fi
+    echo "decoding ${type} done."
 done
 
-i=0; for pid in "${pids[@]}"; do wait ${pid} || ((++i)); done
-[ ${i} -gt 0 ] && echo "$0: ${i} background jobs are failed." && false
 echo "Finished"
 
 exit 0

examples/librispeech/s1/run.sh

@@ -7,7 +7,7 @@ set -e
 stage=0
 stop_stage=100
 conf_path=conf/transformer.yaml
-avg_num=5
+avg_num=30
 
 . ${MAIN_ROOT}/utils/parse_options.sh || exit 1;