refactor g2p egs (#630)

* refactor g2p egs

* add sha-bone; remove avg.sh from egs;

examples/aishell/s0/local/avg.sh

-#! /usr/bin/env bash
-
-if [ $# != 2 ];then
-    echo "usage: ${0} ckpt_dir avg_num"
-    exit -1
-fi
-
-ckpt_dir=${1}
-average_num=${2}
-decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
-
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
---dst_model ${decode_checkpoint} \
---ckpt_dir ${ckpt_dir}  \
---num ${average_num} \
---val_best
-            
-if [ $? -ne 0 ]; then
-    echo "Failed in avg ckpt!"
-    exit 1
-fi
-
-exit 0
\ No newline at end of file

examples/aishell/s1/local/test.sh

@@ -15,6 +15,10 @@ fi
 config_path=$1
 ckpt_prefix=$2
 
+ckpt_name=$(basename ${ckpt_prefxi})
+
+mkdir -p exp
+
 # download language model
 #bash local/download_lm_ch.sh
 #if [ $? -ne 0 ]; then
@@ -25,11 +29,13 @@ ckpt_prefix=$2
 for type in attention ctc_greedy_search; do
     echo "decoding ${type}"
     batch_size=64
+    output_dir=${ckpt_prefix}
+    mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
     --device ${device} \
     --nproc 1 \
     --config ${config_path} \
-    --result_file ${ckpt_prefix}.${type}.rsl \
+    --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
     --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
 
@@ -42,11 +48,13 @@ done
 for type in ctc_prefix_beam_search attention_rescoring; do
     echo "decoding ${type}"
     batch_size=1
+    output_dir=${ckpt_prefix}
+    mkdir -p ${output_dir}
     python3 -u ${BIN_DIR}/test.py \
     --device ${device} \
     --nproc 1 \
     --config ${config_path} \
-    --result_file ${ckpt_prefix}.${type}.rsl \
+    --result_file ${output_dir}/${type}.rsl \
     --checkpoint_path ${ckpt_prefix} \
     --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size}
 

examples/chinese_g2p/.gitignore

+data
+exp

examples/chinese_g2p/local/prepare_dataset.sh

-echo "Extracting Prosody Labeling"
+#!/bin/bash
 
 exp_dir="exp"
 data_dir="data"
+
 source ${MAIN_ROOT}/utils/parse_options.sh || exit -1
 
 archive=${data_dir}/"BZNSYP.rar"
@@ -17,7 +18,7 @@ if [ ${md5_result} != ${MD5} ]; then
     exit -1
 fi
 
-   
+
 label_file='ProsodyLabeling/000001-010000.txt'
 filename='000001-010000.txt'
 unrar e ${archive} ${label_file}

examples/chinese_g2p/run.sh

 #!/usr/bin/env bash
+
 source path.sh
 
 stage=-1
 stop_stage=100
 
-exp_dir="exp"
-data_dir="data"
+exp_dir=exp
+data_dir=data
+
 source ${MAIN_ROOT}/utils/parse_options.sh || exit -1
+
 mkdir -p ${exp_dir}
-bash local/prepare_dataset.sh --exp-dir ${exp_dir} --data-dir ${data_dir}
+
+
+if [ $stage -le 0 ] && [ $stop_stage -ge 0 ];then
+    echo "stage 0: Extracting Prosody Labeling"
+    bash local/prepare_dataset.sh --exp-dir ${exp_dir} --data-dir ${data_dir}
+fi
 
 # convert transcription in chinese into pinyin with pypinyin or jieba+pypinyin
 filename="000001-010000.txt"
-echo "Processing transcriptions..."
 
-python3 local/extract_pinyin_label.py ${exp_dir}/${filename} ${exp_dir}/"pinyin_baker.py"
-python3 local/convert_transcription.py ${exp_dir}/${filename} ${exp_dir}/"result_pypinyin.txt"
-python3 local/convert_transcription.py --use-jieba ${exp_dir}/${filename} ${exp_dir}/"result_pypinyin_with_jieba.txt"
+if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then
+    echo "stage 1: Processing transcriptions..."
+    python3 local/extract_pinyin_label.py ${exp_dir}/${filename} ${exp_dir}/ref.pinyin
+
+    python3 local/convert_transcription.py ${exp_dir}/${filename} ${exp_dir}/trans.pinyin
+    python3 local/convert_transcription.py --use-jieba ${exp_dir}/${filename} ${exp_dir}/trans.jieba.pinyin
+fi
 
 echo "done"
 exit 0

examples/librispeech/s0/local/avg.sh

-#! /usr/bin/env bash
-
-if [ $# != 2 ];then
-    echo "usage: ${0} ckpt_dir avg_num"
-    exit -1
-fi
-
-ckpt_dir=${1}
-average_num=${2}
-decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
-
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
---dst_model ${decode_checkpoint} \
---ckpt_dir ${ckpt_dir}  \
---num ${average_num} \
---val_best
-            
-if [ $? -ne 0 ]; then
-    echo "Failed in avg ckpt!"
-    exit 1
-fi
-
-exit 0
\ No newline at end of file

examples/librispeech/s1/conf/conformer.yaml

@@ -14,7 +14,7 @@ data:
   min_output_len: 0.0 # tokens
   max_output_len: 400.0 # tokens
   min_output_input_ratio: 0.05
-  max_output_input_ratio: 10.0 
+  max_output_input_ratio: 10.0
   raw_wav: True  # use raw_wav or kaldi feature
   specgram_type: fbank #linear, mfcc, fbank
   feat_dim: 80
@@ -77,7 +77,7 @@ model:
 training:
   n_epoch: 120
   accum_grad: 8
-  global_grad_clip: 5.0
+  global_grad_clip: 3.0
   optim: adam
   optim_conf:
     lr: 0.004

examples/librispeech/s1/local/avg.sh

-#! /usr/bin/env bash
-
-if [ $# != 2 ]; then
-    echo "usage: ${0} ckpt_dir avg_num"
-    exit -1
-fi
-
-ckpt_dir=${1}
-average_num=${2}
-decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
-
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
---dst_model ${decode_checkpoint} \
---ckpt_dir ${ckpt_dir}  \
---num ${average_num} \
---val_best
-
-if [ $? -ne 0 ]; then
-    echo "Failed in avg ckpt!"
-    exit 1
-fi
-
-exit 0

examples/tiny/s0/local/avg.sh

-#! /usr/bin/env bash
-
-if [ $# != 2 ];then
-    echo "usage: ${0} ckpt_dir avg_num"
-    exit -1
-fi
-
-ckpt_dir=${1}
-average_num=${2}
-decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
-
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
---dst_model ${decode_checkpoint} \
---ckpt_dir ${ckpt_dir}  \
---num ${average_num} \
---val_best
-            
-if [ $? -ne 0 ]; then
-    echo "Failed in avg ckpt!"
-    exit 1
-fi
-
-exit 0
\ No newline at end of file

examples/tiny/s1/local/avg.sh

-#! /usr/bin/env bash
-
-if [ $# != 2 ];then
-    echo "usage: ${0} ckpt_dir avg_num"
-    exit -1
-fi
-
-ckpt_dir=${1}
-average_num=${2}
-decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
-
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
---dst_model ${decode_checkpoint} \
---ckpt_dir ${ckpt_dir}  \
---num ${average_num} \
---val_best
-            
-if [ $? -ne 0 ]; then
-    echo "Failed in avg ckpt!"
-    exit 1
-fi
-
-exit 0
\ No newline at end of file

examples/aishell/s1/local/avg.sh → utils/avg.sh

@@ -9,7 +9,7 @@ ckpt_dir=${1}
 average_num=${2}
 decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams
 
-python3 -u ${MAIN_ROOT}/utils/avg_model.py \
+avg_model.py \
 --dst_model ${decode_checkpoint} \
 --ckpt_dir ${ckpt_dir}  \
 --num ${average_num} \

utils/avg_model.py

+#!/usr/bin/env python3
 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

utils/build_vocab.py

+#!/usr/bin/env python3
 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -70,8 +71,8 @@ def main():
     fout.write(UNK + '\n')  # <unk> must be 1
 
     if args.unit_type == 'spm':
-        # tools/spm_train --input=$wave_data/lang_char/input.txt 
-        # --vocab_size=${nbpe} --model_type=${bpemode} 
+        # tools/spm_train --input=$wave_data/lang_char/input.txt
+        # --vocab_size=${nbpe} --model_type=${bpemode}
         # --model_prefix=${bpemodel} --input_sentence_size=100000000
         import sentencepiece as spm
 

utils/compute_mean_std.py

+#!/usr/bin/env python3
 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

utils/format_data.py

+#!/usr/bin/env python3
 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

utils/spm_decode

-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) Facebook, Inc. and its affiliates.
 # All rights reserved.
 #

utils/spm_encode

-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright (c) Facebook, Inc. and its affiliates.
 # All rights reserved.
 #