diff --git a/examples/aishell/s0/local/avg.sh b/examples/aishell/s0/local/avg.sh deleted file mode 100755 index 8589e35308c97fdcb1abbb13bcf35ca4042c3ae2..0000000000000000000000000000000000000000 --- a/examples/aishell/s0/local/avg.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env bash - -if [ $# != 2 ];then - echo "usage: ${0} ckpt_dir avg_num" - exit -1 -fi - -ckpt_dir=${1} -average_num=${2} -decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams - -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ ---dst_model ${decode_checkpoint} \ ---ckpt_dir ${ckpt_dir} \ ---num ${average_num} \ ---val_best - -if [ $? -ne 0 ]; then - echo "Failed in avg ckpt!" - exit 1 -fi - -exit 0 \ No newline at end of file diff --git a/examples/aishell/s1/local/test.sh b/examples/aishell/s1/local/test.sh index 0dfabc6e4b5d86eb262580edb03dbdc4b98117fa..073aaffd4ddfe1a982280c7c0891a05b10f2ad12 100755 --- a/examples/aishell/s1/local/test.sh +++ b/examples/aishell/s1/local/test.sh @@ -15,6 +15,10 @@ fi config_path=$1 ckpt_prefix=$2 +ckpt_name=$(basename ${ckpt_prefxi}) + +mkdir -p exp + # download language model #bash local/download_lm_ch.sh #if [ $? -ne 0 ]; then @@ -25,11 +29,13 @@ ckpt_prefix=$2 for type in attention ctc_greedy_search; do echo "decoding ${type}" batch_size=64 + output_dir=${ckpt_prefix} + mkdir -p ${output_dir} python3 -u ${BIN_DIR}/test.py \ --device ${device} \ --nproc 1 \ --config ${config_path} \ - --result_file ${ckpt_prefix}.${type}.rsl \ + --result_file ${output_dir}/${type}.rsl \ --checkpoint_path ${ckpt_prefix} \ --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size} @@ -42,11 +48,13 @@ done for type in ctc_prefix_beam_search attention_rescoring; do echo "decoding ${type}" batch_size=1 + output_dir=${ckpt_prefix} + mkdir -p ${output_dir} python3 -u ${BIN_DIR}/test.py \ --device ${device} \ --nproc 1 \ --config ${config_path} \ - --result_file ${ckpt_prefix}.${type}.rsl \ + --result_file ${output_dir}/${type}.rsl \ --checkpoint_path ${ckpt_prefix} \ --opts decoding.decoding_method ${type} decoding.batch_size ${batch_size} diff --git a/examples/chinese_g2p/.gitignore b/examples/chinese_g2p/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..bbd86a25b018bc611bf6ae52cbb6afa5f60bce62 --- /dev/null +++ b/examples/chinese_g2p/.gitignore @@ -0,0 +1,2 @@ +data +exp diff --git a/examples/chinese_g2p/local/prepare_dataset.sh b/examples/chinese_g2p/local/prepare_dataset.sh index fe9948ed3f9cfcf4e8b61ad9100c41439710126a..3933d96cfbd5f8c18bb3b423cbdf2c8619273d66 100644 --- a/examples/chinese_g2p/local/prepare_dataset.sh +++ b/examples/chinese_g2p/local/prepare_dataset.sh @@ -1,7 +1,8 @@ -echo "Extracting Prosody Labeling" +#!/bin/bash exp_dir="exp" data_dir="data" + source ${MAIN_ROOT}/utils/parse_options.sh || exit -1 archive=${data_dir}/"BZNSYP.rar" @@ -17,7 +18,7 @@ if [ ${md5_result} != ${MD5} ]; then exit -1 fi - + label_file='ProsodyLabeling/000001-010000.txt' filename='000001-010000.txt' unrar e ${archive} ${label_file} diff --git a/examples/chinese_g2p/run.sh b/examples/chinese_g2p/run.sh old mode 100644 new mode 100755 index 6bde2e264a0d4411625b7d1bbf496d3034e395ee..8197dce4bea7f39182d40b510883be222939ddeb --- a/examples/chinese_g2p/run.sh +++ b/examples/chinese_g2p/run.sh @@ -1,22 +1,33 @@ #!/usr/bin/env bash + source path.sh stage=-1 stop_stage=100 -exp_dir="exp" -data_dir="data" +exp_dir=exp +data_dir=data + source ${MAIN_ROOT}/utils/parse_options.sh || exit -1 + mkdir -p ${exp_dir} -bash local/prepare_dataset.sh --exp-dir ${exp_dir} --data-dir ${data_dir} + + +if [ $stage -le 0 ] && [ $stop_stage -ge 0 ];then + echo "stage 0: Extracting Prosody Labeling" + bash local/prepare_dataset.sh --exp-dir ${exp_dir} --data-dir ${data_dir} +fi # convert transcription in chinese into pinyin with pypinyin or jieba+pypinyin filename="000001-010000.txt" -echo "Processing transcriptions..." -python3 local/extract_pinyin_label.py ${exp_dir}/${filename} ${exp_dir}/"pinyin_baker.py" -python3 local/convert_transcription.py ${exp_dir}/${filename} ${exp_dir}/"result_pypinyin.txt" -python3 local/convert_transcription.py --use-jieba ${exp_dir}/${filename} ${exp_dir}/"result_pypinyin_with_jieba.txt" +if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then + echo "stage 1: Processing transcriptions..." + python3 local/extract_pinyin_label.py ${exp_dir}/${filename} ${exp_dir}/ref.pinyin + + python3 local/convert_transcription.py ${exp_dir}/${filename} ${exp_dir}/trans.pinyin + python3 local/convert_transcription.py --use-jieba ${exp_dir}/${filename} ${exp_dir}/trans.jieba.pinyin +fi echo "done" exit 0 diff --git a/examples/librispeech/s0/local/avg.sh b/examples/librispeech/s0/local/avg.sh deleted file mode 100755 index 8589e35308c97fdcb1abbb13bcf35ca4042c3ae2..0000000000000000000000000000000000000000 --- a/examples/librispeech/s0/local/avg.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env bash - -if [ $# != 2 ];then - echo "usage: ${0} ckpt_dir avg_num" - exit -1 -fi - -ckpt_dir=${1} -average_num=${2} -decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams - -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ ---dst_model ${decode_checkpoint} \ ---ckpt_dir ${ckpt_dir} \ ---num ${average_num} \ ---val_best - -if [ $? -ne 0 ]; then - echo "Failed in avg ckpt!" - exit 1 -fi - -exit 0 \ No newline at end of file diff --git a/examples/librispeech/s1/conf/conformer.yaml b/examples/librispeech/s1/conf/conformer.yaml index fdc435b8a7b55590022c23c6e748cb86fc0a92a2..8f8bf45398813179db88781dcfc5c71356295934 100644 --- a/examples/librispeech/s1/conf/conformer.yaml +++ b/examples/librispeech/s1/conf/conformer.yaml @@ -14,7 +14,7 @@ data: min_output_len: 0.0 # tokens max_output_len: 400.0 # tokens min_output_input_ratio: 0.05 - max_output_input_ratio: 10.0 + max_output_input_ratio: 10.0 raw_wav: True # use raw_wav or kaldi feature specgram_type: fbank #linear, mfcc, fbank feat_dim: 80 @@ -77,7 +77,7 @@ model: training: n_epoch: 120 accum_grad: 8 - global_grad_clip: 5.0 + global_grad_clip: 3.0 optim: adam optim_conf: lr: 0.004 diff --git a/examples/librispeech/s1/local/avg.sh b/examples/librispeech/s1/local/avg.sh deleted file mode 100755 index 3d15ce2372917ac135ebeaedd8d2ea98c0289969..0000000000000000000000000000000000000000 --- a/examples/librispeech/s1/local/avg.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env bash - -if [ $# != 2 ]; then - echo "usage: ${0} ckpt_dir avg_num" - exit -1 -fi - -ckpt_dir=${1} -average_num=${2} -decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams - -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ ---dst_model ${decode_checkpoint} \ ---ckpt_dir ${ckpt_dir} \ ---num ${average_num} \ ---val_best - -if [ $? -ne 0 ]; then - echo "Failed in avg ckpt!" - exit 1 -fi - -exit 0 diff --git a/examples/tiny/s0/local/avg.sh b/examples/tiny/s0/local/avg.sh deleted file mode 100755 index 8589e35308c97fdcb1abbb13bcf35ca4042c3ae2..0000000000000000000000000000000000000000 --- a/examples/tiny/s0/local/avg.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env bash - -if [ $# != 2 ];then - echo "usage: ${0} ckpt_dir avg_num" - exit -1 -fi - -ckpt_dir=${1} -average_num=${2} -decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams - -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ ---dst_model ${decode_checkpoint} \ ---ckpt_dir ${ckpt_dir} \ ---num ${average_num} \ ---val_best - -if [ $? -ne 0 ]; then - echo "Failed in avg ckpt!" - exit 1 -fi - -exit 0 \ No newline at end of file diff --git a/examples/tiny/s1/local/avg.sh b/examples/tiny/s1/local/avg.sh deleted file mode 100755 index 8589e35308c97fdcb1abbb13bcf35ca4042c3ae2..0000000000000000000000000000000000000000 --- a/examples/tiny/s1/local/avg.sh +++ /dev/null @@ -1,23 +0,0 @@ -#! /usr/bin/env bash - -if [ $# != 2 ];then - echo "usage: ${0} ckpt_dir avg_num" - exit -1 -fi - -ckpt_dir=${1} -average_num=${2} -decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams - -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ ---dst_model ${decode_checkpoint} \ ---ckpt_dir ${ckpt_dir} \ ---num ${average_num} \ ---val_best - -if [ $? -ne 0 ]; then - echo "Failed in avg ckpt!" - exit 1 -fi - -exit 0 \ No newline at end of file diff --git a/examples/aishell/s1/local/avg.sh b/utils/avg.sh similarity index 88% rename from examples/aishell/s1/local/avg.sh rename to utils/avg.sh index 3d15ce2372917ac135ebeaedd8d2ea98c0289969..c8a6ddfea7b819001c5359605a969f48554af6f0 100755 --- a/examples/aishell/s1/local/avg.sh +++ b/utils/avg.sh @@ -9,7 +9,7 @@ ckpt_dir=${1} average_num=${2} decode_checkpoint=${ckpt_dir}/avg_${average_num}.pdparams -python3 -u ${MAIN_ROOT}/utils/avg_model.py \ +avg_model.py \ --dst_model ${decode_checkpoint} \ --ckpt_dir ${ckpt_dir} \ --num ${average_num} \ diff --git a/utils/avg_model.py b/utils/avg_model.py index 0794c4b8f1f9ea9d784fdb7fefebf6edd4795ee9..8ec792f5222bf9df9ae8576d7cce4c4bc7290538 100755 --- a/utils/avg_model.py +++ b/utils/avg_model.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/build_vocab.py b/utils/build_vocab.py index 609ce8e7e2e45a16e8078cb97e46024ed5646168..76092b25dbbded9354c3f6f1e70a058fbd50e913 100755 --- a/utils/build_vocab.py +++ b/utils/build_vocab.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,8 +71,8 @@ def main(): fout.write(UNK + '\n') # must be 1 if args.unit_type == 'spm': - # tools/spm_train --input=$wave_data/lang_char/input.txt - # --vocab_size=${nbpe} --model_type=${bpemode} + # tools/spm_train --input=$wave_data/lang_char/input.txt + # --vocab_size=${nbpe} --model_type=${bpemode} # --model_prefix=${bpemodel} --input_sentence_size=100000000 import sentencepiece as spm diff --git a/utils/compute_mean_std.py b/utils/compute_mean_std.py index aff6f47c6ee04629bc56ee44ab56294a272f044d..a468153d3d2bba4a9afee02e933e6bf3c942606f 100755 --- a/utils/compute_mean_std.py +++ b/utils/compute_mean_std.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/format_data.py b/utils/format_data.py index 9565da14a8a2fd47fb898c7cdbbb7053edc101a2..72262b2ffe6203e9efc475ff0efef49b9e8b5b45 100755 --- a/utils/format_data.py +++ b/utils/format_data.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/utils/spm_decode b/utils/spm_decode index 8497ad8110f5b4a5834aab99320329a3a2675fc3..f22c5bb30961c447f654d7ef75c58ade44168e9e 100755 --- a/utils/spm_decode +++ b/utils/spm_decode @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. # All rights reserved. # diff --git a/utils/spm_encode b/utils/spm_encode index 99357045a4abd123764bf836eda0ffe7485985d0..92ac5c1d9f49284e47d9bd5aaf5e451fff1beb03 100755 --- a/utils/spm_encode +++ b/utils/spm_encode @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. # All rights reserved. #