diff --git a/examples/aishell/asr0/conf/deepspeech2.yaml b/examples/aishell/asr0/conf/deepspeech2.yaml index 8fb4e6e8dff1b9d5606bc56b1320d00d7d3a309b..bdfa42199011973cdef445583663822273daac8f 100644 --- a/examples/aishell/asr0/conf/deepspeech2.yaml +++ b/examples/aishell/asr0/conf/deepspeech2.yaml @@ -14,7 +14,7 @@ collator: batch_size: 64 # one gpu mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/aishell/asr0/conf/deepspeech2_online.yaml b/examples/aishell/asr0/conf/deepspeech2_online.yaml index 29ec2379fad09ae9f517159c5059ac929fbb0006..010d8f155e48558b830f15af1a3b24aa2113d1e9 100644 --- a/examples/aishell/asr0/conf/deepspeech2_online.yaml +++ b/examples/aishell/asr0/conf/deepspeech2_online.yaml @@ -14,7 +14,7 @@ collator: batch_size: 64 # one gpu mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/aishell/asr0/local/data.sh b/examples/aishell/asr0/local/data.sh index 23f04f2a640b5cbeb0ac43fbd07877642dee38ae..1032cedc828932d76e3d71951a745af5ad0d6213 100755 --- a/examples/aishell/asr0/local/data.sh +++ b/examples/aishell/asr0/local/data.sh @@ -3,9 +3,12 @@ stage=-1 stop_stage=100 -source ${MAIN_ROOT}/utils/parse_options.sh +dict_dir=data/lang_char + +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -52,7 +55,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type="char" \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths "data/manifest.train.raw" "data/manifest.dev.raw" if [ $? -ne 0 ]; then @@ -68,7 +71,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type "char" \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${dataset}.raw" \ --output_path="data/manifest.${dataset}" diff --git a/examples/aishell/asr1/conf/chunk_conformer.yaml b/examples/aishell/asr1/conf/chunk_conformer.yaml index 336a6c46224d6be6cd0d1846a742de66973f5109..e07cd07c5b99a5bb1ed166315df4a0df3350f882 100644 --- a/examples/aishell/asr1/conf/chunk_conformer.yaml +++ b/examples/aishell/asr1/conf/chunk_conformer.yaml @@ -12,7 +12,7 @@ data: collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml diff --git a/examples/aishell/asr1/conf/conformer.yaml b/examples/aishell/asr1/conf/conformer.yaml index 0e9d79d8b25e0a19955db8d7c3f9f79c25501ae8..154f44a25ca31a69a68fc195a2a4e3211632e333 100644 --- a/examples/aishell/asr1/conf/conformer.yaml +++ b/examples/aishell/asr1/conf/conformer.yaml @@ -12,7 +12,7 @@ data: collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml diff --git a/examples/aishell/asr1/conf/transformer.yaml b/examples/aishell/asr1/conf/transformer.yaml index c021f66b71513b300b98d6f50fcc39573cc85dca..d13f9e2f36d5ba1335749542073b21e184829030 100644 --- a/examples/aishell/asr1/conf/transformer.yaml +++ b/examples/aishell/asr1/conf/transformer.yaml @@ -12,7 +12,7 @@ data: collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml diff --git a/examples/aishell/asr1/local/data.sh b/examples/aishell/asr1/local/data.sh index 76e28075298c4817e6b553fcb0870cf145fa06f0..41843231846fde71a4a657599e081ff1f6acd281 100755 --- a/examples/aishell/asr1/local/data.sh +++ b/examples/aishell/asr1/local/data.sh @@ -2,10 +2,12 @@ stage=-1 stop_stage=100 +dict_dir=data/lang_char -source ${MAIN_ROOT}/utils/parse_options.sh +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -53,7 +55,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type="char" \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths "data/manifest.train.raw" if [ $? -ne 0 ]; then @@ -69,7 +71,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type "char" \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${dataset}.raw" \ --output_path="data/manifest.${dataset}" diff --git a/examples/callcenter/asr1/conf/chunk_conformer.yaml b/examples/callcenter/asr1/conf/chunk_conformer.yaml index b18b46fe6aa0a91476f5b6fcac0c8e03d3745f42..d20d2b9a6c191ef91481cd52c21fb622f2418b05 100644 --- a/examples/callcenter/asr1/conf/chunk_conformer.yaml +++ b/examples/callcenter/asr1/conf/chunk_conformer.yaml @@ -12,7 +12,7 @@ data: collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml diff --git a/examples/callcenter/asr1/conf/conformer.yaml b/examples/callcenter/asr1/conf/conformer.yaml index 47c438a6d1b2f453500540e102e54d48dbe8cd5f..f86cd4a368486d3989d57ce28c74caef645b0473 100644 --- a/examples/callcenter/asr1/conf/conformer.yaml +++ b/examples/callcenter/asr1/conf/conformer.yaml @@ -12,7 +12,7 @@ data: collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml diff --git a/examples/callcenter/asr1/local/data.sh b/examples/callcenter/asr1/local/data.sh index c40c752abc981ee8354d32d2ad99c1326173bcf8..fe2d3429cd05650a75a313b78880b40c9ba72ca3 100755 --- a/examples/callcenter/asr1/local/data.sh +++ b/examples/callcenter/asr1/local/data.sh @@ -2,10 +2,12 @@ stage=-1 stop_stage=100 +dict_dir=data/lang_char source ${MAIN_ROOT}/utils/parse_options.sh mkdir -p data +mkdir -p ${dict_dir} if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then for dataset in train dev test; do @@ -41,7 +43,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type="char" \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths "data/manifest.train.raw" if [ $? -ne 0 ]; then @@ -57,7 +59,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type "char" \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${dataset}.raw" \ --output_path="data/manifest.${dataset}" diff --git a/examples/librispeech/asr0/conf/deepspeech2.yaml b/examples/librispeech/asr0/conf/deepspeech2.yaml index 8afaabf429e0b243908029eeb5bae2ab7b0c07d6..70fa3fcb28775faa80ff5ab66f3b1075c33e6a9c 100644 --- a/examples/librispeech/asr0/conf/deepspeech2.yaml +++ b/examples/librispeech/asr0/conf/deepspeech2.yaml @@ -14,7 +14,7 @@ collator: batch_size: 20 mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/librispeech/asr0/conf/deepspeech2_online.yaml b/examples/librispeech/asr0/conf/deepspeech2_online.yaml index d6ab95237cc6b4a02dd5aed4ad6886b6a534b6ed..3e07862d606a161f79d92baf0c329132e42a1b38 100644 --- a/examples/librispeech/asr0/conf/deepspeech2_online.yaml +++ b/examples/librispeech/asr0/conf/deepspeech2_online.yaml @@ -14,7 +14,7 @@ collator: batch_size: 15 mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/librispeech/asr0/local/data.sh b/examples/librispeech/asr0/local/data.sh index 0f276cecad316bcb444fddd36df988c0618b7152..fa2c9b2f7df9833ffa08d78accf8af9745ee186c 100755 --- a/examples/librispeech/asr0/local/data.sh +++ b/examples/librispeech/asr0/local/data.sh @@ -4,10 +4,12 @@ stage=-1 stop_stage=100 unit_type=char +dict_dir=data/lang_char source ${MAIN_ROOT}/utils/parse_options.sh mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -67,7 +69,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type ${unit_type} \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths="data/manifest.train.raw" if [ $? -ne 0 ]; then @@ -83,7 +85,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type ${unit_type} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${set}.raw" \ --output_path="data/manifest.${set}" diff --git a/examples/librispeech/asr1/conf/chunk_conformer.yaml b/examples/librispeech/asr1/conf/chunk_conformer.yaml index 2bfb0fb6f7b939c69372cd6be0bc676edcf92880..4a5741904b740940cef6fc98015e82c52601630f 100644 --- a/examples/librispeech/asr1/conf/chunk_conformer.yaml +++ b/examples/librispeech/asr1/conf/chunk_conformer.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 100.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: 'data/bpe_unigram_5000' + spm_model_prefix: 'data/lang_char/bpe_unigram_5000' mean_std_filepath: "" augmentation_config: conf/preprocess.yaml batch_size: 16 diff --git a/examples/librispeech/asr1/conf/conformer.yaml b/examples/librispeech/asr1/conf/conformer.yaml index c844baaafb3ab4c55a95baa7f4b2a43c9ec40f7c..684b6297699051b217af15a018db362e4c401792 100644 --- a/examples/librispeech/asr1/conf/conformer.yaml +++ b/examples/librispeech/asr1/conf/conformer.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 100.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: 'data/bpe_unigram_5000' + spm_model_prefix: 'data/lang_char/bpe_unigram_5000' mean_std_filepath: "" augmentation_config: conf/preprocess.yaml batch_size: 16 diff --git a/examples/librispeech/asr1/conf/transformer.yaml b/examples/librispeech/asr1/conf/transformer.yaml index 5a158f3ed69ee90f2936cff5016937f7b20932b7..1806f3fd67a212f7bfaf370a9ce2bce72e960290 100644 --- a/examples/librispeech/asr1/conf/transformer.yaml +++ b/examples/librispeech/asr1/conf/transformer.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 100.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: 'data/bpe_unigram_5000' + spm_model_prefix: 'data/lang_char/bpe_unigram_5000' mean_std_filepath: "" augmentation_config: conf/preprocess.yaml batch_size: 32 diff --git a/examples/librispeech/asr1/local/data.sh b/examples/librispeech/asr1/local/data.sh index 35f4e635fa26b99f89498c761dc1e6906a24899a..a0bf9a2d3fe02e8cebfdd9a868d62532a6a26543 100755 --- a/examples/librispeech/asr1/local/data.sh +++ b/examples/librispeech/asr1/local/data.sh @@ -2,11 +2,12 @@ stage=-1 stop_stage=100 +dict_dir=data/lang_char # bpemode (unigram or bpe) nbpe=5000 bpemode=unigram -bpeprefix="data/bpe_${bpemode}_${nbpe}" +bpeprefix="${dict_dir}/bpe_${bpemode}_${nbpe}" stride_ms=10 window_ms=25 @@ -17,6 +18,7 @@ source ${MAIN_ROOT}/utils/parse_options.sh mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -79,7 +81,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then --spm_vocab_size=${nbpe} \ --spm_mode ${bpemode} \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths="data/manifest.train.raw" if [ $? -ne 0 ]; then @@ -96,7 +98,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then --cmvn_path "data/mean_std.json" \ --unit_type "spm" \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${sub}.raw" \ --output_path="data/manifest.${sub}" diff --git a/examples/ted_en_zh/st0/conf/transformer.yaml b/examples/ted_en_zh/st0/conf/transformer.yaml index 8a7e10f0a4820e1135499d5e396619b51a018660..5a05fa46ea598c89b4ca80e5384364e069c1cba3 100644 --- a/examples/ted_en_zh/st0/conf/transformer.yaml +++ b/examples/ted_en_zh/st0/conf/transformer.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 20.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: data/bpe_unigram_8000 + spm_model_prefix: data/lang_char/bpe_unigram_8000 mean_std_filepath: "" # augmentation_config: conf/augmentation.json batch_size: 10 diff --git a/examples/ted_en_zh/st0/conf/transformer_joint_noam.yaml b/examples/ted_en_zh/st0/conf/transformer_joint_noam.yaml index 9c1ac91a9e232e9d48336eb2300e2568812e801b..8256f7160598b882eae9ca06a476ca3ffd85742e 100644 --- a/examples/ted_en_zh/st0/conf/transformer_joint_noam.yaml +++ b/examples/ted_en_zh/st0/conf/transformer_joint_noam.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 20.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: data/bpe_unigram_8000 + spm_model_prefix: data/lang_char/bpe_unigram_8000 mean_std_filepath: "" # augmentation_config: conf/augmentation.json batch_size: 10 diff --git a/examples/ted_en_zh/st0/local/data.sh b/examples/ted_en_zh/st0/local/data.sh index d3acbd4486b3753e70fe7d0c3f71b4f1b3576583..fb4efbe3572400363e2da16d1ba888840686d7a3 100755 --- a/examples/ted_en_zh/st0/local/data.sh +++ b/examples/ted_en_zh/st0/local/data.sh @@ -4,19 +4,22 @@ set -e stage=-1 stop_stage=100 +dict_dir=data/lang_char # bpemode (unigram or bpe) nbpe=8000 bpemode=unigram -bpeprefix="data/bpe_${bpemode}_${nbpe}" +bpeprefix="${dict_dir}/bpe_${bpemode}_${nbpe}" data_dir=./TED-En-Zh -source ${MAIN_ROOT}/utils/parse_options.sh +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; + TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} mkdir -p data +mkdir -p ${dict_dir} if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then @@ -73,11 +76,10 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then --spm_vocab_size=${nbpe} \ --spm_mode ${bpemode} \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --text_keys 'text' 'text1' \ --manifest_paths="data/manifest.train.raw" - if [ $? -ne 0 ]; then echo "Build vocabulary failed. Terminated." exit 1 @@ -92,7 +94,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then --cmvn_path "data/mean_std.json" \ --unit_type "spm" \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${set}.raw" \ --output_path="data/manifest.${set}" diff --git a/examples/ted_en_zh/st1/conf/transformer.yaml b/examples/ted_en_zh/st1/conf/transformer.yaml index d96372866fce4d48a67b9d86539ce4f76fbac6d8..d553bde77f732195f86d26c4b6352273a28133ca 100644 --- a/examples/ted_en_zh/st1/conf/transformer.yaml +++ b/examples/ted_en_zh/st1/conf/transformer.yaml @@ -11,9 +11,9 @@ data: max_output_input_ratio: 20.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' - spm_model_prefix: data/bpe_unigram_8000 + spm_model_prefix: data/lang_char/bpe_unigram_8000 mean_std_filepath: "" # augmentation_config: conf/augmentation.json batch_size: 10 diff --git a/examples/ted_en_zh/st1/conf/transformer_mtl_noam.yaml b/examples/ted_en_zh/st1/conf/transformer_mtl_noam.yaml index ea38d6eecbb09089424acb5161433c72e0395b92..b4fb510754713f6c63f5bce9561032225b31c6c5 100644 --- a/examples/ted_en_zh/st1/conf/transformer_mtl_noam.yaml +++ b/examples/ted_en_zh/st1/conf/transformer_mtl_noam.yaml @@ -11,7 +11,7 @@ data: max_output_input_ratio: 20.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' spm_model_prefix: data/train_sp.en-zh-nlpr.zh-nlpr_bpe8000_tc mean_std_filepath: "" diff --git a/examples/ted_en_zh/st1/local/data.sh b/examples/ted_en_zh/st1/local/data.sh index b080a5b497e703c6b2f1c2d385315f771b33a1a0..2e9d05d102e6d892eca263ffb41a001087923425 100755 --- a/examples/ted_en_zh/st1/local/data.sh +++ b/examples/ted_en_zh/st1/local/data.sh @@ -4,11 +4,12 @@ set -e stage=-1 stop_stage=100 +dict_dir=data/lang_char # bpemode (unigram or bpe) nbpe=8000 bpemode=unigram -bpeprefix="data/bpe_${bpemode}_${nbpe}" +bpeprefix="${dict_dir}/bpe_${bpemode}_${nbpe}" data_dir=./TED_EnZh @@ -17,6 +18,7 @@ source ${MAIN_ROOT}/utils/parse_options.sh TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} mkdir -p data +mkdir -p ${dict_dir} if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then @@ -73,7 +75,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then --spm_vocab_size=${nbpe} \ --spm_mode ${bpemode} \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --text_keys 'text' 'text1' \ --manifest_paths="data/manifest.train.raw" @@ -93,7 +95,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then --cmvn_path "data/mean_std.json" \ --unit_type "spm" \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${set}.raw" \ --output_path="data/manifest.${set}" diff --git a/examples/timit/asr1/conf/transformer.yaml b/examples/timit/asr1/conf/transformer.yaml index 1d18468b80025b5ced93c08db0e7f38acc2eb937..89ae2fd3de5595db93056cfb9c5c391afbc1a09b 100644 --- a/examples/timit/asr1/conf/transformer.yaml +++ b/examples/timit/asr1/conf/transformer.yaml @@ -11,7 +11,7 @@ data: max_output_input_ratio: 1000.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: "word" mean_std_filepath: "" augmentation_config: conf/preprocess.yaml diff --git a/examples/timit/asr1/local/data.sh b/examples/timit/asr1/local/data.sh index e588e48df112c604878b0128251f867ce90905b2..fb720932d6665dc99c5e930a03714c06c05bdb47 100755 --- a/examples/timit/asr1/local/data.sh +++ b/examples/timit/asr1/local/data.sh @@ -3,15 +3,19 @@ stage=-1 stop_stage=100 +dict_dir=data/lang_char + unit_type=word TIMIT_path= -source ${MAIN_ROOT}/utils/parse_options.sh +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} + if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then # download data, generate manifests python3 ${TARGET_DIR}/timit/timit_kaldi_standard_split.py \ @@ -52,7 +56,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type ${unit_type} \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths="data/manifest.train.raw" if [ $? -ne 0 ]; then @@ -68,7 +72,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type ${unit_type} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.${set}.raw" \ --output_path="data/manifest.${set}" diff --git a/examples/tiny/asr0/conf/deepspeech2.yaml b/examples/tiny/asr0/conf/deepspeech2.yaml index 58899a1568e3fd61ba23aaf1cb83347428a7f40d..ba453aad76976bdf76cfc7805a29707e371cc5ad 100644 --- a/examples/tiny/asr0/conf/deepspeech2.yaml +++ b/examples/tiny/asr0/conf/deepspeech2.yaml @@ -14,7 +14,7 @@ data: collator: mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/tiny/asr0/conf/deepspeech2_online.yaml b/examples/tiny/asr0/conf/deepspeech2_online.yaml index 334b1d31ce21ab95c3099c76caf9cdd36c61cd92..36c774e374e92ed35227cb9a8ddd667ebe4775ac 100644 --- a/examples/tiny/asr0/conf/deepspeech2_online.yaml +++ b/examples/tiny/asr0/conf/deepspeech2_online.yaml @@ -14,7 +14,7 @@ data: collator: mean_std_filepath: data/mean_std.json unit_type: char - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt augmentation_config: conf/augmentation.json random_seed: 0 spm_model_prefix: diff --git a/examples/tiny/asr0/local/data.sh b/examples/tiny/asr0/local/data.sh index f1fb8cb1d093a3adde7f71cebc1ecee50bff7238..2a544ef89ee269ded158295f4ea58f8e9dbe5776 100755 --- a/examples/tiny/asr0/local/data.sh +++ b/examples/tiny/asr0/local/data.sh @@ -4,10 +4,12 @@ stage=-1 stop_stage=100 unit_type=char +dict_dir=data/lang_char -source ${MAIN_ROOT}/utils/parse_options.sh +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -51,7 +53,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then python3 ${MAIN_ROOT}/utils/build_vocab.py \ --unit_type ${unit_type} \ --count_threshold=0 \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths="data/manifest.tiny.raw" if [ $? -ne 0 ]; then @@ -65,7 +67,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then python3 ${MAIN_ROOT}/utils/format_data.py \ --cmvn_path "data/mean_std.json" \ --unit_type ${unit_type} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.tiny.raw" \ --output_path="data/manifest.tiny" diff --git a/examples/tiny/asr1/conf/chunk_confermer.yaml b/examples/tiny/asr1/conf/chunk_confermer.yaml index 6bed27f5c9caba478f127064d2fcce102eccf1f7..6183a903bde34783a15dc58d9c3f5ce47b3553d7 100644 --- a/examples/tiny/asr1/conf/chunk_confermer.yaml +++ b/examples/tiny/asr1/conf/chunk_confermer.yaml @@ -12,7 +12,7 @@ data: collator: mean_std_filepath: "" - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' spm_model_prefix: 'data/bpe_unigram_200' augmentation_config: conf/preprocess.yaml diff --git a/examples/tiny/asr1/conf/chunk_transformer.yaml b/examples/tiny/asr1/conf/chunk_transformer.yaml index 7aed1b1933ca1edcf34e6c45a49dbc68eed91527..01d383fb82d01ceb424d1dad3d48a6c401e1ce82 100644 --- a/examples/tiny/asr1/conf/chunk_transformer.yaml +++ b/examples/tiny/asr1/conf/chunk_transformer.yaml @@ -12,7 +12,7 @@ data: collator: mean_std_filepath: "" - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' spm_model_prefix: 'data/bpe_unigram_200' augmentation_config: conf/preprocess.yaml diff --git a/examples/tiny/asr1/conf/conformer.yaml b/examples/tiny/asr1/conf/conformer.yaml index 2c09b3ae6954cff537a7f1c934b4193e56f3243f..a3fee6901df062e04e91f54f478910ef5c716a93 100644 --- a/examples/tiny/asr1/conf/conformer.yaml +++ b/examples/tiny/asr1/conf/conformer.yaml @@ -12,7 +12,7 @@ data: collator: mean_std_filepath: "" - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' spm_model_prefix: 'data/bpe_unigram_200' augmentation_config: conf/preprocess.yaml diff --git a/examples/tiny/asr1/conf/transformer.yaml b/examples/tiny/asr1/conf/transformer.yaml index 1378e848dceee2565e1d4de1b31d6e887ba65103..5a87d6d245a36e30f50ced4c7777ff90ef1c9f28 100644 --- a/examples/tiny/asr1/conf/transformer.yaml +++ b/examples/tiny/asr1/conf/transformer.yaml @@ -12,7 +12,7 @@ data: collator: mean_std_filepath: data/mean_std.json - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'spm' spm_model_prefix: 'data/bpe_unigram_200' augmentation_config: conf/preprocess.yaml diff --git a/examples/tiny/asr1/local/data.sh b/examples/tiny/asr1/local/data.sh index 87539d5ed33b3ebbb21d398dc78349ead6cd27e3..1ef9f7768aede38d6d192e8c458b87433aa364c1 100755 --- a/examples/tiny/asr1/local/data.sh +++ b/examples/tiny/asr1/local/data.sh @@ -3,14 +3,17 @@ stage=-1 stop_stage=100 +dict_dir=data/lang_char + # bpemode (unigram or bpe) nbpe=200 bpemode=unigram -bpeprefix="data/bpe_${bpemode}_${nbpe}" +bpeprefix="${dict_dir}/bpe_${bpemode}_${nbpe}" -source ${MAIN_ROOT}/utils/parse_options.sh +. ${MAIN_ROOT}/utils/parse_options.sh || exit -1; mkdir -p data +mkdir -p ${dict_dir} TARGET_DIR=${MAIN_ROOT}/examples/dataset mkdir -p ${TARGET_DIR} @@ -57,7 +60,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then --spm_vocab_size=${nbpe} \ --spm_mode ${bpemode} \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_paths="data/manifest.tiny.raw" if [ $? -ne 0 ]; then @@ -72,7 +75,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then --cmvn_path "data/mean_std.json" \ --unit_type "spm" \ --spm_model_prefix ${bpeprefix} \ - --vocab_path="data/vocab.txt" \ + --vocab_path="${dict_dir}/vocab.txt" \ --manifest_path="data/manifest.tiny.raw" \ --output_path="data/manifest.tiny" diff --git a/examples/wenetspeech/asr1/conf/conformer.yaml b/examples/wenetspeech/asr1/conf/conformer.yaml index 0340dc85dcd6d4ca7ca0eedd08b70d16c3846e01..a3a42ec63f6d95738a2cfb890999a6677856fd1a 100644 --- a/examples/wenetspeech/asr1/conf/conformer.yaml +++ b/examples/wenetspeech/asr1/conf/conformer.yaml @@ -51,7 +51,7 @@ data: max_output_input_ratio: 10.0 collator: - vocab_filepath: data/vocab.txt + vocab_filepath: data/lang_char/vocab.txt unit_type: 'char' spm_model_prefix: '' augmentation_config: conf/preprocess.yaml