diff --git a/data/librispeech/librispeech.py b/data/librispeech/librispeech.py deleted file mode 100644 index ae1bae2defee069b0a228c08deebeb9d54548c67..0000000000000000000000000000000000000000 --- a/data/librispeech/librispeech.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Prepare Librispeech ASR datasets. - -Download, unpack and create manifest files. -Manifest file is a json-format file with each line containing the -meta data (i.e. audio filepath, transcript and audio duration) -of each audio file in the data set. -""" - -import distutils.util -import os -import sys -import argparse -import soundfile -import json -import codecs -import io -from data_utils.utility import download, unpack - -URL_ROOT = "http://www.openslr.org/resources/12" -URL_ROOT = "https://openslr.magicdatatech.com/resources/12" -URL_TEST_CLEAN = URL_ROOT + "/test-clean.tar.gz" -URL_TEST_OTHER = URL_ROOT + "/test-other.tar.gz" -URL_DEV_CLEAN = URL_ROOT + "/dev-clean.tar.gz" -URL_DEV_OTHER = URL_ROOT + "/dev-other.tar.gz" -URL_TRAIN_CLEAN_100 = URL_ROOT + "/train-clean-100.tar.gz" -URL_TRAIN_CLEAN_360 = URL_ROOT + "/train-clean-360.tar.gz" -URL_TRAIN_OTHER_500 = URL_ROOT + "/train-other-500.tar.gz" - -MD5_TEST_CLEAN = "32fa31d27d2e1cad72775fee3f4849a9" -MD5_TEST_OTHER = "fb5a50374b501bb3bac4815ee91d3135" -MD5_DEV_CLEAN = "42e2234ba48799c1f50f24a7926300a1" -MD5_DEV_OTHER = "c8d0bcc9cca99d4f8b62fcc847357931" -MD5_TRAIN_CLEAN_100 = "2a93770f6d5c6c964bc36631d331a522" -MD5_TRAIN_CLEAN_360 = "c0e676e450a7ff2f54aeade5171606fa" -MD5_TRAIN_OTHER_500 = "d1a0fd59409feb2c614ce4d30c387708" - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument( - "--target_dir", - default='~/.cache/paddle/dataset/speech/libri', - type=str, - help="Directory to save the dataset. (default: %(default)s)") -parser.add_argument( - "--manifest_prefix", - default="manifest", - type=str, - help="Filepath prefix for output manifests. (default: %(default)s)") -parser.add_argument( - "--full_download", - default="True", - type=distutils.util.strtobool, - help="Download all datasets for Librispeech." - " If False, only download a minimal requirement (test-clean, dev-clean" - " train-clean-100). (default: %(default)s)") -args = parser.parse_args() - - -def create_manifest(data_dir, manifest_path): - """Create a manifest json file summarizing the data set, with each line - containing the meta data (i.e. audio filepath, transcription text, audio - duration) of each audio file within the data set. - """ - print("Creating manifest %s ..." % manifest_path) - json_lines = [] - for subfolder, _, filelist in sorted(os.walk(data_dir)): - text_filelist = [ - filename for filename in filelist if filename.endswith('trans.txt') - ] - if len(text_filelist) > 0: - text_filepath = os.path.join(subfolder, text_filelist[0]) - for line in io.open(text_filepath, encoding="utf8"): - segments = line.strip().split() - text = ' '.join(segments[1:]).lower() - audio_filepath = os.path.join(subfolder, segments[0] + '.flac') - audio_data, samplerate = soundfile.read(audio_filepath) - duration = float(len(audio_data)) / samplerate - json_lines.append( - json.dumps({ - 'audio_filepath': audio_filepath, - 'duration': duration, - 'text': text - })) - with codecs.open(manifest_path, 'w', 'utf-8') as out_file: - for line in json_lines: - out_file.write(line + '\n') - - -def prepare_dataset(url, md5sum, target_dir, manifest_path): - """Download, unpack and create summmary manifest file. - """ - if not os.path.exists(os.path.join(target_dir, "LibriSpeech")): - # download - filepath = download(url, md5sum, target_dir) - # unpack - unpack(filepath, target_dir) - else: - print("Skip downloading and unpacking. Data already exists in %s." % - target_dir) - # create manifest json file - create_manifest(target_dir, manifest_path) - - -def main(): - if args.target_dir.startswith('~'): - args.target_dir = os.path.expanduser(args.target_dir) - - prepare_dataset( - url=URL_TEST_CLEAN, - md5sum=MD5_TEST_CLEAN, - target_dir=os.path.join(args.target_dir, "test-clean"), - manifest_path=args.manifest_prefix + ".test-clean") - prepare_dataset( - url=URL_DEV_CLEAN, - md5sum=MD5_DEV_CLEAN, - target_dir=os.path.join(args.target_dir, "dev-clean"), - manifest_path=args.manifest_prefix + ".dev-clean") - if args.full_download: - prepare_dataset( - url=URL_TRAIN_CLEAN_100, - md5sum=MD5_TRAIN_CLEAN_100, - target_dir=os.path.join(args.target_dir, "train-clean-100"), - manifest_path=args.manifest_prefix + ".train-clean-100") - prepare_dataset( - url=URL_TEST_OTHER, - md5sum=MD5_TEST_OTHER, - target_dir=os.path.join(args.target_dir, "test-other"), - manifest_path=args.manifest_prefix + ".test-other") - prepare_dataset( - url=URL_DEV_OTHER, - md5sum=MD5_DEV_OTHER, - target_dir=os.path.join(args.target_dir, "dev-other"), - manifest_path=args.manifest_prefix + ".dev-other") - prepare_dataset( - url=URL_TRAIN_CLEAN_360, - md5sum=MD5_TRAIN_CLEAN_360, - target_dir=os.path.join(args.target_dir, "train-clean-360"), - manifest_path=args.manifest_prefix + ".train-clean-360") - prepare_dataset( - url=URL_TRAIN_OTHER_500, - md5sum=MD5_TRAIN_OTHER_500, - target_dir=os.path.join(args.target_dir, "train-other-500"), - manifest_path=args.manifest_prefix + ".train-other-500") - - -if __name__ == '__main__': - main() diff --git a/data/aishell/aishell.py b/examples/aishell/local/aishell.py similarity index 100% rename from data/aishell/aishell.py rename to examples/aishell/local/aishell.py diff --git a/examples/aishell/run_data.sh b/examples/aishell/local/run_data.sh similarity index 50% rename from examples/aishell/run_data.sh rename to examples/aishell/local/run_data.sh index fc002fcd2884de64e8aa7c06f0214afe44f9193b..c4fa11c4cbeef49e37f9c97c543f63948b0d89ca 100644 --- a/examples/aishell/run_data.sh +++ b/examples/aishell/local/run_data.sh @@ -1,11 +1,9 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # download data, generate manifests -PYTHONPATH=.:$PYTHONPATH python3 data/aishell/aishell.py \ ---manifest_prefix='data/aishell/manifest' \ ---target_dir='./dataset/aishell' +PYTHONPATH=.:$PYTHONPATH python3 local/aishell.py \ +--manifest_prefix="data/manifest" \ +--target_dir="./dataset/aishell" if [ $? -ne 0 ]; then echo "Prepare Aishell failed. Terminated." @@ -14,10 +12,10 @@ fi # build vocabulary -python3 tools/build_vocab.py \ +python3 ${MAIN_ROOT}/tools/build_vocab.py \ --count_threshold=0 \ ---vocab_path='data/aishell/vocab.txt' \ ---manifest_paths 'data/aishell/manifest.train' 'data/aishell/manifest.dev' +--vocab_path="data/vocab.txt" \ +--manifest_paths "data/manifest.train" "data/manifest.dev" if [ $? -ne 0 ]; then echo "Build vocabulary failed. Terminated." @@ -26,11 +24,11 @@ fi # compute mean and stddev for normalizer -python3 tools/compute_mean_std.py \ ---manifest_path='data/aishell/manifest.train' \ +python3 ${MAIN_ROOT}/tools/compute_mean_std.py \ +--manifest_path="data/manifest.train" \ --num_samples=2000 \ ---specgram_type='linear' \ ---output_path='data/aishell/mean_std.npz' +--specgram_type="linear" \ +--output_path="data/mean_std.npz" if [ $? -ne 0 ]; then echo "Compute mean and stddev failed. Terminated." diff --git a/examples/aishell/run_infer.sh b/examples/aishell/local/run_infer.sh similarity index 62% rename from examples/aishell/run_infer.sh rename to examples/aishell/local/run_infer.sh index ef382e310ebe0dba8317585c755444c834022bcd..7a0d796910aa95d5f184c8189629ce0988802e47 100644 --- a/examples/aishell/run_infer.sh +++ b/examples/aishell/local/run_infer.sh @@ -27,14 +27,14 @@ python3 -u infer.py \ --use_gru=True \ --use_gpu=True \ --share_rnn_weights=False \ ---infer_manifest='data/aishell/manifest.test' \ ---mean_std_path='data/aishell/mean_std.npz' \ ---vocab_path='data/aishell/vocab.txt' \ ---model_path='checkpoints/aishell/step_final' \ ---lang_model_path='models/lm/zh_giga.no_cna_cmn.prune01244.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='cer' \ ---specgram_type='linear' +--infer_manifest="data/aishell/manifest.test" \ +--mean_std_path="data/aishell/mean_std.npz" \ +--vocab_path="data/aishell/vocab.txt" \ +--model_path="checkpoints/aishell/step_final" \ +--lang_model_path="models/lm/zh_giga.no_cna_cmn.prune01244.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="cer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/aishell/run_infer_golden.sh b/examples/aishell/local/run_infer_golden.sh similarity index 68% rename from examples/aishell/run_infer_golden.sh rename to examples/aishell/local/run_infer_golden.sh index dabdc0c68f5a295d5d0517a51de5c26889431511..7e0ede661628412223866986b7bf81e34b897a21 100644 --- a/examples/aishell/run_infer_golden.sh +++ b/examples/aishell/local/run_infer_golden.sh @@ -36,14 +36,14 @@ python3 -u infer.py \ --use_gru=True \ --use_gpu=False \ --share_rnn_weights=False \ ---infer_manifest='data/aishell/manifest.test' \ ---mean_std_path='models/aishell/mean_std.npz' \ ---vocab_path='models/aishell/vocab.txt' \ ---model_path='models/aishell' \ ---lang_model_path='models/lm/zh_giga.no_cna_cmn.prune01244.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='cer' \ ---specgram_type='linear' +--infer_manifest="data/aishell/manifest.test" \ +--mean_std_path="models/aishell/mean_std.npz" \ +--vocab_path="models/aishell/vocab.txt" \ +--model_path="models/aishell" \ +--lang_model_path="models/lm/zh_giga.no_cna_cmn.prune01244.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="cer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/aishell/run_test.sh b/examples/aishell/local/run_test.sh similarity index 63% rename from examples/aishell/run_test.sh rename to examples/aishell/local/run_test.sh index b56d164b99cabd8fe319d1456b0e6b106e9feef7..31be69feb5ef4ab5c070f2061f8e2cf31dc2a5e6 100644 --- a/examples/aishell/run_test.sh +++ b/examples/aishell/local/run_test.sh @@ -27,14 +27,14 @@ python3 -u test.py \ --use_gru=True \ --use_gpu=True \ --share_rnn_weights=False \ ---test_manifest='data/aishell/manifest.test' \ ---mean_std_path='data/aishell/mean_std.npz' \ ---vocab_path='data/aishell/vocab.txt' \ ---model_path='checkpoints/aishell/step_final' \ ---lang_model_path='models/lm/zh_giga.no_cna_cmn.prune01244.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='cer' \ ---specgram_type='linear' +--test_manifest="data/aishell/manifest.test" \ +--mean_std_path="data/aishell/mean_std.npz" \ +--vocab_path="data/aishell/vocab.txt" \ +--model_path="checkpoints/aishell/step_final" \ +--lang_model_path="models/lm/zh_giga.no_cna_cmn.prune01244.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="cer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/aishell/run_test_golden.sh b/examples/aishell/local/run_test_golden.sh similarity index 69% rename from examples/aishell/run_test_golden.sh rename to examples/aishell/local/run_test_golden.sh index 2f79cc64797231c04899b9c726af53eab8f1c246..ea423c0481890a1424b2106d46818a9ae0a0abfb 100644 --- a/examples/aishell/run_test_golden.sh +++ b/examples/aishell/local/run_test_golden.sh @@ -36,14 +36,14 @@ python3 -u test.py \ --use_gru=True \ --use_gpu=True \ --share_rnn_weights=False \ ---test_manifest='data/aishell/manifest.test' \ ---mean_std_path='models/aishell/mean_std.npz' \ ---vocab_path='models/aishell/vocab.txt' \ ---model_path='models/aishell' \ ---lang_model_path='models/lm/zh_giga.no_cna_cmn.prune01244.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='cer' \ ---specgram_type='linear' +--test_manifest="data/aishell/manifest.test" \ +--mean_std_path="models/aishell/mean_std.npz" \ +--vocab_path="models/aishell/vocab.txt" \ +--model_path="models/aishell" \ +--lang_model_path="models/lm/zh_giga.no_cna_cmn.prune01244.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="cer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/aishell/run_train.sh b/examples/aishell/local/run_train.sh similarity index 66% rename from examples/aishell/run_train.sh rename to examples/aishell/local/run_train.sh index 889e1904735a88b8b2a63c2160da54eff9ae5bbe..6faed6c6202fdc72eaca69b434ae0678ad0a53dc 100644 --- a/examples/aishell/run_train.sh +++ b/examples/aishell/local/run_train.sh @@ -24,14 +24,14 @@ python3 -u train.py \ --use_gpu=True \ --is_local=True \ --share_rnn_weights=False \ ---train_manifest='data/aishell/manifest.train' \ ---dev_manifest='data/aishell/manifest.dev' \ ---mean_std_path='data/aishell/mean_std.npz' \ ---vocab_path='data/aishell/vocab.txt' \ ---output_model_dir='./checkpoints/aishell' \ ---augment_conf_path='conf/augmentation.config' \ ---specgram_type='linear' \ ---shuffle_method='batch_shuffle_clipped' \ +--train_manifest="data/aishell/manifest.train" \ +--dev_manifest="data/aishell/manifest.dev" \ +--mean_std_path="data/aishell/mean_std.npz" \ +--vocab_path="data/aishell/vocab.txt" \ +--output_model_dir="./checkpoints/aishell" \ +--augment_conf_path="conf/augmentation.config" \ +--specgram_type="linear" \ +--shuffle_method="batch_shuffle_clipped" \ if [ $? -ne 0 ]; then echo "Failed in training!" diff --git a/examples/aishell/path.sh b/examples/aishell/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..fd1cebba8ea9313ca30ee8bece2f6ddb0f29112d --- /dev/null +++ b/examples/aishell/path.sh @@ -0,0 +1,8 @@ +export MAIN_ROOT=${PWD}/../../ + +export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export LC_ALL=C + +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} diff --git a/examples/aishell/run.sh b/examples/aishell/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..c8e58913900677679a40210c1f2dae658c435db6 --- /dev/null +++ b/examples/aishell/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +source path.sh + +# prepare data +bash ./local/run_data.sh + +# test pretrain model +bash ./local/run_test_golden.sh + +# test pretain model +bash ./local/run_infer_golden.sh + +# train model +bash ./local/run_train.sh + +# test model +bash ./local/run_test.sh + +# infer model +bash ./local/run_infer.sh + +# tune model +bash ./local/run_tune.sh diff --git a/examples/baidu_en8k/path.sh b/examples/baidu_en8k/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..fd1cebba8ea9313ca30ee8bece2f6ddb0f29112d --- /dev/null +++ b/examples/baidu_en8k/path.sh @@ -0,0 +1,8 @@ +export MAIN_ROOT=${PWD}/../../ + +export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export LC_ALL=C + +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} diff --git a/examples/baidu_en8k/run_infer_golden.sh b/examples/baidu_en8k/run_infer_golden.sh index 85fea5f479cebd01a9cc615ab059738f953acbcc..11d7541eec9e309fe999a8c9956c6176015ae2b0 100644 --- a/examples/baidu_en8k/run_infer_golden.sh +++ b/examples/baidu_en8k/run_infer_golden.sh @@ -1,9 +1,9 @@ #! /usr/bin/env bash -cd ../.. > /dev/null +source path.sh # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -12,7 +12,7 @@ cd - > /dev/null # download well-trained model -cd models/baidu_en8k > /dev/null +cd ${MAIN_ROOT}/models/baidu_en8k > /dev/null bash download_model.sh if [ $? -ne 0 ]; then exit 1 @@ -22,7 +22,7 @@ cd - > /dev/null # infer CUDA_VISIBLE_DEVICES=0 \ -python3 -u infer.py \ +python3 -u ${MAIN_ROOT}/infer.py \ --num_samples=10 \ --beam_size=500 \ --num_proc_bsearch=5 \ @@ -36,14 +36,14 @@ python3 -u infer.py \ --use_gru=True \ --use_gpu=False \ --share_rnn_weights=False \ ---infer_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='models/baidu_en8k/mean_std.npz' \ ---vocab_path='models/baidu_en8k/vocab.txt' \ ---model_path='models/baidu_en8k' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--infer_manifest="${MAIN_ROOT}/examples/librispeech/data/manifest.test-clean" \ +--mean_std_path="${MAIN_ROOT}/models/baidu_en8k/mean_std.npz" \ +--vocab_path="${MAIN_ROOT}/models/baidu_en8k/vocab.txt" \ +--model_path="${MAIN_ROOT}/models/baidu_en8k" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/baidu_en8k/run_test_golden.sh b/examples/baidu_en8k/run_test_golden.sh index 1ed63362477e9746665b0beb547ef6ebad10c311..f77f934cc8e7ec1391f259eccc5b28064bd8e780 100644 --- a/examples/baidu_en8k/run_test_golden.sh +++ b/examples/baidu_en8k/run_test_golden.sh @@ -37,14 +37,14 @@ python3 -u test.py \ --use_gru=True \ --use_gpu=False \ --share_rnn_weights=False \ ---test_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='models/baidu_en8k/mean_std.npz' \ ---vocab_path='models/baidu_en8k/vocab.txt' \ ---model_path='models/baidu_en8k' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--test_manifest="data/librispeech/manifest.test-clean" \ +--mean_std_path="models/baidu_en8k/mean_std.npz" \ +--vocab_path="models/baidu_en8k/vocab.txt" \ +--model_path="models/baidu_en8k" \ +--lang_model_path="models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/deploy_demo/path.sh b/examples/deploy_demo/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..fd1cebba8ea9313ca30ee8bece2f6ddb0f29112d --- /dev/null +++ b/examples/deploy_demo/path.sh @@ -0,0 +1,8 @@ +export MAIN_ROOT=${PWD}/../../ + +export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export LC_ALL=C + +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} diff --git a/examples/deploy_demo/run_demo_client.sh b/examples/deploy_demo/run_demo_client.sh index 7c2609511afc5760865fcd49ab2d515b1b2cc5d3..60581c6615d078da237fc45cf8263b269c5d3b06 100644 --- a/examples/deploy_demo/run_demo_client.sh +++ b/examples/deploy_demo/run_demo_client.sh @@ -1,11 +1,11 @@ #! /usr/bin/env bash -cd ../.. > /dev/null +source path.sh # start demo client CUDA_VISIBLE_DEVICES=0 \ -python3 -u deploy/demo_client.py \ ---host_ip='localhost' \ +python3 -u ${MAIN_ROOT}/deploy/demo_client.py \ +--host_ip="localhost" \ --host_port=8086 \ if [ $? -ne 0 ]; then diff --git a/examples/deploy_demo/run_english_demo_server.sh b/examples/deploy_demo/run_english_demo_server.sh index 5d86ab6e10f6bb08b993ee1ef961c7985ff90057..ae092dbce81626e1228992603d303df5148111d0 100644 --- a/examples/deploy_demo/run_english_demo_server.sh +++ b/examples/deploy_demo/run_english_demo_server.sh @@ -1,10 +1,10 @@ #! /usr/bin/env bash # TODO: replace the model with a mandarin model -cd ../.. > /dev/null +source path.sh # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -13,7 +13,7 @@ cd - > /dev/null # download well-trained model -cd models/baidu_en8k > /dev/null +cd ${MAIN_ROOT}/models/baidu_en8k > /dev/null bash download_model.sh if [ $? -ne 0 ]; then exit 1 @@ -23,8 +23,8 @@ cd - > /dev/null # start demo server CUDA_VISIBLE_DEVICES=0 \ -python3 -u deploy/demo_server.py \ ---host_ip='localhost' \ +python3 -u ${MAIN_ROOT}/deploy/demo_server.py \ +--host_ip="localhost" \ --host_port=8086 \ --num_conv_layers=2 \ --num_rnn_layers=3 \ @@ -36,14 +36,14 @@ python3 -u deploy/demo_server.py \ --use_gru=True \ --use_gpu=True \ --share_rnn_weights=False \ ---speech_save_dir='demo_cache' \ ---warmup_manifest='data/tiny/manifest.test-clean' \ ---mean_std_path='models/baidu_en8k/mean_std.npz' \ ---vocab_path='models/baidu_en8k/vocab.txt' \ ---model_path='models/baidu_en8k' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---specgram_type='linear' +--speech_save_dir="demo_cache" \ +--warmup_manifest="${MAIN_ROOT}/examples/tiny/data/manifest.test-clean" \ +--mean_std_path="${MAIN_ROOT}/models/baidu_en8k/mean_std.npz" \ +--vocab_path="${MAIN_ROOT}/models/baidu_en8k/vocab.txt" \ +--model_path="${MAIN_ROOT}/models/baidu_en8k" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in starting demo server!" diff --git a/examples/librispeech/local/run_data.sh b/examples/librispeech/local/run_data.sh new file mode 100644 index 0000000000000000000000000000000000000000..1e8fd6a52e11b8cf23922c434f271161a0cb9d8c --- /dev/null +++ b/examples/librispeech/local/run_data.sh @@ -0,0 +1,43 @@ +#! /usr/bin/env bash + +# download data, generate manifests +PYTHONPATH=.:$PYTHONPATH python3 local/librispeech.py \ +--manifest_prefix="data/manifest" \ +--target_dir="./dataset/librispeech" \ +--full_download="True" + +if [ $? -ne 0 ]; then + echo "Prepare LibriSpeech failed. Terminated." + exit 1 +fi + +cat data/manifest.train-* | shuf > data/manifest.train + + +# build vocabulary +python3 ${MAIN_ROOT}/tools/build_vocab.py \ +--count_threshold=0 \ +--vocab_path="data/vocab.txt" \ +--manifest_paths="data/manifest.train" + +if [ $? -ne 0 ]; then + echo "Build vocabulary failed. Terminated." + exit 1 +fi + + +# compute mean and stddev for normalizer +python3 ${MAIN_ROOT}/tools/compute_mean_std.py \ +--manifest_path="data/manifest.train" \ +--num_samples=2000 \ +--specgram_type="linear" \ +--output_path="data/mean_std.npz" + +if [ $? -ne 0 ]; then + echo "Compute mean and stddev failed. Terminated." + exit 1 +fi + + +echo "LibriSpeech Data preparation done." +exit 0 diff --git a/examples/librispeech/run_infer.sh b/examples/librispeech/local/run_infer.sh similarity index 53% rename from examples/librispeech/run_infer.sh rename to examples/librispeech/local/run_infer.sh index 2452b45e3da7460ddce9477afa9e01e3bc974172..33959b38107e9f2e49ee1288c82204d2b2dfef0b 100644 --- a/examples/librispeech/run_infer.sh +++ b/examples/librispeech/local/run_infer.sh @@ -1,9 +1,7 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -13,7 +11,7 @@ cd - > /dev/null # infer CUDA_VISIBLE_DEVICES=0 \ -python3 -u infer.py \ +python3 -u ${MAIN_ROOT}/infer.py \ --num_samples=10 \ --beam_size=500 \ --num_proc_bsearch=8 \ @@ -27,14 +25,14 @@ python3 -u infer.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---infer_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='data/librispeech/mean_std.npz' \ ---vocab_path='data/librispeech/vocab.txt' \ ---model_path='checkpoints/libri/step_final' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--infer_manifest="data/manifest.test-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--model_path="checkpoints/step_final" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/librispeech/run_infer_golden.sh b/examples/librispeech/local/run_infer_golden.sh similarity index 54% rename from examples/librispeech/run_infer_golden.sh rename to examples/librispeech/local/run_infer_golden.sh index 6004c6af6d8c3911487f3c9c2fefa11c04c73b14..21663681ba1813c797aedac24bb9a7eb74f0d0c2 100644 --- a/examples/librispeech/run_infer_golden.sh +++ b/examples/librispeech/local/run_infer_golden.sh @@ -1,9 +1,7 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -12,7 +10,7 @@ cd - > /dev/null # download well-trained model -cd models/librispeech > /dev/null +cd ${MAIN_ROOT}/models/librispeech > /dev/null bash download_model.sh if [ $? -ne 0 ]; then exit 1 @@ -22,7 +20,7 @@ cd - > /dev/null # infer CUDA_VISIBLE_DEVICES=0 \ -python3 -u infer.py \ +python3 -u ${MAIN_ROOT}/infer.py \ --num_samples=10 \ --beam_size=500 \ --num_proc_bsearch=8 \ @@ -36,14 +34,14 @@ python3 -u infer.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---infer_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='models/librispeech/mean_std.npz' \ ---vocab_path='models/librispeech/vocab.txt' \ ---model_path='models/librispeech' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--infer_manifest="data/manifest.test-clean" \ +--mean_std_path="${MAIN_ROOT}/models/librispeech/mean_std.npz" \ +--vocab_path="${MAIN_ROOT}/models/librispeech/vocab.txt" \ +--model_path="${MAIN_ROOT}/models/librispeech" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/librispeech/run_test.sh b/examples/librispeech/local/run_test.sh similarity index 55% rename from examples/librispeech/run_test.sh rename to examples/librispeech/local/run_test.sh index e8e65c39ed1730ca029d3a7b56009f179fc9a669..cd8c07542d387c9ff04a568dec2c24f11355a369 100644 --- a/examples/librispeech/run_test.sh +++ b/examples/librispeech/local/run_test.sh @@ -1,9 +1,7 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -13,7 +11,7 @@ cd - > /dev/null # evaluate model CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ -python3 -u test.py \ +python3 -u ${MAIN_ROOT}/test.py \ --batch_size=128 \ --beam_size=500 \ --num_proc_bsearch=8 \ @@ -27,14 +25,14 @@ python3 -u test.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---test_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='data/librispeech/mean_std.npz' \ ---vocab_path='data/librispeech/vocab.txt' \ ---model_path='checkpoints/libri/step_final' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--test_manifest="data/manifest.test-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--model_path="checkpoints/step_final" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/librispeech/run_test_golden.sh b/examples/librispeech/local/run_test_golden.sh similarity index 55% rename from examples/librispeech/run_test_golden.sh rename to examples/librispeech/local/run_test_golden.sh index 1db36758a3e9546e68fb7c78e5b1c5d47183f792..54ec6ad03bc0c41f30703ae6d8bdab198d58b7fa 100644 --- a/examples/librispeech/run_test_golden.sh +++ b/examples/librispeech/local/run_test_golden.sh @@ -1,9 +1,7 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # download language model -cd models/lm > /dev/null +cd ${MAIN_ROOT}/models/lm > /dev/null bash download_lm_en.sh if [ $? -ne 0 ]; then exit 1 @@ -12,7 +10,7 @@ cd - > /dev/null # download well-trained model -cd models/librispeech > /dev/null +cd ${MAIN_ROOT}/models/librispeech > /dev/null bash download_model.sh if [ $? -ne 0 ]; then exit 1 @@ -22,7 +20,7 @@ cd - > /dev/null # evaluate model CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ -python3 -u test.py \ +python3 -u ${MAIN_ROOT}/test.py \ --batch_size=128 \ --beam_size=500 \ --num_proc_bsearch=8 \ @@ -36,14 +34,14 @@ python3 -u test.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---test_manifest='data/librispeech/manifest.test-clean' \ ---mean_std_path='models/librispeech/mean_std.npz' \ ---vocab_path='models/librispeech/vocab.txt' \ ---model_path='models/librispeech' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--test_manifest="data/manifest.test-clean" \ +--mean_std_path="${MAIN_ROOT}/models/librispeech/mean_std.npz" \ +--vocab_path="${MAIN_ROOT}/models/librispeech/vocab.txt" \ +--model_path="${MAIN_ROOT}/models/librispeech" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/librispeech/run_train.sh b/examples/librispeech/local/run_train.sh similarity index 60% rename from examples/librispeech/run_train.sh rename to examples/librispeech/local/run_train.sh index 6b8982d78ecdc7674191e7de2e5c6f9c82f9cf56..32aa2657b725ddae546f4a3c090a1d27399a61e5 100644 --- a/examples/librispeech/run_train.sh +++ b/examples/librispeech/local/run_train.sh @@ -1,13 +1,11 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # train model # if you wish to resume from an exists model, uncomment --init_from_pretrained_model export FLAGS_sync_nccl_allreduce=0 CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \ -python3 -u train.py \ +python3 -u ${MAIN_ROOT}/train.py \ --batch_size=20 \ --num_epoch=50 \ --num_conv_layers=2 \ @@ -25,14 +23,14 @@ python3 -u train.py \ --use_gpu=True \ --is_local=True \ --share_rnn_weights=True \ ---train_manifest='data/librispeech/manifest.train' \ ---dev_manifest='data/librispeech/manifest.dev-clean' \ ---mean_std_path='data/librispeech/mean_std.npz' \ ---vocab_path='data/librispeech/vocab.txt' \ ---output_model_dir='./checkpoints/libri' \ ---augment_conf_path='conf/augmentation.config' \ ---specgram_type='linear' \ ---shuffle_method='batch_shuffle_clipped' \ +--train_manifest="data/manifest.train" \ +--dev_manifest="data/manifest.dev-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--output_model_dir="./checkpoints/libri" \ +--augment_conf_path="${MAIN_ROOT}/conf/augmentation.config" \ +--specgram_type="linear" \ +--shuffle_method="batch_shuffle_clipped" \ if [ $? -ne 0 ]; then echo "Failed in training!" diff --git a/examples/librispeech/run_tune.sh b/examples/librispeech/local/run_tune.sh similarity index 63% rename from examples/librispeech/run_tune.sh rename to examples/librispeech/local/run_tune.sh index 834f060fa4ef175ca7482d96da60ab31fe308a56..80390c3599003b12112b0b9870550a5076388316 100644 --- a/examples/librispeech/run_tune.sh +++ b/examples/librispeech/local/run_tune.sh @@ -1,7 +1,5 @@ #! /usr/bin/env bash -cd ../.. > /dev/null - # grid-search for hyper-parameters in language model CUDA_VISIBLE_DEVICES=0,1,2,3 \ python3 -u tools/tune.py \ @@ -23,13 +21,13 @@ python3 -u tools/tune.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---tune_manifest='data/librispeech/manifest.dev-clean' \ ---mean_std_path='data/librispeech/mean_std.npz' \ ---vocab_path='models/librispeech/vocab.txt' \ ---model_path='models/librispeech' \ ---lang_model_path='models/lm/common_crawl_00.prune01111.trie.klm' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--tune_manifest="data/manifest.dev-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="${MAIN_ROOT}/models/librispeech/vocab.txt" \ +--model_path="${MAIN_ROOT}/models/librispeech" \ +--lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in tuning!" diff --git a/examples/librispeech/path.sh b/examples/librispeech/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..fd1cebba8ea9313ca30ee8bece2f6ddb0f29112d --- /dev/null +++ b/examples/librispeech/path.sh @@ -0,0 +1,8 @@ +export MAIN_ROOT=${PWD}/../../ + +export PATH=${MAIN_ROOT}:${PWD}/tools:${PATH} +export LC_ALL=C + +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} diff --git a/examples/librispeech/run.sh b/examples/librispeech/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..c8e58913900677679a40210c1f2dae658c435db6 --- /dev/null +++ b/examples/librispeech/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +source path.sh + +# prepare data +bash ./local/run_data.sh + +# test pretrain model +bash ./local/run_test_golden.sh + +# test pretain model +bash ./local/run_infer_golden.sh + +# train model +bash ./local/run_train.sh + +# test model +bash ./local/run_test.sh + +# infer model +bash ./local/run_infer.sh + +# tune model +bash ./local/run_tune.sh diff --git a/examples/librispeech/run_data.sh b/examples/librispeech/run_data.sh deleted file mode 100644 index 788b415de41909c7f60e0b29e9002d860ad160e2..0000000000000000000000000000000000000000 --- a/examples/librispeech/run_data.sh +++ /dev/null @@ -1,45 +0,0 @@ -#! /usr/bin/env bash - -cd ../.. > /dev/null - -# download data, generate manifests -PYTHONPATH=.:$PYTHONPATH python3 data/librispeech/librispeech.py \ ---manifest_prefix='data/librispeech/manifest' \ ---target_dir='./dataset/librispeech' \ ---full_download='True' - -if [ $? -ne 0 ]; then - echo "Prepare LibriSpeech failed. Terminated." - exit 1 -fi - -cat data/librispeech/manifest.train-* | shuf > data/librispeech/manifest.train - - -# build vocabulary -python3 tools/build_vocab.py \ ---count_threshold=0 \ ---vocab_path='data/librispeech/vocab.txt' \ ---manifest_paths='data/librispeech/manifest.train' - -if [ $? -ne 0 ]; then - echo "Build vocabulary failed. Terminated." - exit 1 -fi - - -# compute mean and stddev for normalizer -python3 tools/compute_mean_std.py \ ---manifest_path='data/librispeech/manifest.train' \ ---num_samples=2000 \ ---specgram_type='linear' \ ---output_path='data/librispeech/mean_std.npz' - -if [ $? -ne 0 ]; then - echo "Compute mean and stddev failed. Terminated." - exit 1 -fi - - -echo "LibriSpeech Data preparation done." -exit 0 diff --git a/examples/tiny/local/run_data.sh b/examples/tiny/local/run_data.sh index 2acbe97be0b68919f81167d3cb1b82df991b51ea..3ad387dbcc966468dedf0b0935f030089e4209e5 100644 --- a/examples/tiny/local/run_data.sh +++ b/examples/tiny/local/run_data.sh @@ -7,9 +7,9 @@ fi # download data, generate manifests PYTHONPATH=.:$PYTHONPATH python3 ../librispeech/local/librispeech.py \ ---manifest_prefix='data/manifest' \ +--manifest_prefix="data/manifest" \ --target_dir="${MAIN_ROOT}/dataset/librispeech" \ ---full_download='False' +--full_download="False" if [ $? -ne 0 ]; then echo "Prepare LibriSpeech failed. Terminated." @@ -21,8 +21,8 @@ head -n 64 data/manifest.dev-clean > data/manifest.tiny # build vocabulary python3 ${MAIN_ROOT}/tools/build_vocab.py \ --count_threshold=0 \ ---vocab_path='data/vocab.txt' \ ---manifest_paths='data/manifest.tiny' +--vocab_path="data/vocab.txt" \ +--manifest_paths="data/manifest.tiny" if [ $? -ne 0 ]; then echo "Build vocabulary failed. Terminated." @@ -32,10 +32,10 @@ fi # compute mean and stddev for normalizer python3 ${MAIN_ROOT}/tools/compute_mean_std.py \ ---manifest_path='data/manifest.tiny' \ +--manifest_path="data/manifest.tiny" \ --num_samples=64 \ ---specgram_type='linear' \ ---output_path='data/mean_std.npz' +--specgram_type="linear" \ +--output_path="data/mean_std.npz" if [ $? -ne 0 ]; then echo "Compute mean and stddev failed. Terminated." diff --git a/examples/tiny/local/run_infer.sh b/examples/tiny/local/run_infer.sh index 09e86079d185288bb1a4ffca2188d2e8185384a0..bbaa094e9d52fe960250b2130d01a3b504acc17f 100644 --- a/examples/tiny/local/run_infer.sh +++ b/examples/tiny/local/run_infer.sh @@ -25,14 +25,14 @@ python3 -u $MAIN_ROOT/infer.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---infer_manifest='data/manifest.test-clean' \ ---mean_std_path='data/mean_std.npz' \ ---vocab_path='data/vocab.txt' \ ---model_path='checkpoints/step_final' \ +--infer_manifest="data/manifest.test-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--model_path="checkpoints/step_final" \ --lang_model_path="$MAIN_ROOT/models/lm/common_crawl_00.prune01111.trie.klm" \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/tiny/local/run_infer_golden.sh b/examples/tiny/local/run_infer_golden.sh index 4d5bbecc189f780c8b279cf0ad7453664a98ad2a..21663681ba1813c797aedac24bb9a7eb74f0d0c2 100644 --- a/examples/tiny/local/run_infer_golden.sh +++ b/examples/tiny/local/run_infer_golden.sh @@ -34,14 +34,14 @@ python3 -u ${MAIN_ROOT}/infer.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---infer_manifest='data/manifest.test-clean' \ +--infer_manifest="data/manifest.test-clean" \ --mean_std_path="${MAIN_ROOT}/models/librispeech/mean_std.npz" \ --vocab_path="${MAIN_ROOT}/models/librispeech/vocab.txt" \ --model_path="${MAIN_ROOT}/models/librispeech" \ --lang_model_path="${MAIN_ROOT}/models/lm/common_crawl_00.prune01111.trie.klm" \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in inference!" diff --git a/examples/tiny/local/run_test.sh b/examples/tiny/local/run_test.sh index bdc955125a24d5cf102b5fbe535d39bfaddefa58..ef1fa5a2dd094ae14e3781318d869bb135ad0ad8 100644 --- a/examples/tiny/local/run_test.sh +++ b/examples/tiny/local/run_test.sh @@ -25,14 +25,14 @@ python3 -u $MAIN_ROOT/test.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---test_manifest='data/manifest.test-clean' \ ---mean_std_path='data/mean_std.npz' \ ---vocab_path='data/vocab.txt' \ ---model_path='checkpoints/step_final' \ +--test_manifest="data/manifest.test-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--model_path="checkpoints/step_final" \ --lang_model_path="$MAIN_ROOT/models/lm/common_crawl_00.prune01111.trie.klm" \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/tiny/local/run_test_golden.sh b/examples/tiny/local/run_test_golden.sh index e818a932d2aec6b67346780030d7d929ef83164a..9983fade8e7e40a29f4423e126aabda1fab469f4 100644 --- a/examples/tiny/local/run_test_golden.sh +++ b/examples/tiny/local/run_test_golden.sh @@ -34,14 +34,14 @@ python3 -u $MAIN_ROOT/test.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---test_manifest='data/manifest.test-clean' \ +--test_manifest="data/manifest.test-clean" \ --mean_std_path="$MAIN_ROOT/models/librispeech/mean_std.npz" \ --vocab_path="$MAIN_ROOT/models/librispeech/vocab.txt" \ --model_path="$MAIN_ROOT/models/librispeech" \ --lang_model_path="$MAIN_ROOT/models/lm/common_crawl_00.prune01111.trie.klm" \ ---decoding_method='ctc_beam_search' \ ---error_rate_type='wer' \ ---specgram_type='linear' +--decoding_method="ctc_beam_search" \ +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in evaluation!" diff --git a/examples/tiny/local/run_train.sh b/examples/tiny/local/run_train.sh index 88c6fec3d5014830e21f6a6e4aa1cddd6ede4fe3..de9dcbd74138f6e81bfd950ec7389ca34d8152ac 100644 --- a/examples/tiny/local/run_train.sh +++ b/examples/tiny/local/run_train.sh @@ -22,14 +22,14 @@ python3 -u ${MAIN_ROOT}/train.py \ --use_gpu=True \ --is_local=True \ --share_rnn_weights=True \ ---train_manifest='data/manifest.tiny' \ ---dev_manifest='data/manifest.tiny' \ ---mean_std_path='data/mean_std.npz' \ ---vocab_path='data/vocab.txt' \ ---output_model_dir='./checkpoints/' \ +--train_manifest="data/manifest.tiny" \ +--dev_manifest="data/manifest.tiny" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ +--output_model_dir="./checkpoints/" \ --augment_conf_path="${MAIN_ROOT}/conf/augmentation.config" \ ---specgram_type='linear' \ ---shuffle_method='batch_shuffle_clipped' \ +--specgram_type="linear" \ +--shuffle_method="batch_shuffle_clipped" \ if [ $? -ne 0 ]; then echo "Failed in training!" diff --git a/examples/tiny/local/run_tune.sh b/examples/tiny/local/run_tune.sh index 67501acf96e99fccf0a1432bf99b37b2ad81fd6f..b5cc4d6a1b49ab770b87c08a0dd83cc736228285 100644 --- a/examples/tiny/local/run_tune.sh +++ b/examples/tiny/local/run_tune.sh @@ -21,13 +21,13 @@ python3 -u $MAIN_ROOT/tools/tune.py \ --use_gru=False \ --use_gpu=True \ --share_rnn_weights=True \ ---tune_manifest='data/manifest.dev-clean' \ ---mean_std_path='data/mean_std.npz' \ ---vocab_path='data/vocab.txt' \ +--tune_manifest="data/manifest.dev-clean" \ +--mean_std_path="data/mean_std.npz" \ +--vocab_path="data/vocab.txt" \ --model_path="$MAIN_ROOT/models/librispeech" \ --lang_model_path="$MAIN_ROOT/models/lm/common_crawl_00.prune01111.trie.klm" \ ---error_rate_type='wer' \ ---specgram_type='linear' +--error_rate_type="wer" \ +--specgram_type="linear" if [ $? -ne 0 ]; then echo "Failed in tuning!" diff --git a/requirements.txt b/requirements.txt index b480fdd705e20a5c4c85ecf5200e43483aef8323..8c57208a60cc0c3ab2f7b046f7a932a041013fbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ scipy==1.2.1 resampy==0.1.5 SoundFile==0.9.0.post1 python_speech_features -paddlepaddle-gpu==1.8.5