diff --git a/speechx/.gitignore b/speechx/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e0c618470756b87fe51bfb0d34a7e7f7e92003df --- /dev/null +++ b/speechx/.gitignore @@ -0,0 +1 @@ +tools/valgrind* diff --git a/speechx/README.md b/speechx/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bee320289f6d234497d9e0c9220f738a93a89936 --- /dev/null +++ b/speechx/README.md @@ -0,0 +1,24 @@ +# SpeechX -- Speech Inference All in One + +> Test under `Ubuntu 16.04.7 LTS`. + +## Build + +``` +./build.sh +```l + +## Valgrind + +> If using docker please check `--privileged` is set when `docker run`. + +1. Fatal error at startup: a function redirection which is mandatory for this platform-tool combination cannot be set up +``` +apt-get install libc6-dbg +``` + +``` +pushd tools +./setup_valgrind.sh +popd +``` diff --git a/speechx/examples/.gitignore b/speechx/examples/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b7075fa56c3540c9a7cfea427d9b57b69a1866af --- /dev/null +++ b/speechx/examples/.gitignore @@ -0,0 +1,2 @@ +*.ark +paddle_asr_model/ diff --git a/speechx/examples/path.sh b/speechx/examples/decoder/path.sh similarity index 83% rename from speechx/examples/path.sh rename to speechx/examples/decoder/path.sh index 1f62ed7eaefe2e4d2a7319077e16239692f3143f..7b4b7545b38b3daf0eefb64b577f6698e6b9c9b1 100644 --- a/speechx/examples/path.sh +++ b/speechx/examples/decoder/path.sh @@ -2,7 +2,6 @@ SPEECHX_ROOT=$PWD/../.. SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples -SPEECHX_BIN=$SPEECHX_EXAMPLES/nnet:$SPEECHX_EXAMPLES/decoder:$SPEECHX_EXAMPLES/feat SPEECHX_TOOLS=$SPEECHX_ROOT/tools TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin @@ -11,4 +10,5 @@ TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin export LC_AL=C +SPEECHX_BIN=$SPEECHX_EXAMPLES/decoder export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN diff --git a/speechx/examples/decoder/run.sh b/speechx/examples/decoder/run.sh old mode 100644 new mode 100755 index 4251cc04565a52cd7a04edb1dcb42c3789f566af..fc5e91824633beaf9d87eda2a255e3bb4ddb9903 --- a/speechx/examples/decoder/run.sh +++ b/speechx/examples/decoder/run.sh @@ -2,18 +2,19 @@ set +x set -e +. path.sh + # 1. compile -if [ ! -d ../../build/examples ]; then - cd ../.. +if [ ! -d ${SPEECHX_EXAMPLES} ]; then + pushd ${SPEECHX_ROOT} bash build.sh - cd - + popd fi -. ../path.sh # 2. download model if [ ! -d ../paddle_asr_model ]; then - wget https://paddlespeech.bj.bcebos.com/s2t/paddle_asr_online/paddle_asr_model.tar.gz + wget -c https://paddlespeech.bj.bcebos.com/s2t/paddle_asr_online/paddle_asr_model.tar.gz tar xzfv paddle_asr_model.tar.gz mv ./paddle_asr_model ../ # produce wav scp @@ -25,8 +26,15 @@ feat_wspecifier=./feats.ark cmvn=./cmvn.ark # 3. run feat -linear_spectrogram_main --wav_rspecifier=scp:$model_dir/wav.scp --feature_wspecifier=ark:$feat_wspecifier --cmvn_write_path=$cmvn +linear_spectrogram_main \ + --wav_rspecifier=scp:$model_dir/wav.scp \ + --feature_wspecifier=ark,t:$feat_wspecifier \ + --cmvn_write_path=$cmvn # 4. run decoder -offline_decoder_main --feature_respecifier=ark:$feat_wspecifier --model_path=$model_dir/avg_1.jit.pdmodel --param_path=$model_dir/avg_1.jit.pdparams --dict_file=$model_dir/vocab.txt --lm_path=$model_dir/avg_1.jit.klm - +offline_decoder_main \ + --feature_respecifier=ark:$feat_wspecifier \ + --model_path=$model_dir/avg_1.jit.pdmodel \ + --param_path=$model_dir/avg_1.jit.pdparams \ + --dict_file=$model_dir/vocab.txt \ + --lm_path=$model_dir/avg_1.jit.klm \ No newline at end of file diff --git a/speechx/examples/decoder/valgrind.sh b/speechx/examples/decoder/valgrind.sh old mode 100644 new mode 100755 index 3e90299ebe3eeffa18027cb4066912a9ff472cef..14efe0ba42bc98d2fa4915c776303422902bb2dd --- a/speechx/examples/decoder/valgrind.sh +++ b/speechx/examples/decoder/valgrind.sh @@ -5,16 +5,22 @@ set +x set -e -if [ ! -d ../../tools/valgrind/install ]; then +. ./path.sh + +if [ ! -d ${SPEECHX_TOOLS}/valgrind/install ]; then echo "please install valgrind in the speechx tools dir.\n" exit 1 fi -. ../path.sh - model_dir=../paddle_asr_model feat_wspecifier=./feats.ark cmvn=./cmvn.ark -valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all offline_decoder_main --feature_respecifier=ark:$feat_wspecifier --model_path=$model_dir/avg_1.jit.pdmodel --param_path=$model_dir/avg_1.jit.pdparams --dict_file=$model_dir/vocab.txt --lm_path=$model_dir/avg_1.jit.klm +valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all \ + offline_decoder_main \ + --feature_respecifier=ark:$feat_wspecifier \ + --model_path=$model_dir/avg_1.jit.pdmodel \ + --param_path=$model_dir/avg_1.jit.pdparams \ + --dict_file=$model_dir/vocab.txt \ + --lm_path=$model_dir/avg_1.jit.klm diff --git a/speechx/examples/feat/path.sh b/speechx/examples/feat/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..8ab7ee2991824e7be78f3c09217e9fd5da95015b --- /dev/null +++ b/speechx/examples/feat/path.sh @@ -0,0 +1,14 @@ +# This contains the locations of binarys build required for running the examples. + +SPEECHX_ROOT=$PWD/../.. +SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples + +SPEECHX_TOOLS=$SPEECHX_ROOT/tools +TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin + +[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; } + +export LC_AL=C + +SPEECHX_BIN=$SPEECHX_EXAMPLES/feat +export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN diff --git a/speechx/examples/feat/run.sh b/speechx/examples/feat/run.sh old mode 100644 new mode 100755 index 276fefa3aff413affcc62624bb597d6ef4502c27..bd21bd7f4e1c4262a5af339c68d92e989f0bd5c2 --- a/speechx/examples/feat/run.sh +++ b/speechx/examples/feat/run.sh @@ -2,15 +2,15 @@ set +x set -e +. ./path.sh + # 1. compile -if [ ! -d ../../build/examples ]; then - cd ../.. +if [ ! -d ${SPEECHX_EXAMPLES} ]; then + pushd ${SPEECHX_ROOT} bash build.sh - cd - + popd fi -. ../path.sh - # 2. download model if [ ! -d ../paddle_asr_model ]; then wget https://paddlespeech.bj.bcebos.com/s2t/paddle_asr_online/paddle_asr_model.tar.gz @@ -25,4 +25,7 @@ feat_wspecifier=./feats.ark cmvn=./cmvn.ark # 3. run feat -linear_spectrogram_main --wav_rspecifier=scp:$model_dir/wav.scp --feature_wspecifier=ark,t:$feat_wspecifier --cmvn_write_path=$cmvn +linear_spectrogram_main \ + --wav_rspecifier=scp:$model_dir/wav.scp \ + --feature_wspecifier=ark,t:$feat_wspecifier \ + --cmvn_write_path=$cmvn diff --git a/speechx/examples/feat/valgrind.sh b/speechx/examples/feat/valgrind.sh old mode 100644 new mode 100755 index 5a6a538a64a1dc5b6a03ffac490c8bd4a54b87f2..f8aab63f8c99add2c7fd8c66db7f9a71ef0c6bf3 --- a/speechx/examples/feat/valgrind.sh +++ b/speechx/examples/feat/valgrind.sh @@ -5,16 +5,20 @@ set +x set -e -if [ ! -d ../../tools/valgrind/install ]; then +. ./path.sh + +if [ ! -d ${SPEECHX_TOOLS}/valgrind/install ]; then echo "please install valgrind in the speechx tools dir.\n" exit 1 fi -. ../path.sh - model_dir=../paddle_asr_model feat_wspecifier=./feats.ark cmvn=./cmvn.ark -valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all linear_spectrogram_main --wav_rspecifier=scp:$model_dir/wav.scp --feature_wspecifier=ark,t:$feat_wspecifier --cmvn_write_path=$cmvn +valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all \ + linear_spectrogram_main \ + --wav_rspecifier=scp:$model_dir/wav.scp \ + --feature_wspecifier=ark,t:$feat_wspecifier \ + --cmvn_write_path=$cmvn diff --git a/speechx/examples/nnet/path.sh b/speechx/examples/nnet/path.sh new file mode 100644 index 0000000000000000000000000000000000000000..f70e70eeaa18f050db440daa195673b6d6d62828 --- /dev/null +++ b/speechx/examples/nnet/path.sh @@ -0,0 +1,14 @@ +# This contains the locations of binarys build required for running the examples. + +SPEECHX_ROOT=$PWD/../.. +SPEECHX_EXAMPLES=$SPEECHX_ROOT/build/examples + +SPEECHX_TOOLS=$SPEECHX_ROOT/tools +TOOLS_BIN=$SPEECHX_TOOLS/valgrind/install/bin + +[ -d $SPEECHX_EXAMPLES ] || { echo "Error: 'build/examples' directory not found. please ensure that the project build successfully"; } + +export LC_AL=C + +SPEECHX_BIN=$SPEECHX_EXAMPLES/nnet +export PATH=$PATH:$SPEECHX_BIN:$TOOLS_BIN diff --git a/speechx/examples/nnet/run.sh b/speechx/examples/nnet/run.sh old mode 100644 new mode 100755 index 6615082a7847709bb3a7324e61870c1c95a9cb04..4d67d198842e0aa70945786d43dd6e674ec30b08 --- a/speechx/examples/nnet/run.sh +++ b/speechx/examples/nnet/run.sh @@ -2,15 +2,15 @@ set +x set -e +. path.sh + # 1. compile -if [ ! -d ../../build/examples ]; then - cd ../.. +if [ ! -d ${SPEECHX_EXAMPLES} ]; then + pushd ${SPEECHX_ROOT} bash build.sh - cd - + popd fi -. ../path.sh - # 2. download model if [ ! -d ../paddle_asr_model ]; then wget https://paddlespeech.bj.bcebos.com/s2t/paddle_asr_online/paddle_asr_model.tar.gz @@ -23,5 +23,7 @@ fi model_dir=../paddle_asr_model # 4. run decoder -pp-model-test --model_path=$model_dir/avg_1.jit.pdmodel --param_path=$model_dir/avg_1.jit.pdparams +pp-model-test \ + --model_path=$model_dir/avg_1.jit.pdmodel \ + --param_path=$model_dir/avg_1.jit.pdparams diff --git a/speechx/examples/nnet/valgrind.sh b/speechx/examples/nnet/valgrind.sh old mode 100644 new mode 100755 index d692db504b2cd8119ea9f8f9fbf9149e5c185acb..2a08c6082f7803538a26b97dc34d968642b16f81 --- a/speechx/examples/nnet/valgrind.sh +++ b/speechx/examples/nnet/valgrind.sh @@ -5,14 +5,16 @@ set +x set -e -if [ ! -d ../../tools/valgrind/install ]; then +. ./path.sh + +if [ ! -d ${SPEECHX_TOOLS}/valgrind/install ]; then echo "please install valgrind in the speechx tools dir.\n" exit 1 fi -. ../path.sh - model_dir=../paddle_asr_model -valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all pp-model-test --model_path=$model_dir/avg_1.jit.pdmodel --param_path=$model_dir/avg_1.jit.pdparams - +valgrind --tool=memcheck --track-origins=yes --leak-check=full --show-leak-kinds=all \ + pp-model-test \ + --model_path=$model_dir/avg_1.jit.pdmodel \ + --param_path=$model_dir/avg_1.jit.pdparams \ No newline at end of file diff --git a/speechx/tools/setup_valgrind.sh b/speechx/tools/setup_valgrind.sh index 9e9f07168361bd5d95cc8ac7caed28a5692eca2a..b887087e45f735af9b347aec3f460501fab7f572 100755 --- a/speechx/tools/setup_valgrind.sh +++ b/speechx/tools/setup_valgrind.sh @@ -11,7 +11,7 @@ url=https://sourceware.org/pub/valgrind/valgrind-3.18.1.tar.bz2 if [ -f $tarball ]; then echo "use the $tarball have downloaded." else - wget -t3 --no-check-certificate $url + wget -c -t3 --no-check-certificate $url fi tar xjfv $tarball