Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
DeepSpeech
提交
88a19897
D
DeepSpeech
项目概览
PaddlePaddle
/
DeepSpeech
大约 2 年 前同步成功
通知
210
Star
8425
Fork
1598
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
245
列表
看板
标记
里程碑
合并请求
3
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
DeepSpeech
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
245
Issue
245
列表
看板
标记
里程碑
合并请求
3
合并请求
3
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
88a19897
编写于
9月 25, 2021
作者:
H
Hui Zhang
提交者:
GitHub
9月 25, 2021
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #851 from Jackwaterveg/release_model
Optimize the 1xt2x code
上级
09d5209f
30b3e237
变更
24
隐藏空白更改
内联
并排
Showing
24 changed file
with
96 addition
and
119 deletion
+96
-119
deepspeech/frontend/featurizer/text_featurizer.py
deepspeech/frontend/featurizer/text_featurizer.py
+0
-1
examples/1xt2x/aishell/.gitignore
examples/1xt2x/aishell/.gitignore
+1
-0
examples/1xt2x/aishell/local/data.sh
examples/1xt2x/aishell/local/data.sh
+12
-28
examples/1xt2x/aishell/local/download_model.sh
examples/1xt2x/aishell/local/download_model.sh
+9
-3
examples/1xt2x/aishell/path.sh
examples/1xt2x/aishell/path.sh
+1
-1
examples/1xt2x/aishell/run.sh
examples/1xt2x/aishell/run.sh
+3
-2
examples/1xt2x/baidu_en8k/.gitignore
examples/1xt2x/baidu_en8k/.gitignore
+1
-0
examples/1xt2x/baidu_en8k/local/data.sh
examples/1xt2x/baidu_en8k/local/data.sh
+14
-29
examples/1xt2x/baidu_en8k/local/download_model.sh
examples/1xt2x/baidu_en8k/local/download_model.sh
+9
-3
examples/1xt2x/baidu_en8k/path.sh
examples/1xt2x/baidu_en8k/path.sh
+1
-1
examples/1xt2x/baidu_en8k/run.sh
examples/1xt2x/baidu_en8k/run.sh
+3
-2
examples/1xt2x/librispeech/.gitignore
examples/1xt2x/librispeech/.gitignore
+1
-0
examples/1xt2x/librispeech/local/data.sh
examples/1xt2x/librispeech/local/data.sh
+14
-31
examples/1xt2x/librispeech/local/download_model.sh
examples/1xt2x/librispeech/local/download_model.sh
+9
-3
examples/1xt2x/librispeech/path.sh
examples/1xt2x/librispeech/path.sh
+1
-2
examples/1xt2x/librispeech/run.sh
examples/1xt2x/librispeech/run.sh
+3
-2
examples/1xt2x/src_deepspeech2x/__init__.py
examples/1xt2x/src_deepspeech2x/__init__.py
+0
-0
examples/1xt2x/src_deepspeech2x/bin/test.py
examples/1xt2x/src_deepspeech2x/bin/test.py
+1
-1
examples/1xt2x/src_deepspeech2x/models/__init__.py
examples/1xt2x/src_deepspeech2x/models/__init__.py
+0
-0
examples/1xt2x/src_deepspeech2x/models/ds2/__init__.py
examples/1xt2x/src_deepspeech2x/models/ds2/__init__.py
+0
-0
examples/1xt2x/src_deepspeech2x/models/ds2/deepspeech2.py
examples/1xt2x/src_deepspeech2x/models/ds2/deepspeech2.py
+1
-1
examples/1xt2x/src_deepspeech2x/models/ds2/rnn.py
examples/1xt2x/src_deepspeech2x/models/ds2/rnn.py
+0
-0
examples/1xt2x/src_deepspeech2x/test_model.py
examples/1xt2x/src_deepspeech2x/test_model.py
+10
-8
utils/format_data.py
utils/format_data.py
+2
-1
未找到文件。
deepspeech/frontend/featurizer/text_featurizer.py
浏览文件 @
88a19897
...
@@ -199,7 +199,6 @@ class TextFeaturizer():
...
@@ -199,7 +199,6 @@ class TextFeaturizer():
"""Load vocabulary from file."""
"""Load vocabulary from file."""
vocab_list
=
load_dict
(
vocab_filepath
,
maskctc
)
vocab_list
=
load_dict
(
vocab_filepath
,
maskctc
)
assert
vocab_list
is
not
None
assert
vocab_list
is
not
None
assert
SPACE
in
vocab_list
id2token
=
dict
(
id2token
=
dict
(
[(
idx
,
token
)
for
(
idx
,
token
)
in
enumerate
(
vocab_list
)])
[(
idx
,
token
)
for
(
idx
,
token
)
in
enumerate
(
vocab_list
)])
...
...
examples/1xt2x/aishell/.gitignore
浏览文件 @
88a19897
...
@@ -2,3 +2,4 @@ exp
...
@@ -2,3 +2,4 @@ exp
data
data
*log
*log
tmp
tmp
nohup*
examples/1xt2x/aishell/local/data.sh
浏览文件 @
88a19897
#!/bin/bash
#!/bin/bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
stage
=
-1
stage
=
-1
stop_stage
=
100
stop_stage
=
100
...
@@ -9,17 +15,16 @@ mkdir -p data
...
@@ -9,17 +15,16 @@ mkdir -p data
TARGET_DIR
=
${
MAIN_ROOT
}
/examples/dataset
TARGET_DIR
=
${
MAIN_ROOT
}
/examples/dataset
mkdir
-p
${
TARGET_DIR
}
mkdir
-p
${
TARGET_DIR
}
bash
local
/download_model.sh
bash
local
/download_model.sh
${
ckpt_dir
}
if
[
$?
-ne
0
]
;
then
if
[
$?
-ne
0
]
;
then
exit
1
exit
1
fi
fi
cd
${
ckpt_dir
}
tar
xzvf aishell_model_v1.8_to_v2.x.tar.gz
tar
xzvf aishell_model_v1.8_to_v2.x.tar.gz
mv
aishell_v1.8.pdparams exp/deepspeech2/checkpoints/
cd
-
mv
README.md exp/deepspeech2/
mv
${
ckpt_dir
}
/mean_std.npz data/
mv
mean_std.npz data/
mv
${
ckpt_dir
}
/vocab.txt data/
mv
vocab.txt data/
rm
aishell_model_v1.8_to_v2.x.tar.gz
-f
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
...
@@ -39,27 +44,6 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
...
@@ -39,27 +44,6 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# compute mean and stddev for normalizer
num_workers
=
$(
nproc
)
python3
${
MAIN_ROOT
}
/utils/compute_mean_std.py
\
--manifest_path
=
"data/manifest.train.raw"
\
--specgram_type
=
"linear"
\
--delta_delta
=
false
\
--stride_ms
=
10.0
\
--window_ms
=
20.0
\
--sample_rate
=
16000
\
--use_dB_normalization
=
True
\
--num_samples
=
2000
\
--num_workers
=
${
num_workers
}
\
--output_path
=
"data/mean_std.json"
if
[
$?
-ne
0
]
;
then
echo
"Compute mean and stddev failed. Terminated."
exit
1
fi
fi
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
# format manifest with tokenids, vocab size
# format manifest with tokenids, vocab size
...
@@ -67,7 +51,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
...
@@ -67,7 +51,7 @@ if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
{
{
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
--feat_type
"raw"
\
--feat_type
"raw"
\
--cmvn_path
"data/mean_std.
json
"
\
--cmvn_path
"data/mean_std.
npz
"
\
--unit_type
"char"
\
--unit_type
"char"
\
--vocab_path
=
"data/vocab.txt"
\
--vocab_path
=
"data/vocab.txt"
\
--manifest_path
=
"data/manifest.
${
dataset
}
.raw"
\
--manifest_path
=
"data/manifest.
${
dataset
}
.raw"
\
...
...
examples/1xt2x/aishell/local/download_model.sh
浏览文件 @
88a19897
#! /usr/bin/env bash
#! /usr/bin/env bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
.
${
MAIN_ROOT
}
/utils/utility.sh
.
${
MAIN_ROOT
}
/utils/utility.sh
URL
=
'https://deepspeech.bj.bcebos.com/mandarin_models/aishell_model_v1.8_to_v2.x.tar.gz'
URL
=
'https://deepspeech.bj.bcebos.com/mandarin_models/aishell_model_v1.8_to_v2.x.tar.gz'
MD5
=
4ade113c69ea291b8ce5ec6a03296659
MD5
=
87e7577d4bea737dbf3e8daab37aa808
TARGET
=
.
/aishell_model_v1.8_to_v2.x.tar.gz
TARGET
=
${
ckpt_dir
}
/aishell_model_v1.8_to_v2.x.tar.gz
echo
"Download Aishell model ..."
echo
"Download Aishell model ..."
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
echo
"Fail to download Aishell model!"
echo
"Fail to download Aishell model!"
exit
1
exit
1
fi
fi
tar
-zxvf
$TARGET
exit
0
exit
0
examples/1xt2x/aishell/path.sh
浏览文件 @
88a19897
...
@@ -12,5 +12,5 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
...
@@ -12,5 +12,5 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
MODEL
=
deepspeech2
MODEL
=
deepspeech2
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/deepspeech2x/bin
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/
src_
deepspeech2x/bin
echo
"BIN_DIR "
${
BIN_DIR
}
echo
"BIN_DIR "
${
BIN_DIR
}
examples/1xt2x/aishell/run.sh
浏览文件 @
88a19897
...
@@ -7,6 +7,7 @@ stop_stage=100
...
@@ -7,6 +7,7 @@ stop_stage=100
conf_path
=
conf/deepspeech2.yaml
conf_path
=
conf/deepspeech2.yaml
avg_num
=
1
avg_num
=
1
model_type
=
offline
model_type
=
offline
gpus
=
2
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
...
@@ -17,11 +18,11 @@ echo "checkpoint name ${ckpt}"
...
@@ -17,11 +18,11 @@ echo "checkpoint name ${ckpt}"
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
# prepare data
# prepare data
mkdir
-p
exp/
${
ckpt
}
/checkpoints
mkdir
-p
exp/
${
ckpt
}
/checkpoints
bash ./local/data.sh
||
exit
-1
bash ./local/data.sh
exp/
${
ckpt
}
/checkpoints
||
exit
-1
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# test ckpt avg_n
# test ckpt avg_n
CUDA_VISIBLE_DEVICES
=
1
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
CUDA_VISIBLE_DEVICES
=
${
gpus
}
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
fi
fi
examples/1xt2x/baidu_en8k/.gitignore
浏览文件 @
88a19897
...
@@ -2,3 +2,4 @@ exp
...
@@ -2,3 +2,4 @@ exp
data
data
*log
*log
tmp
tmp
nohup*
examples/1xt2x/baidu_en8k/local/data.sh
浏览文件 @
88a19897
#!/bin/bash
#!/bin/bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
stage
=
-1
stage
=
-1
stop_stage
=
100
stop_stage
=
100
unit_type
=
char
source
${
MAIN_ROOT
}
/utils/parse_options.sh
source
${
MAIN_ROOT
}
/utils/parse_options.sh
...
@@ -10,17 +17,17 @@ TARGET_DIR=${MAIN_ROOT}/examples/dataset
...
@@ -10,17 +17,17 @@ TARGET_DIR=${MAIN_ROOT}/examples/dataset
mkdir
-p
${
TARGET_DIR
}
mkdir
-p
${
TARGET_DIR
}
bash
local
/download_model.sh
bash
local
/download_model.sh
${
ckpt_dir
}
if
[
$?
-ne
0
]
;
then
if
[
$?
-ne
0
]
;
then
exit
1
exit
1
fi
fi
cd
${
ckpt_dir
}
tar
xzvf baidu_en8k_v1.8_to_v2.x.tar.gz
tar
xzvf baidu_en8k_v1.8_to_v2.x.tar.gz
mv
baidu_en8k_v1.8.pdparams exp/deepspeech2/checkpoints/
cd
-
mv
README.md exp/deepspeech2/
mv
${
ckpt_dir
}
/mean_std.npz data/
mv
mean_std.npz data/
mv
${
ckpt_dir
}
/vocab.txt data/
mv
vocab.txt data/
rm
baidu_en8k_v1.8_to_v2.x.tar.gz
-f
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
# download data, generate manifests
# download data, generate manifests
...
@@ -53,35 +60,13 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
...
@@ -53,35 +60,13 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# compute mean and stddev for normalizer
num_workers
=
$(
nproc
)
python3
${
MAIN_ROOT
}
/utils/compute_mean_std.py
\
--manifest_path
=
"data/manifest.train.raw"
\
--num_samples
=
2000
\
--specgram_type
=
"linear"
\
--delta_delta
=
false
\
--sample_rate
=
16000
\
--stride_ms
=
10.0
\
--window_ms
=
20.0
\
--use_dB_normalization
=
True
\
--num_workers
=
${
num_workers
}
\
--output_path
=
"data/mean_std.json"
if
[
$?
-ne
0
]
;
then
echo
"Compute mean and stddev failed. Terminated."
exit
1
fi
fi
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
# format manifest with tokenids, vocab size
# format manifest with tokenids, vocab size
for
set
in
train dev
test
dev-clean dev-other test-clean test-other
;
do
for
set
in
train dev
test
dev-clean dev-other test-clean test-other
;
do
{
{
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
--feat_type
"raw"
\
--feat_type
"raw"
\
--cmvn_path
"data/mean_std.
json
"
\
--cmvn_path
"data/mean_std.
npz
"
\
--unit_type
${
unit_type
}
\
--unit_type
${
unit_type
}
\
--vocab_path
=
"data/vocab.txt"
\
--vocab_path
=
"data/vocab.txt"
\
--manifest_path
=
"data/manifest.
${
set
}
.raw"
\
--manifest_path
=
"data/manifest.
${
set
}
.raw"
\
...
...
examples/1xt2x/baidu_en8k/local/download_model.sh
浏览文件 @
88a19897
#! /usr/bin/env bash
#! /usr/bin/env bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
.
${
MAIN_ROOT
}
/utils/utility.sh
.
${
MAIN_ROOT
}
/utils/utility.sh
URL
=
'https://deepspeech.bj.bcebos.com/eng_models/baidu_en8k_v1.8_to_v2.x.tar.gz'
URL
=
'https://deepspeech.bj.bcebos.com/eng_models/baidu_en8k_v1.8_to_v2.x.tar.gz'
MD5
=
fdabeb6c96963ac85d9188f0275c6a1b
MD5
=
c1676be8505cee436e6f312823e9008c
TARGET
=
.
/baidu_en8k_v1.8_to_v2.x.tar.gz
TARGET
=
${
ckpt_dir
}
/baidu_en8k_v1.8_to_v2.x.tar.gz
echo
"Download BaiduEn8k model ..."
echo
"Download BaiduEn8k model ..."
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
echo
"Fail to download BaiduEn8k model!"
echo
"Fail to download BaiduEn8k model!"
exit
1
exit
1
fi
fi
tar
-zxvf
$TARGET
exit
0
exit
0
examples/1xt2x/baidu_en8k/path.sh
浏览文件 @
88a19897
...
@@ -12,5 +12,5 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
...
@@ -12,5 +12,5 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
MODEL
=
deepspeech2
MODEL
=
deepspeech2
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/deepspeech2x/bin
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/
src_
deepspeech2x/bin
echo
"BIN_DIR "
${
BIN_DIR
}
echo
"BIN_DIR "
${
BIN_DIR
}
examples/1xt2x/baidu_en8k/run.sh
浏览文件 @
88a19897
...
@@ -7,6 +7,7 @@ stop_stage=100
...
@@ -7,6 +7,7 @@ stop_stage=100
conf_path
=
conf/deepspeech2.yaml
conf_path
=
conf/deepspeech2.yaml
avg_num
=
1
avg_num
=
1
model_type
=
offline
model_type
=
offline
gpus
=
0
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
...
@@ -17,11 +18,11 @@ echo "checkpoint name ${ckpt}"
...
@@ -17,11 +18,11 @@ echo "checkpoint name ${ckpt}"
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
# prepare data
# prepare data
mkdir
-p
exp/
${
ckpt
}
/checkpoints
mkdir
-p
exp/
${
ckpt
}
/checkpoints
bash ./local/data.sh
||
exit
-1
bash ./local/data.sh
exp/
${
ckpt
}
/checkpoints
||
exit
-1
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# test ckpt avg_n
# test ckpt avg_n
CUDA_VISIBLE_DEVICES
=
0
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
CUDA_VISIBLE_DEVICES
=
${
gpus
}
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
fi
fi
examples/1xt2x/librispeech/.gitignore
浏览文件 @
88a19897
...
@@ -2,3 +2,4 @@ exp
...
@@ -2,3 +2,4 @@ exp
data
data
*log
*log
tmp
tmp
nohup*
examples/1xt2x/librispeech/local/data.sh
浏览文件 @
88a19897
#!/bin/bash
#!/bin/bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
stage
=
-1
stage
=
-1
stop_stage
=
100
stop_stage
=
100
unit_type
=
char
source
${
MAIN_ROOT
}
/utils/parse_options.sh
source
${
MAIN_ROOT
}
/utils/parse_options.sh
...
@@ -9,18 +17,16 @@ mkdir -p data
...
@@ -9,18 +17,16 @@ mkdir -p data
TARGET_DIR
=
${
MAIN_ROOT
}
/examples/dataset
TARGET_DIR
=
${
MAIN_ROOT
}
/examples/dataset
mkdir
-p
${
TARGET_DIR
}
mkdir
-p
${
TARGET_DIR
}
bash
local
/download_model.sh
${
ckpt_dir
}
bash
local
/download_model.sh
if
[
$?
-ne
0
]
;
then
if
[
$?
-ne
0
]
;
then
exit
1
exit
1
fi
fi
cd
${
ckpt_dir
}
tar
xzvf librispeech_v1.8_to_v2.x.tar.gz
tar
xzvf librispeech_v1.8_to_v2.x.tar.gz
mv
librispeech_v1.8.pdparams exp/deepspeech2/checkpoints/
cd
-
mv
README.md exp/deepspeech2/
mv
${
ckpt_dir
}
/mean_std.npz data/
mv
mean_std.npz data/
mv
${
ckpt_dir
}
/vocab.txt data/
mv
vocab.txt data/
rm
librispeech_v1.8_to_v2.x.tar.gz
-f
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
if
[
${
stage
}
-le
-1
]
&&
[
${
stop_stage
}
-ge
-1
]
;
then
# download data, generate manifests
# download data, generate manifests
...
@@ -52,36 +58,13 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
...
@@ -52,36 +58,13 @@ if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
done
done
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# compute mean and stddev for normalizer
num_workers
=
$(
nproc
)
python3
${
MAIN_ROOT
}
/utils/compute_mean_std.py
\
--manifest_path
=
"data/manifest.train.raw"
\
--num_samples
=
2000
\
--specgram_type
=
"linear"
\
--delta_delta
=
false
\
--sample_rate
=
16000
\
--stride_ms
=
10.0
\
--window_ms
=
20.0
\
--use_dB_normalization
=
True
\
--num_workers
=
${
num_workers
}
\
--output_path
=
"data/mean_std.json"
if
[
$?
-ne
0
]
;
then
echo
"Compute mean and stddev failed. Terminated."
exit
1
fi
fi
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
if
[
${
stage
}
-le
2
]
&&
[
${
stop_stage
}
-ge
2
]
;
then
# format manifest with tokenids, vocab size
# format manifest with tokenids, vocab size
for
set
in
train dev
test
dev-clean dev-other test-clean test-other
;
do
for
set
in
train dev
test
dev-clean dev-other test-clean test-other
;
do
{
{
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
python3
${
MAIN_ROOT
}
/utils/format_data.py
\
--feat_type
"raw"
\
--feat_type
"raw"
\
--cmvn_path
"data/mean_std.
json
"
\
--cmvn_path
"data/mean_std.
npz
"
\
--unit_type
${
unit_type
}
\
--unit_type
${
unit_type
}
\
--vocab_path
=
"data/vocab.txt"
\
--vocab_path
=
"data/vocab.txt"
\
--manifest_path
=
"data/manifest.
${
set
}
.raw"
\
--manifest_path
=
"data/manifest.
${
set
}
.raw"
\
...
...
examples/1xt2x/librispeech/local/download_model.sh
浏览文件 @
88a19897
#! /usr/bin/env bash
#! /usr/bin/env bash
if
[
$#
!=
1
]
;
then
echo
"usage:
${
0
}
ckpt_dir"
exit
-1
fi
ckpt_dir
=
$1
.
${
MAIN_ROOT
}
/utils/utility.sh
.
${
MAIN_ROOT
}
/utils/utility.sh
URL
=
'https://deepspeech.bj.bcebos.com/eng_models/librispeech_v1.8_to_v2.x.tar.gz'
URL
=
'https://deepspeech.bj.bcebos.com/eng_models/librispeech_v1.8_to_v2.x.tar.gz'
MD5
=
7b0f582fe2f5a840b840e7ee52246bc5
MD5
=
a06d9aadb560ea113984dc98d67232c8
TARGET
=
.
/librispeech_v1.8_to_v2.x.tar.gz
TARGET
=
${
ckpt_dir
}
/librispeech_v1.8_to_v2.x.tar.gz
echo
"Download LibriSpeech model ..."
echo
"Download LibriSpeech model ..."
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
...
@@ -13,7 +20,6 @@ if [ $? -ne 0 ]; then
echo
"Fail to download LibriSpeech model!"
echo
"Fail to download LibriSpeech model!"
exit
1
exit
1
fi
fi
tar
-zxvf
$TARGET
exit
0
exit
0
examples/1xt2x/librispeech/path.sh
浏览文件 @
88a19897
...
@@ -12,5 +12,4 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
...
@@ -12,5 +12,4 @@ export PYTHONPATH=${LOCAL_DEEPSPEECH2}:${PYTHONPATH}
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
export
LD_LIBRARY_PATH
=
${
LD_LIBRARY_PATH
}
:/usr/local/lib/
MODEL
=
deepspeech2
MODEL
=
deepspeech2
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/deepspeech2x/bin
export
BIN_DIR
=
${
LOCAL_DEEPSPEECH2
}
/src_deepspeech2x/bin
echo
"BIN_DIR "
${
BIN_DIR
}
examples/1xt2x/librispeech/run.sh
浏览文件 @
88a19897
...
@@ -7,6 +7,7 @@ stop_stage=100
...
@@ -7,6 +7,7 @@ stop_stage=100
conf_path
=
conf/deepspeech2.yaml
conf_path
=
conf/deepspeech2.yaml
avg_num
=
1
avg_num
=
1
model_type
=
offline
model_type
=
offline
gpus
=
1
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
source
${
MAIN_ROOT
}
/utils/parse_options.sh
||
exit
1
;
...
@@ -17,10 +18,10 @@ echo "checkpoint name ${ckpt}"
...
@@ -17,10 +18,10 @@ echo "checkpoint name ${ckpt}"
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
if
[
${
stage
}
-le
0
]
&&
[
${
stop_stage
}
-ge
0
]
;
then
# prepare data
# prepare data
mkdir
-p
exp/
${
ckpt
}
/checkpoints
mkdir
-p
exp/
${
ckpt
}
/checkpoints
bash ./local/data.sh
||
exit
-1
bash ./local/data.sh
exp/
${
ckpt
}
/checkpoints
||
exit
-1
fi
fi
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
if
[
${
stage
}
-le
1
]
&&
[
${
stop_stage
}
-ge
1
]
;
then
# test ckpt avg_n
# test ckpt avg_n
CUDA_VISIBLE_DEVICES
=
0
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
CUDA_VISIBLE_DEVICES
=
${
gpus
}
./local/test.sh
${
conf_path
}
exp/
${
ckpt
}
/checkpoints/
${
v18_ckpt
}
${
model_type
}
||
exit
-1
fi
fi
examples/1xt2x/deepspeech2x/__init__.py
→
examples/1xt2x/
src_
deepspeech2x/__init__.py
浏览文件 @
88a19897
文件已移动
examples/1xt2x/deepspeech2x/bin/test.py
→
examples/1xt2x/
src_
deepspeech2x/bin/test.py
浏览文件 @
88a19897
...
@@ -12,7 +12,7 @@
...
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# See the License for the specific language governing permissions and
# limitations under the License.
# limitations under the License.
"""Evaluation for DeepSpeech2 model."""
"""Evaluation for DeepSpeech2 model."""
from
deepspeech2x.
model
import
DeepSpeech2Tester
as
Tester
from
src_deepspeech2x.test_
model
import
DeepSpeech2Tester
as
Tester
from
deepspeech.exps.deepspeech2.config
import
get_cfg_defaults
from
deepspeech.exps.deepspeech2.config
import
get_cfg_defaults
from
deepspeech.training.cli
import
default_argument_parser
from
deepspeech.training.cli
import
default_argument_parser
...
...
examples/1xt2x/deepspeech2x/models/__init__.py
→
examples/1xt2x/
src_
deepspeech2x/models/__init__.py
浏览文件 @
88a19897
文件已移动
examples/1xt2x/deepspeech2x/models/ds2/__init__.py
→
examples/1xt2x/
src_
deepspeech2x/models/ds2/__init__.py
浏览文件 @
88a19897
文件已移动
examples/1xt2x/deepspeech2x/models/ds2/deepspeech2.py
→
examples/1xt2x/
src_
deepspeech2x/models/ds2/deepspeech2.py
浏览文件 @
88a19897
...
@@ -15,8 +15,8 @@
...
@@ -15,8 +15,8 @@
from
typing
import
Optional
from
typing
import
Optional
import
paddle
import
paddle
from
deepspeech2x.models.ds2.rnn
import
RNNStack
from
paddle
import
nn
from
paddle
import
nn
from
src_deepspeech2x.models.ds2.rnn
import
RNNStack
from
yacs.config
import
CfgNode
from
yacs.config
import
CfgNode
from
deepspeech.models.ds2.conv
import
ConvStack
from
deepspeech.models.ds2.conv
import
ConvStack
...
...
examples/1xt2x/deepspeech2x/models/ds2/rnn.py
→
examples/1xt2x/
src_
deepspeech2x/models/ds2/rnn.py
浏览文件 @
88a19897
文件已移动
examples/1xt2x/
deepspeech2x/
model.py
→
examples/1xt2x/
src_deepspeech2x/test_
model.py
浏览文件 @
88a19897
...
@@ -20,12 +20,13 @@ from typing import Optional
...
@@ -20,12 +20,13 @@ from typing import Optional
import
numpy
as
np
import
numpy
as
np
import
paddle
import
paddle
from
deepspeech2x.models.ds2
import
DeepSpeech2InferModel
from
deepspeech2x.models.ds2
import
DeepSpeech2Model
from
paddle
import
distributed
as
dist
from
paddle
import
distributed
as
dist
from
paddle.io
import
DataLoader
from
paddle.io
import
DataLoader
from
src_deepspeech2x.models.ds2
import
DeepSpeech2InferModel
from
src_deepspeech2x.models.ds2
import
DeepSpeech2Model
from
yacs.config
import
CfgNode
from
yacs.config
import
CfgNode
from
deepspeech.frontend.featurizer.text_featurizer
import
TextFeaturizer
from
deepspeech.io.collator
import
SpeechCollator
from
deepspeech.io.collator
import
SpeechCollator
from
deepspeech.io.dataset
import
ManifestDataset
from
deepspeech.io.dataset
import
ManifestDataset
from
deepspeech.io.sampler
import
SortagradBatchSampler
from
deepspeech.io.sampler
import
SortagradBatchSampler
...
@@ -38,7 +39,6 @@ from deepspeech.utils import error_rate
...
@@ -38,7 +39,6 @@ from deepspeech.utils import error_rate
from
deepspeech.utils
import
layer_tools
from
deepspeech.utils
import
layer_tools
from
deepspeech.utils
import
mp_tools
from
deepspeech.utils
import
mp_tools
from
deepspeech.utils.log
import
Log
from
deepspeech.utils.log
import
Log
#from deepspeech.utils.log import Autolog
logger
=
Log
(
__name__
).
getlog
()
logger
=
Log
(
__name__
).
getlog
()
...
@@ -272,6 +272,9 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
...
@@ -272,6 +272,9 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
return
default
return
default
def
__init__
(
self
,
config
,
args
):
def
__init__
(
self
,
config
,
args
):
self
.
_text_featurizer
=
TextFeaturizer
(
unit_type
=
config
.
collator
.
unit_type
,
vocab_filepath
=
None
)
super
().
__init__
(
config
,
args
)
super
().
__init__
(
config
,
args
)
def
ordid2token
(
self
,
texts
,
texts_len
):
def
ordid2token
(
self
,
texts
,
texts_len
):
...
@@ -296,9 +299,6 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
...
@@ -296,9 +299,6 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
error_rate_func
=
error_rate
.
cer
if
cfg
.
error_rate_type
==
'cer'
else
error_rate
.
wer
error_rate_func
=
error_rate
.
cer
if
cfg
.
error_rate_type
==
'cer'
else
error_rate
.
wer
vocab_list
=
self
.
test_loader
.
collate_fn
.
vocab_list
vocab_list
=
self
.
test_loader
.
collate_fn
.
vocab_list
if
""
in
vocab_list
:
space_id
=
vocab_list
.
index
(
""
)
vocab_list
[
space_id
]
=
" "
target_transcripts
=
self
.
ordid2token
(
texts
,
texts_len
)
target_transcripts
=
self
.
ordid2token
(
texts
,
texts_len
)
...
@@ -337,6 +337,10 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
...
@@ -337,6 +337,10 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
cutoff_prob
=
cfg
.
cutoff_prob
,
cutoff_prob
=
cfg
.
cutoff_prob
,
cutoff_top_n
=
cfg
.
cutoff_top_n
,
cutoff_top_n
=
cfg
.
cutoff_top_n
,
num_processes
=
cfg
.
num_proc_bsearch
)
num_processes
=
cfg
.
num_proc_bsearch
)
result_transcripts
=
[
self
.
_text_featurizer
.
detokenize
(
item
)
for
item
in
result_transcripts
]
return
result_transcripts
return
result_transcripts
@
mp_tools
.
rank_zero_only
@
mp_tools
.
rank_zero_only
...
@@ -367,8 +371,6 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
...
@@ -367,8 +371,6 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
error_rate_type
,
num_ins
,
num_ins
,
errors_sum
/
len_refs
)
error_rate_type
,
num_ins
,
num_ins
,
errors_sum
/
len_refs
)
logger
.
info
(
msg
)
logger
.
info
(
msg
)
# self.autolog.report()
def
run_test
(
self
):
def
run_test
(
self
):
self
.
resume_or_scratch
()
self
.
resume_or_scratch
()
try
:
try
:
...
...
utils/format_data.py
浏览文件 @
88a19897
...
@@ -53,7 +53,8 @@ def main():
...
@@ -53,7 +53,8 @@ def main():
fout
=
open
(
args
.
output_path
,
'w'
,
encoding
=
'utf-8'
)
fout
=
open
(
args
.
output_path
,
'w'
,
encoding
=
'utf-8'
)
# get feat dim
# get feat dim
mean
,
std
=
load_cmvn
(
args
.
cmvn_path
,
filetype
=
'json'
)
filetype
=
args
.
cmvn_path
.
split
(
"."
)[
-
1
]
mean
,
istd
=
load_cmvn
(
args
.
cmvn_path
,
filetype
=
filetype
)
feat_dim
=
mean
.
shape
[
0
]
#(D)
feat_dim
=
mean
.
shape
[
0
]
#(D)
print
(
f
"Feature dim:
{
feat_dim
}
"
)
print
(
f
"Feature dim:
{
feat_dim
}
"
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录