Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
DeepSpeech
提交
c0295aa1
D
DeepSpeech
项目概览
PaddlePaddle
/
DeepSpeech
大约 2 年 前同步成功
通知
210
Star
8425
Fork
1598
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
245
列表
看板
标记
里程碑
合并请求
3
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
DeepSpeech
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
245
Issue
245
列表
看板
标记
里程碑
合并请求
3
合并请求
3
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
c0295aa1
编写于
10月 24, 2021
作者:
H
Hui Zhang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
wer/cer/bleu Calculator, label smoothing func
上级
6a9daa80
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
269 addition
and
8 deletion
+269
-8
deepspeech/utils/asr_utils.py
deepspeech/utils/asr_utils.py
+52
-0
deepspeech/utils/bleu_score.py
deepspeech/utils/bleu_score.py
+67
-5
deepspeech/utils/error_rate.py
deepspeech/utils/error_rate.py
+150
-3
未找到文件。
deepspeech/utils/asr_utils.py
0 → 100644
浏览文件 @
c0295aa1
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
json
import
numpy
as
np
__all__
=
[
"label_smoothing_dist"
]
# TODO(takaaki-hori): add different smoothing methods
def
label_smoothing_dist
(
odim
,
lsm_type
,
transcript
=
None
,
blank
=
0
):
"""Obtain label distribution for loss smoothing.
:param odim:
:param lsm_type:
:param blank:
:param transcript:
:return:
"""
if
transcript
is
not
None
:
with
open
(
transcript
,
"rb"
)
as
f
:
trans_json
=
json
.
load
(
f
)[
"utts"
]
if
lsm_type
==
"unigram"
:
assert
transcript
is
not
None
,
(
"transcript is required for %s label smoothing"
%
lsm_type
)
labelcount
=
np
.
zeros
(
odim
)
for
k
,
v
in
trans_json
.
items
():
ids
=
np
.
array
([
int
(
n
)
for
n
in
v
[
"output"
][
0
][
"tokenid"
].
split
()])
# to avoid an error when there is no text in an uttrance
if
len
(
ids
)
>
0
:
labelcount
[
ids
]
+=
1
labelcount
[
odim
-
1
]
=
len
(
transcript
)
# count <eos>
labelcount
[
labelcount
==
0
]
=
1
# flooring
labelcount
[
blank
]
=
0
# remove counts for blank
labeldist
=
labelcount
.
astype
(
np
.
float32
)
/
np
.
sum
(
labelcount
)
else
:
logging
.
error
(
"Error: unexpected label smoothing type: %s"
%
lsm_type
)
sys
.
exit
()
return
labeldist
deepspeech/utils/bleu_score.py
浏览文件 @
c0295aa1
...
@@ -15,16 +15,16 @@
...
@@ -15,16 +15,16 @@
e.g. wer for word-level, cer for char-level.
e.g. wer for word-level, cer for char-level.
"""
"""
import
sacrebleu
import
sacrebleu
import
nltk
import
numpy
as
np
__all__
=
[
'bleu'
,
'char_bleu'
]
__all__
=
[
'bleu'
,
'char_bleu'
,
"ErrorCalculator"
]
def
bleu
(
hypothesis
,
reference
):
def
bleu
(
hypothesis
,
reference
):
"""Calculate BLEU. BLEU compares reference text and
"""Calculate BLEU. BLEU compares reference text and
hypothesis text in word-level using scarebleu.
hypothesis text in word-level using scarebleu.
:param reference: The reference sentences.
:param reference: The reference sentences.
:type reference: list[list[str]]
:type reference: list[list[str]]
:param hypothesis: The hypothesis sentence.
:param hypothesis: The hypothesis sentence.
...
@@ -39,8 +39,6 @@ def char_bleu(hypothesis, reference):
...
@@ -39,8 +39,6 @@ def char_bleu(hypothesis, reference):
"""Calculate BLEU. BLEU compares reference text and
"""Calculate BLEU. BLEU compares reference text and
hypothesis text in char-level using scarebleu.
hypothesis text in char-level using scarebleu.
:param reference: The reference sentences.
:param reference: The reference sentences.
:type reference: list[list[str]]
:type reference: list[list[str]]
:param hypothesis: The hypothesis sentence.
:param hypothesis: The hypothesis sentence.
...
@@ -52,3 +50,67 @@ def char_bleu(hypothesis, reference):
...
@@ -52,3 +50,67 @@ def char_bleu(hypothesis, reference):
for
ref
in
reference
]
for
ref
in
reference
]
return
sacrebleu
.
corpus_bleu
(
hypothesis
,
reference
)
return
sacrebleu
.
corpus_bleu
(
hypothesis
,
reference
)
class
ErrorCalculator
():
"""Calculate BLEU for ST and MT models during training.
:param y_hats: numpy array with predicted text
:param y_pads: numpy array with true (target) text
:param char_list: vocabulary list
:param sym_space: space symbol
:param sym_pad: pad symbol
:param report_bleu: report BLUE score if True
"""
def
__init__
(
self
,
char_list
,
sym_space
,
sym_pad
,
report_bleu
=
False
):
"""Construct an ErrorCalculator object."""
super
().
__init__
()
self
.
char_list
=
char_list
self
.
space
=
sym_space
self
.
pad
=
sym_pad
self
.
report_bleu
=
report_bleu
if
self
.
space
in
self
.
char_list
:
self
.
idx_space
=
self
.
char_list
.
index
(
self
.
space
)
else
:
self
.
idx_space
=
None
def
__call__
(
self
,
ys_hat
,
ys_pad
):
"""Calculate corpus-level BLEU score.
:param torch.Tensor ys_hat: prediction (batch, seqlen)
:param torch.Tensor ys_pad: reference (batch, seqlen)
:return: corpus-level BLEU score in a mini-batch
:rtype float
"""
bleu
=
None
if
not
self
.
report_bleu
:
return
bleu
bleu
=
self
.
calculate_corpus_bleu
(
ys_hat
,
ys_pad
)
return
bleu
def
calculate_corpus_bleu
(
self
,
ys_hat
,
ys_pad
):
"""Calculate corpus-level BLEU score in a mini-batch.
:param torch.Tensor seqs_hat: prediction (batch, seqlen)
:param torch.Tensor seqs_true: reference (batch, seqlen)
:return: corpus-level BLEU score
:rtype float
"""
seqs_hat
,
seqs_true
=
[],
[]
for
i
,
y_hat
in
enumerate
(
ys_hat
):
y_true
=
ys_pad
[
i
]
eos_true
=
np
.
where
(
y_true
==
-
1
)[
0
]
ymax
=
eos_true
[
0
]
if
len
(
eos_true
)
>
0
else
len
(
y_true
)
# NOTE: padding index (-1) in y_true is used to pad y_hat
# because y_hats is not padded with -1
seq_hat
=
[
self
.
char_list
[
int
(
idx
)]
for
idx
in
y_hat
[:
ymax
]]
seq_true
=
[
self
.
char_list
[
int
(
idx
)]
for
idx
in
y_true
if
int
(
idx
)
!=
-
1
]
seq_hat_text
=
""
.
join
(
seq_hat
).
replace
(
self
.
space
,
" "
)
seq_hat_text
=
seq_hat_text
.
replace
(
self
.
pad
,
""
)
seq_true_text
=
""
.
join
(
seq_true
).
replace
(
self
.
space
,
" "
)
seqs_hat
.
append
(
seq_hat_text
)
seqs_true
.
append
(
seq_true_text
)
bleu
=
nltk
.
bleu_score
.
corpus_bleu
([[
ref
]
for
ref
in
seqs_true
],
seqs_hat
)
return
bleu
*
100
deepspeech/utils/error_rate.py
浏览文件 @
c0295aa1
...
@@ -16,10 +16,11 @@ e.g. wer for word-level, cer for char-level.
...
@@ -16,10 +16,11 @@ e.g. wer for word-level, cer for char-level.
"""
"""
import
editdistance
import
editdistance
import
numpy
as
np
import
numpy
as
np
import
logging
import
sys
from
itertools
import
groupby
__all__
=
[
'word_errors'
,
'char_errors'
,
'wer'
,
'cer'
]
__all__
=
[
'word_errors'
,
'char_errors'
,
'wer'
,
'cer'
,
"ErrorCalculator"
]
editdistance
.
eval
(
"a"
,
"b"
)
def
_levenshtein_distance
(
ref
,
hyp
):
def
_levenshtein_distance
(
ref
,
hyp
):
...
@@ -211,3 +212,149 @@ def cer(reference, hypothesis, ignore_case=False, remove_space=False):
...
@@ -211,3 +212,149 @@ def cer(reference, hypothesis, ignore_case=False, remove_space=False):
cer
=
float
(
edit_distance
)
/
ref_len
cer
=
float
(
edit_distance
)
/
ref_len
return
cer
return
cer
class
ErrorCalculator
():
"""Calculate CER and WER for E2E_ASR and CTC models during training.
:param y_hats: numpy array with predicted text
:param y_pads: numpy array with true (target) text
:param char_list: List[str]
:param sym_space: <space>
:param sym_blank: <blank>
:return:
"""
def
__init__
(
self
,
char_list
,
sym_space
,
sym_blank
,
report_cer
=
False
,
report_wer
=
False
):
"""Construct an ErrorCalculator object."""
super
().
__init__
()
self
.
report_cer
=
report_cer
self
.
report_wer
=
report_wer
self
.
char_list
=
char_list
self
.
space
=
sym_space
self
.
blank
=
sym_blank
self
.
idx_blank
=
self
.
char_list
.
index
(
self
.
blank
)
if
self
.
space
in
self
.
char_list
:
self
.
idx_space
=
self
.
char_list
.
index
(
self
.
space
)
else
:
self
.
idx_space
=
None
def
__call__
(
self
,
ys_hat
,
ys_pad
,
is_ctc
=
False
):
"""Calculate sentence-level WER/CER score.
:param paddle.Tensor ys_hat: prediction (batch, seqlen)
:param paddle.Tensor ys_pad: reference (batch, seqlen)
:param bool is_ctc: calculate CER score for CTC
:return: sentence-level WER score
:rtype float
:return: sentence-level CER score
:rtype float
"""
cer
,
wer
=
None
,
None
if
is_ctc
:
return
self
.
calculate_cer_ctc
(
ys_hat
,
ys_pad
)
elif
not
self
.
report_cer
and
not
self
.
report_wer
:
return
cer
,
wer
seqs_hat
,
seqs_true
=
self
.
convert_to_char
(
ys_hat
,
ys_pad
)
if
self
.
report_cer
:
cer
=
self
.
calculate_cer
(
seqs_hat
,
seqs_true
)
if
self
.
report_wer
:
wer
=
self
.
calculate_wer
(
seqs_hat
,
seqs_true
)
return
cer
,
wer
def
calculate_cer_ctc
(
self
,
ys_hat
,
ys_pad
):
"""Calculate sentence-level CER score for CTC.
:param paddle.Tensor ys_hat: prediction (batch, seqlen)
:param paddle.Tensor ys_pad: reference (batch, seqlen)
:return: average sentence-level CER score
:rtype float
"""
cers
,
char_ref_lens
=
[],
[]
for
i
,
y
in
enumerate
(
ys_hat
):
y_hat
=
[
x
[
0
]
for
x
in
groupby
(
y
)]
y_true
=
ys_pad
[
i
]
seq_hat
,
seq_true
=
[],
[]
for
idx
in
y_hat
:
idx
=
int
(
idx
)
if
idx
!=
-
1
and
idx
!=
self
.
idx_blank
and
idx
!=
self
.
idx_space
:
seq_hat
.
append
(
self
.
char_list
[
int
(
idx
)])
for
idx
in
y_true
:
idx
=
int
(
idx
)
if
idx
!=
-
1
and
idx
!=
self
.
idx_blank
and
idx
!=
self
.
idx_space
:
seq_true
.
append
(
self
.
char_list
[
int
(
idx
)])
hyp_chars
=
""
.
join
(
seq_hat
)
ref_chars
=
""
.
join
(
seq_true
)
if
len
(
ref_chars
)
>
0
:
cers
.
append
(
editdistance
.
eval
(
hyp_chars
,
ref_chars
))
char_ref_lens
.
append
(
len
(
ref_chars
))
cer_ctc
=
float
(
sum
(
cers
))
/
sum
(
char_ref_lens
)
if
cers
else
None
return
cer_ctc
def
convert_to_char
(
self
,
ys_hat
,
ys_pad
):
"""Convert index to character.
:param paddle.Tensor seqs_hat: prediction (batch, seqlen)
:param paddle.Tensor seqs_true: reference (batch, seqlen)
:return: token list of prediction
:rtype list
:return: token list of reference
:rtype list
"""
seqs_hat
,
seqs_true
=
[],
[]
for
i
,
y_hat
in
enumerate
(
ys_hat
):
y_true
=
ys_pad
[
i
]
eos_true
=
np
.
where
(
y_true
==
-
1
)[
0
]
ymax
=
eos_true
[
0
]
if
len
(
eos_true
)
>
0
else
len
(
y_true
)
# NOTE: padding index (-1) in y_true is used to pad y_hat
seq_hat
=
[
self
.
char_list
[
int
(
idx
)]
for
idx
in
y_hat
[:
ymax
]]
seq_true
=
[
self
.
char_list
[
int
(
idx
)]
for
idx
in
y_true
if
int
(
idx
)
!=
-
1
]
seq_hat_text
=
""
.
join
(
seq_hat
).
replace
(
self
.
space
,
" "
)
seq_hat_text
=
seq_hat_text
.
replace
(
self
.
blank
,
""
)
seq_true_text
=
""
.
join
(
seq_true
).
replace
(
self
.
space
,
" "
)
seqs_hat
.
append
(
seq_hat_text
)
seqs_true
.
append
(
seq_true_text
)
return
seqs_hat
,
seqs_true
def
calculate_cer
(
self
,
seqs_hat
,
seqs_true
):
"""Calculate sentence-level CER score.
:param list seqs_hat: prediction
:param list seqs_true: reference
:return: average sentence-level CER score
:rtype float
"""
char_eds
,
char_ref_lens
=
[],
[]
for
i
,
seq_hat_text
in
enumerate
(
seqs_hat
):
seq_true_text
=
seqs_true
[
i
]
hyp_chars
=
seq_hat_text
.
replace
(
" "
,
""
)
ref_chars
=
seq_true_text
.
replace
(
" "
,
""
)
char_eds
.
append
(
editdistance
.
eval
(
hyp_chars
,
ref_chars
))
char_ref_lens
.
append
(
len
(
ref_chars
))
return
float
(
sum
(
char_eds
))
/
sum
(
char_ref_lens
)
def
calculate_wer
(
self
,
seqs_hat
,
seqs_true
):
"""Calculate sentence-level WER score.
:param list seqs_hat: prediction
:param list seqs_true: reference
:return: average sentence-level WER score
:rtype float
"""
word_eds
,
word_ref_lens
=
[],
[]
for
i
,
seq_hat_text
in
enumerate
(
seqs_hat
):
seq_true_text
=
seqs_true
[
i
]
hyp_words
=
seq_hat_text
.
split
()
ref_words
=
seq_true_text
.
split
()
word_eds
.
append
(
editdistance
.
eval
(
hyp_words
,
ref_words
))
word_ref_lens
.
append
(
len
(
ref_words
))
return
float
(
sum
(
word_eds
))
/
sum
(
word_ref_lens
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录