Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
DeepSpeech
提交
8641608f
D
DeepSpeech
项目概览
PaddlePaddle
/
DeepSpeech
大约 1 年 前同步成功
通知
206
Star
8425
Fork
1598
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
245
列表
看板
标记
里程碑
合并请求
3
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
D
DeepSpeech
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
245
Issue
245
列表
看板
标记
里程碑
合并请求
3
合并请求
3
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
8641608f
编写于
6月 08, 2022
作者:
Y
YangZhou
提交者:
GitHub
6月 08, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #2015 from zh794390558/endpoint
[server][asr] support endpoint for conformer streaming model
上级
f3132ce2
dfdf450b
变更
52
展开全部
隐藏空白更改
内联
并排
Showing
52 changed file
with
547 addition
and
369 deletion
+547
-369
.pre-commit-config.yaml
.pre-commit-config.yaml
+6
-6
demos/streaming_asr_server/conf/application.yaml
demos/streaming_asr_server/conf/application.yaml
+2
-0
demos/streaming_asr_server/conf/ws_conformer_application.yaml
...s/streaming_asr_server/conf/ws_conformer_application.yaml
+3
-0
demos/streaming_asr_server/conf/ws_conformer_wenetspeech_application.yaml
...asr_server/conf/ws_conformer_wenetspeech_application.yaml
+2
-0
demos/streaming_asr_server/server.sh
demos/streaming_asr_server/server.sh
+2
-1
demos/streaming_asr_server/test.sh
demos/streaming_asr_server/test.sh
+2
-1
examples/wenetspeech/asr1/local/extract_meta.py
examples/wenetspeech/asr1/local/extract_meta.py
+0
-1
paddlespeech/cli/base_commands.py
paddlespeech/cli/base_commands.py
+0
-1
paddlespeech/cli/cls/infer.py
paddlespeech/cli/cls/infer.py
+2
-2
paddlespeech/cli/vector/infer.py
paddlespeech/cli/vector/infer.py
+2
-2
paddlespeech/resource/model_alias.py
paddlespeech/resource/model_alias.py
+1
-2
paddlespeech/s2t/decoders/scorers/ctc_prefix_score.py
paddlespeech/s2t/decoders/scorers/ctc_prefix_score.py
+2
-1
paddlespeech/s2t/exps/deepspeech2/model.py
paddlespeech/s2t/exps/deepspeech2/model.py
+4
-6
paddlespeech/s2t/models/ds2/__init__.py
paddlespeech/s2t/models/ds2/__init__.py
+2
-1
paddlespeech/s2t/models/ds2/deepspeech2.py
paddlespeech/s2t/models/ds2/deepspeech2.py
+8
-4
paddlespeech/s2t/models/lm/transformer.py
paddlespeech/s2t/models/lm/transformer.py
+1
-1
paddlespeech/s2t/models/u2/updater.py
paddlespeech/s2t/models/u2/updater.py
+0
-1
paddlespeech/s2t/modules/ctc.py
paddlespeech/s2t/modules/ctc.py
+1
-1
paddlespeech/s2t/utils/tensor_utils.py
paddlespeech/s2t/utils/tensor_utils.py
+2
-1
paddlespeech/server/conf/ws_application.yaml
paddlespeech/server/conf/ws_application.yaml
+1
-0
paddlespeech/server/conf/ws_conformer_application.yaml
paddlespeech/server/conf/ws_conformer_application.yaml
+2
-0
paddlespeech/server/engine/asr/online/asr_engine.py
paddlespeech/server/engine/asr/online/asr_engine.py
+233
-211
paddlespeech/server/engine/asr/online/ctc_endpoint.py
paddlespeech/server/engine/asr/online/ctc_endpoint.py
+118
-0
paddlespeech/server/engine/asr/online/ctc_search.py
paddlespeech/server/engine/asr/online/ctc_search.py
+35
-15
paddlespeech/server/engine/tts/online/python/tts_engine.py
paddlespeech/server/engine/tts/online/python/tts_engine.py
+0
-1
paddlespeech/server/ws/asr_api.py
paddlespeech/server/ws/asr_api.py
+28
-5
paddlespeech/t2s/exps/synthesize.py
paddlespeech/t2s/exps/synthesize.py
+2
-8
paddlespeech/t2s/exps/synthesize_e2e.py
paddlespeech/t2s/exps/synthesize_e2e.py
+2
-8
paddlespeech/t2s/exps/voice_cloning.py
paddlespeech/t2s/exps/voice_cloning.py
+2
-8
paddlespeech/t2s/models/vits/__init__.py
paddlespeech/t2s/models/vits/__init__.py
+1
-1
paddlespeech/t2s/models/vits/vits_updater.py
paddlespeech/t2s/models/vits/vits_updater.py
+4
-2
paddlespeech/t2s/modules/losses.py
paddlespeech/t2s/modules/losses.py
+9
-9
speechx/examples/README.md
speechx/examples/README.md
+0
-1
speechx/examples/ds2_ol/README.md
speechx/examples/ds2_ol/README.md
+1
-1
speechx/speechx/codelab/README.md
speechx/speechx/codelab/README.md
+0
-1
speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc
...hx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc
+2
-2
speechx/speechx/decoder/ctc_tlg_decoder.cc
speechx/speechx/decoder/ctc_tlg_decoder.cc
+1
-1
speechx/speechx/decoder/param.h
speechx/speechx/decoder/param.h
+1
-1
speechx/speechx/decoder/tlg_decoder_main.cc
speechx/speechx/decoder/tlg_decoder_main.cc
+2
-2
speechx/speechx/frontend/audio/assembler.cc
speechx/speechx/frontend/audio/assembler.cc
+15
-14
speechx/speechx/frontend/audio/assembler.h
speechx/speechx/frontend/audio/assembler.h
+3
-7
speechx/speechx/frontend/audio/audio_cache.h
speechx/speechx/frontend/audio/audio_cache.h
+1
-1
speechx/speechx/frontend/audio/fbank.cc
speechx/speechx/frontend/audio/fbank.cc
+4
-4
speechx/speechx/frontend/audio/feature_cache.cc
speechx/speechx/frontend/audio/feature_cache.cc
+2
-2
speechx/speechx/frontend/audio/feature_cache.h
speechx/speechx/frontend/audio/feature_cache.h
+1
-3
speechx/speechx/frontend/audio/feature_common.h
speechx/speechx/frontend/audio/feature_common.h
+4
-3
speechx/speechx/frontend/audio/feature_common_inl.h
speechx/speechx/frontend/audio/feature_common_inl.h
+12
-10
speechx/speechx/frontend/audio/feature_pipeline.h
speechx/speechx/frontend/audio/feature_pipeline.h
+1
-1
speechx/speechx/frontend/audio/linear_spectrogram.cc
speechx/speechx/frontend/audio/linear_spectrogram.cc
+4
-5
speechx/speechx/nnet/nnet_forward_main.cc
speechx/speechx/nnet/nnet_forward_main.cc
+8
-5
speechx/speechx/protocol/websocket/websocket_client.h
speechx/speechx/protocol/websocket/websocket_client.h
+2
-2
speechx/speechx/protocol/websocket/websocket_server.cc
speechx/speechx/protocol/websocket/websocket_server.cc
+4
-3
未找到文件。
.pre-commit-config.yaml
浏览文件 @
8641608f
...
...
@@ -51,12 +51,12 @@ repos:
language
:
system
files
:
\.(c|cc|cxx|cpp|cu|h|hpp|hxx|cuh|proto)$
exclude
:
(?=speechx/speechx/kaldi|speechx/patch|speechx/tools/fstbin|speechx/tools/lmbin).*(\.cpp|\.cc|\.h|\.py)$
-
id
:
copyright_checker
name
:
copyright_checker
entry
:
python .pre-commit-hooks/copyright-check.hook
language
:
system
files
:
\.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$
exclude
:
(?=third_party|pypinyin|speechx/speechx/kaldi|speechx/patch|speechx/tools/fstbin|speechx/tools/lmbin).*(\.cpp|\.cc|\.h|\.py)$
#
- id: copyright_checker
#
name: copyright_checker
#
entry: python .pre-commit-hooks/copyright-check.hook
#
language: system
#
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$
#
exclude: (?=third_party|pypinyin|speechx/speechx/kaldi|speechx/patch|speechx/tools/fstbin|speechx/tools/lmbin).*(\.cpp|\.cc|\.h|\.py)$
-
repo
:
https://github.com/asottile/reorder_python_imports
rev
:
v2.4.0
hooks
:
...
...
demos/streaming_asr_server/conf/application.yaml
浏览文件 @
8641608f
...
...
@@ -31,6 +31,8 @@ asr_online:
force_yes
:
True
device
:
'
cpu'
# cpu or gpu:id
decode_method
:
"
attention_rescoring"
continuous_decoding
:
True
# enable continue decoding when endpoint detected
am_predictor_conf
:
device
:
# set 'gpu:id' or 'cpu'
switch_ir_optim
:
True
...
...
demos/streaming_asr_server/conf/ws_conformer_application.yaml
浏览文件 @
8641608f
...
...
@@ -30,6 +30,9 @@ asr_online:
decode_method
:
force_yes
:
True
device
:
'
cpu'
# cpu or gpu:id
decode_method
:
"
attention_rescoring"
continuous_decoding
:
True
# enable continue decoding when endpoint detected
am_predictor_conf
:
device
:
# set 'gpu:id' or 'cpu'
switch_ir_optim
:
True
...
...
demos/streaming_asr_server/conf/ws_conformer_wenetspeech_application.yaml
浏览文件 @
8641608f
...
...
@@ -31,6 +31,8 @@ asr_online:
force_yes
:
True
device
:
'
cpu'
# cpu or gpu:id
decode_method
:
"
attention_rescoring"
continuous_decoding
:
True
# enable continue decoding when endpoint detected
am_predictor_conf
:
device
:
# set 'gpu:id' or 'cpu'
switch_ir_optim
:
True
...
...
demos/streaming_asr_server/server.sh
浏览文件 @
8641608f
...
...
@@ -5,4 +5,5 @@ export CUDA_VISIBLE_DEVICE=0,1,2,3
paddlespeech_server start
--config_file
conf/punc_application.yaml &> punc.log &
# nohup python3 streaming_asr_server.py --config_file conf/ws_conformer_application.yaml > streaming_asr.log 2>&1 &
paddlespeech_server start
--config_file
conf/ws_conformer_application.yaml &> streaming_asr.log &
\ No newline at end of file
paddlespeech_server start
--config_file
conf/ws_conformer_application.yaml &> streaming_asr.log &
demos/streaming_asr_server/test.sh
浏览文件 @
8641608f
...
...
@@ -9,4 +9,5 @@ paddlespeech_client asr_online --server_ip 127.0.0.1 --port 8290 --input ./zh.wa
# read the wav and call streaming and punc service
# If `127.0.0.1` is not accessible, you need to use the actual service IP address.
# python3 websocket_client.py --server_ip 127.0.0.1 --port 8290 --punc.server_ip 127.0.0.1 --punc.port 8190 --wavfile ./zh.wav
paddlespeech_client asr_online
--server_ip
127.0.0.1
--port
8290
--punc
.server_ip 127.0.0.1
--punc
.port 8190
--input
./zh.wav
\ No newline at end of file
paddlespeech_client asr_online
--server_ip
127.0.0.1
--port
8290
--punc
.server_ip 127.0.0.1
--punc
.port 8190
--input
./zh.wav
examples/wenetspeech/asr1/local/extract_meta.py
浏览文件 @
8641608f
...
...
@@ -13,7 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
argparse
import
json
import
os
...
...
paddlespeech/cli/base_commands.py
浏览文件 @
8641608f
...
...
@@ -145,4 +145,3 @@ for com, info in _commands.items():
name
=
'paddlespeech.{}'
.
format
(
com
),
description
=
info
[
0
],
cls
=
'paddlespeech.cli.{}.{}'
.
format
(
com
,
info
[
1
]))
\ No newline at end of file
paddlespeech/cli/cls/infer.py
浏览文件 @
8641608f
...
...
@@ -21,12 +21,12 @@ from typing import Union
import
numpy
as
np
import
paddle
import
yaml
from
paddleaudio
import
load
from
paddleaudio.features
import
LogMelSpectrogram
from
..executor
import
BaseExecutor
from
..log
import
logger
from
..utils
import
stats_wrapper
from
paddleaudio
import
load
from
paddleaudio.features
import
LogMelSpectrogram
__all__
=
[
'CLSExecutor'
]
...
...
paddlespeech/cli/vector/infer.py
浏览文件 @
8641608f
...
...
@@ -22,13 +22,13 @@ from typing import Union
import
paddle
import
soundfile
from
paddleaudio.backends
import
load
as
load_audio
from
paddleaudio.compliance.librosa
import
melspectrogram
from
yacs.config
import
CfgNode
from
..executor
import
BaseExecutor
from
..log
import
logger
from
..utils
import
stats_wrapper
from
paddleaudio.backends
import
load
as
load_audio
from
paddleaudio.compliance.librosa
import
melspectrogram
from
paddlespeech.vector.io.batch
import
feature_normalize
from
paddlespeech.vector.modules.sid_model
import
SpeakerIdetification
...
...
paddlespeech/resource/model_alias.py
浏览文件 @
8641608f
...
...
@@ -22,8 +22,7 @@ model_alias = {
# -------------- ASR --------------
# ---------------------------------
"deepspeech2offline"
:
[
"paddlespeech.s2t.models.ds2:DeepSpeech2Model"
],
"deepspeech2online"
:
[
"paddlespeech.s2t.models.ds2:DeepSpeech2Model"
],
"deepspeech2online"
:
[
"paddlespeech.s2t.models.ds2:DeepSpeech2Model"
],
"conformer"
:
[
"paddlespeech.s2t.models.u2:U2Model"
],
"conformer_online"
:
[
"paddlespeech.s2t.models.u2:U2Model"
],
"transformer"
:
[
"paddlespeech.s2t.models.u2:U2Model"
],
...
...
paddlespeech/s2t/decoders/scorers/ctc_prefix_score.py
浏览文件 @
8641608f
...
...
@@ -76,7 +76,8 @@ class CTCPrefixScorePD():
last_ids
=
[
yi
[
-
1
]
for
yi
in
y
]
# last output label ids
n_bh
=
len
(
last_ids
)
# batch * hyps
n_hyps
=
n_bh
//
self
.
batch
# assuming each utterance has the same # of hyps
self
.
scoring_num
=
paddle
.
shape
(
scoring_ids
)[
-
1
]
if
scoring_ids
is
not
None
else
0
self
.
scoring_num
=
paddle
.
shape
(
scoring_ids
)[
-
1
]
if
scoring_ids
is
not
None
else
0
# prepare state info
if
state
is
None
:
r_prev
=
paddle
.
full
(
...
...
paddlespeech/s2t/exps/deepspeech2/model.py
浏览文件 @
8641608f
...
...
@@ -22,11 +22,9 @@ import numpy as np
import
paddle
from
paddle
import
distributed
as
dist
from
paddle
import
inference
from
paddle.io
import
DataLoader
from
paddlespeech.s2t.io.dataloader
import
BatchDataLoader
from
paddlespeech.s2t.frontend.featurizer.text_featurizer
import
TextFeaturizer
from
paddlespeech.s2t.io.data
set
import
ManifestDataset
from
paddlespeech.s2t.io.data
loader
import
BatchDataLoader
from
paddlespeech.s2t.models.ds2
import
DeepSpeech2InferModel
from
paddlespeech.s2t.models.ds2
import
DeepSpeech2Model
from
paddlespeech.s2t.training.gradclip
import
ClipGradByGlobalNormWithLog
...
...
@@ -238,8 +236,7 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
def
__init__
(
self
,
config
,
args
):
super
().
__init__
(
config
,
args
)
self
.
_text_featurizer
=
TextFeaturizer
(
unit_type
=
config
.
unit_type
,
vocab
=
config
.
vocab_filepath
)
unit_type
=
config
.
unit_type
,
vocab
=
config
.
vocab_filepath
)
self
.
vocab_list
=
self
.
_text_featurizer
.
vocab_list
def
ordid2token
(
self
,
texts
,
texts_len
):
...
...
@@ -248,7 +245,8 @@ class DeepSpeech2Tester(DeepSpeech2Trainer):
for
text
,
n
in
zip
(
texts
,
texts_len
):
n
=
n
.
numpy
().
item
()
ids
=
text
[:
n
]
trans
.
append
(
self
.
_text_featurizer
.
defeaturize
(
ids
.
numpy
().
tolist
()))
trans
.
append
(
self
.
_text_featurizer
.
defeaturize
(
ids
.
numpy
().
tolist
()))
return
trans
def
compute_metrics
(
self
,
...
...
paddlespeech/s2t/models/ds2/__init__.py
浏览文件 @
8641608f
...
...
@@ -11,10 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
sys
from
.deepspeech2
import
DeepSpeech2InferModel
from
.deepspeech2
import
DeepSpeech2Model
from
paddlespeech.s2t.utils
import
dynamic_pip_install
import
sys
try
:
import
paddlespeech_ctcdecoders
...
...
paddlespeech/s2t/models/ds2/deepspeech2.py
浏览文件 @
8641608f
...
...
@@ -372,11 +372,15 @@ class DeepSpeech2InferModel(DeepSpeech2Model):
def
__init__
(
self
,
*
args
,
**
kwargs
):
super
().
__init__
(
*
args
,
**
kwargs
)
def
forward
(
self
,
audio_chunk
,
audio_chunk_lens
,
chunk_state_h_box
=
None
,
def
forward
(
self
,
audio_chunk
,
audio_chunk_lens
,
chunk_state_h_box
=
None
,
chunk_state_c_box
=
None
):
if
self
.
encoder
.
rnn_direction
==
"forward"
:
eouts_chunk
,
eouts_chunk_lens
,
final_state_h_box
,
final_state_c_box
=
self
.
encoder
(
audio_chunk
,
audio_chunk_lens
,
chunk_state_h_box
,
chunk_state_c_box
)
audio_chunk
,
audio_chunk_lens
,
chunk_state_h_box
,
chunk_state_c_box
)
probs_chunk
=
self
.
decoder
.
softmax
(
eouts_chunk
)
return
probs_chunk
,
eouts_chunk_lens
,
final_state_h_box
,
final_state_c_box
elif
self
.
encoder
.
rnn_direction
==
"bidirect"
:
...
...
@@ -392,8 +396,8 @@ class DeepSpeech2InferModel(DeepSpeech2Model):
self
,
input_spec
=
[
paddle
.
static
.
InputSpec
(
shape
=
[
None
,
None
,
self
.
encoder
.
feat_size
],
#[B, chunk_size, feat_dim]
shape
=
[
None
,
None
,
self
.
encoder
.
feat_size
],
#[B, chunk_size, feat_dim]
dtype
=
'float32'
),
paddle
.
static
.
InputSpec
(
shape
=
[
None
],
dtype
=
'int64'
),
# audio_length, [B]
...
...
paddlespeech/s2t/models/lm/transformer.py
浏览文件 @
8641608f
...
...
@@ -90,7 +90,7 @@ class TransformerLM(nn.Layer, LMInterface, BatchScorerInterface):
def
_target_mask
(
self
,
ys_in_pad
):
ys_mask
=
ys_in_pad
!=
0
m
=
subsequent_mask
(
paddle
.
shape
(
ys_mask
)[
-
1
])
)
.
unsqueeze
(
0
)
m
=
subsequent_mask
(
paddle
.
shape
(
ys_mask
)[
-
1
]).
unsqueeze
(
0
)
return
ys_mask
.
unsqueeze
(
-
2
)
&
m
def
forward
(
self
,
x
:
paddle
.
Tensor
,
t
:
paddle
.
Tensor
...
...
paddlespeech/s2t/models/u2/updater.py
浏览文件 @
8641608f
...
...
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from
contextlib
import
nullcontext
import
paddle
...
...
paddlespeech/s2t/modules/ctc.py
浏览文件 @
8641608f
...
...
@@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import
sys
from
typing
import
Union
import
paddle
...
...
@@ -22,7 +23,6 @@ from paddlespeech.s2t.modules.align import Linear
from
paddlespeech.s2t.modules.loss
import
CTCLoss
from
paddlespeech.s2t.utils
import
ctc_utils
from
paddlespeech.s2t.utils.log
import
Log
import
sys
logger
=
Log
(
__name__
).
getlog
()
...
...
paddlespeech/s2t/utils/tensor_utils.py
浏览文件 @
8641608f
...
...
@@ -82,7 +82,8 @@ def pad_sequence(sequences: List[paddle.Tensor],
max_size
=
paddle
.
shape
(
sequences
[
0
])
# (TODO Hui Zhang): slice not supprot `end==start`
# trailing_dims = max_size[1:]
trailing_dims
=
tuple
(
max_size
[
1
:].
numpy
().
tolist
())
if
sequences
[
0
].
ndim
>=
2
else
()
trailing_dims
=
tuple
(
max_size
[
1
:].
numpy
().
tolist
())
if
sequences
[
0
].
ndim
>=
2
else
()
max_len
=
max
([
s
.
shape
[
0
]
for
s
in
sequences
])
if
batch_first
:
out_dims
=
(
len
(
sequences
),
max_len
)
+
trailing_dims
...
...
paddlespeech/server/conf/ws_application.yaml
浏览文件 @
8641608f
...
...
@@ -29,6 +29,7 @@ asr_online:
cfg_path
:
decode_method
:
force_yes
:
True
device
:
# cpu or gpu:id
am_predictor_conf
:
device
:
# set 'gpu:id' or 'cpu'
...
...
paddlespeech/server/conf/ws_conformer_application.yaml
浏览文件 @
8641608f
...
...
@@ -30,6 +30,8 @@ asr_online:
decode_method
:
force_yes
:
True
device
:
# cpu or gpu:id
continuous_decoding
:
True
# enable continue decoding when endpoint detected
am_predictor_conf
:
device
:
# set 'gpu:id' or 'cpu'
switch_ir_optim
:
True
...
...
paddlespeech/server/engine/asr/online/asr_engine.py
浏览文件 @
8641608f
此差异已折叠。
点击以展开。
paddlespeech/server/engine/asr/online/ctc_endpoint.py
0 → 100644
浏览文件 @
8641608f
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from
dataclasses
import
dataclass
import
numpy
as
np
from
paddlespeech.cli.log
import
logger
@
dataclass
class
OnlineCTCEndpointRule
:
must_contain_nonsilence
:
bool
=
True
min_trailing_silence
:
int
=
1000
min_utterance_length
:
int
=
0
@
dataclass
class
OnlineCTCEndpoingOpt
:
frame_shift_in_ms
:
int
=
10
blank
:
int
=
0
# blank id, that we consider as silence for purposes of endpointing.
blank_threshold
:
float
=
0.8
# above blank threshold is silence
# We support three rules. We terminate decoding if ANY of these rules
# evaluates to "true". If you want to add more rules, do it by changing this
# code. If you want to disable a rule, you can set the silence-timeout for
# that rule to a very large number.
# rule1 times out after 5 seconds of silence, even if we decoded nothing.
rule1
:
OnlineCTCEndpointRule
=
OnlineCTCEndpointRule
(
False
,
5000
,
0
)
# rule4 times out after 1.0 seconds of silence after decoding something,
# even if we did not reach a final-state at all.
rule2
:
OnlineCTCEndpointRule
=
OnlineCTCEndpointRule
(
True
,
1000
,
0
)
# rule5 times out after the utterance is 20 seconds long, regardless of
# anything else.
rule3
:
OnlineCTCEndpointRule
=
OnlineCTCEndpointRule
(
False
,
0
,
20000
)
class
OnlineCTCEndpoint
:
"""
[END-TO-END AUTOMATIC SPEECH RECOGNITION INTEGRATED WITH CTC-BASED VOICE ACTIVITY DETECTION](https://arxiv.org/pdf/2002.00551.pdf)
"""
def
__init__
(
self
,
opts
:
OnlineCTCEndpoingOpt
):
self
.
opts
=
opts
logger
.
info
(
f
"Endpont Opts:
{
opts
}
"
)
self
.
frame_shift_in_ms
=
opts
.
frame_shift_in_ms
self
.
num_frames_decoded
=
0
self
.
trailing_silence_frames
=
0
self
.
reset
()
def
reset
(
self
):
self
.
num_frames_decoded
=
0
self
.
trailing_silence_frames
=
0
def
rule_activated
(
self
,
rule
:
OnlineCTCEndpointRule
,
rule_name
:
str
,
decoding_something
:
bool
,
trailine_silence
:
int
,
utterance_length
:
int
)
->
bool
:
ans
=
(
decoding_something
or
(
not
rule
.
must_contain_nonsilence
)
)
and
trailine_silence
>=
rule
.
min_trailing_silence
and
utterance_length
>=
rule
.
min_utterance_length
if
(
ans
):
logger
.
info
(
f
"Endpoint Rule:
{
rule_name
}
activated:
{
rule
}
"
)
return
ans
def
endpoint_detected
(
self
,
ctc_log_probs
:
np
.
ndarray
,
decoding_something
:
bool
)
->
bool
:
"""detect endpoint.
Args:
ctc_log_probs (np.ndarray): (T, D)
decoding_something (bool): contain nonsilince.
Returns:
bool: whether endpoint detected.
"""
for
logprob
in
ctc_log_probs
:
blank_prob
=
np
.
exp
(
logprob
[
self
.
opts
.
blank
])
self
.
num_frames_decoded
+=
1
if
blank_prob
>
self
.
opts
.
blank_threshold
:
self
.
trailing_silence_frames
+=
1
else
:
self
.
trailing_silence_frames
=
0
assert
self
.
num_frames_decoded
>=
self
.
trailing_silence_frames
assert
self
.
frame_shift_in_ms
>
0
utterance_length
=
self
.
num_frames_decoded
*
self
.
frame_shift_in_ms
trailing_silence
=
self
.
trailing_silence_frames
*
self
.
frame_shift_in_ms
if
self
.
rule_activated
(
self
.
opts
.
rule1
,
'rule1'
,
decoding_something
,
trailing_silence
,
utterance_length
):
return
True
if
self
.
rule_activated
(
self
.
opts
.
rule2
,
'rule2'
,
decoding_something
,
trailing_silence
,
utterance_length
):
return
True
if
self
.
rule_activated
(
self
.
opts
.
rule3
,
'rule3'
,
decoding_something
,
trailing_silence
,
utterance_length
):
return
True
return
False
paddlespeech/server/engine/asr/online/ctc_search.py
浏览文件 @
8641608f
...
...
@@ -30,8 +30,29 @@ class CTCPrefixBeamSearch:
config (yacs.config.CfgNode): the ctc prefix beam search configuration
"""
self
.
config
=
config
# beam size
self
.
first_beam_size
=
self
.
config
.
beam_size
# TODO(support second beam size)
self
.
second_beam_size
=
int
(
self
.
first_beam_size
*
1.0
)
logger
.
info
(
f
"first and second beam size:
{
self
.
first_beam_size
}
,
{
self
.
second_beam_size
}
"
)
# state
self
.
cur_hyps
=
None
self
.
hyps
=
None
self
.
abs_time_step
=
0
self
.
reset
()
def
reset
(
self
):
"""Rest the search cache value
"""
self
.
cur_hyps
=
None
self
.
hyps
=
None
self
.
abs_time_step
=
0
@
paddle
.
no_grad
()
def
search
(
self
,
ctc_probs
,
device
,
blank_id
=
0
):
"""ctc prefix beam search method decode a chunk feature
...
...
@@ -47,12 +68,17 @@ class CTCPrefixBeamSearch:
"""
# decode
logger
.
info
(
"start to ctc prefix search"
)
assert
len
(
ctc_probs
.
shape
)
==
2
batch_size
=
1
beam_size
=
self
.
config
.
beam_size
maxlen
=
ctc_probs
.
shape
[
0
]
assert
len
(
ctc_probs
.
shape
)
==
2
vocab_size
=
ctc_probs
.
shape
[
1
]
first_beam_size
=
min
(
self
.
first_beam_size
,
vocab_size
)
second_beam_size
=
min
(
self
.
second_beam_size
,
vocab_size
)
logger
.
info
(
f
"effect first and second beam size:
{
self
.
first_beam_size
}
,
{
self
.
second_beam_size
}
"
)
maxlen
=
ctc_probs
.
shape
[
0
]
# cur_hyps: (prefix, (blank_ending_score, none_blank_ending_score))
# 0. blank_ending_score,
...
...
@@ -75,7 +101,8 @@ class CTCPrefixBeamSearch:
# 2.1 First beam prune: select topk best
# do token passing process
top_k_logp
,
top_k_index
=
logp
.
topk
(
beam_size
)
# (beam_size,)
top_k_logp
,
top_k_index
=
logp
.
topk
(
first_beam_size
)
# (first_beam_size,)
for
s
in
top_k_index
:
s
=
s
.
item
()
ps
=
logp
[
s
].
item
()
...
...
@@ -148,7 +175,7 @@ class CTCPrefixBeamSearch:
next_hyps
.
items
(),
key
=
lambda
x
:
log_add
([
x
[
1
][
0
],
x
[
1
][
1
]]),
reverse
=
True
)
self
.
cur_hyps
=
next_hyps
[:
beam_size
]
self
.
cur_hyps
=
next_hyps
[:
second_
beam_size
]
# 2.3 update the absolute time step
self
.
abs_time_step
+=
1
...
...
@@ -163,7 +190,7 @@ class CTCPrefixBeamSearch:
"""Return the one best result
Returns:
list: the one best result
list: the one best result
, List[str]
"""
return
[
self
.
hyps
[
0
][
0
]]
...
...
@@ -171,17 +198,10 @@ class CTCPrefixBeamSearch:
"""Return the search hyps
Returns:
list: return the search hyps
list: return the search hyps
, List[Tuple[str, float, ...]]
"""
return
self
.
hyps
def
reset
(
self
):
"""Rest the search cache value
"""
self
.
cur_hyps
=
None
self
.
hyps
=
None
self
.
abs_time_step
=
0
def
finalize_search
(
self
):
"""do nothing in ctc_prefix_beam_search
"""
...
...
paddlespeech/server/engine/tts/online/python/tts_engine.py
浏览文件 @
8641608f
...
...
@@ -42,7 +42,6 @@ class TTSServerExecutor(TTSExecutor):
self
.
task_resource
=
CommonTaskResource
(
task
=
'tts'
,
model_format
=
'dynamic'
,
inference_mode
=
'online'
)
def
get_model_info
(
self
,
field
:
str
,
model_name
:
str
,
...
...
paddlespeech/server/ws/asr_api.py
浏览文件 @
8641608f
...
...
@@ -19,7 +19,6 @@ from fastapi import WebSocketDisconnect
from
starlette.websockets
import
WebSocketState
as
WebSocketState
from
paddlespeech.cli.log
import
logger
from
paddlespeech.server.engine.asr.online.asr_engine
import
PaddleASRConnectionHanddler
from
paddlespeech.server.engine.engine_pool
import
get_engine_pool
router
=
APIRouter
()
...
...
@@ -38,7 +37,7 @@ async def websocket_endpoint(websocket: WebSocket):
#2. if we accept the websocket headers, we will get the online asr engine instance
engine_pool
=
get_engine_pool
()
asr_
engine
=
engine_pool
[
'asr'
]
asr_
model
=
engine_pool
[
'asr'
]
#3. each websocket connection, we will create an PaddleASRConnectionHanddler to process such audio
# and each connection has its own connection instance to process the request
...
...
@@ -70,7 +69,8 @@ async def websocket_endpoint(websocket: WebSocket):
resp
=
{
"status"
:
"ok"
,
"signal"
:
"server_ready"
}
# do something at begining here
# create the instance to process the audio
connection_handler
=
PaddleASRConnectionHanddler
(
asr_engine
)
#connection_handler = PaddleASRConnectionHanddler(asr_model)
connection_handler
=
asr_model
.
new_handler
()
await
websocket
.
send_json
(
resp
)
elif
message
[
'signal'
]
==
'end'
:
# reset single engine for an new connection
...
...
@@ -100,11 +100,34 @@ async def websocket_endpoint(websocket: WebSocket):
# and decode for the result in this package data
connection_handler
.
extract_feat
(
message
)
connection_handler
.
decode
(
is_finished
=
False
)
if
connection_handler
.
endpoint_state
:
logger
.
info
(
"endpoint: detected and rescoring."
)
connection_handler
.
rescoring
()
word_time_stamp
=
connection_handler
.
get_word_time_stamp
()
asr_results
=
connection_handler
.
get_result
()
# return the current period result
# if the engine create the vad instance, this connection will have many period results
if
connection_handler
.
endpoint_state
:
if
connection_handler
.
continuous_decoding
:
logger
.
info
(
"endpoint: continue decoding"
)
connection_handler
.
reset_continuous_decoding
()
else
:
logger
.
info
(
"endpoint: exit decoding"
)
# ending by endpoint
resp
=
{
"status"
:
"ok"
,
"signal"
:
"finished"
,
'result'
:
asr_results
,
'times'
:
word_time_stamp
}
await
websocket
.
send_json
(
resp
)
break
# return the current partial result
# if the engine create the vad instance, this connection will have many partial results
resp
=
{
'result'
:
asr_results
}
await
websocket
.
send_json
(
resp
)
except
WebSocketDisconnect
as
e
:
logger
.
error
(
e
)
paddlespeech/t2s/exps/synthesize.py
浏览文件 @
8641608f
...
...
@@ -140,10 +140,7 @@ def parse_args():
],
help
=
'Choose acoustic model type of tts task.'
)
parser
.
add_argument
(
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
parser
.
add_argument
(
'--am_ckpt'
,
type
=
str
,
...
...
@@ -179,10 +176,7 @@ def parse_args():
],
help
=
'Choose vocoder type of tts task.'
)
parser
.
add_argument
(
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
parser
.
add_argument
(
'--voc_ckpt'
,
type
=
str
,
default
=
None
,
help
=
'Checkpoint file of voc.'
)
parser
.
add_argument
(
...
...
paddlespeech/t2s/exps/synthesize_e2e.py
浏览文件 @
8641608f
...
...
@@ -174,10 +174,7 @@ def parse_args():
],
help
=
'Choose acoustic model type of tts task.'
)
parser
.
add_argument
(
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
parser
.
add_argument
(
'--am_ckpt'
,
type
=
str
,
...
...
@@ -220,10 +217,7 @@ def parse_args():
],
help
=
'Choose vocoder type of tts task.'
)
parser
.
add_argument
(
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
parser
.
add_argument
(
'--voc_ckpt'
,
type
=
str
,
default
=
None
,
help
=
'Checkpoint file of voc.'
)
parser
.
add_argument
(
...
...
paddlespeech/t2s/exps/voice_cloning.py
浏览文件 @
8641608f
...
...
@@ -131,10 +131,7 @@ def parse_args():
choices
=
[
'fastspeech2_aishell3'
,
'tacotron2_aishell3'
],
help
=
'Choose acoustic model type of tts task.'
)
parser
.
add_argument
(
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
'--am_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of acoustic model.'
)
parser
.
add_argument
(
'--am_ckpt'
,
type
=
str
,
...
...
@@ -160,10 +157,7 @@ def parse_args():
help
=
'Choose vocoder type of tts task.'
)
parser
.
add_argument
(
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
'--voc_config'
,
type
=
str
,
default
=
None
,
help
=
'Config of voc.'
)
parser
.
add_argument
(
'--voc_ckpt'
,
type
=
str
,
default
=
None
,
help
=
'Checkpoint file of voc.'
)
parser
.
add_argument
(
...
...
paddlespeech/t2s/models/vits/__init__.py
浏览文件 @
8641608f
...
...
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from
.vits
import
*
from
.vits_updater
import
*
\ No newline at end of file
from
.vits_updater
import
*
paddlespeech/t2s/models/vits/vits_updater.py
浏览文件 @
8641608f
...
...
@@ -56,7 +56,8 @@ class VITSUpdater(StandardUpdater):
self
.
models
:
Dict
[
str
,
Layer
]
=
models
# self.model = model
self
.
model
=
model
.
_layers
if
isinstance
(
model
,
paddle
.
DataParallel
)
else
model
self
.
model
=
model
.
_layers
if
isinstance
(
model
,
paddle
.
DataParallel
)
else
model
self
.
optimizers
=
optimizers
self
.
optimizer_g
:
Optimizer
=
optimizers
[
'generator'
]
...
...
@@ -225,7 +226,8 @@ class VITSEvaluator(StandardEvaluator):
models
=
{
"main"
:
model
}
self
.
models
:
Dict
[
str
,
Layer
]
=
models
# self.model = model
self
.
model
=
model
.
_layers
if
isinstance
(
model
,
paddle
.
DataParallel
)
else
model
self
.
model
=
model
.
_layers
if
isinstance
(
model
,
paddle
.
DataParallel
)
else
model
self
.
criterions
=
criterions
self
.
criterion_mel
=
criterions
[
'mel'
]
...
...
paddlespeech/t2s/modules/losses.py
浏览文件 @
8641608f
...
...
@@ -971,18 +971,18 @@ class FeatureMatchLoss(nn.Layer):
return
feat_match_loss
# loss for VITS
class
KLDivergenceLoss
(
nn
.
Layer
):
"""KL divergence loss."""
def
forward
(
self
,
z_p
:
paddle
.
Tensor
,
logs_q
:
paddle
.
Tensor
,
m_p
:
paddle
.
Tensor
,
logs_p
:
paddle
.
Tensor
,
z_mask
:
paddle
.
Tensor
,
)
->
paddle
.
Tensor
:
self
,
z_p
:
paddle
.
Tensor
,
logs_q
:
paddle
.
Tensor
,
m_p
:
paddle
.
Tensor
,
logs_p
:
paddle
.
Tensor
,
z_mask
:
paddle
.
Tensor
,
)
->
paddle
.
Tensor
:
"""Calculate KL divergence loss.
Args:
...
...
@@ -1002,8 +1002,8 @@ class KLDivergenceLoss(nn.Layer):
logs_p
=
paddle
.
cast
(
logs_p
,
'float32'
)
z_mask
=
paddle
.
cast
(
z_mask
,
'float32'
)
kl
=
logs_p
-
logs_q
-
0.5
kl
+=
0.5
*
((
z_p
-
m_p
)
**
2
)
*
paddle
.
exp
(
-
2.0
*
logs_p
)
kl
+=
0.5
*
((
z_p
-
m_p
)
**
2
)
*
paddle
.
exp
(
-
2.0
*
logs_p
)
kl
=
paddle
.
sum
(
kl
*
z_mask
)
loss
=
kl
/
paddle
.
sum
(
z_mask
)
return
loss
\ No newline at end of file
return
loss
speechx/examples/README.md
浏览文件 @
8641608f
...
...
@@ -25,4 +25,3 @@ netron exp/deepspeech2_online/checkpoints/avg_1.jit.pdmodel --port 8022 --host
> Reminder: Only for developer, make sure you know what's it.
*
codelab - for speechx developer, using for test.
speechx/examples/ds2_ol/README.md
浏览文件 @
8641608f
...
...
@@ -3,4 +3,4 @@
## Examples
*
`websocket`
- Streaming ASR with websocket for deepspeech2_aishell.
*
`aishell`
- Streaming Decoding under aishell dataset, for local WER test.
\ No newline at end of file
*
`aishell`
- Streaming Decoding under aishell dataset, for local WER test.
speechx/speechx/codelab/README.md
浏览文件 @
8641608f
...
...
@@ -4,4 +4,3 @@
> Reminder: Only for developer.
*
codelab - for speechx developer, using for test.
speechx/speechx/decoder/ctc_prefix_beam_search_decoder_main.cc
浏览文件 @
8641608f
...
...
@@ -91,8 +91,8 @@ int main(int argc, char* argv[]) {
std
::
shared_ptr
<
ppspeech
::
Decodable
>
decodable
(
new
ppspeech
::
Decodable
(
nnet
,
raw_data
));
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_stride
=
FLAGS_downsampling_rate
*
FLAGS_nnet_decoder_chunk
;
int32
receptive_field_length
=
FLAGS_receptive_field_length
;
LOG
(
INFO
)
<<
"chunk size (frame): "
<<
chunk_size
;
...
...
speechx/speechx/decoder/ctc_tlg_decoder.cc
浏览文件 @
8641608f
...
...
@@ -64,7 +64,7 @@ std::string TLGDecoder::GetPartialResult() {
std
::
string
word
=
word_symbol_table_
->
Find
(
words_id
[
idx
]);
words
+=
word
;
}
return
words
;
return
words
;
}
std
::
string
TLGDecoder
::
GetFinalBestPath
()
{
...
...
speechx/speechx/decoder/param.h
浏览文件 @
8641608f
...
...
@@ -82,7 +82,7 @@ FeaturePipelineOptions InitFeaturePipelineOptions() {
opts
.
assembler_opts
.
subsampling_rate
=
FLAGS_downsampling_rate
;
opts
.
assembler_opts
.
receptive_filed_length
=
FLAGS_receptive_field_length
;
opts
.
assembler_opts
.
nnet_decoder_chunk
=
FLAGS_nnet_decoder_chunk
;
return
opts
;
}
...
...
speechx/speechx/decoder/tlg_decoder_main.cc
浏览文件 @
8641608f
...
...
@@ -93,8 +93,8 @@ int main(int argc, char* argv[]) {
std
::
shared_ptr
<
ppspeech
::
Decodable
>
decodable
(
new
ppspeech
::
Decodable
(
nnet
,
raw_data
,
FLAGS_acoustic_scale
));
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_stride
=
FLAGS_downsampling_rate
*
FLAGS_nnet_decoder_chunk
;
int32
receptive_field_length
=
FLAGS_receptive_field_length
;
LOG
(
INFO
)
<<
"chunk size (frame): "
<<
chunk_size
;
...
...
speechx/speechx/frontend/audio/assembler.cc
浏览文件 @
8641608f
...
...
@@ -24,7 +24,8 @@ using std::unique_ptr;
Assembler
::
Assembler
(
AssemblerOptions
opts
,
unique_ptr
<
FrontendInterface
>
base_extractor
)
{
frame_chunk_stride_
=
opts
.
subsampling_rate
*
opts
.
nnet_decoder_chunk
;
frame_chunk_size_
=
(
opts
.
nnet_decoder_chunk
-
1
)
*
opts
.
subsampling_rate
+
opts
.
receptive_filed_length
;
frame_chunk_size_
=
(
opts
.
nnet_decoder_chunk
-
1
)
*
opts
.
subsampling_rate
+
opts
.
receptive_filed_length
;
receptive_filed_length_
=
opts
.
receptive_filed_length
;
base_extractor_
=
std
::
move
(
base_extractor
);
dim_
=
base_extractor_
->
Dim
();
...
...
@@ -50,8 +51,8 @@ bool Assembler::Compute(Vector<BaseFloat>* feats) {
Vector
<
BaseFloat
>
feature
;
result
=
base_extractor_
->
Read
(
&
feature
);
if
(
result
==
false
||
feature
.
Dim
()
==
0
)
{
if
(
IsFinished
()
==
false
)
return
false
;
break
;
if
(
IsFinished
()
==
false
)
return
false
;
break
;
}
feature_cache_
.
push
(
feature
);
}
...
...
@@ -61,22 +62,22 @@ bool Assembler::Compute(Vector<BaseFloat>* feats) {
}
while
(
feature_cache_
.
size
()
<
frame_chunk_size_
)
{
Vector
<
BaseFloat
>
feature
(
dim_
,
kaldi
::
kSetZero
);
feature_cache_
.
push
(
feature
);
Vector
<
BaseFloat
>
feature
(
dim_
,
kaldi
::
kSetZero
);
feature_cache_
.
push
(
feature
);
}
int32
counter
=
0
;
int32
counter
=
0
;
int32
cache_size
=
frame_chunk_size_
-
frame_chunk_stride_
;
int32
elem_dim
=
base_extractor_
->
Dim
();
while
(
counter
<
frame_chunk_size_
)
{
Vector
<
BaseFloat
>&
val
=
feature_cache_
.
front
();
int32
start
=
counter
*
elem_dim
;
feats
->
Range
(
start
,
elem_dim
).
CopyFromVec
(
val
);
if
(
frame_chunk_size_
-
counter
<=
cache_size
)
{
feature_cache_
.
push
(
val
);
}
feature_cache_
.
pop
();
counter
++
;
Vector
<
BaseFloat
>&
val
=
feature_cache_
.
front
();
int32
start
=
counter
*
elem_dim
;
feats
->
Range
(
start
,
elem_dim
).
CopyFromVec
(
val
);
if
(
frame_chunk_size_
-
counter
<=
cache_size
)
{
feature_cache_
.
push
(
val
);
}
feature_cache_
.
pop
();
counter
++
;
}
return
result
;
...
...
speechx/speechx/frontend/audio/assembler.h
浏览文件 @
8641608f
...
...
@@ -25,7 +25,7 @@ struct AssemblerOptions {
int32
receptive_filed_length
;
int32
subsampling_rate
;
int32
nnet_decoder_chunk
;
AssemblerOptions
()
:
receptive_filed_length
(
1
),
subsampling_rate
(
1
),
...
...
@@ -47,15 +47,11 @@ class Assembler : public FrontendInterface {
// feat dim
virtual
size_t
Dim
()
const
{
return
dim_
;
}
virtual
void
SetFinished
()
{
base_extractor_
->
SetFinished
();
}
virtual
void
SetFinished
()
{
base_extractor_
->
SetFinished
();
}
virtual
bool
IsFinished
()
const
{
return
base_extractor_
->
IsFinished
();
}
virtual
void
Reset
()
{
base_extractor_
->
Reset
();
}
virtual
void
Reset
()
{
base_extractor_
->
Reset
();
}
private:
bool
Compute
(
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
feats
);
...
...
speechx/speechx/frontend/audio/audio_cache.h
浏览文件 @
8641608f
...
...
@@ -30,7 +30,7 @@ class AudioCache : public FrontendInterface {
virtual
bool
Read
(
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
waves
);
// the audio dim is 1, one sample, which is useless,
// the audio dim is 1, one sample, which is useless,
// so we return size_(cache samples) instead.
virtual
size_t
Dim
()
const
{
return
size_
;
}
...
...
speechx/speechx/frontend/audio/fbank.cc
浏览文件 @
8641608f
...
...
@@ -29,19 +29,19 @@ using kaldi::Matrix;
using
std
::
vector
;
FbankComputer
::
FbankComputer
(
const
Options
&
opts
)
:
opts_
(
opts
),
computer_
(
opts
)
{}
:
opts_
(
opts
),
computer_
(
opts
)
{}
int32
FbankComputer
::
Dim
()
const
{
return
opts_
.
mel_opts
.
num_bins
+
(
opts_
.
use_energy
?
1
:
0
);
}
bool
FbankComputer
::
NeedRawLogEnergy
()
{
return
opts_
.
use_energy
&&
opts_
.
raw_energy
;
return
opts_
.
use_energy
&&
opts_
.
raw_energy
;
}
// Compute feat
bool
FbankComputer
::
Compute
(
Vector
<
BaseFloat
>*
window
,
Vector
<
BaseFloat
>*
feat
)
{
bool
FbankComputer
::
Compute
(
Vector
<
BaseFloat
>*
window
,
Vector
<
BaseFloat
>*
feat
)
{
RealFft
(
window
,
true
);
kaldi
::
ComputePowerSpectrum
(
window
);
const
kaldi
::
MelBanks
&
mel_bank
=
*
(
computer_
.
GetMelBanks
(
1.0
));
...
...
speechx/speechx/frontend/audio/feature_cache.cc
浏览文件 @
8641608f
...
...
@@ -72,9 +72,9 @@ bool FeatureCache::Compute() {
bool
result
=
base_extractor_
->
Read
(
&
feature
);
if
(
result
==
false
||
feature
.
Dim
()
==
0
)
return
false
;
int32
num_chunk
=
feature
.
Dim
()
/
dim_
;
int32
num_chunk
=
feature
.
Dim
()
/
dim_
;
for
(
int
chunk_idx
=
0
;
chunk_idx
<
num_chunk
;
++
chunk_idx
)
{
int32
start
=
chunk_idx
*
dim_
;
int32
start
=
chunk_idx
*
dim_
;
Vector
<
BaseFloat
>
feature_chunk
(
dim_
);
SubVector
<
BaseFloat
>
tmp
(
feature
.
Data
()
+
start
,
dim_
);
feature_chunk
.
CopyFromVec
(
tmp
);
...
...
speechx/speechx/frontend/audio/feature_cache.h
浏览文件 @
8641608f
...
...
@@ -22,9 +22,7 @@ namespace ppspeech {
struct
FeatureCacheOptions
{
int32
max_size
;
int32
timeout
;
// ms
FeatureCacheOptions
()
:
max_size
(
kint16max
),
timeout
(
1
)
{}
FeatureCacheOptions
()
:
max_size
(
kint16max
),
timeout
(
1
)
{}
};
class
FeatureCache
:
public
FrontendInterface
{
...
...
speechx/speechx/frontend/audio/feature_common.h
浏览文件 @
8641608f
...
...
@@ -23,11 +23,11 @@ template <class F>
class
StreamingFeatureTpl
:
public
FrontendInterface
{
public:
typedef
typename
F
::
Options
Options
;
StreamingFeatureTpl
(
const
Options
&
opts
,
StreamingFeatureTpl
(
const
Options
&
opts
,
std
::
unique_ptr
<
FrontendInterface
>
base_extractor
);
virtual
void
Accept
(
const
kaldi
::
VectorBase
<
kaldi
::
BaseFloat
>&
waves
);
virtual
bool
Read
(
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
feats
);
// the dim_ is the dim of single frame feature
virtual
size_t
Dim
()
const
{
return
computer_
.
Dim
();
}
...
...
@@ -39,8 +39,9 @@ class StreamingFeatureTpl : public FrontendInterface {
base_extractor_
->
Reset
();
remained_wav_
.
Resize
(
0
);
}
private:
bool
Compute
(
const
kaldi
::
Vector
<
kaldi
::
BaseFloat
>&
waves
,
bool
Compute
(
const
kaldi
::
Vector
<
kaldi
::
BaseFloat
>&
waves
,
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
feats
);
Options
opts_
;
std
::
unique_ptr
<
FrontendInterface
>
base_extractor_
;
...
...
speechx/speechx/frontend/audio/feature_common_inl.h
浏览文件 @
8641608f
...
...
@@ -16,16 +16,15 @@
namespace
ppspeech
{
template
<
class
F
>
StreamingFeatureTpl
<
F
>::
StreamingFeatureTpl
(
const
Options
&
opts
,
std
::
unique_ptr
<
FrontendInterface
>
base_extractor
)
:
opts_
(
opts
),
computer_
(
opts
),
window_function_
(
opts
.
frame_opts
)
{
StreamingFeatureTpl
<
F
>::
StreamingFeatureTpl
(
const
Options
&
opts
,
std
::
unique_ptr
<
FrontendInterface
>
base_extractor
)
:
opts_
(
opts
),
computer_
(
opts
),
window_function_
(
opts
.
frame_opts
)
{
base_extractor_
=
std
::
move
(
base_extractor
);
}
template
<
class
F
>
void
StreamingFeatureTpl
<
F
>::
Accept
(
const
kaldi
::
VectorBase
<
kaldi
::
BaseFloat
>&
waves
)
{
void
StreamingFeatureTpl
<
F
>::
Accept
(
const
kaldi
::
VectorBase
<
kaldi
::
BaseFloat
>&
waves
)
{
base_extractor_
->
Accept
(
waves
);
}
...
...
@@ -58,8 +57,9 @@ bool StreamingFeatureTpl<F>::Read(kaldi::Vector<kaldi::BaseFloat>* feats) {
// Compute feat
template
<
class
F
>
bool
StreamingFeatureTpl
<
F
>::
Compute
(
const
kaldi
::
Vector
<
kaldi
::
BaseFloat
>&
waves
,
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
feats
)
{
bool
StreamingFeatureTpl
<
F
>::
Compute
(
const
kaldi
::
Vector
<
kaldi
::
BaseFloat
>&
waves
,
kaldi
::
Vector
<
kaldi
::
BaseFloat
>*
feats
)
{
const
kaldi
::
FrameExtractionOptions
&
frame_opts
=
computer_
.
GetFrameOptions
();
int32
num_samples
=
waves
.
Dim
();
...
...
@@ -84,9 +84,11 @@ bool StreamingFeatureTpl<F>::Compute(const kaldi::Vector<kaldi::BaseFloat>& wave
&
window
,
need_raw_log_energy
?
&
raw_log_energy
:
NULL
);
kaldi
::
Vector
<
kaldi
::
BaseFloat
>
this_feature
(
computer_
.
Dim
(),
kaldi
::
kUndefined
);
kaldi
::
Vector
<
kaldi
::
BaseFloat
>
this_feature
(
computer_
.
Dim
(),
kaldi
::
kUndefined
);
computer_
.
Compute
(
&
window
,
&
this_feature
);
kaldi
::
SubVector
<
kaldi
::
BaseFloat
>
output_row
(
feats
->
Data
()
+
frame
*
Dim
(),
Dim
());
kaldi
::
SubVector
<
kaldi
::
BaseFloat
>
output_row
(
feats
->
Data
()
+
frame
*
Dim
(),
Dim
());
output_row
.
CopyFromVec
(
this_feature
);
}
return
true
;
...
...
speechx/speechx/frontend/audio/feature_pipeline.h
浏览文件 @
8641608f
...
...
@@ -16,6 +16,7 @@
#pragma once
#include "frontend/audio/assembler.h"
#include "frontend/audio/audio_cache.h"
#include "frontend/audio/data_cache.h"
#include "frontend/audio/fbank.h"
...
...
@@ -23,7 +24,6 @@
#include "frontend/audio/frontend_itf.h"
#include "frontend/audio/linear_spectrogram.h"
#include "frontend/audio/normalizer.h"
#include "frontend/audio/assembler.h"
namespace
ppspeech
{
...
...
speechx/speechx/frontend/audio/linear_spectrogram.cc
浏览文件 @
8641608f
...
...
@@ -28,22 +28,21 @@ using kaldi::VectorBase;
using
kaldi
::
Matrix
;
using
std
::
vector
;
LinearSpectrogramComputer
::
LinearSpectrogramComputer
(
const
Options
&
opts
)
LinearSpectrogramComputer
::
LinearSpectrogramComputer
(
const
Options
&
opts
)
:
opts_
(
opts
)
{
kaldi
::
FeatureWindowFunction
feature_window_function
(
opts
.
frame_opts
);
int32
window_size
=
opts
.
frame_opts
.
WindowSize
();
frame_length_
=
window_size
;
dim_
=
window_size
/
2
+
1
;
BaseFloat
hanning_window_energy
=
kaldi
::
VecVec
(
feature_window_function
.
window
,
feature_window_function
.
window
);
BaseFloat
hanning_window_energy
=
kaldi
::
VecVec
(
feature_window_function
.
window
,
feature_window_function
.
window
);
int32
sample_rate
=
opts
.
frame_opts
.
samp_freq
;
scale_
=
2.0
/
(
hanning_window_energy
*
sample_rate
);
}
// Compute spectrogram feat
bool
LinearSpectrogramComputer
::
Compute
(
Vector
<
BaseFloat
>*
window
,
Vector
<
BaseFloat
>*
feat
)
{
Vector
<
BaseFloat
>*
feat
)
{
window
->
Resize
(
frame_length_
,
kaldi
::
kCopyData
);
RealFft
(
window
,
true
);
kaldi
::
ComputePowerSpectrum
(
window
);
...
...
speechx/speechx/nnet/nnet_forward_main.cc
浏览文件 @
8641608f
...
...
@@ -14,8 +14,8 @@
#include "base/flags.h"
#include "base/log.h"
#include "frontend/audio/data_cache.h"
#include "frontend/audio/assembler.h"
#include "frontend/audio/data_cache.h"
#include "kaldi/util/table-types.h"
#include "nnet/decodable.h"
#include "nnet/paddle_nnet.h"
...
...
@@ -75,8 +75,8 @@ int main(int argc, char* argv[]) {
std
::
shared_ptr
<
ppspeech
::
Decodable
>
decodable
(
new
ppspeech
::
Decodable
(
nnet
,
raw_data
,
FLAGS_acoustic_scale
));
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_size
=
FLAGS_receptive_field_length
+
(
FLAGS_nnet_decoder_chunk
-
1
)
*
FLAGS_downsampling_rate
;
int32
chunk_stride
=
FLAGS_downsampling_rate
*
FLAGS_nnet_decoder_chunk
;
int32
receptive_field_length
=
FLAGS_receptive_field_length
;
LOG
(
INFO
)
<<
"chunk size (frame): "
<<
chunk_size
;
...
...
@@ -130,7 +130,9 @@ int main(int argc, char* argv[]) {
vector
<
kaldi
::
BaseFloat
>
prob
;
while
(
decodable
->
FrameLikelihood
(
frame_idx
,
&
prob
))
{
kaldi
::
Vector
<
kaldi
::
BaseFloat
>
vec_tmp
(
prob
.
size
());
std
::
memcpy
(
vec_tmp
.
Data
(),
prob
.
data
(),
sizeof
(
kaldi
::
BaseFloat
)
*
prob
.
size
());
std
::
memcpy
(
vec_tmp
.
Data
(),
prob
.
data
(),
sizeof
(
kaldi
::
BaseFloat
)
*
prob
.
size
());
prob_vec
.
push_back
(
vec_tmp
);
frame_idx
++
;
}
...
...
@@ -142,7 +144,8 @@ int main(int argc, char* argv[]) {
KALDI_LOG
<<
" the nnet prob of "
<<
utt
<<
" is empty"
;
continue
;
}
kaldi
::
Matrix
<
kaldi
::
BaseFloat
>
result
(
prob_vec
.
size
(),
prob_vec
[
0
].
Dim
());
kaldi
::
Matrix
<
kaldi
::
BaseFloat
>
result
(
prob_vec
.
size
(),
prob_vec
[
0
].
Dim
());
for
(
int32
row_idx
=
0
;
row_idx
<
prob_vec
.
size
();
++
row_idx
)
{
for
(
int32
col_idx
=
0
;
col_idx
<
prob_vec
[
0
].
Dim
();
++
col_idx
)
{
result
(
row_idx
,
col_idx
)
=
prob_vec
[
row_idx
](
col_idx
);
...
...
speechx/speechx/protocol/websocket/websocket_client.h
浏览文件 @
8641608f
...
...
@@ -40,8 +40,8 @@ class WebSocketClient {
void
SendEndSignal
();
void
SendDataEnd
();
bool
Done
()
const
{
return
done_
;
}
std
::
string
GetResult
()
const
{
return
result_
;
}
std
::
string
GetPartialResult
()
const
{
return
partial_result_
;}
std
::
string
GetResult
()
const
{
return
result_
;
}
std
::
string
GetPartialResult
()
const
{
return
partial_result_
;
}
private:
void
Connect
();
...
...
speechx/speechx/protocol/websocket/websocket_server.cc
浏览文件 @
8641608f
...
...
@@ -76,9 +76,10 @@ void ConnectionHandler::OnSpeechData(const beast::flat_buffer& buffer) {
recognizer_
->
Accept
(
pcm_data
);
std
::
string
partial_result
=
recognizer_
->
GetPartialResult
();
json
::
value
rv
=
{
{
"status"
,
"ok"
},
{
"type"
,
"partial_result"
},
{
"result"
,
partial_result
}};
json
::
value
rv
=
{{
"status"
,
"ok"
},
{
"type"
,
"partial_result"
},
{
"result"
,
partial_result
}};
ws_
.
text
(
true
);
ws_
.
write
(
asio
::
buffer
(
json
::
serialize
(
rv
)));
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录