diff --git a/examples/csmsc/tts3_rhy/README.md b/examples/csmsc/tts3_rhy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..855aa885c39849a8f948f3329892cea6ac3eb807 --- /dev/null +++ b/examples/csmsc/tts3_rhy/README.md @@ -0,0 +1,74 @@ +# This example mainly follows the FastSpeech2 with CSMSC +This example contains code used to train a rhythm version of [Fastspeech2](https://arxiv.org/abs/2006.04558) model with [Chinese Standard Mandarin Speech Copus](https://www.data-baker.com/open_source.html). + +## Dataset +### Download and Extract +Download CSMSC from it's [Official Website](https://test.data-baker.com/data/index/TNtts/) and extract it to `~/datasets`. Then the dataset is in the directory `~/datasets/BZNSYP`. + +### Get MFA Result and Extract +We use [MFA](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) to get durations for fastspeech2. +You can directly download the rhythm version of MFA result from here [baker_alignment_tone.zip](https://paddlespeech.bj.bcebos.com/Rhy_e2e/baker_alignment_tone.zip), or train your MFA model reference to [mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/mfa) of our repo. +Remember in our repo, you should add `--rhy-with-duration` flag to obtain the rhythm information. + +## Get Started +Assume the path to the dataset is `~/datasets/BZNSYP`. +Assume the path to the MFA result of CSMSC is `./baker_alignment_tone`. +Run the command below to +1. **source path**. +2. preprocess the dataset. +3. train the model. +4. synthesize wavs. + - synthesize waveform from `metadata.jsonl`. + - synthesize waveform from a text file. +5. inference using the static model. +```bash +./run.sh +``` +You can choose a range of stages you want to run, or set `stage` equal to `stop-stage` to use only one stage, for example, running the following command will only preprocess the dataset. +```bash +./run.sh --stage 0 --stop-stage 0 +``` +### Data Preprocessing +```bash +./local/preprocess.sh ${conf_path} +``` +When it is done. A `dump` folder is created in the current directory. The structure of the dump folder is listed below. + +```text +dump +├── dev +│ ├── norm +│ └── raw +├── phone_id_map.txt +├── speaker_id_map.txt +├── test +│ ├── norm +│ └── raw +└── train + ├── energy_stats.npy + ├── norm + ├── pitch_stats.npy + ├── raw + └── speech_stats.npy +``` +The dataset is split into 3 parts, namely `train`, `dev`, and` test`, each of which contains a `norm` and `raw` subfolder. The raw folder contains speech、pitch and energy features of each utterance, while the norm folder contains normalized ones. The statistics used to normalize features are computed from the training set, which is located in `dump/train/*_stats.npy`. + +Also, there is a `metadata.jsonl` in each subfolder. It is a table-like file that contains phones, text_lengths, speech_lengths, durations, the path of speech features, the path of pitch features, the path of energy features, speaker, and the id of each utterance. + +# For more details, You can refer to [FastSpeech2 with CSMSC](../tts3) + +## Pretrained Model +Pretrained FastSpeech2 model for end-to-end rhythm version: +- [fastspeech2_rhy_csmsc_ckpt_1.3.0.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_rhy_csmsc_ckpt_1.3.0.zip) + +This FastSpeech2 checkpoint contains files listed below. +```text +fastspeech2_rhy_csmsc_ckpt_1.3.0 +├── default.yaml # default config used to train fastspeech2 +├── phone_id_map.txt # phone vocabulary file when training fastspeech2 +├── snapshot_iter_153000.pdz # model parameters and optimizer states +├── durations.txt # the intermediate output of preprocess.sh +├── energy_stats.npy +├── pitch_stats.npy +└── speech_stats.npy # statistics used to normalize spectrogram when training fastspeech2 +``` diff --git a/examples/csmsc/tts3_rhy/conf/default.yaml b/examples/csmsc/tts3_rhy/conf/default.yaml new file mode 120000 index 0000000000000000000000000000000000000000..3f69c4ebb7d1710c088899ea587e6bba05463903 --- /dev/null +++ b/examples/csmsc/tts3_rhy/conf/default.yaml @@ -0,0 +1 @@ +../../tts3/conf/default.yaml \ No newline at end of file diff --git a/examples/csmsc/tts3_rhy/local/preprocess.sh b/examples/csmsc/tts3_rhy/local/preprocess.sh new file mode 120000 index 0000000000000000000000000000000000000000..f4d0955e671a5b751c5cb2d8856bcaa218dea08c --- /dev/null +++ b/examples/csmsc/tts3_rhy/local/preprocess.sh @@ -0,0 +1 @@ +../../tts3/local/preprocess.sh \ No newline at end of file diff --git a/examples/csmsc/tts3_rhy/local/synthesize.sh b/examples/csmsc/tts3_rhy/local/synthesize.sh new file mode 120000 index 0000000000000000000000000000000000000000..f36d41990ec14a546a083a6a089d43c6c1ff2372 --- /dev/null +++ b/examples/csmsc/tts3_rhy/local/synthesize.sh @@ -0,0 +1 @@ +../../tts3/local/synthesize.sh \ No newline at end of file diff --git a/examples/csmsc/tts3_rhy/local/synthesize_e2e.sh b/examples/csmsc/tts3_rhy/local/synthesize_e2e.sh new file mode 100755 index 0000000000000000000000000000000000000000..8f5d801045dc46de7698a85aa78935c1b0346fa0 --- /dev/null +++ b/examples/csmsc/tts3_rhy/local/synthesize_e2e.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 +ckpt_name=$3 + +stage=0 +stop_stage=0 + +# pwgan +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_csmsc \ + --voc_config=pwg_baker_ckpt_0.4/pwg_default.yaml \ + --voc_ckpt=pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \ + --voc_stat=pwg_baker_ckpt_0.4/pwg_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference \ + --use_rhy=True +fi + +# for more GAN Vocoders +# multi band melgan +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=mb_melgan_csmsc \ + --voc_config=mb_melgan_csmsc_ckpt_0.1.1/default.yaml \ + --voc_ckpt=mb_melgan_csmsc_ckpt_0.1.1/snapshot_iter_1000000.pdz\ + --voc_stat=mb_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference \ + --use_rhy=True +fi + +# the pretrained models haven't release now +# style melgan +# style melgan's Dygraph to Static Graph is not ready now +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=style_melgan_csmsc \ + --voc_config=style_melgan_csmsc_ckpt_0.1.1/default.yaml \ + --voc_ckpt=style_melgan_csmsc_ckpt_0.1.1/snapshot_iter_1500000.pdz \ + --voc_stat=style_melgan_csmsc_ckpt_0.1.1/feats_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --use_rhy=True + # --inference_dir=${train_output_path}/inference +fi + +# hifigan +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + echo "in hifigan syn_e2e" + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=hifigan_csmsc \ + --voc_config=hifigan_csmsc_ckpt_0.1.1/default.yaml \ + --voc_ckpt=hifigan_csmsc_ckpt_0.1.1/snapshot_iter_2500000.pdz \ + --voc_stat=hifigan_csmsc_ckpt_0.1.1/feats_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference \ + --use_rhy=True +fi + + +# wavernn +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + echo "in wavernn syn_e2e" + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=wavernn_csmsc \ + --voc_config=wavernn_csmsc_ckpt_0.2.0/default.yaml \ + --voc_ckpt=wavernn_csmsc_ckpt_0.2.0/snapshot_iter_400000.pdz \ + --voc_stat=wavernn_csmsc_ckpt_0.2.0/feats_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference \ + --use_rhy=True +fi diff --git a/examples/csmsc/tts3_rhy/local/train.sh b/examples/csmsc/tts3_rhy/local/train.sh new file mode 120000 index 0000000000000000000000000000000000000000..11a597e85fa66662eadd0a7b43faf5d05a69a764 --- /dev/null +++ b/examples/csmsc/tts3_rhy/local/train.sh @@ -0,0 +1 @@ +../../tts3/local/train.sh \ No newline at end of file diff --git a/examples/csmsc/tts3_rhy/path.sh b/examples/csmsc/tts3_rhy/path.sh new file mode 120000 index 0000000000000000000000000000000000000000..394bed7e72752d07bfb90782cbb4ed4f8cf7c8a8 --- /dev/null +++ b/examples/csmsc/tts3_rhy/path.sh @@ -0,0 +1 @@ +../tts3/path.sh \ No newline at end of file diff --git a/examples/csmsc/tts3_rhy/run.sh b/examples/csmsc/tts3_rhy/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..e49f43ee665bde4f2c8e379a472c6486f6cac8dd --- /dev/null +++ b/examples/csmsc/tts3_rhy/run.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e +source path.sh + +gpus=0,1 +stage=0 +stop_stage=100 + +conf_path=conf/default.yaml +train_output_path=exp/default +ckpt_name=snapshot_iter_153.pdz + +# with the following command, you can choose the stage range you want to run +# such as `./run.sh --stage 0 --stop-stage 0` +# this can not be mixed use with `$1`, `$2` ... +source ${MAIN_ROOT}/utils/parse_options.sh || exit 1 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # prepare data + ### please place the mfa result of rhythm here + ./local/preprocess.sh ${conf_path} || exit -1 +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # train model, all `ckpt` under `train_output_path/checkpoints/` dir + CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1 +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # synthesize, vocoder is pwgan by default + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # synthesize_e2e, vocoder is pwgan by default + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize_e2e.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 +fi diff --git a/paddlespeech/resource/pretrained_models.py b/paddlespeech/resource/pretrained_models.py index d3b5a8f300cc400554e7b035be91e1ebee02adeb..3c5aa1f901974fa105df5f722757ef3b46304e6d 100644 --- a/paddlespeech/resource/pretrained_models.py +++ b/paddlespeech/resource/pretrained_models.py @@ -1690,3 +1690,16 @@ g2pw_onnx_models = { }, }, } + +# --------------------------------- +# ------------- Rhy_frontend --------------- +# --------------------------------- +rhy_frontend_models = { + 'rhy_e2e': { + '1.0': { + 'url': + 'https://paddlespeech.bj.bcebos.com/Rhy_e2e/rhy_frontend.zip', + 'md5': '6624a77393de5925d5a84400b363d8ef', + }, + }, +} diff --git a/paddlespeech/t2s/exps/syn_utils.py b/paddlespeech/t2s/exps/syn_utils.py index 41663891e122390592de60c7418c32ed98627af0..82b7184882f5e8bd693d1bd6d2ea705c322127f1 100644 --- a/paddlespeech/t2s/exps/syn_utils.py +++ b/paddlespeech/t2s/exps/syn_utils.py @@ -161,10 +161,13 @@ def get_test_dataset(test_metadata: List[Dict[str, Any]], # frontend def get_frontend(lang: str='zh', phones_dict: Optional[os.PathLike]=None, - tones_dict: Optional[os.PathLike]=None): + tones_dict: Optional[os.PathLike]=None, + use_rhy=False): if lang == 'zh': frontend = Frontend( - phone_vocab_path=phones_dict, tone_vocab_path=tones_dict) + phone_vocab_path=phones_dict, + tone_vocab_path=tones_dict, + use_rhy=use_rhy) elif lang == 'en': frontend = English(phone_vocab_path=phones_dict) elif lang == 'mix': diff --git a/paddlespeech/t2s/exps/synthesize_e2e.py b/paddlespeech/t2s/exps/synthesize_e2e.py index 9ce8286fbf14a2813f511924be92b0131db5df58..6250024770f33936712a2445bfbfe34f26d0398c 100644 --- a/paddlespeech/t2s/exps/synthesize_e2e.py +++ b/paddlespeech/t2s/exps/synthesize_e2e.py @@ -27,6 +27,7 @@ from paddlespeech.t2s.exps.syn_utils import get_sentences from paddlespeech.t2s.exps.syn_utils import get_voc_inference from paddlespeech.t2s.exps.syn_utils import run_frontend from paddlespeech.t2s.exps.syn_utils import voc_to_static +from paddlespeech.t2s.utils import str2bool def evaluate(args): @@ -49,7 +50,8 @@ def evaluate(args): frontend = get_frontend( lang=args.lang, phones_dict=args.phones_dict, - tones_dict=args.tones_dict) + tones_dict=args.tones_dict, + use_rhy=args.use_rhy) print("frontend done!") # acoustic model @@ -240,6 +242,11 @@ def parse_args(): type=str, help="text to synthesize, a 'utt_id sentence' pair per line.") parser.add_argument("--output_dir", type=str, help="output dir.") + parser.add_argument( + "--use_rhy", + type=str2bool, + default=False, + help="run rhythm frontend or not") args = parser.parse_args() return args diff --git a/paddlespeech/t2s/frontend/rhy_prediction/__init__.py b/paddlespeech/t2s/frontend/rhy_prediction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..62e98f80524e962c60e4c7a7eb413b00c4478054 --- /dev/null +++ b/paddlespeech/t2s/frontend/rhy_prediction/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from .rhy_predictor import * diff --git a/paddlespeech/t2s/frontend/rhy_prediction/rhy_predictor.py b/paddlespeech/t2s/frontend/rhy_prediction/rhy_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..a2a6b8a698c9436e6ab3464c77c66977a1d7986c --- /dev/null +++ b/paddlespeech/t2s/frontend/rhy_prediction/rhy_predictor.py @@ -0,0 +1,106 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re + +import paddle +import yaml +from paddlenlp.transformers import ErnieTokenizer +from yacs.config import CfgNode + +from paddlespeech.cli.utils import download_and_decompress +from paddlespeech.resource.pretrained_models import rhy_frontend_models +from paddlespeech.text.models.ernie_linear import ErnieLinear +from paddlespeech.utils.env import MODEL_HOME + +DefinedClassifier = { + 'ErnieLinear': ErnieLinear, +} + +model_version = '1.0' + + +class RhyPredictor(): + def __init__( + self, + model_dir: os.PathLike=MODEL_HOME, ): + uncompress_path = download_and_decompress( + rhy_frontend_models['rhy_e2e'][model_version], model_dir) + with open(os.path.join(uncompress_path, 'rhy_default.yaml')) as f: + config = CfgNode(yaml.safe_load(f)) + self.punc_list = [] + with open(os.path.join(uncompress_path, 'rhy_token'), 'r') as f: + for line in f: + self.punc_list.append(line.strip()) + self.punc_list = [0] + self.punc_list + self.make_rhy_dict() + self.model = DefinedClassifier["ErnieLinear"](**config["model"]) + pretrained_token = config['data_params']['pretrained_token'] + self.tokenizer = ErnieTokenizer.from_pretrained(pretrained_token) + state_dict = paddle.load( + os.path.join(uncompress_path, 'snapshot_iter_2600_main_params.pdz')) + self.model.set_state_dict(state_dict) + self.model.eval() + + def _clean_text(self, text): + text = text.lower() + text = re.sub('[^A-Za-z0-9\u4e00-\u9fa5]', '', text) + text = re.sub(f'[{"".join([p for p in self.punc_list][1:])}]', '', text) + return text + + def preprocess(self, text, tokenizer): + clean_text = self._clean_text(text) + assert len(clean_text) > 0, f'Invalid input string: {text}' + tokenized_input = tokenizer( + list(clean_text), return_length=True, is_split_into_words=True) + _inputs = dict() + _inputs['input_ids'] = tokenized_input['input_ids'] + _inputs['seg_ids'] = tokenized_input['token_type_ids'] + _inputs['seq_len'] = tokenized_input['seq_len'] + return _inputs + + def get_prediction(self, raw_text): + _inputs = self.preprocess(raw_text, self.tokenizer) + seq_len = _inputs['seq_len'] + input_ids = paddle.to_tensor(_inputs['input_ids']).unsqueeze(0) + seg_ids = paddle.to_tensor(_inputs['seg_ids']).unsqueeze(0) + logits, _ = self.model(input_ids, seg_ids) + preds = paddle.argmax(logits, axis=-1).squeeze(0) + tokens = self.tokenizer.convert_ids_to_tokens( + _inputs['input_ids'][1:seq_len - 1]) + labels = preds[1:seq_len - 1].tolist() + assert len(tokens) == len(labels) + # add 0 for non punc + text = '' + for t, l in zip(tokens, labels): + text += t + if l != 0: # Non punc. + text += self.punc_list[l] + return text + + def make_rhy_dict(self): + self.rhy_dict = {} + for i, p in enumerate(self.punc_list[1:]): + self.rhy_dict[p] = 'sp' + str(i + 1) + + def pinyin_align(self, pinyins, rhy_pre): + final_py = [] + j = 0 + for i in range(len(rhy_pre)): + if rhy_pre[i] in self.rhy_dict: + final_py.append(self.rhy_dict[rhy_pre[i]]) + else: + final_py.append(pinyins[j]) + j += 1 + return final_py diff --git a/paddlespeech/t2s/frontend/zh_frontend.py b/paddlespeech/t2s/frontend/zh_frontend.py index e30286986ac2b4c58bb76c98da08268b0531f446..ddd8cf5c7a39b79eaeeee83fc51a63d7203064c1 100644 --- a/paddlespeech/t2s/frontend/zh_frontend.py +++ b/paddlespeech/t2s/frontend/zh_frontend.py @@ -30,6 +30,7 @@ from pypinyin_dict.phrase_pinyin_data import large_pinyin from paddlespeech.t2s.frontend.g2pw import G2PWOnnxConverter from paddlespeech.t2s.frontend.generate_lexicon import generate_lexicon +from paddlespeech.t2s.frontend.rhy_prediction.rhy_predictor import RhyPredictor from paddlespeech.t2s.frontend.tone_sandhi import ToneSandhi from paddlespeech.t2s.frontend.zh_normalization.text_normlization import TextNormalizer from paddlespeech.t2s.ssml.xml_processor import MixTextProcessor @@ -82,11 +83,13 @@ class Frontend(): def __init__(self, g2p_model="g2pW", phone_vocab_path=None, - tone_vocab_path=None): + tone_vocab_path=None, + use_rhy=False): self.mix_ssml_processor = MixTextProcessor() self.tone_modifier = ToneSandhi() self.text_normalizer = TextNormalizer() self.punc = "：，；。？！“”‘’':,;.?!" + self.rhy_phns = ['sp1', 'sp2', 'sp3', 'sp4'] self.phrases_dict = { '开户行': [['ka1i'], ['hu4'], ['hang2']], '发卡行': [['fa4'], ['ka3'], ['hang2']], @@ -105,6 +108,10 @@ class Frontend(): '嘞': [['lei5']], '掺和': [['chan1'], ['huo5']] } + self.use_rhy = use_rhy + if use_rhy: + self.rhy_predictor = RhyPredictor() + print("Rhythm predictor loaded.") # g2p_model can be pypinyin and g2pM and g2pW self.g2p_model = g2p_model if self.g2p_model == "g2pM": @@ -195,9 +202,13 @@ class Frontend(): segments = sentences phones_list = [] for seg in segments: + if self.use_rhy: + seg = self.rhy_predictor._clean_text(seg) phones = [] # Replace all English words in the sentence seg = re.sub('[a-zA-Z]+', '', seg) + if self.use_rhy: + seg = self.rhy_predictor.get_prediction(seg) seg_cut = psg.lcut(seg) initials = [] finals = [] @@ -205,11 +216,18 @@ class Frontend(): # 为了多音词获得更好的效果，这里采用整句预测 if self.g2p_model == "g2pW": try: + if self.use_rhy: + seg = self.rhy_predictor._clean_text(seg) pinyins = self.g2pW_model(seg)[0] except Exception: # g2pW采用模型采用繁体输入，如果有cover不了的简体词，采用g2pM预测 print("[%s] not in g2pW dict,use g2pM" % seg) pinyins = self.g2pM_model(seg, tone=True, char_split=False) + if self.use_rhy: + rhy_text = self.rhy_predictor.get_prediction(seg) + final_py = self.rhy_predictor.pinyin_align(pinyins, + rhy_text) + pinyins = final_py pre_word_length = 0 for word, pos in seg_cut: sub_initials = [] @@ -271,7 +289,7 @@ class Frontend(): phones.append(c) if c and c in self.punc: phones.append('sp') - if v and v not in self.punc: + if v and v not in self.punc and v not in self.rhy_phns: phones.append(v) phones_list.append(phones) if merge_sentences: @@ -330,7 +348,7 @@ class Frontend(): phones.append(c) if c and c in self.punc: phones.append('sp') - if v and v not in self.punc: + if v and v not in self.punc and v not in self.rhy_phns: phones.append(v) phones_list.append(phones) if merge_sentences: @@ -504,6 +522,11 @@ class Frontend(): print("----------------------------") return [sum(all_phonemes, [])] + def add_sp_if_no(self, phonemes): + if not phonemes[-1][-1].startswith('sp'): + phonemes[-1].append('sp4') + return phonemes + def get_input_ids(self, sentence: str, merge_sentences: bool=True, @@ -519,6 +542,8 @@ class Frontend(): merge_sentences=merge_sentences, print_info=print_info, robot=robot) + if self.use_rhy: + phonemes = self.add_sp_if_no(phonemes) result = {} phones = [] tones = []