diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2f80e46b5689c369c91a3f16d00930135e354a19..60f0b92f6025d78908cf5043161c6b21771aaa95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,11 +1,12 @@ +repos: - repo: https://github.com/pre-commit/mirrors-yapf.git - sha: v0.16.0 + rev: v0.16.0 hooks: - id: yapf files: \.py$ exclude: (?=third_party).*(\.py)$ - repo: https://github.com/pre-commit/pre-commit-hooks - sha: a11d9314b22d8f8c7556443875b731ef05965464 + rev: a11d9314b22d8f8c7556443875b731ef05965464 hooks: - id: check-merge-conflict - id: check-symlinks @@ -31,7 +32,7 @@ - --jobs=1 exclude: (?=third_party).*(\.py)$ - repo : https://github.com/Lucas-C/pre-commit-hooks - sha: v1.0.1 + rev: v1.0.1 hooks: - id: forbid-crlf files: \.md$ diff --git a/README.md b/README.md index d5e56d37a37bbdd7f377b1e635e17cae25a7d30b..23124231d601c1ce1b6d36c36bcbd6079d3ae198 100644 --- a/README.md +++ b/README.md @@ -539,6 +539,7 @@ You are warmly welcome to submit questions in [discussions](https://github.com/P - Many thanks to [mymagicpower](https://github.com/mymagicpower) for the Java implementation of ASR upon [short](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_sdk) and [long](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_long_audio_sdk) audio files. - Many thanks to [JiehangXie](https://github.com/JiehangXie)/[PaddleBoBo](https://github.com/JiehangXie/PaddleBoBo) for developing Virtual Uploader(VUP)/Virtual YouTuber(VTuber) with PaddleSpeech TTS function. - Many thanks to [745165806](https://github.com/745165806)/[PaddleSpeechTask](https://github.com/745165806/PaddleSpeechTask) for contributing Punctuation Restoration model. +- Many thanks to [kslz](https://github.com/745165806) for supplementary Chinese documents. Besides, PaddleSpeech depends on a lot of open source repositories. See [references](./docs/source/reference.md) for more information. diff --git a/README_cn.md b/README_cn.md index 1ba1e8a3b7c9c6287e65c04aca31b4790fa51f98..4ce4ade9bef2b5972f3c283fd57bfeb9527934ca 100644 --- a/README_cn.md +++ b/README_cn.md @@ -548,6 +548,7 @@ year={2021} - 非常感谢 [mymagicpower](https://github.com/mymagicpower) 采用PaddleSpeech 对 ASR 的[短语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_sdk)及[长语音](https://github.com/mymagicpower/AIAS/tree/main/3_audio_sdks/asr_long_audio_sdk)进行 Java 实现。 - 非常感谢 [JiehangXie](https://github.com/JiehangXie)/[PaddleBoBo](https://github.com/JiehangXie/PaddleBoBo) 采用 PaddleSpeech 语音合成功能实现 Virtual Uploader(VUP)/Virtual YouTuber(VTuber) 虚拟主播。 - 非常感谢 [745165806](https://github.com/745165806)/[PaddleSpeechTask](https://github.com/745165806/PaddleSpeechTask) 贡献标点重建相关模型。 +- 非常感谢 [kslz](https://github.com/kslz) 补充中文文档。 此外,PaddleSpeech 依赖于许多开源存储库。有关更多信息,请参阅 [references](./docs/source/reference.md)。 diff --git a/dataset/voxceleb/README.md b/dataset/voxceleb/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3efb3519b94a75cc45ddb49aa5cc49b243018a88 --- /dev/null +++ b/dataset/voxceleb/README.md @@ -0,0 +1,10 @@ +# [VoxCeleb](http://www.robots.ox.ac.uk/~vgg/data/voxceleb/) +VoxCeleb is an audio-visual dataset consisting of short clips of human speech, extracted from interview videos uploaded to YouTube。 + +VoxCeleb contains speech from speakers spanning a wide range of different ethnicities, accents, professions and ages. +All speaking face-tracks are captured "in the wild", with background chatter, laughter, overlapping speech, pose variation and different lighting conditions. +VoxCeleb consists of both audio and video. Each segment is at least 3 seconds long. + +The dataset consists of two versions, VoxCeleb1 and VoxCeleb2. Each version has it's own train/test split. For each we provide YouTube URLs, face detections and tracks, audio files, cropped face videos and speaker meta-data. There is no overlap between the two versions. + +more info in details refers to http://www.robots.ox.ac.uk/~vgg/data/voxceleb/ diff --git a/dataset/voxceleb/voxceleb1.py b/dataset/voxceleb/voxceleb1.py new file mode 100644 index 0000000000000000000000000000000000000000..ce744751679d6e6ae756c8119cd4388adbebe404 --- /dev/null +++ b/dataset/voxceleb/voxceleb1.py @@ -0,0 +1,188 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Prepare VoxCeleb1 dataset + +create manifest files. +Manifest file is a json-format file with each line containing the +meta data (i.e. audio filepath, transcript and audio duration) +of each audio file in the data set. + +researchers should download the voxceleb1 dataset yourselves +through google form to get the username & password and unpack the data +""" +import argparse +import codecs +import glob +import json +import os +import subprocess +from pathlib import Path + +import soundfile + +from utils.utility import check_md5sum +from utils.utility import download +from utils.utility import unzip + +# all the data will be download in the current data/voxceleb directory default +DATA_HOME = os.path.expanduser('.') + +# if you use the http://www.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a/ as the download base url +# you need to get the username & password via the google form + +# if you use the https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a as the download base url, +# you need use --no-check-certificate to connect the target download url + +BASE_URL = "https://thor.robots.ox.ac.uk/~vgg/data/voxceleb/vox1a" + +# dev data +DEV_LIST = { + "vox1_dev_wav_partaa": "e395d020928bc15670b570a21695ed96", + "vox1_dev_wav_partab": "bbfaaccefab65d82b21903e81a8a8020", + "vox1_dev_wav_partac": "017d579a2a96a077f40042ec33e51512", + "vox1_dev_wav_partad": "7bb1e9f70fddc7a678fa998ea8b3ba19", +} +DEV_TARGET_DATA = "vox1_dev_wav_parta* vox1_dev_wav.zip ae63e55b951748cc486645f532ba230b" + +# test data +TEST_LIST = {"vox1_test_wav.zip": "185fdc63c3c739954633d50379a3d102"} +TEST_TARGET_DATA = "vox1_test_wav.zip vox1_test_wav.zip 185fdc63c3c739954633d50379a3d102" + +# kaldi trial +# this trial file is organized by kaldi according the official file, +# which is a little different with the official trial veri_test2.txt +KALDI_BASE_URL = "http://www.openslr.org/resources/49/" +TRIAL_LIST = {"voxceleb1_test_v2.txt": "29fc7cc1c5d59f0816dc15d6e8be60f7"} +TRIAL_TARGET_DATA = "voxceleb1_test_v2.txt voxceleb1_test_v2.txt 29fc7cc1c5d59f0816dc15d6e8be60f7" + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--target_dir", + default=DATA_HOME + "/voxceleb1/", + type=str, + help="Directory to save the voxceleb1 dataset. (default: %(default)s)") +parser.add_argument( + "--manifest_prefix", + default="manifest", + type=str, + help="Filepath prefix for output manifests. (default: %(default)s)") + +args = parser.parse_args() + +def create_manifest(data_dir, manifest_path_prefix): + print("Creating manifest %s ..." % manifest_path_prefix) + json_lines = [] + data_path = os.path.join(data_dir, "wav", "**", "*.wav") + total_sec = 0.0 + total_text = 0.0 + total_num = 0 + speakers = set() + for audio_path in glob.glob(data_path, recursive=True): + audio_id = "-".join(audio_path.split("/")[-3:]) + utt2spk = audio_path.split("/")[-3] + duration = soundfile.info(audio_path).duration + text = "" + json_lines.append( + json.dumps( + { + "utt": audio_id, + "utt2spk": str(utt2spk), + "feat": audio_path, + "feat_shape": (duration, ), + "text": text # compatible with asr data format + }, + ensure_ascii=False)) + + total_sec += duration + total_text += len(text) + total_num += 1 + speakers.add(utt2spk) + + # data_dir_name refer to dev or test + # voxceleb1 is given explicit in the path + data_dir_name = Path(data_dir).name + manifest_path_prefix = manifest_path_prefix + "." + data_dir_name + with codecs.open(manifest_path_prefix, 'w', encoding='utf-8') as f: + for line in json_lines: + f.write(line + "\n") + + manifest_dir = os.path.dirname(manifest_path_prefix) + meta_path = os.path.join(manifest_dir, "voxceleb1." + + data_dir_name) + ".meta" + with codecs.open(meta_path, 'w', encoding='utf-8') as f: + print(f"{total_num} utts", file=f) + print(f"{len(speakers)} speakers", file=f) + print(f"{total_sec / (60 * 60)} h", file=f) + print(f"{total_text} text", file=f) + print(f"{total_text / total_sec} text/sec", file=f) + print(f"{total_sec / total_num} sec/utt", file=f) + +def prepare_dataset(base_url, data_list, target_dir, manifest_path, + target_data): + if not os.path.exists(target_dir): + os.mkdir(target_dir) + + # wav directory already exists, it need do nothing + if not os.path.exists(os.path.join(target_dir, "wav")): + # download all dataset part + for zip_part in data_list.keys(): + download_url = " --no-check-certificate " + base_url + "/" + zip_part + download( + url=download_url, + md5sum=data_list[zip_part], + target_dir=target_dir) + + # pack the all part to target zip file + all_target_part, target_name, target_md5sum = target_data.split() + target_name = os.path.join(target_dir, target_name) + if not os.path.exists(target_name): + pack_part_cmd = "cat {}/{} > {}".format(target_dir, all_target_part, + target_name) + subprocess.call(pack_part_cmd, shell=True) + + # check the target zip file md5sum + if not check_md5sum(target_name, target_md5sum): + raise RuntimeError("{} MD5 checkssum failed".format(target_name)) + else: + print("Check {} md5sum successfully".format(target_name)) + + # unzip the all zip file + if target_name.endswith(".zip"): + unzip(target_name, target_dir) + + # create the manifest file + create_manifest(data_dir=target_dir, manifest_path_prefix=manifest_path) + +def main(): + if args.target_dir.startswith('~'): + args.target_dir = os.path.expanduser(args.target_dir) + + prepare_dataset( + base_url=BASE_URL, + data_list=DEV_LIST, + target_dir=os.path.join(args.target_dir, "dev"), + manifest_path=args.manifest_prefix, + target_data=DEV_TARGET_DATA) + + prepare_dataset( + base_url=BASE_URL, + data_list=TEST_LIST, + target_dir=os.path.join(args.target_dir, "test"), + manifest_path=args.manifest_prefix, + target_data=TEST_TARGET_DATA) + + print("Manifest prepare done!") + +if __name__ == '__main__': + main() diff --git a/docs/source/released_model.md b/docs/source/released_model.md index 3310bfb23cfb56acd3a9e84a41bbed2272dcc913..23309d8eb4b1bd904478d427025fb8fbe2692188 100644 --- a/docs/source/released_model.md +++ b/docs/source/released_model.md @@ -9,9 +9,10 @@ Acoustic Model | Training Data | Token-based | Size | Descriptions | CER | WER | [Ds2 Offline Aishell ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr0/asr0_deepspeech2_aishell_ckpt_0.1.1.model.tar.gz)| Aishell Dataset | Char-based | 306 MB | 2 Conv + 3 bidirectional GRU layers| 0.064 |-| 151 h | [Ds2 Offline Aishell ASR0](../../examples/aishell/asr0) [Conformer Offline Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_conformer_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 284 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.056 |-| 151 h | [Conformer Offline Aishell ASR1](../../examples/aishell/asr1) [Transformer Aishell ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/aishell/asr1/asr1_transformer_aishell_ckpt_0.1.1.model.tar.gz) | Aishell Dataset | Char-based | 128 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: Attention rescoring | 0.0523 || 151 h | [Transformer Aishell ASR1](../../examples/aishell/asr1) -[Conformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_conformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 191 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0337 | 960 h | [Conformer Librispeech ASR1](../../example/librispeech/asr1) -[Transformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_transformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 131 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0381 | 960 h | [Transformer Librispeech ASR1](../../example/librispeech/asr1) -[Transformer Librispeech ASR2 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr2/asr2_transformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 131 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: JoinCTC w/ LM |-| 0.0240 | 960 h | [Transformer Librispeech ASR2](../../example/librispeech/asr2) +[Ds2 Offline Librispeech ASR0 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr0/asr0_deepspeech2_librispeech_ckpt_0.1.1.model.tar.gz)| Librispeech Dataset | Char-based | 518 MB | 2 Conv + 3 bidirectional LSTM layers| - |0.0725| 960 h | [Ds2 Offline Librispeech ASR0](../../examples/librispeech/asr0) +[Conformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_conformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 191 MB | Encoder:Conformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0337 | 960 h | [Conformer Librispeech ASR1](../../examples/librispeech/asr1) +[Transformer Librispeech ASR1 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr1/asr1_transformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 131 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: Attention rescoring |-| 0.0381 | 960 h | [Transformer Librispeech ASR1](../../examples/librispeech/asr1) +[Transformer Librispeech ASR2 Model](https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr2/asr2_transformer_librispeech_ckpt_0.1.1.model.tar.gz) | Librispeech Dataset | subword-based | 131 MB | Encoder:Transformer, Decoder:Transformer, Decoding method: JoinCTC w/ LM |-| 0.0240 | 960 h | [Transformer Librispeech ASR2](../../examples/librispeech/asr2) ### Language Model based on NGram Language Model | Training Data | Token-based | Size | Descriptions @@ -65,7 +66,7 @@ GE2E + FastSpeech2 | AISHELL-3 |[ge2e-fastspeech2-aishell3](https://github.com/ Model Type | Dataset| Example Link | Pretrained Models :-------------:| :------------:| :-----: | :-----: PANN | Audioset| [audioset_tagging_cnn](https://github.com/qiuqiangkong/audioset_tagging_cnn) | [panns_cnn6.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn6.pdparams), [panns_cnn10.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn10.pdparams), [panns_cnn14.pdparams](https://bj.bcebos.com/paddleaudio/models/panns_cnn14.pdparams) -PANN | ESC-50 |[pann-esc50]("./examples/esc50/cls0")|[esc50_cnn6.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn6.tar.gz), [esc50_cnn10.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn10.tar.gz), [esc50_cnn14.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn14.tar.gz) +PANN | ESC-50 |[pann-esc50](../../examples/esc50/cls0)|[esc50_cnn6.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn6.tar.gz), [esc50_cnn10.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn10.tar.gz), [esc50_cnn14.tar.gz](https://paddlespeech.bj.bcebos.com/cls/esc50/esc50_cnn14.tar.gz) ## Punctuation Restoration Models Model Type | Dataset| Example Link | Pretrained Models diff --git a/docs/source/tts/README.md b/docs/source/tts/README.md index 3de8901bfb52d7803fc804b42c1f3c9b8b04ab85..835db08eeb4000635dbc80a5d84501742ae7764f 100644 --- a/docs/source/tts/README.md +++ b/docs/source/tts/README.md @@ -71,7 +71,3 @@ Check our [website](https://paddlespeech.readthedocs.io/en/latest/tts/demo.html) #### GE2E 1. [ge2e_ckpt_0.3.zip](https://paddlespeech.bj.bcebos.com/Parakeet/ge2e_ckpt_0.3.zip) - -## License - -Parakeet is provided under the [Apache-2.0 license](LICENSE). diff --git a/docs/source/tts/quick_start.md b/docs/source/tts/quick_start.md index 3180d80ad4dc6603d51b16f64f54c6c5c45c0aac..bddee7786634d7e9e891f97249692b45d235e850 100644 --- a/docs/source/tts/quick_start.md +++ b/docs/source/tts/quick_start.md @@ -1,3 +1,4 @@ +([简体中文](./quick_start_cn.md)|English) # Quick Start of Text-to-Speech The examples in PaddleSpeech are mainly classified by datasets, the TTS datasets we mainly used are: * CSMCS (Mandarin single speaker) diff --git a/docs/source/tts/quick_start_cn.md b/docs/source/tts/quick_start_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..39bf3d0a11b0992b6f4c15e27e1a01d8b5468095 --- /dev/null +++ b/docs/source/tts/quick_start_cn.md @@ -0,0 +1,205 @@ +(简体中文|[English](./quick_start.md)) +# 语音合成快速开始 +这些PaddleSpeech中的样例主要按数据集分类,我们主要使用的TTS数据集有: + +* CSMCS (普通话单发音人) +* AISHELL3 (普通话多发音人) +* LJSpeech (英文单发音人) +* VCTK (英文多发音人) + +PaddleSpeech 的 TTS 模型具有以下映射关系: + +* tts0 - Tactron2 +* tts1 - TransformerTTS +* tts2 - SpeedySpeech +* tts3 - FastSpeech2 +* voc0 - WaveFlow +* voc1 - Parallel WaveGAN +* voc2 - MelGAN +* voc3 - MultiBand MelGAN +* voc4 - Style MelGAN +* voc5 - HiFiGAN +* vc0 - Tactron2 Voice Clone with GE2E +* vc1 - FastSpeech2 Voice Clone with GE2E + +## 快速开始 + +让我们以 FastSpeech2 + Parallel WaveGAN 和 CSMSC 数据集 为例. [examples/csmsc](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc) + +### 用 CSMSC 数据集训练 Parallel WaveGAN + +- 进入目录 + ```bash + cd examples/csmsc/voc1 + ``` +- 设置环境变量 + ```bash + source path.sh + ``` + **在你开始做任何事情之前,必须先做这步** + 将 `MAIN_ROOT` 设置为项目目录. 使用 `parallelwave_gan` 模型作为 `MODEL`. + +- 运行 + ```bash + bash run.sh + ``` + 这只是一个演示,请确保源数据已经准备好,并且在下一个 `step` 之前每个 `step` 都运行正常. +### 用CSMSC数据集训练FastSpeech2 + +- 进入目录 + ```bash + cd examples/csmsc/tts3 + ``` + +- 设置环境变量 + ```bash + source path.sh + ``` + **在你开始做任何事情之前,必须先做这步** + 将 `MAIN_ROOT` 设置为项目目录. 使用 `fastspeech2` 模型作为 `MODEL` 。 + +- 运行 + ```bash + bash run.sh + ``` + 这只是一个演示,请确保源数据已经准备好,并且在下一个 `step` 之前每个 `step` 都运行正常。 + +`run.sh` 中主要包括以下步骤: + +- 设置路径。 +- 预处理数据集, +- 训练模型。 +- 从 `metadata.jsonl` 中合成波形 +- 从文本文件合成波形。(在声学模型中) +- 使用静态模型进行推理。(可选) + +有关更多详细信息,请参见 examples 中的 `README.md` + +## TTS 流水线 +本节介绍如何使用 TTS 提供的预训练模型,并对其进行推理。 + +TTS中的预训练模型在压缩包中提供。将其解压缩以获得如下文件夹: +**Acoustic Models:** + +```text +checkpoint_name +├── default.yaml +├── snapshot_iter_*.pdz +├── speech_stats.npy +├── phone_id_map.txt +├── spk_id_map.txt (optimal) +└── tone_id_map.txt (optimal) +``` +**Vocoders:** +```text +checkpoint_name +├── default.yaml +├── snapshot_iter_*.pdz +└── stats.npy +``` +- `default.yaml` 存储用于训练模型的配置。 +- `snapshot_iter_*.pdz` 是检查点文件,其中`*`是它经过训练的步骤。 +- `*_stats.npy` 是特征的统计文件,如果它在训练前已被标准化。 +- `phone_id_map.txt` 是音素到音素 ID 的映射关系。 +- `tone_id_map.txt` 是在训练声学模型之前分割音调和拼音时,音调到音调 ID 的映射关系。(例如在 csmsc/speedyspeech 的示例中) +- `spk_id_map.txt` 是多发音人声学模型中 "发音人" 到 "spk_ids" 的映射关系。 + +下面的示例代码显示了如何使用模型进行预测。 +### Acoustic Models 声学模型(文本到频谱图) +下面的代码显示了如何使用 `FastSpeech2` 模型。加载预训练模型后,使用它和 normalizer 对象构建预测对象,然后使用 `fastspeech2_inferencet(phone_ids)` 生成频谱图,频谱图可进一步用于使用声码器合成原始音频。 + +```python +from pathlib import Path +import numpy as np +import paddle +import yaml +from yacs.config import CfgNode +from paddlespeech.t2s.models.fastspeech2 import FastSpeech2 +from paddlespeech.t2s.models.fastspeech2 import FastSpeech2Inference +from paddlespeech.t2s.modules.normalizer import ZScore +# examples/fastspeech2/baker/frontend.py +from frontend import Frontend + +# 加载预训练模型 +checkpoint_dir = Path("fastspeech2_nosil_baker_ckpt_0.4") +with open(checkpoint_dir / "phone_id_map.txt", "r") as f: + phn_id = [line.strip().split() for line in f.readlines()] +vocab_size = len(phn_id) +with open(checkpoint_dir / "default.yaml") as f: + fastspeech2_config = CfgNode(yaml.safe_load(f)) +odim = fastspeech2_config.n_mels +model = FastSpeech2( + idim=vocab_size, odim=odim, **fastspeech2_config["model"]) +model.set_state_dict( + paddle.load(args.fastspeech2_checkpoint)["main_params"]) +model.eval() + +# 加载特征文件 +stat = np.load(checkpoint_dir / "speech_stats.npy") +mu, std = stat +mu = paddle.to_tensor(mu) +std = paddle.to_tensor(std) +fastspeech2_normalizer = ZScore(mu, std) + +# 构建预测对象 +fastspeech2_inference = FastSpeech2Inference(fastspeech2_normalizer, model) + +# load Chinese Frontend +frontend = Frontend(checkpoint_dir / "phone_id_map.txt") + +# 构建一个中文前端 +sentence = "你好吗?" +input_ids = frontend.get_input_ids(sentence, merge_sentences=True) +phone_ids = input_ids["phone_ids"] +flags = 0 +# 构建预测对象加载中文前端,对中文文本前端的输出进行分段 +for part_phone_ids in phone_ids: + with paddle.no_grad(): + temp_mel = fastspeech2_inference(part_phone_ids) + if flags == 0: + mel = temp_mel + flags = 1 + else: + mel = paddle.concat([mel, temp_mel]) +``` + +### Vcoder声码器(谱图到波形) +下面的代码显示了如何使用 `Parallel WaveGAN` 模型。像上面的例子一样,加载预训练模型后,使用它和 normalizer 对象构建预测对象,然后使用 `pwg_inference(mel)` 生成原始音频( wav 格式)。 + +```python +from pathlib import Path +import numpy as np +import paddle +import soundfile as sf +import yaml +from yacs.config import CfgNode +from paddlespeech.t2s.models.parallel_wavegan import PWGGenerator +from paddlespeech.t2s.models.parallel_wavegan import PWGInference +from paddlespeech.t2s.modules.normalizer import ZScore + +# 加载预训练模型 +checkpoint_dir = Path("parallel_wavegan_baker_ckpt_0.4") +with open(checkpoint_dir / "pwg_default.yaml") as f: + pwg_config = CfgNode(yaml.safe_load(f)) +vocoder = PWGGenerator(**pwg_config["generator_params"]) +vocoder.set_state_dict(paddle.load(args.pwg_params)) +vocoder.remove_weight_norm() +vocoder.eval() + +# 加载特征文件 +stat = np.load(checkpoint_dir / "pwg_stats.npy") +mu, std = stat +mu = paddle.to_tensor(mu) +std = paddle.to_tensor(std) +pwg_normalizer = ZScore(mu, std) + +# 加载预训练模型构造预测对象 +pwg_inference = PWGInference(pwg_normalizer, vocoder) + +# 频谱图到波形 +wav = pwg_inference(mel) +sf.write( + audio_path, + wav.numpy(), + samplerate=fastspeech2_config.fs) +``` \ No newline at end of file diff --git a/examples/aishell3/vc0/README.md b/examples/aishell3/vc0/README.md index 91d32619bfd2dc6328f0cb7df73e229ef1859e88..21cd0aa20fcaeb09ba05a88a092ebb8e3f8bc6c1 100644 --- a/examples/aishell3/vc0/README.md +++ b/examples/aishell3/vc0/README.md @@ -1,94 +1,119 @@ + # Tacotron2 + AISHELL-3 Voice Cloning -This example contains code used to train a [Tacotron2 ](https://arxiv.org/abs/1712.05884) model with [AISHELL-3](http://www.aishelltech.com/aishell_3). The trained model can be used in Voice Cloning Task, We refer to the model structure of [Transfer Learning from Speaker Verification to Multispeaker Text-To-Speech Synthesis](https://arxiv.org/pdf/1806.04558.pdf). The general steps are as follows: -1. Speaker Encoder: We use Speaker Verification to train a speaker encoder. Datasets used in this task are different from those used in Tacotron2 because the transcriptions are not needed, we use more datasets, refer to [ge2e](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/ge2e). -2. Synthesizer: We use the trained speaker encoder to generate speaker embedding for each sentence in AISHELL-3. This embedding is an extra input of Tacotron2 which will be concated with encoder outputs. -3. Vocoder: We use WaveFlow as the neural Vocoder, refer to [waveflow](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/ljspeech/voc0). +This example contains code used to train a [Tacotron2](https://arxiv.org/abs/1712.05884) model with [AISHELL-3](http://www.aishelltech.com/aishell_3). The trained model can be used in Voice Cloning Task, We refer to the model structure of [Transfer Learning from Speaker Verification to Multispeaker Text-To-Speech Synthesis](https://arxiv.org/pdf/1806.04558.pdf). The general steps are as follows: +1. Speaker Encoder: We use Speaker Verification to train a speaker encoder. Datasets used in this task are different from those used in `Tacotron2` because the transcriptions are not needed, we use more datasets, refer to [ge2e](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/ge2e). +2. Synthesizer: We use the trained speaker encoder to generate speaker embedding for each sentence in AISHELL-3. This embedding is an extra input of `Tacotron2` which will be concated with encoder outputs. +3. Vocoder: We use [Parallel Wave GAN](http://arxiv.org/abs/1910.11480) as the neural Vocoder, refer to [voc1](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc1). + +## Dataset +### Download and Extract +Download AISHELL-3. +```bash +wget https://www.openslr.org/resources/93/data_aishell3.tgz +``` +Extract AISHELL-3. +```bash +mkdir data_aishell3 +tar zxvf data_aishell3.tgz -C data_aishell3 +``` +### Get MFA Result and Extract +We use [MFA2.x](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) to get durations for aishell3_fastspeech2. +You can download from here [aishell3_alignment_tone.tar.gz](https://paddlespeech.bj.bcebos.com/MFA/AISHELL-3/with_tone/aishell3_alignment_tone.tar.gz), or train your MFA model reference to [mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/mfa) (use MFA1.x now) of our repo. + +## Pretrained GE2E Model +We use pretrained GE2E model to generate speaker embedding for each sentence. + +Download pretrained GE2E model from here [ge2e_ckpt_0.3.zip](https://bj.bcebos.com/paddlespeech/Parakeet/released_models/ge2e/ge2e_ckpt_0.3.zip), and `unzip` it. ## Get Started Assume the path to the dataset is `~/datasets/data_aishell3`. -Assume the path to the MFA result of AISHELL-3 is `./alignment`. -Assume the path to the pretrained ge2e model is `ge2e_ckpt_path=./ge2e_ckpt_0.3/step-3000000` +Assume the path to the MFA result of AISHELL-3 is `./aishell3_alignment_tone`. +Assume the path to the pretrained ge2e model is `./ge2e_ckpt_0.3`. + Run the command below to 1. **source path**. 2. preprocess the dataset. 3. train the model. -4. start a voice cloning inference. +4. synthesize waveform from `metadata.jsonl`. +5. start a voice cloning inference. ```bash ./run.sh ``` -You can choose a range of stages you want to run, or set `stage` equal to `stop-stage` to use only one stage, for example, run the following command will only preprocess the dataset. +You can choose a range of stages you want to run, or set `stage` equal to `stop-stage` to use only one stage, for example, running the following command will only preprocess the dataset. ```bash ./run.sh --stage 0 --stop-stage 0 ``` ### Data Preprocessing ```bash -CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${input} ${preprocess_path} ${alignment} ${ge2e_ckpt_path} +CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${conf_path} ${ge2e_ckpt_path} ``` -#### Generate Speaker Embedding - Use pretrained GE2E (speaker encoder) to generate speaker embedding for each sentence in AISHELL-3, which has the same file structure with wav files and the format is `.npy`. - -```bash -if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then - python3 ${BIN_DIR}/../ge2e/inference.py \ - --input=${input} \ - --output=${preprocess_path}/embed \ - --ngpu=1 \ - --checkpoint_path=${ge2e_ckpt_path} -fi +When it is done. A `dump` folder is created in the current directory. The structure of the dump folder is listed below. +```text +dump +├── dev +│ ├── norm +│ └── raw +├── embed +│ ├── SSB0005 +│ ├── SSB0009 +│ ├── ... +│ └── ... +├── phone_id_map.txt +├── speaker_id_map.txt +├── test +│ ├── norm +│ └── raw +└── train + ├── norm + ├── raw + └── speech_stats.npy ``` +The `embed` contains the generated speaker embedding for each sentence in AISHELL-3, which has the same file structure with wav files and the format is `.npy`. The computing time of utterance embedding can be x hours. -#### Process Wav -There is silence in the edge of AISHELL-3's wavs, and the audio amplitude is very small, so, we need to remove the silence and normalize the audio. You can the silence remove method based on volume or energy, but the effect is not very good, We use [MFA](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) to get the alignment of text and speech, then utilize the alignment results to remove the silence. - -We use Montreal Force Aligner 1.0. The label in aishell3 includes pinyin,so the lexicon we provided to MFA is pinyin rather than Chinese characters. And the prosody marks(`$` and `%`) need to be removed. You should preprocess the dataset into the format which MFA needs, the texts have the same name with wavs and have the suffix `.lab`. -We use [lexicon.txt](https://github.com/PaddlePaddle/PaddleSpeech/blob/develop/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/lexicon.txt) as the lexicon. +The dataset is split into 3 parts, namely `train`, `dev`, and` test`, each of which contains a `norm` and `raw` subfolder. The raw folder contains speech features of each utterance, while the norm folder contains normalized ones. The statistics used to normalize features are computed from the training set, which is located in `dump/train/*_stats.npy`. -You can download the alignment results from here [alignment_aishell3.tar.gz](https://paddlespeech.bj.bcebos.com/MFA/AISHELL-3/alignment_aishell3.tar.gz), or train your MFA model reference to [mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/mfa) (use MFA1.x now) of our repo. +Also, there is a `metadata.jsonl` in each subfolder. It is a table-like file that contains phones, text_lengths, speech_lengths, durations, the path of speech features, speaker, and id of each utterance. +The preprocessing step is very similar to that one of [tts0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/tts0), but there is one more `ge2e/inference` step here. +### Model Training +`./local/train.sh` calls `${BIN_DIR}/train.py`. ```bash -if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then - echo "Process wav ..." - python3 ${BIN_DIR}/process_wav.py \ - --input=${input}/wav \ - --output=${preprocess_path}/normalized_wav \ - --alignment=${alignment} -fi +CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} ``` +The training step is very similar to that one of [tts0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/tts0), but we should set `--voice-cloning=True` when calling `${BIN_DIR}/train.py`. -#### Preprocess Transcription -We revert the transcription into `phones` and `tones`. It is worth noting that our processing here is different from that used for MFA, we separated the tones. This is a processing method, of course, you can only segment initials and vowels. - +### Synthesizing +We use [parallel wavegan](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/aishell3/voc1) as the neural vocoder. +Download pretrained parallel wavegan model from [pwg_aishell3_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/pwgan/pwg_aishell3_ckpt_0.5.zip) and unzip it. ```bash -if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then - python3 ${BIN_DIR}/preprocess_transcription.py \ - --input=${input} \ - --output=${preprocess_path} -fi +unzip pwg_aishell3_ckpt_0.5.zip ``` -The default input is `~/datasets/data_aishell3/train`,which contains `label_train-set.txt`, the processed results are `metadata.yaml` and `metadata.pickle`. the former is a text format for easy viewing, and the latter is a binary format for direct reading. -#### Extract Mel -```python -if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then - python3 ${BIN_DIR}/extract_mel.py \ - --input=${preprocess_path}/normalized_wav \ - --output=${preprocess_path}/mel -fi +Parallel WaveGAN checkpoint contains files listed below. +```text +pwg_aishell3_ckpt_0.5 +├── default.yaml # default config used to train parallel wavegan +├── feats_stats.npy # statistics used to normalize spectrogram when training parallel wavegan +└── snapshot_iter_1000000.pdz # generator parameters of parallel wavegan ``` - -### Model Training +`./local/synthesize.sh` calls `${BIN_DIR}/../synthesize.py`, which can synthesize waveform from `metadata.jsonl`. ```bash -CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${preprocess_path} ${train_output_path} +CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} ``` +The synthesizing step is very similar to that one of [tts0](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/tts0), but we should set `--voice-cloning=True` when calling `${BIN_DIR}/../synthesize.py`. -Our model removes stop token prediction in Tacotron2, because of the problem of the extremely unbalanced proportion of positive and negative samples of stop token prediction, and it's very sensitive to the clip of audio silence. We use the last symbol from the highest point of attention to the encoder side as the termination condition. - -In addition, to accelerate the convergence of the model, we add `guided attention loss` to induce the alignment between encoder and decoder to show diagonal lines faster. ### Voice Cloning +Assume there are some reference audios in `./ref_audio` +```text +ref_audio +├── 001238.wav +├── LJ015-0254.wav +└── audio_self_test.mp3 +``` +`./local/voice_cloning.sh` calls `${BIN_DIR}/../voice_cloning.py` + ```bash -CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${ge2e_params_path} ${tacotron2_params_path} ${waveflow_params_path} ${vc_input} ${vc_output} +CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_output_path} ${ckpt_name} ${ge2e_params_path} ${ref_audio_dir} ``` -## Pretrained Model -[tacotron2_aishell3_ckpt_0.3.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/tacotron2/tacotron2_aishell3_ckpt_0.3.zip). diff --git a/examples/aishell3/vc0/conf/default.yaml b/examples/aishell3/vc0/conf/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16a4a60c68c2a063f2e4b410a4257b00070de95f --- /dev/null +++ b/examples/aishell3/vc0/conf/default.yaml @@ -0,0 +1,86 @@ +########################################################### +# FEATURE EXTRACTION SETTING # +########################################################### + +fs: 24000 # sr +n_fft: 2048 # FFT size (samples). +n_shift: 300 # Hop size (samples). 12.5ms +win_length: 1200 # Window length (samples). 50ms + # If set to null, it will be the same as fft_size. +window: "hann" # Window function. + +# Only used for feats_type != raw + +fmin: 80 # Minimum frequency of Mel basis. +fmax: 7600 # Maximum frequency of Mel basis. +n_mels: 80 # The number of mel basis. + +########################################################### +# DATA SETTING # +########################################################### +batch_size: 64 +num_workers: 2 + +########################################################### +# MODEL SETTING # +########################################################### +model: # keyword arguments for the selected model + embed_dim: 512 # char or phn embedding dimension + elayers: 1 # number of blstm layers in encoder + eunits: 512 # number of blstm units + econv_layers: 3 # number of convolutional layers in encoder + econv_chans: 512 # number of channels in convolutional layer + econv_filts: 5 # filter size of convolutional layer + atype: location # attention function type + adim: 512 # attention dimension + aconv_chans: 32 # number of channels in convolutional layer of attention + aconv_filts: 15 # filter size of convolutional layer of attention + cumulate_att_w: True # whether to cumulate attention weight + dlayers: 2 # number of lstm layers in decoder + dunits: 1024 # number of lstm units in decoder + prenet_layers: 2 # number of layers in prenet + prenet_units: 256 # number of units in prenet + postnet_layers: 5 # number of layers in postnet + postnet_chans: 512 # number of channels in postnet + postnet_filts: 5 # filter size of postnet layer + output_activation: null # activation function for the final output + use_batch_norm: True # whether to use batch normalization in encoder + use_concate: True # whether to concatenate encoder embedding with decoder outputs + use_residual: False # whether to use residual connection in encoder + dropout_rate: 0.5 # dropout rate + zoneout_rate: 0.1 # zoneout rate + reduction_factor: 1 # reduction factor + spk_embed_dim: 256 # speaker embedding dimension + spk_embed_integration_type: concat # how to integrate speaker embedding + + +########################################################### +# UPDATER SETTING # +########################################################### +updater: + use_masking: True # whether to apply masking for padded part in loss calculation + bce_pos_weight: 5.0 # weight of positive sample in binary cross entropy calculation + use_guided_attn_loss: True # whether to use guided attention loss + guided_attn_loss_sigma: 0.4 # sigma of guided attention loss + guided_attn_loss_lambda: 1.0 # strength of guided attention loss + + +########################################################## +# OPTIMIZER SETTING # +########################################################## +optimizer: + optim: adam # optimizer type + learning_rate: 1.0e-03 # learning rate + epsilon: 1.0e-06 # epsilon + weight_decay: 0.0 # weight decay coefficient + +########################################################### +# TRAINING SETTING # +########################################################### +max_epoch: 200 +num_snapshots: 5 + +########################################################### +# OTHER SETTING # +########################################################### +seed: 42 \ No newline at end of file diff --git a/examples/aishell3/vc0/local/preprocess.sh b/examples/aishell3/vc0/local/preprocess.sh index 5bf880667eecdb284bb2a4c64b5f7fe6722eb5d7..069cf94c4eef312a212ada244554029bea5d27f1 100755 --- a/examples/aishell3/vc0/local/preprocess.sh +++ b/examples/aishell3/vc0/local/preprocess.sh @@ -1,36 +1,72 @@ #!/bin/bash -stage=0 +stage=3 stop_stage=100 -input=$1 -preprocess_path=$2 -alignment=$3 -ge2e_ckpt_path=$4 +config_path=$1 +ge2e_ckpt_path=$2 +# gen speaker embedding if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then python3 ${MAIN_ROOT}/paddlespeech/vector/exps/ge2e/inference.py \ - --input=${input}/wav \ - --output=${preprocess_path}/embed \ + --input=~/datasets/data_aishell3/train/wav/ \ + --output=dump/embed \ --checkpoint_path=${ge2e_ckpt_path} fi +# copy from tts3/preprocess if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then - echo "Process wav ..." - python3 ${BIN_DIR}/process_wav.py \ - --input=${input}/wav \ - --output=${preprocess_path}/normalized_wav \ - --alignment=${alignment} + # get durations from MFA's result + echo "Generate durations.txt from MFA results ..." + python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \ + --inputdir=./aishell3_alignment_tone \ + --output durations.txt \ + --config=${config_path} fi if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then - python3 ${BIN_DIR}/preprocess_transcription.py \ - --input=${input} \ - --output=${preprocess_path} + # extract features + echo "Extract features ..." + python3 ${BIN_DIR}/preprocess.py \ + --dataset=aishell3 \ + --rootdir=~/datasets/data_aishell3/ \ + --dumpdir=dump \ + --dur-file=durations.txt \ + --config=${config_path} \ + --num-cpu=20 \ + --cut-sil=True \ + --spk_emb_dir=dump/embed fi if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then - python3 ${BIN_DIR}/extract_mel.py \ - --input=${preprocess_path}/normalized_wav \ - --output=${preprocess_path}/mel + # get features' stats(mean and std) + echo "Get features' stats ..." + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="speech" +fi + +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + # normalize and covert phone to id, dev and test should use train's stats + echo "Normalize ..." + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --dumpdir=dump/train/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/dev/raw/metadata.jsonl \ + --dumpdir=dump/dev/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/test/raw/metadata.jsonl \ + --dumpdir=dump/test/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt fi diff --git a/examples/aishell3/vc0/local/synthesize.sh b/examples/aishell3/vc0/local/synthesize.sh new file mode 100755 index 0000000000000000000000000000000000000000..9843028034ab935cf0bf4aff9d82407092fb8268 --- /dev/null +++ b/examples/aishell3/vc0/local/synthesize.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 +ckpt_name=$3 + +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../synthesize.py \ + --am=tacotron2_aishell3 \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_aishell3 \ + --voc_config=pwg_aishell3_ckpt_0.5/default.yaml \ + --voc_ckpt=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ + --voc_stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ + --test_metadata=dump/test/norm/metadata.jsonl \ + --output_dir=${train_output_path}/test \ + --phones_dict=dump/phone_id_map.txt \ + --speaker_dict=dump/speaker_id_map.txt \ + --voice-cloning=True diff --git a/examples/aishell3/vc0/local/train.sh b/examples/aishell3/vc0/local/train.sh index f062cbbf3e36a72ba9b9d5aa6dfe809341b1d4ea..c775fcadcceef12e05225c46aa53812e22aa2ee4 100755 --- a/examples/aishell3/vc0/local/train.sh +++ b/examples/aishell3/vc0/local/train.sh @@ -1,9 +1,13 @@ #!/bin/bash -preprocess_path=$1 +config_path=$1 train_output_path=$2 python3 ${BIN_DIR}/train.py \ - --data=${preprocess_path} \ - --output=${train_output_path} \ - --ngpu=1 \ No newline at end of file + --train-metadata=dump/train/norm/metadata.jsonl \ + --dev-metadata=dump/dev/norm/metadata.jsonl \ + --config=${config_path} \ + --output-dir=${train_output_path} \ + --ngpu=2 \ + --phones-dict=dump/phone_id_map.txt \ + --voice-cloning=True \ No newline at end of file diff --git a/examples/aishell3/vc0/local/voice_cloning.sh b/examples/aishell3/vc0/local/voice_cloning.sh index 3fe3de767dce9e3ca4f0b990eb2af9fa0e2aad7d..79831f3ff26a468dadd525a8ccd9c143ef912d8d 100755 --- a/examples/aishell3/vc0/local/voice_cloning.sh +++ b/examples/aishell3/vc0/local/voice_cloning.sh @@ -1,14 +1,24 @@ #!/bin/bash -ge2e_params_path=$1 -tacotron2_params_path=$2 -waveflow_params_path=$3 -vc_input=$4 -vc_output=$5 +config_path=$1 +train_output_path=$2 +ckpt_name=$3 +ge2e_params_path=$4 +ref_audio_dir=$5 -python3 ${BIN_DIR}/voice_cloning.py \ +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../voice_cloning.py \ + --am=tacotron2_aishell3 \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_aishell3 \ + --voc_config=pwg_aishell3_ckpt_0.5/default.yaml \ + --voc_ckpt=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ + --voc_stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ --ge2e_params_path=${ge2e_params_path} \ - --tacotron2_params_path=${tacotron2_params_path} \ - --waveflow_params_path=${waveflow_params_path} \ - --input-dir=${vc_input} \ - --output-dir=${vc_output} \ No newline at end of file + --text="凯莫瑞安联合体的经济崩溃迫在眉睫。" \ + --input-dir=${ref_audio_dir} \ + --output-dir=${train_output_path}/vc_syn \ + --phones-dict=dump/phone_id_map.txt diff --git a/examples/aishell3/vc0/path.sh b/examples/aishell3/vc0/path.sh index dfae49afe31b475d1916729732146b9469f4b5f1..9cdbe256e11a068a5471fed1ac392804cf615ac1 100755 --- a/examples/aishell3/vc0/path.sh +++ b/examples/aishell3/vc0/path.sh @@ -9,5 +9,5 @@ export PYTHONDONTWRITEBYTECODE=1 export PYTHONIOENCODING=UTF-8 export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} -MODEL=voice_cloning/tacotron2_ge2e +MODEL=new_tacotron2 export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL} diff --git a/examples/aishell3/vc0/run.sh b/examples/aishell3/vc0/run.sh index 870360c1c3372d58f75384f921504d8ff0f4e2df..64f4ee3bc89b2c14e0850cbe22b1bbd85962a4a3 100755 --- a/examples/aishell3/vc0/run.sh +++ b/examples/aishell3/vc0/run.sh @@ -3,25 +3,20 @@ set -e source path.sh -gpus=0 +gpus=0,1 stage=0 stop_stage=100 -input=~/datasets/data_aishell3/train -preprocess_path=dump -alignment=./alignment +conf_path=conf/default.yaml +train_output_path=exp/default +ckpt_name=snapshot_iter_482.pdz +ref_audio_dir=ref_audio # not include ".pdparams" here ge2e_ckpt_path=./ge2e_ckpt_0.3/step-3000000 -train_output_path=output + # include ".pdparams" here ge2e_params_path=${ge2e_ckpt_path}.pdparams -tacotron2_params_path=${train_output_path}/checkpoints/step-1000.pdparams -# pretrained model -# tacotron2_params_path=./tacotron2_aishell3_ckpt_0.3/step-450000.pdparams -waveflow_params_path=./waveflow_ljspeech_ckpt_0.3/step-2000000.pdparams -vc_input=ref_audio -vc_output=syn_audio # with the following command, you can choose the stage range you want to run # such as `./run.sh --stage 0 --stop-stage 0` @@ -30,15 +25,20 @@ source ${MAIN_ROOT}/utils/parse_options.sh || exit 1 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then # prepare data - CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${input} ${preprocess_path} ${alignment} ${ge2e_ckpt_path} || exit -1 + CUDA_VISIBLE_DEVICES=${gpus} ./local/preprocess.sh ${conf_path} ${ge2e_ckpt_path} || exit -1 fi if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then - CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${preprocess_path} ${train_output_path} || exit -1 + # train model, all `ckpt` under `train_output_path/checkpoints/` dir + CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1 fi if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then - CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${ge2e_params_path} ${tacotron2_params_path} ${waveflow_params_path} ${vc_input} ${vc_output} || exit -1 + # synthesize, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 fi - +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # synthesize, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_output_path} ${ckpt_name} ${ge2e_params_path} ${ref_audio_dir} || exit -1 +fi diff --git a/examples/aishell3/vc1/README.md b/examples/aishell3/vc1/README.md index d5745bc32cef8b54894b807c39767fb64246665b..8a566089171d9ed46b62bf87b8aecfa8aeb1f10e 100644 --- a/examples/aishell3/vc1/README.md +++ b/examples/aishell3/vc1/README.md @@ -114,7 +114,7 @@ ref_audio ├── LJ015-0254.wav └── audio_self_test.mp3 ``` -`./local/voice_cloning.sh` calls `${BIN_DIR}/voice_cloning.py` +`./local/voice_cloning.sh` calls `${BIN_DIR}/../voice_cloning.py` ```bash CUDA_VISIBLE_DEVICES=${gpus} ./local/voice_cloning.sh ${conf_path} ${train_output_path} ${ckpt_name} ${ge2e_params_path} ${ref_audio_dir} diff --git a/examples/aishell3/vc1/local/voice_cloning.sh b/examples/aishell3/vc1/local/voice_cloning.sh index 6a50826e843c6d7bb983bd1d4cfdbf2767886126..2a8864bab4c351319d9325730da744c770e8a503 100755 --- a/examples/aishell3/vc1/local/voice_cloning.sh +++ b/examples/aishell3/vc1/local/voice_cloning.sh @@ -8,13 +8,15 @@ ref_audio_dir=$5 FLAGS_allocator_strategy=naive_best_fit \ FLAGS_fraction_of_gpu_memory_to_use=0.01 \ -python3 ${BIN_DIR}/voice_cloning.py \ - --fastspeech2-config=${config_path} \ - --fastspeech2-checkpoint=${train_output_path}/checkpoints/${ckpt_name} \ - --fastspeech2-stat=dump/train/speech_stats.npy \ - --pwg-config=pwg_aishell3_ckpt_0.5/default.yaml \ - --pwg-checkpoint=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ - --pwg-stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ +python3 ${BIN_DIR}/../voice_cloning.py \ + --am=fastspeech2_aishell3 \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_aishell3 \ + --voc_config=pwg_aishell3_ckpt_0.5/default.yaml \ + --voc_ckpt=pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \ + --voc_stat=pwg_aishell3_ckpt_0.5/feats_stats.npy \ --ge2e_params_path=${ge2e_params_path} \ --text="凯莫瑞安联合体的经济崩溃迫在眉睫。" \ --input-dir=${ref_audio_dir} \ diff --git a/examples/csmsc/tts0/README.md b/examples/csmsc/tts0/README.md index 3f3b4a3949a2fb9eae8fc71543e740f1a9ad1430..b030a51cbc03aaa843d120f9aa68b6f401fccbda 100644 --- a/examples/csmsc/tts0/README.md +++ b/examples/csmsc/tts0/README.md @@ -44,15 +44,13 @@ dump │ ├── norm │ └── raw └── train - ├── energy_stats.npy ├── norm - ├── pitch_stats.npy ├── raw └── speech_stats.npy ``` -The dataset is split into 3 parts, namely `train`, `dev`, and` test`, each of which contains a `norm` and `raw` subfolder. The raw folder contains speech、pitch and energy features of each utterance, while the norm folder contains normalized ones. The statistics used to normalize features are computed from the training set, which is located in `dump/train/*_stats.npy`. +The dataset is split into 3 parts, namely `train`, `dev`, and` test`, each of which contains a `norm` and `raw` subfolder. The raw folder contains speech features of each utterance, while the norm folder contains normalized ones. The statistics used to normalize features are computed from the training set, which is located in `dump/train/*_stats.npy`. -Also, there is a `metadata.jsonl` in each subfolder. It is a table-like file that contains phones, text_lengths, speech_lengths, durations, the path of speech features, the path of pitch features, the path of energy features, speaker, and the id of each utterance. +Also, there is a `metadata.jsonl` in each subfolder. It is a table-like file that contains phones, text_lengths, speech_lengths, durations, the path of speech features, speaker, and the id of each utterance. ### Model Training ```bash diff --git a/examples/csmsc/tts0/local/inference.sh b/examples/csmsc/tts0/local/inference.sh new file mode 100755 index 0000000000000000000000000000000000000000..e417d748e494ea67203c506b8c8e39ca6eb2815b --- /dev/null +++ b/examples/csmsc/tts0/local/inference.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +train_output_path=$1 + +stage=0 +stop_stage=0 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + python3 ${BIN_DIR}/../inference.py \ + --inference_dir=${train_output_path}/inference \ + --am=tacotron2_csmsc \ + --voc=pwgan_csmsc \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/pd_infer_out \ + --phones_dict=dump/phone_id_map.txt +fi + +# for more GAN Vocoders +# multi band melgan +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + python3 ${BIN_DIR}/../inference.py \ + --inference_dir=${train_output_path}/inference \ + --am=tacotron2_csmsc \ + --voc=mb_melgan_csmsc \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/pd_infer_out \ + --phones_dict=dump/phone_id_map.txt +fi + +# style melgan +# style melgan's Dygraph to Static Graph is not ready now +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + python3 ${BIN_DIR}/../inference.py \ + --inference_dir=${train_output_path}/inference \ + --am=tacotron2_csmsc \ + --voc=style_melgan_csmsc \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/pd_infer_out \ + --phones_dict=dump/phone_id_map.txt +fi + +# hifigan +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + python3 ${BIN_DIR}/../inference.py \ + --inference_dir=${train_output_path}/inference \ + --am=tacotron2_csmsc \ + --voc=hifigan_csmsc \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/pd_infer_out \ + --phones_dict=dump/phone_id_map.txt +fi \ No newline at end of file diff --git a/examples/csmsc/tts0/local/synthesize_e2e.sh b/examples/csmsc/tts0/local/synthesize_e2e.sh index fe5d11d4400876c050fcc4ac48c7dc1286a65778..c957df876e2b8c92ff21d3bcf28bb5b6055e9b58 100755 --- a/examples/csmsc/tts0/local/synthesize_e2e.sh +++ b/examples/csmsc/tts0/local/synthesize_e2e.sh @@ -22,8 +22,9 @@ if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then --lang=zh \ --text=${BIN_DIR}/../sentences.txt \ --output_dir=${train_output_path}/test_e2e \ - --inference_dir=${train_output_path}/inference \ - --phones_dict=dump/phone_id_map.txt + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference + fi # for more GAN Vocoders diff --git a/examples/csmsc/tts3/README.md b/examples/csmsc/tts3/README.md index 13d291b5c390328ada4cbbbf580633ed4a194ffb..04c6a5da62a712f37fb3b3dae9ff309a55009511 100644 --- a/examples/csmsc/tts3/README.md +++ b/examples/csmsc/tts3/README.md @@ -1,3 +1,4 @@ +([简体中文](./README_cn.md)|English) # FastSpeech2 with CSMSC This example contains code used to train a [Fastspeech2](https://arxiv.org/abs/2006.04558) model with [Chinese Standard Mandarin Speech Copus](https://www.data-baker.com/open_source.html). diff --git a/examples/csmsc/tts3/README_cn.md b/examples/csmsc/tts3/README_cn.md new file mode 100644 index 0000000000000000000000000000000000000000..25931ecb182189e4ab76d3befb9b5185bca71caf --- /dev/null +++ b/examples/csmsc/tts3/README_cn.md @@ -0,0 +1,273 @@ +(简体中文|[English](./README.md)) +# 用 CSMSC 数据集训练 FastSpeech2 模型 + +本用例包含用于训练 [Fastspeech2](https://arxiv.org/abs/2006.04558) 模型的代码,使用 [Chinese Standard Mandarin Speech Copus](https://www.data-baker.com/open_source.html) 数据集。 + +## 数据集 +### 下载并解压 +从 [官方网站](https://test.data-baker.com/data/index/source) 下载数据集 + +### 获取MFA结果并解压 +我们使用 [MFA](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) 去获得 fastspeech2 的音素持续时间。 +你们可以从这里下载 [baker_alignment_tone.tar.gz](https://paddlespeech.bj.bcebos.com/MFA/BZNSYP/with_tone/baker_alignment_tone.tar.gz), 或参考 [mfa example](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/other/mfa) 训练你自己的模型。 + +## 开始 +假设数据集的路径是 `~/datasets/BZNSYP`. +假设CSMSC的MFA结果路径为 `./baker_alignment_tone`. +运行下面的命令会进行如下操作: + +1. **设置原路径**。 +2. 对数据集进行预处理。 +3. 训练模型 +4. 合成波形 + - 从 `metadata.jsonl` 合成波形。 + - 从文本文件合成波形。 +5. 使用静态模型进行推理。 +```bash +./run.sh +``` +您可以选择要运行的一系列阶段,或者将 `stage` 设置为 `stop-stage` 以仅使用一个阶段,例如,运行以下命令只会预处理数据集。 +```bash +./run.sh --stage 0 --stop-stage 0 +``` +### 数据预处理 +```bash +./local/preprocess.sh ${conf_path} +``` +当它完成时。将在当前目录中创建 `dump` 文件夹。转储文件夹的结构如下所示。 + +```text +dump +├── dev +│ ├── norm +│ └── raw +├── phone_id_map.txt +├── speaker_id_map.txt +├── test +│ ├── norm +│ └── raw +└── train + ├── energy_stats.npy + ├── norm + ├── pitch_stats.npy + ├── raw + └── speech_stats.npy +``` + +数据集分为三个部分,即 `train` 、 `dev` 和 `test` ,每个部分都包含一个 `norm` 和 `raw` 子文件夹。原始文件夹包含每个话语的语音、音调和能量特征,而 `norm` 文件夹包含规范化的特征。用于规范化特征的统计数据是从 `dump/train/*_stats.npy` 中的训练集计算出来的。 + +此外,还有一个 `metadata.jsonl` 在每个子文件夹中。它是一个类似表格的文件,包含音素、文本长度、语音长度、持续时间、语音特征路径、音调特征路径、能量特征路径、说话人和每个话语的 id。 + +### 模型训练 +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} +``` +`./local/train.sh` 调用 `${BIN_DIR}/train.py` 。 +以下是完整的帮助信息。 + +```text +usage: train.py [-h] [--config CONFIG] [--train-metadata TRAIN_METADATA] + [--dev-metadata DEV_METADATA] [--output-dir OUTPUT_DIR] + [--ngpu NGPU] [--phones-dict PHONES_DICT] + [--speaker-dict SPEAKER_DICT] [--voice-cloning VOICE_CLONING] + +Train a FastSpeech2 model. + +optional arguments: + -h, --help show this help message and exit + --config CONFIG fastspeech2 config file. + --train-metadata TRAIN_METADATA + training data. + --dev-metadata DEV_METADATA + dev data. + --output-dir OUTPUT_DIR + output dir. + --ngpu NGPU if ngpu=0, use cpu. + --phones-dict PHONES_DICT + phone vocabulary file. + --speaker-dict SPEAKER_DICT + speaker id map file for multiple speaker model. + --voice-cloning VOICE_CLONING + whether training voice cloning model. +``` +1. `--config` 是一个 yaml 格式的配置文件,用于覆盖默认配置,位于 `conf/default.yaml`. +2. `--train-metadata` 和 `--dev-metadata` 应为 `dump` 文件夹中 `train` 和 `dev` 下的规范化元数据文件 +3. `--output-dir` 是保存结果的目录。 检查点保存在此目录中的 `checkpoints/` 目录下。 +4. `--ngpu` 要使用的 GPU 数,如果 ngpu==0,则使用 cpu 。 +5. `--phones-dict` 是音素词汇表文件的路径。 + +### 合成 +我们使用 [parallel wavegan](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/csmsc/voc1) 作为神经声码器(vocoder)。 +从 [pwg_baker_ckpt_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/pwgan/pwg_baker_ckpt_0.4.zip) 下载预训练的 parallel wavegan 模型并将其解压。 + +```bash +unzip pwg_baker_ckpt_0.4.zip +``` +Parallel WaveGAN 检查点包含如下文件。 +```text +pwg_baker_ckpt_0.4 +├── pwg_default.yaml # 用于训练 parallel wavegan 的默认配置 +├── pwg_snapshot_iter_400000.pdz # parallel wavegan 的模型参数 +└── pwg_stats.npy # 训练平行波形时用于规范化谱图的统计数据 +``` +`./local/synthesize.sh` 调用 `${BIN_DIR}/../synthesize.py` 即可从 `metadata.jsonl`中合成波形。 + +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} +``` +```text +usage: synthesize.py [-h] + [--am {speedyspeech_csmsc,fastspeech2_csmsc,fastspeech2_ljspeech,fastspeech2_aishell3,fastspeech2_vctk}] + [--am_config AM_CONFIG] [--am_ckpt AM_CKPT] + [--am_stat AM_STAT] [--phones_dict PHONES_DICT] + [--tones_dict TONES_DICT] [--speaker_dict SPEAKER_DICT] + [--voice-cloning VOICE_CLONING] + [--voc {pwgan_csmsc,pwgan_ljspeech,pwgan_aishell3,pwgan_vctk,mb_melgan_csmsc}] + [--voc_config VOC_CONFIG] [--voc_ckpt VOC_CKPT] + [--voc_stat VOC_STAT] [--ngpu NGPU] + [--test_metadata TEST_METADATA] [--output_dir OUTPUT_DIR] + +Synthesize with acoustic model & vocoder + +optional arguments: + -h, --help show this help message and exit + --am {speedyspeech_csmsc,fastspeech2_csmsc,fastspeech2_ljspeech,fastspeech2_aishell3,fastspeech2_vctk} + Choose acoustic model type of tts task. + --am_config AM_CONFIG + Config of acoustic model. Use deault config when it is + None. + --am_ckpt AM_CKPT Checkpoint file of acoustic model. + --am_stat AM_STAT mean and standard deviation used to normalize + spectrogram when training acoustic model. + --phones_dict PHONES_DICT + phone vocabulary file. + --tones_dict TONES_DICT + tone vocabulary file. + --speaker_dict SPEAKER_DICT + speaker id map file. + --voice-cloning VOICE_CLONING + whether training voice cloning model. + --voc {pwgan_csmsc,pwgan_ljspeech,pwgan_aishell3,pwgan_vctk,mb_melgan_csmsc} + Choose vocoder type of tts task. + --voc_config VOC_CONFIG + Config of voc. Use deault config when it is None. + --voc_ckpt VOC_CKPT Checkpoint file of voc. + --voc_stat VOC_STAT mean and standard deviation used to normalize + spectrogram when training voc. + --ngpu NGPU if ngpu == 0, use cpu. + --test_metadata TEST_METADATA + test metadata. + --output_dir OUTPUT_DIR + output dir. +``` +`./local/synthesize_e2e.sh` 调用 `${BIN_DIR}/../synthesize_e2e.py`,即可从文本文件中合成波形。 + +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize_e2e.sh ${conf_path} ${train_output_path} ${ckpt_name} +``` +```text +usage: synthesize_e2e.py [-h] + [--am {speedyspeech_csmsc,fastspeech2_csmsc,fastspeech2_ljspeech,fastspeech2_aishell3,fastspeech2_vctk}] + [--am_config AM_CONFIG] [--am_ckpt AM_CKPT] + [--am_stat AM_STAT] [--phones_dict PHONES_DICT] + [--tones_dict TONES_DICT] + [--speaker_dict SPEAKER_DICT] [--spk_id SPK_ID] + [--voc {pwgan_csmsc,pwgan_ljspeech,pwgan_aishell3,pwgan_vctk,mb_melgan_csmsc}] + [--voc_config VOC_CONFIG] [--voc_ckpt VOC_CKPT] + [--voc_stat VOC_STAT] [--lang LANG] + [--inference_dir INFERENCE_DIR] [--ngpu NGPU] + [--text TEXT] [--output_dir OUTPUT_DIR] + +Synthesize with acoustic model & vocoder + +optional arguments: + -h, --help show this help message and exit + --am {speedyspeech_csmsc,fastspeech2_csmsc,fastspeech2_ljspeech,fastspeech2_aishell3,fastspeech2_vctk} + Choose acoustic model type of tts task. + --am_config AM_CONFIG + Config of acoustic model. Use deault config when it is + None. + --am_ckpt AM_CKPT Checkpoint file of acoustic model. + --am_stat AM_STAT mean and standard deviation used to normalize + spectrogram when training acoustic model. + --phones_dict PHONES_DICT + phone vocabulary file. + --tones_dict TONES_DICT + tone vocabulary file. + --speaker_dict SPEAKER_DICT + speaker id map file. + --spk_id SPK_ID spk id for multi speaker acoustic model + --voc {pwgan_csmsc,pwgan_ljspeech,pwgan_aishell3,pwgan_vctk,mb_melgan_csmsc} + Choose vocoder type of tts task. + --voc_config VOC_CONFIG + Config of voc. Use deault config when it is None. + --voc_ckpt VOC_CKPT Checkpoint file of voc. + --voc_stat VOC_STAT mean and standard deviation used to normalize + spectrogram when training voc. + --lang LANG Choose model language. zh or en + --inference_dir INFERENCE_DIR + dir to save inference models + --ngpu NGPU if ngpu == 0, use cpu. + --text TEXT text to synthesize, a 'utt_id sentence' pair per line. + --output_dir OUTPUT_DIR + output dir. +``` +1. `--am` 声学模型格式是否符合 {model_name}_{dataset} +2. `--am_config`, `--am_checkpoint`, `--am_stat` 和 `--phones_dict` 是声学模型的参数,对应于 fastspeech2 预训练模型中的 4 个文件。 +3. `--voc` 声码器(vocoder)格式是否符合 {model_name}_{dataset} +4. `--voc_config`, `--voc_checkpoint`, `--voc_stat` 是声码器的参数,对应于 parallel wavegan 预训练模型中的 3 个文件。 +5. `--lang` 对应模型的语言可以是 `zh` 或 `en` 。 +6. `--test_metadata` 应为 `dump` 文件夹中 `test` 下的规范化元数据文件、 +7. `--text` 是文本文件,其中包含要合成的句子。 +8. `--output_dir` 是保存合成音频文件的目录。 +9. `--ngpu` 要使用的GPU数,如果 ngpu==0,则使用 cpu 。 + +### 推理 +在合成之后,我们将在 `${train_output_path}/inference` 中得到 fastspeech2 和 pwgan 的静态模型 +`./local/inference.sh` 调用 `${BIN_DIR}/inference.py` 为 fastspeech2 + pwgan 综合提供了一个 paddle 静态模型推理示例。 + +```bash +CUDA_VISIBLE_DEVICES=${gpus} ./local/inference.sh ${train_output_path} +``` + +## 预训练模型 +预先训练的 FastSpeech2 模型,在音频边缘没有空白音频: +- [fastspeech2_nosil_baker_ckpt_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_ckpt_0.4.zip) +- [fastspeech2_conformer_baker_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_conformer_baker_ckpt_0.5.zip) + +静态模型可以在这里下载 [fastspeech2_nosil_baker_static_0.4.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_baker_static_0.4.zip). + +Model | Step | eval/loss | eval/l1_loss | eval/duration_loss | eval/pitch_loss| eval/energy_loss +:-------------:| :------------:| :-----: | :-----: | :--------: |:--------:|:---------: +default| 2(gpu) x 76000|1.0991|0.59132|0.035815|0.31915|0.15287| +conformer| 2(gpu) x 76000|1.0675|0.56103|0.035869|0.31553|0.15509| + +FastSpeech2检查点包含下列文件。 +```text +fastspeech2_nosil_baker_ckpt_0.4 +├── default.yaml # 用于训练 fastspeech2 的默认配置 +├── phone_id_map.txt # 训练 fastspeech2 时的音素词汇文件 +├── snapshot_iter_76000.pdz # 模型参数和优化器状态 +└── speech_stats.npy # 训练 fastspeech2 时用于规范化频谱图的统计数据 +``` +您可以使用以下脚本通过使用预训练的 fastspeech2 和 parallel wavegan 模型为 `${BIN_DIR}/../sentences.txt` 合成句子 +```bash +source path.sh + +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=fastspeech2_nosil_baker_ckpt_0.4/default.yaml \ + --am_ckpt=fastspeech2_nosil_baker_ckpt_0.4/snapshot_iter_76000.pdz \ + --am_stat=fastspeech2_nosil_baker_ckpt_0.4/speech_stats.npy \ + --voc=pwgan_csmsc \ + --voc_config=pwg_baker_ckpt_0.4/pwg_default.yaml \ + --voc_ckpt=pwg_baker_ckpt_0.4/pwg_snapshot_iter_400000.pdz \ + --voc_stat=pwg_baker_ckpt_0.4/pwg_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=exp/default/test_e2e \ + --inference_dir=exp/default/inference \ + --phones_dict=fastspeech2_nosil_baker_ckpt_0.4/phone_id_map.txt +``` diff --git a/examples/csmsc/tts3/local/inference.sh b/examples/csmsc/tts3/local/inference.sh index 7c58980cdd1e9602743b13dccbaac09b0e3f443b..9322cfd697912100663457f2b9bcada543e27733 100755 --- a/examples/csmsc/tts3/local/inference.sh +++ b/examples/csmsc/tts3/local/inference.sh @@ -48,4 +48,15 @@ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then --text=${BIN_DIR}/../sentences.txt \ --output_dir=${train_output_path}/pd_infer_out \ --phones_dict=dump/phone_id_map.txt +fi + +# wavernn +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + python3 ${BIN_DIR}/../inference.py \ + --inference_dir=${train_output_path}/inference \ + --am=fastspeech2_csmsc \ + --voc=wavernn_csmsc \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/pd_infer_out \ + --phones_dict=dump/phone_id_map.txt fi \ No newline at end of file diff --git a/examples/csmsc/tts3/local/synthesize_e2e.sh b/examples/csmsc/tts3/local/synthesize_e2e.sh index d4744486ca634bd85b0381f7e715147b78400d6f..d1fadf77d9e14b0230f5d3547d3dcd7e8a221b7d 100755 --- a/examples/csmsc/tts3/local/synthesize_e2e.sh +++ b/examples/csmsc/tts3/local/synthesize_e2e.sh @@ -89,3 +89,25 @@ if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then --inference_dir=${train_output_path}/inference \ --phones_dict=dump/phone_id_map.txt fi + + +# wavernn +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + echo "in wavernn syn_e2e" + FLAGS_allocator_strategy=naive_best_fit \ + FLAGS_fraction_of_gpu_memory_to_use=0.01 \ + python3 ${BIN_DIR}/../synthesize_e2e.py \ + --am=fastspeech2_csmsc \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=wavernn_csmsc \ + --voc_config=wavernn_test/default.yaml \ + --voc_ckpt=wavernn_test/snapshot_iter_5000.pdz \ + --voc_stat=wavernn_test/feats_stats.npy \ + --lang=zh \ + --text=${BIN_DIR}/../sentences.txt \ + --output_dir=${train_output_path}/test_e2e \ + --phones_dict=dump/phone_id_map.txt \ + --inference_dir=${train_output_path}/inference +fi diff --git a/examples/csmsc/voc6/conf/default.yaml b/examples/csmsc/voc6/conf/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7696cf40d5a5f791b5aa1b18efa7fc9512267f3 --- /dev/null +++ b/examples/csmsc/voc6/conf/default.yaml @@ -0,0 +1,67 @@ + +########################################################### +# FEATURE EXTRACTION SETTING # +########################################################### +fs: 24000 # Sampling rate. +n_fft: 2048 # FFT size (samples). +n_shift: 300 # Hop size (samples). 12.5ms +win_length: 1200 # Window length (samples). 50ms + # If set to null, it will be the same as fft_size. +window: "hann" # Window function. +n_mels: 80 # Number of mel basis. +fmin: 80 # Minimum freq in mel basis calculation. (Hz) +fmax: 7600 # Maximum frequency in mel basis calculation. (Hz) +mu_law: True # Recommended to suppress noise if using raw bitsexit() + + +########################################################### +# MODEL SETTING # +########################################################### +model: + rnn_dims: 512 # Hidden dims of RNN Layers. + fc_dims: 512 + bits: 9 # Bit depth of signal + aux_context_window: 2 # Context window size for auxiliary feature. + # If set to 2, previous 2 and future 2 frames will be considered. + aux_channels: 80 # Number of channels for auxiliary feature conv. + # Must be the same as num_mels. + upsample_scales: [4, 5, 3, 5] # Upsampling scales. Prodcut of these must be the same as hop size, same with pwgan here + compute_dims: 128 # Dims of Conv1D in MelResNet. + res_out_dims: 128 # Dims of output in MelResNet. + res_blocks: 10 # Number of residual blocks. + mode: RAW # either 'raw'(softmax on raw bits) or 'mold' (sample from mixture of logistics) +inference: + gen_batched: True # whether to genenate sample in batch mode + target: 12000 # target number of samples to be generated in each batch entry + overlap: 600 # number of samples for crossfading between batches + + +########################################################### +# DATA LOADER SETTING # +########################################################### +batch_size: 64 # Batch size. +batch_max_steps: 4500 # Length of each audio in batch. Make sure dividable by hop_size. +num_workers: 2 # Number of workers in DataLoader. + +########################################################### +# OPTIMIZER SETTING # +########################################################### +grad_clip: 4.0 +learning_rate: 1.0e-4 + + +########################################################### +# INTERVAL SETTING # +########################################################### + +train_max_steps: 400000 # Number of training steps. +save_interval_steps: 5000 # Interval steps to save checkpoint. +eval_interval_steps: 1000 # Interval steps to evaluate the network. +gen_eval_samples_interval_steps: 5000 # the iteration interval of generating valid samples +generate_num: 5 # number of samples to generate at each checkpoint + +########################################################### +# OTHER SETTING # +########################################################### +num_snapshots: 10 # max number of snapshots to keep while training +seed: 42 # random seed for paddle, random, and np.random diff --git a/examples/csmsc/voc6/local/preprocess.sh b/examples/csmsc/voc6/local/preprocess.sh new file mode 100755 index 0000000000000000000000000000000000000000..2dcc39ac7c31b82fd8eb5c47509df9f4b81fac41 --- /dev/null +++ b/examples/csmsc/voc6/local/preprocess.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +stage=0 +stop_stage=100 + +config_path=$1 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # get durations from MFA's result + echo "Generate durations.txt from MFA results ..." + python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \ + --inputdir=./baker_alignment_tone \ + --output=durations.txt \ + --config=${config_path} +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # extract features + echo "Extract features ..." + python3 ${BIN_DIR}/../gan_vocoder/preprocess.py \ + --rootdir=~/datasets/BZNSYP/ \ + --dataset=baker \ + --dumpdir=dump \ + --dur-file=durations.txt \ + --config=${config_path} \ + --cut-sil=True \ + --num-cpu=20 +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # get features' stats(mean and std) + echo "Get features' stats ..." + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="feats" +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # normalize, dev and test should use train's stats + echo "Normalize ..." + + python3 ${BIN_DIR}/../gan_vocoder/normalize.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --dumpdir=dump/train/norm \ + --stats=dump/train/feats_stats.npy + python3 ${BIN_DIR}/../gan_vocoder/normalize.py \ + --metadata=dump/dev/raw/metadata.jsonl \ + --dumpdir=dump/dev/norm \ + --stats=dump/train/feats_stats.npy + + python3 ${BIN_DIR}/../gan_vocoder/normalize.py \ + --metadata=dump/test/raw/metadata.jsonl \ + --dumpdir=dump/test/norm \ + --stats=dump/train/feats_stats.npy +fi diff --git a/examples/csmsc/voc6/local/synthesize.sh b/examples/csmsc/voc6/local/synthesize.sh new file mode 100755 index 0000000000000000000000000000000000000000..7f0cbe48cc633c7be527fc0166beac8d7be92625 --- /dev/null +++ b/examples/csmsc/voc6/local/synthesize.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 +ckpt_name=$3 + +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/synthesize.py \ + --config=${config_path} \ + --checkpoint=${train_output_path}/checkpoints/${ckpt_name} \ + --test-metadata=dump/test/norm/metadata.jsonl \ + --output-dir=${train_output_path}/test diff --git a/examples/csmsc/voc6/local/train.sh b/examples/csmsc/voc6/local/train.sh new file mode 100755 index 0000000000000000000000000000000000000000..9695631ef023795f6c54e5c11bbcff5b6a6b2998 --- /dev/null +++ b/examples/csmsc/voc6/local/train.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +config_path=$1 +train_output_path=$2 + +FLAGS_cudnn_exhaustive_search=true \ +FLAGS_conv_workspace_size_limit=4000 \ +python ${BIN_DIR}/train.py \ + --train-metadata=dump/train/norm/metadata.jsonl \ + --dev-metadata=dump/dev/norm/metadata.jsonl \ + --config=${config_path} \ + --output-dir=${train_output_path} \ + --ngpu=1 diff --git a/examples/csmsc/voc6/path.sh b/examples/csmsc/voc6/path.sh new file mode 100755 index 0000000000000000000000000000000000000000..b0c98584deab72a8d42290daa10686eea5a766b7 --- /dev/null +++ b/examples/csmsc/voc6/path.sh @@ -0,0 +1,13 @@ +#!/bin/bash +export MAIN_ROOT=`realpath ${PWD}/../../../` + +export PATH=${MAIN_ROOT}:${MAIN_ROOT}/utils:${PATH} +export LC_ALL=C + +export PYTHONDONTWRITEBYTECODE=1 +# Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} + +MODEL=wavernn +export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL} \ No newline at end of file diff --git a/examples/csmsc/voc6/run.sh b/examples/csmsc/voc6/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..5f754fff32cfd5c34f8e0b2156877df60ab951b0 --- /dev/null +++ b/examples/csmsc/voc6/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +source path.sh + +gpus=0,1 +stage=0 +stop_stage=100 + +conf_path=conf/default.yaml +train_output_path=exp/default +test_input=dump/dump_gta_test +ckpt_name=snapshot_iter_100000.pdz + +source ${MAIN_ROOT}/utils/parse_options.sh || exit 1 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # prepare data + ./local/preprocess.sh ${conf_path} || exit -1 +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # prepare data + CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1 +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # synthesize + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 +fi diff --git a/examples/ljspeech/tts0/README.md b/examples/ljspeech/tts0/README.md deleted file mode 100644 index baaec818306b30f2a8a049a5ba2eeb2225b90fff..0000000000000000000000000000000000000000 --- a/examples/ljspeech/tts0/README.md +++ /dev/null @@ -1,89 +0,0 @@ -# Tacotron2 with LJSpeech -PaddlePaddle dynamic graph implementation of Tacotron2, a neural network architecture for speech synthesis directly from the text. The implementation is based on [Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram Predictions](https://arxiv.org/abs/1712.05884). - -## Dataset -We experiment with the LJSpeech dataset. Download and unzip [LJSpeech](https://keithito.com/LJ-Speech-Dataset/). - -```bash -wget https://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 -tar xjvf LJSpeech-1.1.tar.bz2 -``` -## Get Started -Assume the path to the dataset is `~/datasets/LJSpeech-1.1`. -Run the command below to -1. **source path**. -2. preprocess the dataset. -3. train the model. -4. synthesize mels. -```bash -./run.sh -``` -You can choose a range of stages you want to run, or set `stage` equal to `stop-stage` to use only one stage, for example, running the following command will only preprocess the dataset. -```bash -./run.sh --stage 0 --stop-stage 0 -``` -### Data Preprocessing -```bash -./local/preprocess.sh ${conf_path} -``` -### Model Training -`./local/train.sh` calls `${BIN_DIR}/train.py`. -```bash -CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} -``` -Here's the complete help message. -```text -usage: train.py [-h] [--config FILE] [--data DATA_DIR] [--output OUTPUT_DIR] - [--checkpoint_path CHECKPOINT_PATH] [--ngpu NGPU] [--opts ...] - -optional arguments: - -h, --help show this help message and exit - --config FILE path of the config file to overwrite to default config - with. - --data DATA_DIR path to the dataset. - --output OUTPUT_DIR path to save checkpoint and logs. - --checkpoint_path CHECKPOINT_PATH - path of the checkpoint to load - --ngpu NGPU if ngpu == 0, use cpu. - --opts ... options to overwrite --config file and the default - config, passing in KEY VALUE pairs -``` - -If you want to train on CPU, just set `--ngpu=0`. -If you want to train on multiple GPUs, just set `--ngpu` as the num of GPU. -By default, training will be resumed from the latest checkpoint in `--output`, if you want to start a new training, please use a new `${OUTPUTPATH}` with no checkpoint. -And if you want to resume from another existing model, you should set `checkpoint_path` to be the checkpoint path you want to load. -**Note: The checkpoint path cannot contain the file extension.** - -### Synthesizing -`./local/synthesize.sh` calls `${BIN_DIR}/synthesize.py`, which synthesize **mels** from text_list here. -```bash -CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${train_output_path} ${ckpt_name} -``` -```text -usage: synthesize.py [-h] [--config FILE] [--checkpoint_path CHECKPOINT_PATH] - [--input INPUT] [--output OUTPUT] [--ngpu NGPU] - [--opts ...] [-v] - -generate mel spectrogram with TransformerTTS. - -optional arguments: - -h, --help show this help message and exit - --config FILE extra config to overwrite the default config - --checkpoint_path CHECKPOINT_PATH - path of the checkpoint to load. - --input INPUT path of the text sentences - --output OUTPUT path to save outputs - --ngpu NGPU if ngpu == 0, use cpu. - --opts ... options to overwrite --config file and the default - config, passing in KEY VALUE pairs - -v, --verbose print msg -``` -**Ps.** You can use [waveflow](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/examples/ljspeech/voc0) as the neural vocoder to synthesize mels to wavs. (Please refer to `synthesize.sh` in our LJSpeech waveflow example) - -## Pretrained Models -Pretrained Models can be downloaded from the links below. We provide 2 models with different configurations. - -1. This model uses a binary classifier to predict the stop token. [tacotron2_ljspeech_ckpt_0.3.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/tacotron2/tacotron2_ljspeech_ckpt_0.3.zip) - -2. This model does not have a stop token predictor. It uses the attention peak position to decide whether all the contents have been uttered. Also, guided attention loss is used to speed up training. This model is trained with `configs/alternative.yaml`.[tacotron2_ljspeech_ckpt_0.3_alternative.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/tacotron2/tacotron2_ljspeech_ckpt_0.3_alternative.zip) diff --git a/examples/ljspeech/tts0/conf/default.yaml b/examples/ljspeech/tts0/conf/default.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d76ebd43211c9c4428cb714221ae121ff5566a47 --- /dev/null +++ b/examples/ljspeech/tts0/conf/default.yaml @@ -0,0 +1,87 @@ +# This configuration is for Paddle to train Tacotron 2. Compared to the +# original paper, this configuration additionally use the guided attention +# loss to accelerate the learning of the diagonal attention. It requires +# only a single GPU with 12 GB memory and it takes ~1 days to finish the +# training on Titan V. + +########################################################### +# FEATURE EXTRACTION SETTING # +########################################################### +fs: 22050 # Sampling rate. +n_fft: 1024 # FFT size (samples). +n_shift: 256 # Hop size (samples). 11.6ms +win_length: null # Window length (samples). + # If set to null, it will be the same as fft_size. +window: "hann" # Window function. +n_mels: 80 # Number of mel basis. +fmin: 80 # Minimum freq in mel basis calculation. (Hz) +fmax: 7600 # Maximum frequency in mel basis calculation. (Hz) + +########################################################### +# DATA SETTING # +########################################################### +batch_size: 64 +num_workers: 2 + +########################################################### +# MODEL SETTING # +########################################################### +model: # keyword arguments for the selected model + embed_dim: 512 # char or phn embedding dimension + elayers: 1 # number of blstm layers in encoder + eunits: 512 # number of blstm units + econv_layers: 3 # number of convolutional layers in encoder + econv_chans: 512 # number of channels in convolutional layer + econv_filts: 5 # filter size of convolutional layer + atype: location # attention function type + adim: 512 # attention dimension + aconv_chans: 32 # number of channels in convolutional layer of attention + aconv_filts: 15 # filter size of convolutional layer of attention + cumulate_att_w: True # whether to cumulate attention weight + dlayers: 2 # number of lstm layers in decoder + dunits: 1024 # number of lstm units in decoder + prenet_layers: 2 # number of layers in prenet + prenet_units: 256 # number of units in prenet + postnet_layers: 5 # number of layers in postnet + postnet_chans: 512 # number of channels in postnet + postnet_filts: 5 # filter size of postnet layer + output_activation: null # activation function for the final output + use_batch_norm: True # whether to use batch normalization in encoder + use_concate: True # whether to concatenate encoder embedding with decoder outputs + use_residual: False # whether to use residual connection in encoder + dropout_rate: 0.5 # dropout rate + zoneout_rate: 0.1 # zoneout rate + reduction_factor: 1 # reduction factor + spk_embed_dim: null # speaker embedding dimension + + +########################################################### +# UPDATER SETTING # +########################################################### +updater: + use_masking: True # whether to apply masking for padded part in loss calculation + bce_pos_weight: 5.0 # weight of positive sample in binary cross entropy calculation + use_guided_attn_loss: True # whether to use guided attention loss + guided_attn_loss_sigma: 0.4 # sigma of guided attention loss + guided_attn_loss_lambda: 1.0 # strength of guided attention loss + + +########################################################## +# OPTIMIZER SETTING # +########################################################## +optimizer: + optim: adam # optimizer type + learning_rate: 1.0e-03 # learning rate + epsilon: 1.0e-06 # epsilon + weight_decay: 0.0 # weight decay coefficient + +########################################################### +# TRAINING SETTING # +########################################################### +max_epoch: 300 +num_snapshots: 5 + +########################################################### +# OTHER SETTING # +########################################################### +seed: 42 diff --git a/examples/ljspeech/tts0/local/preprocess.sh b/examples/ljspeech/tts0/local/preprocess.sh index c39a3172d044e46c11a7aee53ae51a18d5092fee..e0e4bc7a617962b4c12244491cc3fd4cc0d7e9d6 100755 --- a/examples/ljspeech/tts0/local/preprocess.sh +++ b/examples/ljspeech/tts0/local/preprocess.sh @@ -1,8 +1,62 @@ #!/bin/bash -preprocess_path=$1 +stage=0 +stop_stage=100 -python3 ${BIN_DIR}/preprocess.py \ - --input=~/datasets/LJSpeech-1.1 \ - --output=${preprocess_path} \ - -v \ \ No newline at end of file +config_path=$1 + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + # get durations from MFA's result + echo "Generate durations.txt from MFA results ..." + python3 ${MAIN_ROOT}/utils/gen_duration_from_textgrid.py \ + --inputdir=./ljspeech_alignment \ + --output=durations.txt \ + --config=${config_path} +fi + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + # extract features + echo "Extract features ..." + python3 ${BIN_DIR}/preprocess.py \ + --dataset=ljspeech \ + --rootdir=~/datasets/LJSpeech-1.1/ \ + --dumpdir=dump \ + --dur-file=durations.txt \ + --config=${config_path} \ + --num-cpu=20 \ + --cut-sil=True +fi + +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + # get features' stats(mean and std) + echo "Get features' stats ..." + python3 ${MAIN_ROOT}/utils/compute_statistics.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --field-name="speech" + +fi + +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # normalize and covert phone to id, dev and test should use train's stats + echo "Normalize ..." + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/train/raw/metadata.jsonl \ + --dumpdir=dump/train/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/dev/raw/metadata.jsonl \ + --dumpdir=dump/dev/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt + + python3 ${BIN_DIR}/normalize.py \ + --metadata=dump/test/raw/metadata.jsonl \ + --dumpdir=dump/test/norm \ + --speech-stats=dump/train/speech_stats.npy \ + --phones-dict=dump/phone_id_map.txt \ + --speaker-dict=dump/speaker_id_map.txt +fi diff --git a/examples/ljspeech/tts0/local/synthesize.sh b/examples/ljspeech/tts0/local/synthesize.sh index 3f5f9c067e739bcf40d4e9d53beae3e2e601a714..0d005820dc5dc8197133b06dabb0185b6eb2752e 100755 --- a/examples/ljspeech/tts0/local/synthesize.sh +++ b/examples/ljspeech/tts0/local/synthesize.sh @@ -1,11 +1,20 @@ #!/bin/bash -train_output_path=$1 -ckpt_name=$2 +config_path=$1 +train_output_path=$2 +ckpt_name=$3 -python3 ${BIN_DIR}/synthesize.py \ - --config=${train_output_path}/config.yaml \ - --checkpoint_path=${train_output_path}/checkpoints/${ckpt_name} \ - --input=${BIN_DIR}/../sentences_en.txt \ - --output=${train_output_path}/test \ - --ngpu=1 +FLAGS_allocator_strategy=naive_best_fit \ +FLAGS_fraction_of_gpu_memory_to_use=0.01 \ +python3 ${BIN_DIR}/../synthesize.py \ + --am=tacotron2_ljspeech \ + --am_config=${config_path} \ + --am_ckpt=${train_output_path}/checkpoints/${ckpt_name} \ + --am_stat=dump/train/speech_stats.npy \ + --voc=pwgan_ljspeech \ + --voc_config=pwg_ljspeech_ckpt_0.5/pwg_default.yaml \ + --voc_ckpt=pwg_ljspeech_ckpt_0.5/pwg_snapshot_iter_400000.pdz \ + --voc_stat=pwg_ljspeech_ckpt_0.5/pwg_stats.npy \ + --test_metadata=dump/test/norm/metadata.jsonl \ + --output_dir=${train_output_path}/test \ + --phones_dict=dump/phone_id_map.txt diff --git a/examples/ljspeech/tts0/local/train.sh b/examples/ljspeech/tts0/local/train.sh index a94f955a2abdd6a30373de7be5468525b4f7196e..f90db91505d7ff337824fc716212f566754cb5d8 100755 --- a/examples/ljspeech/tts0/local/train.sh +++ b/examples/ljspeech/tts0/local/train.sh @@ -1,9 +1,12 @@ #!/bin/bash -preprocess_path=$1 +config_path=$1 train_output_path=$2 python3 ${BIN_DIR}/train.py \ - --data=${preprocess_path} \ - --output=${train_output_path} \ - --ngpu=1 \ \ No newline at end of file + --train-metadata=dump/train/norm/metadata.jsonl \ + --dev-metadata=dump/dev/norm/metadata.jsonl \ + --config=${config_path} \ + --output-dir=${train_output_path} \ + --ngpu=1 \ + --phones-dict=dump/phone_id_map.txt \ No newline at end of file diff --git a/examples/ljspeech/tts0/path.sh b/examples/ljspeech/tts0/path.sh index a37cd21e3210967415d6842067f9294bfbce7b5e..9cdbe256e11a068a5471fed1ac392804cf615ac1 100755 --- a/examples/ljspeech/tts0/path.sh +++ b/examples/ljspeech/tts0/path.sh @@ -9,5 +9,5 @@ export PYTHONDONTWRITEBYTECODE=1 export PYTHONIOENCODING=UTF-8 export PYTHONPATH=${MAIN_ROOT}:${PYTHONPATH} -MODEL=tacotron2 +MODEL=new_tacotron2 export BIN_DIR=${MAIN_ROOT}/paddlespeech/t2s/exps/${MODEL} diff --git a/examples/ljspeech/tts0/run.sh b/examples/ljspeech/tts0/run.sh index 47c76c3d2c8b766802b81ca0e159623b709eba84..c64fa8883220db1b019d56056fe7c06033176573 100755 --- a/examples/ljspeech/tts0/run.sh +++ b/examples/ljspeech/tts0/run.sh @@ -3,13 +3,13 @@ set -e source path.sh -gpus=0 +gpus=0,1 stage=0 stop_stage=100 -preprocess_path=preprocessed_ljspeech -train_output_path=output -ckpt_name=step-35000 +conf_path=conf/default.yaml +train_output_path=exp/default +ckpt_name=snapshot_iter_201.pdz # with the following command, you can choose the stage range you want to run # such as `./run.sh --stage 0 --stop-stage 0` @@ -18,16 +18,20 @@ source ${MAIN_ROOT}/utils/parse_options.sh || exit 1 if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then # prepare data - ./local/preprocess.sh ${preprocess_path} || exit -1 + ./local/preprocess.sh ${conf_path} || exit -1 fi if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then # train model, all `ckpt` under `train_output_path/checkpoints/` dir - CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${preprocess_path} ${train_output_path} || exit -1 + CUDA_VISIBLE_DEVICES=${gpus} ./local/train.sh ${conf_path} ${train_output_path} || exit -1 fi if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then - # train model, all `ckpt` under `train_output_path/checkpoints/` dir - CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${train_output_path} ${ckpt_name} || exit -1 + # synthesize, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 fi +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # synthesize_e2e, vocoder is pwgan + CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize_e2e.sh ${conf_path} ${train_output_path} ${ckpt_name} || exit -1 +fi diff --git a/examples/ljspeech/voc0/run.sh b/examples/ljspeech/voc0/run.sh index ddd82cb44936431212aafb7a3b7fbca47eac0860..b040c0b241e110a292011b1a87fbe52cd0d9dce7 100755 --- a/examples/ljspeech/voc0/run.sh +++ b/examples/ljspeech/voc0/run.sh @@ -10,7 +10,7 @@ stop_stage=100 preprocess_path=preprocessed_ljspeech train_output_path=output # mel generated by Tacotron2 -input_mel_path=../tts0/output/test +input_mel_path=${preprocess_path}/mel_test ckpt_name=step-10000 # with the following command, you can choose the stage range you want to run @@ -28,5 +28,7 @@ if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then fi if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + mkdir -p ${preprocess_path}/mel_test + cp ${preprocess_path}/mel/LJ050-001*.npy ${preprocess_path}/mel_test/ CUDA_VISIBLE_DEVICES=${gpus} ./local/synthesize.sh ${input_mel_path} ${train_output_path} ${ckpt_name} || exit -1 fi diff --git a/examples/thchs30/align0/README.md b/examples/thchs30/align0/README.md index da56fffc85bb1f54c66861f68a7fc42544111731..5195ab80b56f044dc1ae246cb176ff9a886fe1ec 100644 --- a/examples/thchs30/align0/README.md +++ b/examples/thchs30/align0/README.md @@ -27,7 +27,7 @@ cd a0 应用程序会自动下载 THCHS-30数据集,处理成 MFA 所需的文件格式并开始训练,您可以修改 `run.sh` 中的参数 `LEXICON_NAME` 来决定您需要强制对齐的级别(word、syllable 和 phone) ## MFA 所使用的字典 --- -MFA 字典的格式请参考: [MFA 官方文档 Dictionary format ](https://montreal-forced-aligner.readthedocs.io/en/latest/dictionary.html) +MFA 字典的格式请参考: [MFA 官方文档](https://montreal-forced-aligner.readthedocs.io/en/latest/) phone.lexicon 直接使用的是 `THCHS-30/data_thchs30/lm_phone/lexicon.txt` word.lexicon 考虑到了中文的多音字,使用**带概率的字典**, 生成规则请参考 `local/gen_word2phone.py` `syllable.lexicon` 获取自 [DNSun/thchs30-pinyin2tone](https://github.com/DNSun/thchs30-pinyin2tone) @@ -39,4 +39,4 @@ word.lexicon 考虑到了中文的多音字,使用**带概率的字典**, 生 **syllabel 级别:** [syllable.lexicon](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/syllable/syllable.lexicon)、[对齐结果](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/syllable/thchs30_alignment.tar.gz)、[模型](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/syllable/thchs30_model.zip) **word 级别:** [word.lexicon](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/word/word.lexicon)、[对齐结果](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/word/thchs30_alignment.tar.gz)、[模型](https://paddlespeech.bj.bcebos.com/MFA/THCHS30/word/thchs30_model.zip) -随后,您可以参考 [MFA 官方文档 Align using pretrained models](https://montreal-forced-aligner.readthedocs.io/en/stable/aligning.html#align-using-pretrained-models) 使用我们给您提供好的模型直接对自己的数据集进行强制对齐,注意,您需要使用和模型对应的 lexicon 文件,当文本是汉字时,您需要用空格把不同的**汉字**(而不是词语)分开 +随后,您可以参考 [MFA 官方文档](https://montreal-forced-aligner.readthedocs.io/en/latest/) 使用我们给您提供好的模型直接对自己的数据集进行强制对齐,注意,您需要使用和模型对应的 lexicon 文件,当文本是汉字时,您需要用空格把不同的**汉字**(而不是词语)分开 diff --git a/paddlespeech/cli/asr/infer.py b/paddlespeech/cli/asr/infer.py index 447b0a1a0f0596b4a2a252ff326f88d2ea0c3960..6e14e0d682f0a3edb3e8908ea9774816b947f6fb 100644 --- a/paddlespeech/cli/asr/infer.py +++ b/paddlespeech/cli/asr/infer.py @@ -91,6 +91,20 @@ pretrained_models = { 'lm_md5': '29e02312deb2e59b3c8686c7966d4fe3' }, + "deepspeech2offline_librispeech-en-16k": { + 'url': + 'https://paddlespeech.bj.bcebos.com/s2t/librispeech/asr0/asr0_deepspeech2_librispeech_ckpt_0.1.1.model.tar.gz', + 'md5': + 'f5666c81ad015c8de03aac2bc92e5762', + 'cfg_path': + 'model.yaml', + 'ckpt_path': + 'exp/deepspeech2/checkpoints/avg_1', + 'lm_url': + 'https://deepspeech.bj.bcebos.com/en_lm/common_crawl_00.prune01111.trie.klm', + 'lm_md5': + '099a601759d467cd0a8523ff939819c5' + }, } model_alias = { @@ -171,8 +185,9 @@ class ASRExecutor(BaseExecutor): """ Download and returns pretrained resources path of current task. """ - assert tag in pretrained_models, 'Can not find pretrained resources of {}.'.format( - tag) + support_models = list(pretrained_models.keys()) + assert tag in pretrained_models, 'The model "{}" you want to use has not been supported, please choose other models.\nThe support models includes:\n\t\t{}\n'.format( + tag, '\n\t\t'.join(support_models)) res_path = os.path.join(MODEL_HOME, tag) decompressed_path = download_and_decompress(pretrained_models[tag], @@ -328,18 +343,15 @@ class ASRExecutor(BaseExecutor): audio = self._inputs["audio"] audio_len = self._inputs["audio_len"] if "deepspeech2online" in model_type or "deepspeech2offline" in model_type: - result_transcripts = self.model.decode( - audio, - audio_len, - self.text_feature.vocab_list, - decoding_method=cfg.decoding_method, - lang_model_path=cfg.lang_model_path, - beam_alpha=cfg.alpha, - beam_beta=cfg.beta, - beam_size=cfg.beam_size, - cutoff_prob=cfg.cutoff_prob, - cutoff_top_n=cfg.cutoff_top_n, - num_processes=cfg.num_proc_bsearch) + decode_batch_size = audio.shape[0] + self.model.decoder.init_decoder( + decode_batch_size, self.text_feature.vocab_list, + cfg.decoding_method, cfg.lang_model_path, cfg.alpha, cfg.beta, + cfg.beam_size, cfg.cutoff_prob, cfg.cutoff_top_n, + cfg.num_proc_bsearch) + + result_transcripts = self.model.decode(audio, audio_len) + self.model.decoder.del_decoder() self._outputs["result"] = result_transcripts[0] elif "conformer" in model_type or "transformer" in model_type: diff --git a/paddlespeech/cli/utils.py b/paddlespeech/cli/utils.py index 4f2c89065993885ecbda1c8312018192ba96ce43..d7dcc90c7abbb4a774f74640061bc3936e440e0d 100644 --- a/paddlespeech/cli/utils.py +++ b/paddlespeech/cli/utils.py @@ -34,7 +34,7 @@ from .entry import commands try: from .. import __version__ except ImportError: - __version__ = 0.0.0 # for develop branch + __version__ = "0.0.0" # for develop branch requests.adapters.DEFAULT_RETRIES = 3 diff --git a/paddlespeech/s2t/io/sampler.py b/paddlespeech/s2t/io/sampler.py index ac55af1236f11d175e9e7717220980cf95c7d79b..89752bb9fdb98faecc0ccc5b8f59ea1f09efc8b6 100644 --- a/paddlespeech/s2t/io/sampler.py +++ b/paddlespeech/s2t/io/sampler.py @@ -51,7 +51,7 @@ def _batch_shuffle(indices, batch_size, epoch, clipped=False): """ rng = np.random.RandomState(epoch) shift_len = rng.randint(0, batch_size - 1) - batch_indices = list(zip(* [iter(indices[shift_len:])] * batch_size)) + batch_indices = list(zip(*[iter(indices[shift_len:])] * batch_size)) rng.shuffle(batch_indices) batch_indices = [item for batch in batch_indices for item in batch] assert clipped is False diff --git a/paddlespeech/t2s/audio/__init__.py b/paddlespeech/t2s/audio/__init__.py index 7747b794536aab1aab1de18dc0b686b8814efdbb..0deefc8bc368fa5e4f6236569d539082fb4c37cb 100644 --- a/paddlespeech/t2s/audio/__init__.py +++ b/paddlespeech/t2s/audio/__init__.py @@ -12,5 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. from .audio import AudioProcessor +from .codec import * from .spec_normalizer import LogMagnitude from .spec_normalizer import NormalizerBase diff --git a/paddlespeech/t2s/audio/codec.py b/paddlespeech/t2s/audio/codec.py new file mode 100644 index 0000000000000000000000000000000000000000..2a759ce4ce231485204ac2b7b88f6d4943cc534a --- /dev/null +++ b/paddlespeech/t2s/audio/codec.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math + +import numpy as np +import paddle + + +# x: [0: 2**bit-1], return: [-1, 1] +def label_2_float(x, bits): + return 2 * x / (2**bits - 1.) - 1. + + +#x: [-1, 1], return: [0, 2**bits-1] +def float_2_label(x, bits): + assert abs(x).max() <= 1.0 + x = (x + 1.) * (2**bits - 1) / 2 + return x.clip(0, 2**bits - 1) + + +# y: [-1, 1], mu: 2**bits, return: [0, 2**bits-1] +# see https://en.wikipedia.org/wiki/%CE%9C-law_algorithm +# be careful the input `mu` here, which is +1 than that of the link above +def encode_mu_law(x, mu): + mu = mu - 1 + fx = np.sign(x) * np.log(1 + mu * np.abs(x)) / np.log(1 + mu) + return np.floor((fx + 1) / 2 * mu + 0.5) + + +# from_labels = True: +# y: [0: 2**bit-1], mu: 2**bits, return: [-1,1] +# from_labels = False: +# y: [-1, 1], return: [-1, 1] +def decode_mu_law(y, mu, from_labels=True): + # TODO: get rid of log2 - makes no sense + if from_labels: + y = label_2_float(y, math.log2(mu)) + mu = mu - 1 + x = paddle.sign(y) / mu * ((1 + mu)**paddle.abs(y) - 1) + return x diff --git a/paddlespeech/t2s/datasets/am_batch_fn.py b/paddlespeech/t2s/datasets/am_batch_fn.py index 2fcb46d9e281ef4ef14d61adbd119e21d9bc94d0..655e06e37a9cd535fd2db7028392e4c5f3fc4765 100644 --- a/paddlespeech/t2s/datasets/am_batch_fn.py +++ b/paddlespeech/t2s/datasets/am_batch_fn.py @@ -46,6 +46,47 @@ def tacotron2_single_spk_batch_fn(examples): return batch +def tacotron2_multi_spk_batch_fn(examples): + # fields = ["text", "text_lengths", "speech", "speech_lengths"] + text = [np.array(item["text"], dtype=np.int64) for item in examples] + speech = [np.array(item["speech"], dtype=np.float32) for item in examples] + text_lengths = [ + np.array(item["text_lengths"], dtype=np.int64) for item in examples + ] + speech_lengths = [ + np.array(item["speech_lengths"], dtype=np.int64) for item in examples + ] + + text = batch_sequences(text) + speech = batch_sequences(speech) + + # convert each batch to paddle.Tensor + text = paddle.to_tensor(text) + speech = paddle.to_tensor(speech) + text_lengths = paddle.to_tensor(text_lengths) + speech_lengths = paddle.to_tensor(speech_lengths) + + batch = { + "text": text, + "text_lengths": text_lengths, + "speech": speech, + "speech_lengths": speech_lengths, + } + # spk_emb has a higher priority than spk_id + if "spk_emb" in examples[0]: + spk_emb = [ + np.array(item["spk_emb"], dtype=np.float32) for item in examples + ] + spk_emb = batch_sequences(spk_emb) + spk_emb = paddle.to_tensor(spk_emb) + batch["spk_emb"] = spk_emb + elif "spk_id" in examples[0]: + spk_id = [np.array(item["spk_id"], dtype=np.int64) for item in examples] + spk_id = paddle.to_tensor(spk_id) + batch["spk_id"] = spk_id + return batch + + def speedyspeech_single_spk_batch_fn(examples): # fields = ["phones", "tones", "num_phones", "num_frames", "feats", "durations"] phones = [np.array(item["phones"], dtype=np.int64) for item in examples] diff --git a/paddlespeech/t2s/datasets/vocoder_batch_fn.py b/paddlespeech/t2s/datasets/vocoder_batch_fn.py index 2e4f740fb6f048dd91a1e799d598261a88a6419c..d969a1d3553712690c588663a10b41f7a07d5fc7 100644 --- a/paddlespeech/t2s/datasets/vocoder_batch_fn.py +++ b/paddlespeech/t2s/datasets/vocoder_batch_fn.py @@ -14,6 +14,10 @@ import numpy as np import paddle +from paddlespeech.t2s.audio.codec import encode_mu_law +from paddlespeech.t2s.audio.codec import float_2_label +from paddlespeech.t2s.audio.codec import label_2_float + class Clip(object): """Collate functor for training vocoders. @@ -49,7 +53,7 @@ class Clip(object): self.end_offset = -(self.batch_max_frames + aux_context_window) self.mel_threshold = self.batch_max_frames + 2 * aux_context_window - def __call__(self, examples): + def __call__(self, batch): """Convert into batch tensors. Parameters @@ -67,11 +71,11 @@ class Clip(object): """ # check length - examples = [ - self._adjust_length(b['wave'], b['feats']) for b in examples + batch = [ + self._adjust_length(b['wave'], b['feats']) for b in batch if b['feats'].shape[0] > self.mel_threshold ] - xs, cs = [b[0] for b in examples], [b[1] for b in examples] + xs, cs = [b[0] for b in batch], [b[1] for b in batch] # make batch with random cut c_lengths = [c.shape[0] for c in cs] @@ -89,7 +93,7 @@ class Clip(object): c_batch = np.stack( [c[start:end] for c, start, end in zip(cs, c_starts, c_ends)]) - # convert each batch to tensor, asuume that each item in batch has the same length + # convert each batch to tensor, assume that each item in batch has the same length y_batch = paddle.to_tensor( y_batch, dtype=paddle.float32).unsqueeze(1) # (B, 1, T) c_batch = paddle.to_tensor( @@ -120,3 +124,113 @@ class Clip(object): 0] * self.hop_size, f"wave length: ({len(x)}), mel length: ({c.shape[0]})" return x, c + + +class WaveRNNClip(Clip): + def __init__(self, + mode: str='RAW', + batch_max_steps: int=4500, + hop_size: int=300, + aux_context_window: int=2, + bits: int=9, + mu_law: bool=True): + self.mode = mode + self.mel_win = batch_max_steps // hop_size + 2 * aux_context_window + self.batch_max_steps = batch_max_steps + self.hop_size = hop_size + self.aux_context_window = aux_context_window + self.mu_law = mu_law + self.batch_max_frames = batch_max_steps // hop_size + self.mel_threshold = self.batch_max_frames + 2 * aux_context_window + if self.mode == 'MOL': + self.bits = 16 + else: + self.bits = bits + + def to_quant(self, wav): + if self.mode == 'RAW': + if self.mu_law: + quant = encode_mu_law(wav, mu=2**self.bits) + else: + quant = float_2_label(wav, bits=self.bits) + elif self.mode == 'MOL': + quant = float_2_label(wav, bits=16) + quant = quant.astype(np.int64) + return quant + + def __call__(self, batch): + # voc_pad = 2 this will pad the input so that the resnet can 'see' wider than input length + # max_offsets = n_frames - 2 - (mel_win + 2 * hp.voc_pad) = n_frames - 15 + """Convert into batch tensors. + + Parameters + ---------- + batch : list + list of tuple of the pair of audio and features. + Audio shape (T, ), features shape(T', C). + + Returns + ---------- + Tensor + Input signal batch (B, 1, T). + Tensor + Target signal batch (B, 1, T). + Tensor + Auxiliary feature batch (B, C, T'), where + T = (T' - 2 * aux_context_window) * hop_size. + + """ + # check length + batch = [ + self._adjust_length(b['wave'], b['feats']) for b in batch + if b['feats'].shape[0] > self.mel_threshold + ] + wav, mel = [b[0] for b in batch], [b[1] for b in batch] + # mel 此处需要转置 + mel = [x.T for x in mel] + max_offsets = [ + x.shape[-1] - 2 - (self.mel_win + 2 * self.aux_context_window) + for x in mel + ] + # the slice point of mel selecting randomly + mel_offsets = [np.random.randint(0, offset) for offset in max_offsets] + # the slice point of wav selecting randomly, which is behind 2(=pad) frames + sig_offsets = [(offset + self.aux_context_window) * self.hop_size + for offset in mel_offsets] + # mels.shape[1] = voc_seq_len // hop_length + 2 * voc_pad + mels = [ + x[:, mel_offsets[i]:mel_offsets[i] + self.mel_win] + for i, x in enumerate(mel) + ] + # label.shape[1] = voc_seq_len + 1 + wav = [self.to_quant(x) for x in wav] + + labels = [ + x[sig_offsets[i]:sig_offsets[i] + self.batch_max_steps + 1] + for i, x in enumerate(wav) + ] + + mels = np.stack(mels).astype(np.float32) + labels = np.stack(labels).astype(np.int64) + + mels = paddle.to_tensor(mels) + labels = paddle.to_tensor(labels, dtype='int64') + # x is input, y is label + x = labels[:, :self.batch_max_steps] + y = labels[:, 1:] + ''' + mode = RAW: + mu_law = True: + quant: bits = 9 0, 1, 2, ..., 509, 510, 511 int + mu_law = False + quant bits = 9 [0, 511] float + mode = MOL: + quant: bits = 16 [0. 65536] float + ''' + # x should be normalizes in.[0, 1] in RAW mode + x = label_2_float(paddle.cast(x, dtype='float32'), self.bits) + # y should be normalizes in.[0, 1] in MOL mode + if self.mode == 'MOL': + y = label_2_float(paddle.cast(y, dtype='float32'), self.bits) + + return x, y, mels diff --git a/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py b/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py index 13569b9995f6b723f3bd7f18617622d65bdf04fb..3fded29b12a180d3153d42ecb4dc810ce25fa900 100644 --- a/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py +++ b/paddlespeech/t2s/exps/fastspeech2/gen_gta_mel.py @@ -29,6 +29,7 @@ from paddlespeech.t2s.datasets.preprocess_utils import merge_silence from paddlespeech.t2s.models.fastspeech2 import FastSpeech2 from paddlespeech.t2s.models.fastspeech2 import StyleFastSpeech2Inference from paddlespeech.t2s.modules.normalizer import ZScore +from paddlespeech.t2s.utils import str2bool def evaluate(args, fastspeech2_config): @@ -196,9 +197,6 @@ def main(): parser.add_argument( "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/fastspeech2/preprocess.py b/paddlespeech/t2s/exps/fastspeech2/preprocess.py index b874b3a70a320bdee2e8ec142fb06e4504cd1ac5..fd6da2cb360af97bafdcc81ec662cb2020302793 100644 --- a/paddlespeech/t2s/exps/fastspeech2/preprocess.py +++ b/paddlespeech/t2s/exps/fastspeech2/preprocess.py @@ -35,6 +35,7 @@ from paddlespeech.t2s.datasets.preprocess_utils import get_input_token from paddlespeech.t2s.datasets.preprocess_utils import get_phn_dur from paddlespeech.t2s.datasets.preprocess_utils import get_spk_id_map from paddlespeech.t2s.datasets.preprocess_utils import merge_silence +from paddlespeech.t2s.utils import str2bool def process_sentence(config: Dict[str, Any], @@ -203,9 +204,6 @@ def main(): parser.add_argument( "--num-cpu", type=int, default=1, help="number of process.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/fastspeech2/train.py b/paddlespeech/t2s/exps/fastspeech2/train.py index 1dfa575a1810075c5dd276b3d65551308def2b3c..d71292b36a415dae33be6c52cccebaa070ab28f6 100644 --- a/paddlespeech/t2s/exps/fastspeech2/train.py +++ b/paddlespeech/t2s/exps/fastspeech2/train.py @@ -38,6 +38,7 @@ from paddlespeech.t2s.training.extensions.visualizer import VisualDL from paddlespeech.t2s.training.optimizer import build_optimizers from paddlespeech.t2s.training.seeding import seed_everything from paddlespeech.t2s.training.trainer import Trainer +from paddlespeech.t2s.utils import str2bool def train_sp(args, config): @@ -182,9 +183,6 @@ def main(): default=None, help="speaker id map file for multiple speaker model.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--voice-cloning", type=str2bool, diff --git a/paddlespeech/t2s/exps/gan_vocoder/parallelwave_gan/train.py b/paddlespeech/t2s/exps/gan_vocoder/parallelwave_gan/train.py index a7881d6bbb2d8dee9e3030124d2f04f97f598570..46ff67e14373a94399c3c92c9a074cd454e29f93 100644 --- a/paddlespeech/t2s/exps/gan_vocoder/parallelwave_gan/train.py +++ b/paddlespeech/t2s/exps/gan_vocoder/parallelwave_gan/train.py @@ -41,6 +41,7 @@ from paddlespeech.t2s.training.extensions.snapshot import Snapshot from paddlespeech.t2s.training.extensions.visualizer import VisualDL from paddlespeech.t2s.training.seeding import seed_everything from paddlespeech.t2s.training.trainer import Trainer +from paddlespeech.t2s.utils import str2bool def train_sp(args, config): @@ -204,8 +205,6 @@ def train_sp(args, config): def main(): # parse args and config and redirect to train_sp - def str2bool(str): - return True if str.lower() == 'true' else False parser = argparse.ArgumentParser( description="Train a ParallelWaveGAN model.") diff --git a/paddlespeech/t2s/exps/gan_vocoder/preprocess.py b/paddlespeech/t2s/exps/gan_vocoder/preprocess.py index 782fbdf212a72ec4eb74c116b978499f46b2c8a6..47d0a29217d6d6ea9a6e0c7fab5164b20bc5e380 100644 --- a/paddlespeech/t2s/exps/gan_vocoder/preprocess.py +++ b/paddlespeech/t2s/exps/gan_vocoder/preprocess.py @@ -30,6 +30,7 @@ from yacs.config import CfgNode from paddlespeech.t2s.data.get_feats import LogMelFBank from paddlespeech.t2s.datasets.preprocess_utils import get_phn_dur from paddlespeech.t2s.datasets.preprocess_utils import merge_silence +from paddlespeech.t2s.utils import str2bool def process_sentence(config: Dict[str, Any], @@ -165,9 +166,6 @@ def main(): parser.add_argument( "--dur-file", default=None, type=str, help="path to durations.txt.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/inference.py b/paddlespeech/t2s/exps/inference.py index 37afd0abcf4bffc67de9a0bc6437e1a316a865c2..26d7e2c089c79f9df3194f3f7f5de34cc58a2fd9 100644 --- a/paddlespeech/t2s/exps/inference.py +++ b/paddlespeech/t2s/exps/inference.py @@ -33,7 +33,7 @@ def main(): default='fastspeech2_csmsc', choices=[ 'speedyspeech_csmsc', 'fastspeech2_csmsc', 'fastspeech2_aishell3', - 'fastspeech2_vctk' + 'fastspeech2_vctk', 'tacotron2_csmsc' ], help='Choose acoustic model type of tts task.') parser.add_argument( @@ -54,7 +54,7 @@ def main(): default='pwgan_csmsc', choices=[ 'pwgan_csmsc', 'mb_melgan_csmsc', 'hifigan_csmsc', 'pwgan_aishell3', - 'pwgan_vctk' + 'pwgan_vctk', 'wavernn_csmsc' ], help='Choose vocoder type of tts task.') # other diff --git a/paddlespeech/t2s/exps/new_tacotron2/preprocess.py b/paddlespeech/t2s/exps/new_tacotron2/preprocess.py index 5fc6b590d3e4b03a5951793b260729a07bff0bc2..ffbeaad92f783b7f78f261cc1a8b4a24ad1f4947 100644 --- a/paddlespeech/t2s/exps/new_tacotron2/preprocess.py +++ b/paddlespeech/t2s/exps/new_tacotron2/preprocess.py @@ -33,6 +33,7 @@ from paddlespeech.t2s.datasets.preprocess_utils import get_input_token from paddlespeech.t2s.datasets.preprocess_utils import get_phn_dur from paddlespeech.t2s.datasets.preprocess_utils import get_spk_id_map from paddlespeech.t2s.datasets.preprocess_utils import merge_silence +from paddlespeech.t2s.utils import str2bool def process_sentence(config: Dict[str, Any], @@ -179,9 +180,6 @@ def main(): parser.add_argument( "--num-cpu", type=int, default=1, help="number of process.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/new_tacotron2/train.py b/paddlespeech/t2s/exps/new_tacotron2/train.py index 20f73f0cedfc7cbea44f952c703434f4e3e0cd60..a77331e746f23fc56025e8a07d1f7c22f0304f2a 100644 --- a/paddlespeech/t2s/exps/new_tacotron2/train.py +++ b/paddlespeech/t2s/exps/new_tacotron2/train.py @@ -27,6 +27,7 @@ from paddle.io import DataLoader from paddle.io import DistributedBatchSampler from yacs.config import CfgNode +from paddlespeech.t2s.datasets.am_batch_fn import tacotron2_multi_spk_batch_fn from paddlespeech.t2s.datasets.am_batch_fn import tacotron2_single_spk_batch_fn from paddlespeech.t2s.datasets.data_table import DataTable from paddlespeech.t2s.models.new_tacotron2 import Tacotron2 @@ -37,6 +38,7 @@ from paddlespeech.t2s.training.extensions.visualizer import VisualDL from paddlespeech.t2s.training.optimizer import build_optimizers from paddlespeech.t2s.training.seeding import seed_everything from paddlespeech.t2s.training.trainer import Trainer +from paddlespeech.t2s.utils import str2bool def train_sp(args, config): @@ -60,33 +62,38 @@ def train_sp(args, config): # dataloader has been too verbose logging.getLogger("DataLoader").disabled = True + fields = [ + "text", + "text_lengths", + "speech", + "speech_lengths", + ] + + converters = { + "speech": np.load, + } + if args.voice_cloning: + print("Training voice cloning!") + collate_fn = tacotron2_multi_spk_batch_fn + fields += ["spk_emb"] + converters["spk_emb"] = np.load + else: + print("single speaker tacotron2!") + collate_fn = tacotron2_single_spk_batch_fn + # construct dataset for training and validation with jsonlines.open(args.train_metadata, 'r') as reader: train_metadata = list(reader) train_dataset = DataTable( data=train_metadata, - fields=[ - "text", - "text_lengths", - "speech", - "speech_lengths", - ], - converters={ - "speech": np.load, - }, ) + fields=fields, + converters=converters, ) with jsonlines.open(args.dev_metadata, 'r') as reader: dev_metadata = list(reader) dev_dataset = DataTable( data=dev_metadata, - fields=[ - "text", - "text_lengths", - "speech", - "speech_lengths", - ], - converters={ - "speech": np.load, - }, ) + fields=fields, + converters=converters, ) # collate function and dataloader train_sampler = DistributedBatchSampler( @@ -100,7 +107,7 @@ def train_sp(args, config): train_dataloader = DataLoader( train_dataset, batch_sampler=train_sampler, - collate_fn=tacotron2_single_spk_batch_fn, + collate_fn=collate_fn, num_workers=config.num_workers) dev_dataloader = DataLoader( @@ -108,7 +115,7 @@ def train_sp(args, config): shuffle=False, drop_last=False, batch_size=config.batch_size, - collate_fn=tacotron2_single_spk_batch_fn, + collate_fn=collate_fn, num_workers=config.num_workers) print("dataloaders done!") @@ -166,6 +173,12 @@ def main(): parser.add_argument( "--phones-dict", type=str, default=None, help="phone vocabulary file.") + parser.add_argument( + "--voice-cloning", + type=str2bool, + default=False, + help="whether training voice cloning model.") + args = parser.parse_args() with open(args.config) as f: diff --git a/paddlespeech/t2s/exps/speedyspeech/gen_gta_mel.py b/paddlespeech/t2s/exps/speedyspeech/gen_gta_mel.py index b6440fd6f27611188cfca95fed7c2c7deac59b1c..31b7d2eac400f5e9172797ec4739eb422d7d9200 100644 --- a/paddlespeech/t2s/exps/speedyspeech/gen_gta_mel.py +++ b/paddlespeech/t2s/exps/speedyspeech/gen_gta_mel.py @@ -30,6 +30,7 @@ from paddlespeech.t2s.frontend.zh_frontend import Frontend from paddlespeech.t2s.models.speedyspeech import SpeedySpeech from paddlespeech.t2s.models.speedyspeech import SpeedySpeechInference from paddlespeech.t2s.modules.normalizer import ZScore +from paddlespeech.t2s.utils import str2bool def evaluate(args, speedyspeech_config): @@ -213,9 +214,6 @@ def main(): parser.add_argument( "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/speedyspeech/normalize.py b/paddlespeech/t2s/exps/speedyspeech/normalize.py index a427c46928cf92ba298b73313d78fe6b52a7fc14..249a4d6d83e59c933994a1532d0e836a0a8679c3 100644 --- a/paddlespeech/t2s/exps/speedyspeech/normalize.py +++ b/paddlespeech/t2s/exps/speedyspeech/normalize.py @@ -23,6 +23,7 @@ from sklearn.preprocessing import StandardScaler from tqdm import tqdm from paddlespeech.t2s.datasets.data_table import DataTable +from paddlespeech.t2s.utils import str2bool def main(): @@ -55,9 +56,6 @@ def main(): default=1, help="logging level. higher is more logging. (default=1)") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--use-relative-path", type=str2bool, diff --git a/paddlespeech/t2s/exps/speedyspeech/preprocess.py b/paddlespeech/t2s/exps/speedyspeech/preprocess.py index 9ff771442e4ef16cd5e9b87df664be7a5306329c..db888fbacd76d806c47fe58bae78e9902d6331b1 100644 --- a/paddlespeech/t2s/exps/speedyspeech/preprocess.py +++ b/paddlespeech/t2s/exps/speedyspeech/preprocess.py @@ -33,6 +33,7 @@ from paddlespeech.t2s.datasets.preprocess_utils import get_phn_dur from paddlespeech.t2s.datasets.preprocess_utils import get_phones_tones from paddlespeech.t2s.datasets.preprocess_utils import get_spk_id_map from paddlespeech.t2s.datasets.preprocess_utils import merge_silence +from paddlespeech.t2s.utils import str2bool def process_sentence(config: Dict[str, Any], @@ -190,9 +191,6 @@ def main(): parser.add_argument( "--num-cpu", type=int, default=1, help="number of process.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--cut-sil", type=str2bool, diff --git a/paddlespeech/t2s/exps/speedyspeech/train.py b/paddlespeech/t2s/exps/speedyspeech/train.py index 448cd7bbf356f1e16dba5bc33464bc0910b2b65f..de0d308bce549dc6f243c2f9db30ae9ef5e7e097 100644 --- a/paddlespeech/t2s/exps/speedyspeech/train.py +++ b/paddlespeech/t2s/exps/speedyspeech/train.py @@ -38,6 +38,7 @@ from paddlespeech.t2s.training.extensions.visualizer import VisualDL from paddlespeech.t2s.training.optimizer import build_optimizers from paddlespeech.t2s.training.seeding import seed_everything from paddlespeech.t2s.training.trainer import Trainer +from paddlespeech.t2s.utils import str2bool def train_sp(args, config): @@ -186,9 +187,6 @@ def main(): parser.add_argument( "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--use-relative-path", type=str2bool, diff --git a/paddlespeech/t2s/exps/synthesize.py b/paddlespeech/t2s/exps/synthesize.py index 02bfcb15d6f4b6e028314d24c21483ee53f5b183..d6dd7af1051b60539f3c7bc5e8867faf8f37b78c 100644 --- a/paddlespeech/t2s/exps/synthesize.py +++ b/paddlespeech/t2s/exps/synthesize.py @@ -25,6 +25,7 @@ from yacs.config import CfgNode from paddlespeech.s2t.utils.dynamic_import import dynamic_import from paddlespeech.t2s.datasets.data_table import DataTable from paddlespeech.t2s.modules.normalizer import ZScore +from paddlespeech.t2s.utils import str2bool model_alias = { # acoustic model @@ -97,6 +98,9 @@ def evaluate(args): fields = ["utt_id", "phones", "tones"] elif am_name == 'tacotron2': fields = ["utt_id", "text"] + if args.voice_cloning: + print("voice cloning!") + fields += ["spk_emb"] test_dataset = DataTable(data=test_metadata, fields=fields) @@ -178,7 +182,11 @@ def evaluate(args): mel = am_inference(phone_ids, tone_ids) elif am_name == 'tacotron2': phone_ids = paddle.to_tensor(datum["text"]) - mel = am_inference(phone_ids) + spk_emb = None + # multi speaker + if args.voice_cloning and "spk_emb" in datum: + spk_emb = paddle.to_tensor(np.load(datum["spk_emb"])) + mel = am_inference(phone_ids, spk_emb=spk_emb) # vocoder wav = voc_inference(mel) sf.write( @@ -199,7 +207,8 @@ def main(): default='fastspeech2_csmsc', choices=[ 'speedyspeech_csmsc', 'fastspeech2_csmsc', 'fastspeech2_ljspeech', - 'fastspeech2_aishell3', 'fastspeech2_vctk', 'tacotron2_csmsc' + 'fastspeech2_aishell3', 'fastspeech2_vctk', 'tacotron2_csmsc', + 'tacotron2_ljspeech', 'tacotron2_aishell3' ], help='Choose acoustic model type of tts task.') parser.add_argument( @@ -225,9 +234,6 @@ def main(): parser.add_argument( "--speaker_dict", type=str, default=None, help="speaker id map file.") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--voice-cloning", type=str2bool, diff --git a/paddlespeech/t2s/exps/synthesize_e2e.py b/paddlespeech/t2s/exps/synthesize_e2e.py index 8ebfcfe7f40a999f44eabe8351bce9aacc86c3e4..0b95a883299cf45c528b54e42be11eaed0d2b5b7 100644 --- a/paddlespeech/t2s/exps/synthesize_e2e.py +++ b/paddlespeech/t2s/exps/synthesize_e2e.py @@ -59,6 +59,10 @@ model_alias = { "paddlespeech.t2s.models.hifigan:HiFiGANGenerator", "hifigan_inference": "paddlespeech.t2s.models.hifigan:HiFiGANInference", + "wavernn": + "paddlespeech.t2s.models.wavernn:WaveRNN", + "wavernn_inference": + "paddlespeech.t2s.models.wavernn:WaveRNNInference", } @@ -151,10 +155,16 @@ def evaluate(args): voc_name = args.voc[:args.voc.rindex('_')] voc_class = dynamic_import(voc_name, model_alias) voc_inference_class = dynamic_import(voc_name + '_inference', model_alias) - voc = voc_class(**voc_config["generator_params"]) - voc.set_state_dict(paddle.load(args.voc_ckpt)["generator_params"]) - voc.remove_weight_norm() - voc.eval() + if voc_name != 'wavernn': + voc = voc_class(**voc_config["generator_params"]) + voc.set_state_dict(paddle.load(args.voc_ckpt)["generator_params"]) + voc.remove_weight_norm() + voc.eval() + else: + voc = voc_class(**voc_config["model"]) + voc.set_state_dict(paddle.load(args.voc_ckpt)["main_params"]) + voc.eval() + voc_mu, voc_std = np.load(args.voc_stat) voc_mu = paddle.to_tensor(voc_mu) voc_std = paddle.to_tensor(voc_std) @@ -178,10 +188,7 @@ def evaluate(args): am_inference = jit.to_static( am_inference, input_spec=[InputSpec([-1], dtype=paddle.int64)]) - paddle.jit.save(am_inference, - os.path.join(args.inference_dir, args.am)) - am_inference = paddle.jit.load( - os.path.join(args.inference_dir, args.am)) + elif am_name == 'speedyspeech': if am_dataset in {"aishell3", "vctk"} and args.speaker_dict: am_inference = jit.to_static( @@ -200,10 +207,13 @@ def evaluate(args): InputSpec([-1], dtype=paddle.int64) ]) - paddle.jit.save(am_inference, - os.path.join(args.inference_dir, args.am)) - am_inference = paddle.jit.load( - os.path.join(args.inference_dir, args.am)) + elif am_name == 'tacotron2': + am_inference = jit.to_static( + am_inference, input_spec=[InputSpec([-1], dtype=paddle.int64)]) + + paddle.jit.save(am_inference, os.path.join(args.inference_dir, args.am)) + am_inference = paddle.jit.load( + os.path.join(args.inference_dir, args.am)) # vocoder voc_inference = jit.to_static( @@ -285,7 +295,7 @@ def main(): choices=[ 'speedyspeech_csmsc', 'speedyspeech_aishell3', 'fastspeech2_csmsc', 'fastspeech2_ljspeech', 'fastspeech2_aishell3', 'fastspeech2_vctk', - 'tacotron2_csmsc' + 'tacotron2_csmsc', 'tacotron2_ljspeech' ], help='Choose acoustic model type of tts task.') parser.add_argument( @@ -322,7 +332,8 @@ def main(): default='pwgan_csmsc', choices=[ 'pwgan_csmsc', 'pwgan_ljspeech', 'pwgan_aishell3', 'pwgan_vctk', - 'mb_melgan_csmsc', 'style_melgan_csmsc', 'hifigan_csmsc' + 'mb_melgan_csmsc', 'style_melgan_csmsc', 'hifigan_csmsc', + 'wavernn_csmsc' ], help='Choose vocoder type of tts task.') diff --git a/paddlespeech/t2s/exps/tacotron2/config.py b/paddlespeech/t2s/exps/tacotron2/config.py deleted file mode 100644 index 0ce2df368424d6b2540278dc7c5daa23d478dbc6..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/config.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from yacs.config import CfgNode as CN - -_C = CN() -_C.data = CN( - dict( - batch_size=32, # batch size - valid_size=64, # the first N examples are reserved for validation - sample_rate=22050, # Hz, sample rate - n_fft=1024, # fft frame size - win_length=1024, # window size - hop_length=256, # hop size between ajacent frame - fmax=8000, # Hz, max frequency when converting to mel - fmin=0, # Hz, min frequency when converting to mel - n_mels=80, # mel bands - padding_idx=0, # text embedding's padding index - )) - -_C.model = CN( - dict( - vocab_size=37, # set this according to the frontend's vocab size - n_tones=None, - reduction_factor=1, # reduction factor - d_encoder=512, # embedding & encoder's internal size - encoder_conv_layers=3, # number of conv layer in tacotron2 encoder - encoder_kernel_size=5, # kernel size of conv layers in tacotron2 encoder - d_prenet=256, # hidden size of decoder prenet - d_attention_rnn=1024, # hidden size of the first rnn layer in tacotron2 decoder - d_decoder_rnn=1024, # hidden size of the second rnn layer in tacotron2 decoder - d_attention=128, # hidden size of decoder location linear layer - attention_filters=32, # number of filter in decoder location conv layer - attention_kernel_size=31, # kernel size of decoder location conv layer - d_postnet=512, # hidden size of decoder postnet - postnet_kernel_size=5, # kernel size of conv layers in postnet - postnet_conv_layers=5, # number of conv layer in decoder postnet - p_encoder_dropout=0.5, # droput probability in encoder - p_prenet_dropout=0.5, # droput probability in decoder prenet - p_attention_dropout=0.1, # droput probability of first rnn layer in decoder - p_decoder_dropout=0.1, # droput probability of second rnn layer in decoder - p_postnet_dropout=0.5, # droput probability in decoder postnet - d_global_condition=None, - use_stop_token=True, # wherther to use binary classifier to predict when to stop - use_guided_attention_loss=False, # whether to use guided attention loss - guided_attention_loss_sigma=0.2 # sigma in guided attention loss - )) - -_C.training = CN( - dict( - lr=1e-3, # learning rate - weight_decay=1e-6, # the coeff of weight decay - grad_clip_thresh=1.0, # the clip norm of grad clip. - plot_interval=1000, # plot attention and spectrogram - valid_interval=1000, # validation - save_interval=1000, # checkpoint - max_iteration=500000, # max iteration to train - )) - - -def get_cfg_defaults(): - """Get a yacs CfgNode object with default values for my_project.""" - # Return a clone so that the defaults will not be altered - # This is for the "local variable" use pattern - return _C.clone() diff --git a/paddlespeech/t2s/exps/tacotron2/ljspeech.py b/paddlespeech/t2s/exps/tacotron2/ljspeech.py deleted file mode 100644 index 08db2a646898db45bcd31594f36b9a26d6c4999c..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/ljspeech.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pickle -from pathlib import Path - -import numpy as np -from paddle.io import Dataset - -from paddlespeech.t2s.data.batch import batch_spec -from paddlespeech.t2s.data.batch import batch_text_id - - -class LJSpeech(Dataset): - """A simple dataset adaptor for the processed ljspeech dataset.""" - - def __init__(self, root): - self.root = Path(root).expanduser() - records = [] - with open(self.root / "metadata.pkl", 'rb') as f: - metadata = pickle.load(f) - for mel_name, text, ids in metadata: - mel_name = self.root / "mel" / (mel_name + ".npy") - records.append((mel_name, text, ids)) - self.records = records - - def __getitem__(self, i): - mel_name, _, ids = self.records[i] - mel = np.load(mel_name) - return ids, mel - - def __len__(self): - return len(self.records) - - -class LJSpeechCollector(object): - """A simple callable to batch LJSpeech examples.""" - - def __init__(self, padding_idx=0, padding_value=0., padding_stop_token=1.0): - self.padding_idx = padding_idx - self.padding_value = padding_value - self.padding_stop_token = padding_stop_token - - def __call__(self, examples): - texts = [] - mels = [] - text_lens = [] - mel_lens = [] - - for data in examples: - text, mel = data - text = np.array(text, dtype=np.int64) - text_lens.append(len(text)) - mels.append(mel) - texts.append(text) - mel_lens.append(mel.shape[1]) - - # Sort by text_len in descending order - texts = [ - i for i, _ in sorted( - zip(texts, text_lens), key=lambda x: x[1], reverse=True) - ] - mels = [ - i for i, _ in sorted( - zip(mels, text_lens), key=lambda x: x[1], reverse=True) - ] - - mel_lens = [ - i for i, _ in sorted( - zip(mel_lens, text_lens), key=lambda x: x[1], reverse=True) - ] - - mel_lens = np.array(mel_lens, dtype=np.int64) - text_lens = np.array(sorted(text_lens, reverse=True), dtype=np.int64) - - # Pad sequence with largest len of the batch - texts, _ = batch_text_id(texts, pad_id=self.padding_idx) - mels, _ = batch_spec(mels, pad_value=self.padding_value) - mels = np.transpose(mels, axes=(0, 2, 1)) - - return texts, mels, text_lens, mel_lens diff --git a/paddlespeech/t2s/exps/tacotron2/preprocess.py b/paddlespeech/t2s/exps/tacotron2/preprocess.py deleted file mode 100644 index 480b33315fdc318fd9c05932d2f30a697ac17d58..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/preprocess.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import os -import pickle -from pathlib import Path - -import numpy as np -import tqdm - -from paddlespeech.t2s.audio import AudioProcessor -from paddlespeech.t2s.audio import LogMagnitude -from paddlespeech.t2s.datasets import LJSpeechMetaData -from paddlespeech.t2s.exps.tacotron2.config import get_cfg_defaults -from paddlespeech.t2s.frontend import EnglishCharacter - - -def create_dataset(config, source_path, target_path, verbose=False): - # create output dir - target_path = Path(target_path).expanduser() - mel_path = target_path / "mel" - os.makedirs(mel_path, exist_ok=True) - - meta_data = LJSpeechMetaData(source_path) - frontend = EnglishCharacter() - processor = AudioProcessor( - sample_rate=config.data.sample_rate, - n_fft=config.data.n_fft, - n_mels=config.data.n_mels, - win_length=config.data.win_length, - hop_length=config.data.hop_length, - fmax=config.data.fmax, - fmin=config.data.fmin) - normalizer = LogMagnitude() - - records = [] - for (fname, text, _) in tqdm.tqdm(meta_data): - wav = processor.read_wav(fname) - mel = processor.mel_spectrogram(wav) - mel = normalizer.transform(mel) - ids = frontend(text) - mel_name = os.path.splitext(os.path.basename(fname))[0] - - # save mel spectrogram - records.append((mel_name, text, ids)) - np.save(mel_path / mel_name, mel) - if verbose: - print("save mel spectrograms into {}".format(mel_path)) - - # save meta data as pickle archive - with open(target_path / "metadata.pkl", 'wb') as f: - pickle.dump(records, f) - if verbose: - print("saved metadata into {}".format(target_path / "metadata.pkl")) - - print("Done.") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="create dataset") - parser.add_argument( - "--config", - type=str, - metavar="FILE", - help="extra config to overwrite the default config") - parser.add_argument( - "--input", type=str, help="path of the ljspeech dataset") - parser.add_argument( - "--output", type=str, help="path to save output dataset") - parser.add_argument( - "--opts", - nargs=argparse.REMAINDER, - help="options to overwrite --config file and the default config, passing in KEY VALUE pairs" - ) - parser.add_argument( - "-v", "--verbose", action="store_true", help="print msg") - - config = get_cfg_defaults() - args = parser.parse_args() - if args.config: - config.merge_from_file(args.config) - if args.opts: - config.merge_from_list(args.opts) - config.freeze() - print(config.data) - - create_dataset(config, args.input, args.output, args.verbose) diff --git a/paddlespeech/t2s/exps/tacotron2/synthesize.ipynb b/paddlespeech/t2s/exps/tacotron2/synthesize.ipynb deleted file mode 100644 index cc424311fb687ce0179911cc1c884a1a266e81c2..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/synthesize.ipynb +++ /dev/null @@ -1,342 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## TTS with Tacotron2 + Waveflow" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import paddle\n", - "from matplotlib import pyplot as plt\n", - "from IPython import display as ipd\n", - "%matplotlib inline\n", - "\n", - "from paddlespeech.t2s.utils import display\n", - "from paddlespeech.t2s.utils import layer_tools\n", - "paddle.set_device(\"gpu:0\")\n", - "\n", - "import sys\n", - "sys.path.append(\"../..\")\n", - "import examples" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tacotron2: synthesizer model\n", - "\n", - "Tacotron2 is used here as a phonemes to spectrogram model. Here we will use an alternative config. In this config, the tacotron2 model does not have a binary classifier to predict whether the generation should stop.\n", - "\n", - "Instead, the peak position is used as the criterion. When the peak position of the attention reaches the end of the encoder outputs, it implies that the content is exhausted. So we stop the generated after 10 frames." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from paddlespeech.t2s.models.tacotron2 import Tacotron2\n", - "from paddlespeech.t2s.frontend import EnglishCharacter" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data:\n", - " batch_size: 32\n", - " fmax: 8000\n", - " fmin: 0\n", - " hop_length: 256\n", - " n_fft: 1024\n", - " n_mels: 80\n", - " padding_idx: 0\n", - " sample_rate: 22050\n", - " valid_size: 64\n", - " win_length: 1024\n", - "model:\n", - " attention_filters: 32\n", - " attention_kernel_size: 31\n", - " d_attention: 128\n", - " d_attention_rnn: 1024\n", - " d_decoder_rnn: 1024\n", - " d_encoder: 512\n", - " d_global_condition: None\n", - " d_postnet: 512\n", - " d_prenet: 256\n", - " encoder_conv_layers: 3\n", - " encoder_kernel_size: 5\n", - " guided_attention_loss_sigma: 0.2\n", - " n_tones: None\n", - " p_attention_dropout: 0.1\n", - " p_decoder_dropout: 0.1\n", - " p_encoder_dropout: 0.5\n", - " p_postnet_dropout: 0.5\n", - " p_prenet_dropout: 0.5\n", - " postnet_conv_layers: 5\n", - " postnet_kernel_size: 5\n", - " reduction_factor: 1\n", - " use_guided_attention_loss: True\n", - " use_stop_token: False\n", - " vocab_size: 37\n", - "training:\n", - " grad_clip_thresh: 1.0\n", - " lr: 0.001\n", - " max_iteration: 500000\n", - " plot_interval: 1000\n", - " save_interval: 1000\n", - " valid_interval: 1000\n", - " weight_decay: 1e-06\n" - ] - } - ], - "source": [ - "from examples.tacotron2 import config as tacotron2_config\n", - "synthesizer_config = tacotron2_config.get_cfg_defaults()\n", - "synthesizer_config.merge_from_file(\"configs/alternative.yaml\")\n", - "print(synthesizer_config)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[checkpoint] Rank 0: loaded model from ../../pretrained/tacotron2/tacotron2_ljspeech_ckpt_0.3_alternative/step-50000.pdparams\n" - ] - } - ], - "source": [ - "frontend = EnglishCharacter()\n", - "model = Tacotron2.from_pretrained(\n", - " synthesizer_config, \"../../pretrained/tacotron2/tacotron2_ljspeech_ckpt_0.3_alternative/step-50000\")\n", - "model.eval()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 36%|███▋ | 365/1000 [00:01<00:02, 256.89it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "content exhausted!\n" - ] - } - ], - "source": [ - "sentence = \"Life was like a box of chocolates, you never know what you're gonna get.\" \n", - "sentence = paddle.to_tensor(frontend(sentence)).unsqueeze(0)\n", - "\n", - "with paddle.no_grad():\n", - " outputs = model.infer(sentence)\n", - "mel_output = outputs[\"mel_outputs_postnet\"][0].numpy().T\n", - "alignment = outputs[\"alignments\"][0].numpy().T" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEYCAYAAAB2qXBEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xcdZ3n/9f7nKrq+yWdhCY3SLgblQAygKCAgAquP5jZUQd0d+M8WJlx1NHxssDM6qi7s4Pjb1DHZXQyiuKsI+JtwBkUWQRUHDDhFm7BhEDIPeTe6U53V9X57B/ndLrS6Ut1d1XX6c7nmUc9+pxT5/Lp09X9zfl8bzIznHPOuYkKah2Ac8656c0LEuecc5PiBYlzzrlJ8YLEOefcpHhB4pxzblK8IHHOOTcpVStIJJ0q6YmS135JH5HUIeleSWuTr7OqFYNzzrnq01T0I5EUApuBc4EPALvN7CZJNwCzzOz6qgfhnHOuKqaqIHkL8JdmdoGk54GLzWyrpHnAA2Z26mjH51Rn9TRVPU7n3FFEIJUkZcKQqCFLoVG0dxwgwAgVESoiMtEU9KH4sEOKiO6oDoCsighDGNs25+naXRAV8NY3Ndmu3cWy9390dd89ZnZ5Ja5drswUXedq4DvJcqeZbU2WtwGdYx1cTxPn6tJqxeacOwopk0G53KH1YFY7Pa9dwI4zs/zeH/ySxqCftkwP7WEPXcV6zmtYT52KhBr8z3dXlOXhgycSWcD87B6yKpBTkQ9e9WLF4ty5u8gj9ywse//svBfmVOziZap6QSIpB1wJ3Dj0PTMzScM+Ekm6DrgOoJ7GqsbonHPpZRQtqnUQo5qKJ5IrgMfMbHuyvl3SvJLU1o7hDjKzFcAKgFZ1+IBgzrnKkFAuR9DcBMfMpvvEWWy+KMTm9ZLZkKFxKzz2nqVobxcEAdbbh8KAO7sWY/15sAiL4j9Jymbi82QyUChAvoAVCuw6uK1i4RoQke4/gVPR/PcaBtNaAHcBy5Pl5cCdUxCDc85NW9E4/tVCVZ9IJDUBbwb+qGTzTcAdkq4FNgDvqmYMzjk3nRlGMeWjtFe1IDGzbmD2kG27AK85d865MqU9tTVVrbacc672JDIL5lOc287+E5t55cyA4JQDhGuamXNnjtbn9hDs2ktx9x4sXwCLYLinASUteyODfAEgrj8BCMPB9yvAgKIXJM455ybDn0icc85NmAH5o7mOxDnnqko6MvWUpJUUhvHXTPxnTg0NbLv6NP7Hx77Bqu4T+MGLy2j4+SwWrNhL4eWnAbAwJEo6KSoQhIMdFoksTnUpgEAQGVbIU9zff0RYZuX3RB+LYZ7acs45NwkGxXSXI16QOOdcmsUdEtPNCxLn3PQ1kNYaaCV1aD0ABYQL53HwxDlsuizHf3nb/dz6+BK+/OplEBnzeQEsolAsHjrOCgWsZJ1CYfTrS0deu+JEkcq1AqsGL0iccy7FjLh6Js28IHHOuZTzJxLnnKuCoL4eMzvsv+tBexvWOZv9r2rj4OyAPWcWIBLZPfDQu8/g1A1rifr74zTUaCmp4VqDlb4HoAAFilNhCuKJSgZadVWu0VbSIdELEuecc5MQmRckzjnnJsifSJxzzk2KIfIW1jqMUXlB4pybdpTJsOHjZ3HJVY/yjo5VBIpYnDnA2nwbTx48nnu2L2X37ll03NtM68sF6h94nKi39/CTjNZcN6nzUBjGvdiLxbgupPRYK3Jo4sLSnuwV7NUO/kTinHNu0kTRpmIOwonzgsQ551Is7tnuBYlzzk2MhMKQ/EXL2HpeHflX97C4cxfZoEj7P0S8eOUsPrfvPCxfGJJ62sxxtunQallDjJQ06yWKz2VRGWmq4eYeqXAHQk9tOeecmzAzT20555ybpMifSJxzrkwSQXMzwZwODp44h6BgbD+nnsuv/nd+sPosWn7TSM+WenL7i7T/+lkKXV2VHyyxnHRWSbwKQwjDeGreQ9srF07casufSJxzzk2Yp7acc85Ngrfacs45NymG6Pee7c45NzJlcyiXRbkcam9l89sXELx5F7ObdtCWO8gLv13M6tfByfboYaPyRplM3FS3Uj3JJ1LXYoYVCkdOgFXpapuUp7aqGp2kdknfl7RG0nOSXi+pQ9K9ktYmX2dVMwbnnJvOBirby33VQrWv+iXgp2Z2GrAMeA64AbjPzE4G7kvWnXPODcMQRSv/VQtVS21JagMuBN4LYGb9QL+kq4CLk91uAx4Arq9WHM65lApCglyWYO4cotmtdC9qpvuYkMxBo/0LTeSe20tPUZy6d3U8gRUcln6yyOKJpWyUSahmiKO5sn0J8ArwDUnLgEeBDwOdZrY12Wcb0DncwZKuA64DqKeximE651x6mZH65r/VjC4DnAV8xczOBLoZksay+L8Zw/5XwsxWmNnZZnZ2lroqhumcc2kmonG8aqGaTySbgE1m9kiy/n3igmS7pHlmtlXSPGBHFWNwzqVNEBI0NVJ87Qm8fHETjefvJJfpZc+BgN5tTSz4OeQef4HCvv3x/iOlrSzConT/T70SjKP4icTMtgEbJZ2abLoUeBa4C1iebFsO3FmtGJxzbiZIe6utavcj+RDwbUk5YD3wh8SF1x2SrgU2AO+qcgzOOTdtGSKqUWusclW1IDGzJ4Czh3nr0mpe1zmXMhJhSwu2eD6985rZ8boc/W1Gbi+0f76J3IZdtHVtwQ72Eh08SLGMVlgKQ1CQ9EeMZmzLLQPylu6+4+mOzjnnjnryia2cc85NnJH+IVK8IHHOuZTzJxLn3FFJ2RxYBGFIeMxcDp7ayf7jc8igbo8x+9kiLat3UNy8lWJk8ZzrNo66jjAZEbcwc+tHIJ5qt5JPJJIuJx6+KgS+ZmY3DXn/OOJRR9qTfW4ws7tHO6cXJM45l3KV6kciKQRuAd5M3NdvpaS7zOzZkt3+O3CHmX1F0lLgbmDxaOdNd+LNOeeOcvHEVhXr2X4OsM7M1ifjH94OXDXMJVuT5TZgy1gn9ScS51xlBSGZ+cey79yF9BwTsGdZkbqOgxQLRWyL0bAjYP4vDhCu3URhz754jnSVOfCiFM9BkrD+/hmd1oqNe6rdOZJWlayvMLMVyfICYGPJe5uAc4cc/2ngZ5I+BDQBl411QS9InHMuxeJWW+OqbN9pZsP13yvXNcA3zexvJb0e+CdJrzGzaKQDvCBxzrmUq+DQJ5uBRSXrC5Ntpa4FLgcws3+XVA/MYZRxEb0gcc5NnkSm8xgKxx1D93GNvPLOgxw3Zwu7drfTvLKFBV8zwk1bKe7aEw+2WChw2AS55aanzA5NrWv9M7u11oAKD5GyEjhZ0hLiAuRq4N1D9nmZePSRb0p6FVBPPCXIiLwgcc65FDODvIUVOpcVJH0QuIe4ae+tZvaMpM8Cq8zsLuBjwD9K+jPizNp7zUYvsb0gcc65lKvkoI1Jn5C7h2z7VMnys8AF4zmnFyTOOZdicWor3T01vCBxzo1PMpIvgC2ez64zZ9H19gN86czbeaj7FFbuPp7oH44juH01xyUtTQ0oVDyOgJk86m8pHyLFOefchE2g+e+U84LEOedSzVNbzrmZQkK5HMHxC+l6zRwOzA/pWhIRZYyOnzVz8zXLkoEXt9Ay9qgakzfQI/4oUMbQJzXlBYlzzqWYGRQ9teWcc24yPLXlnJu+gpCgvo5gdgf9J8xl3+J6upaI3mMKqFhkzqqA2U/shbUbiAqTaJdV7qCNQx0FLbYq3LO9Krwgcc65FDOg4E8kzjnnJsNTW8656SUIUTZD0NwEx8zm5Svn0jfLKMwqEBw02p6HJd/dA1t3EHUfJCrkj0wxjSdVdWiOkaOjc+G4mae2nHPOTcLADIlpVtWCRNJLQBdQBApmdrakDuC7xHMAvwS8y8z2VDMO55ybztL+RDIVibc3mdkZJTN23QDcZ2YnA/cl684554YxMERKua9aqEVq6yrg4mT5NuAB4PoaxOGcA5AI6uoIjj2G3efPZ//xAf2zjHDxAU6fv4Xup2fRuCHDggehccNueGUP0d59cS/2kYynrqNksio3vKP9icSIJ5F/VNJ1ybZOM9uaLG8DOoc7UNJ1klZJWpWnr8phOudcOg30Izman0jeYGabJR0D3CtpTembZmaShv2vi5mtAFYAtKrDm3I4545aR3Vlu5ltTr7ukPQj4Bxgu6R5ZrZV0jxGmVDeOVdFQUjwmpM5/bY1vLphHaHW8vcvXkTPi3PI7QppvbuZA490cOraxw6lsYpmR81AialhR3FqS1KTpJaBZeAtwNPAXcDyZLflwJ3VisE556Y7AwpRUParFqr5RNIJ/Ejx/14ywD+b2U8lrQTukHQtsAF4VxVjcM65ae2oHmvLzNYDy4bZvgu4tFrXdc4NL5zdAZ1z6Dm+ja7jMuw9zYiaigT/8XiePthJtL+L5t4NnMIGsAhIUlkwmM4a6IWevA94b/QpYEdrQeKcc64yZkRlu6RjiSvKDVhpZtuqGpVzzjkgfuCb9qktSf8V+BTwc0DAlyV91sxurXZwzrkKkAgaG+k+/yS2XBBSbIogMpo2Biz86V4KGzbF09aWy9NaU24mpLY+AZyZ1G0gaTbwa8ALEuecq7qZUdm+i3jgxQFdyTbnnHNTYCY8kawDHpF0J3EdyVXAakkfBTCzm6sYn3POHdUGBm1Ms3IKkheS14CBDoQtlQ/HOVcRQUjQUE/QOZeouZ6+Y5t5+fciWp7JMOsho+mFPdjGrUTdPeXXjyjp7DZQRzJa/UgQDi6bT1g1KZb+2zdmQWJmnwGQ1GhmPdUPyTnn3AADiimfanfM6CS9XtKzwJpkfZmkv696ZM4552CGjP77ReCtxGNkYWZPSrqwqlE558ZFmQxWLKIwJOw8hmc/uZBrzn2Y1zU9xbZCG/e+spRX/XErxTXrwIxxz/4x3jlDxtOc2I1p2qe2AMxsow4f8dM/Jc45N0VmQqutjZLOB0xSFvgw8Fx1w3LOOQfJw+AMKEj+GPgSsADYDPwM+JNqBuWcG4M0mO+QUC6H9fYRtLfRdfZCml7K8NjXl/HY47lkLpEdYNvHPpdLpZnQ/PdUM3tP6QZJFwAPVSck55xzpdJezpfTpuzLZW5zzjlXBWYq+1ULIz6RSHo9cD4wd6AXe6IVCIc/yjnnXCUZtSsgyjVaaisHNCf7lPZi3w+8o5pBOedGIaEwhDAkaG2F9hbWfLKdH194C2vzc/noQ4s55X1PYPn+I44bNkeS9ryJI+0/oRELEjN7EHhQ0jfNbAOApABoNrP9UxWgc84d1QwsqtwTiaTLiRtQhcDXzOymYfZ5F/Dp+Oo8aWbvHu2c5dSR/LWkVklNwNPAs5I+Md7gnXPOTUyl6kgkhcAtwBXAUuAaSUuH7HMycCNwgZm9GvjIWPGV02prqZntl/Qe4CfADcCjwOfLONY5N0HKZCAMsf44RZU5biE7bmng9tfeSrdl6AgKbC/mWNvfyd984Wo+/v7LsL4+TgmfPjKtBSOmsJTJYJEdPmHVKPu7qVfBH8U5wDozWw8g6XbiEd2fLdnnfcAtZrYnvrbtGOuk5TyRZJOOiL8L3GVmedKfsnPOuRnBGPcTyRxJq0pe15WcbgGwsWR9U7Kt1CnAKZIekvRwkgobVTlPJP8AvAQ8CfxC0vHEFe7OOeeqzYDxtdraaWZnT+KKGeBk4GJgIfHf/dea2d7RDhiVmf0d8HclmzZIetMkgnTOjUWCMCR/wWvY8oY6eo/rh0ic/L/6+cCjl0JkWCF/KOcxVw8TDeQ/CoVxXcfGs7+riQqmtjYDi0rWFybbSm0CHkmyTy9K+i1xwbJypJOWM4x8p6SvS/pJsr4UWD7O4J1zzk2UjeM1upXAyZKWSMoBV5OM7F7iX4ifRpA0hzjVtX60k5ZTR/JN4B5gfrL+W8qoxR8gKZT0uKR/TdaXSHpE0jpJ302+Geecc8Mqv35krFZbZlYAPkj8N/054A4ze0bSZyVdmex2D7ArmYfqfuATZrZrtPOWU0cyx8zukHTjQCCSxjOM/MBowa3J+ueAL5jZ7ZK+ClwLfGUc53OutgamVCgZNPEQs8PXS6anVRiCAhQGEARYvgAWxYMqJscFdXUEc2ZzcOk8+ltDdvx+L/m9IY0v5Jj7ZJ7wseeJ+vpGjmukHIgUxzK0ZZabHirYvMnM7gbuHrLtUyXLBnw0eZWlnCeSbkmzSb4VSecB+8o5uaSFwH8AvpasC7gE+H6yy23ErcGcc84Nx6bxWFslPkqcQztR0kPAXMofIuWLwH9jcIiV2cDe5PEKhm96BkDSZO06gHoay7ycc87NQNN4rC0AzOwxSRcBpwICnk9q80cl6e3ADjN7VNLF4w3MzFYAKwBa1eH9VpxzR6+U/wUcsyBJutS/DVic7P8WSZjZzWMcegFwpaS3AfXEdSRfAtolZZKnkuGanjmXbkPrIUZbL5nnPK4LKXDov2ESymRRGPcu3/3OM9n5OmPeaTvo7jvA/q4GTvmrPmzNM/F87Llc3AN9qKF1NiPFPJ451126pLwgKaeO5MfAe4nTUi0lr1GZ2Y1mttDMFhM3Mft5MkHW/QymxpYDd44/bOecO0oMdEgs91UD5dSRLDSz0yt4zeuB2yX9T+Bx4OsVPLdzzs04aR/2rJyC5CeS3mJmP5voRczsAeCBZHk98cBhzk0fYzWtHem9IIyb3JqhTDbujQ6ggHBWGxw7F6KI7pNmsfOtvdQ/00DdXbNoe3En83v3Utj+CkRxSsr6+gjq67HCMKms0uVyUl1D4x/P/m7qpfxHU05B8jDwo2QukjxxhbuZWevohznnnKuI6d5qC7gZeD3wVNJRxTnn3BRSyv/yllOQbASe9kLEzVjDpXaSnuAKQxQGqK0V+vNEPT2HHxqGBJ1zsV17IAyJenpQLgdRhBobUVMD9PVjvX1EJy7A6jJk128jmtvOmmvbOfE1m7lo7loe3r2E096bJdqxE6KIohlKptS1JLWFFPdqH0+rsXL4r3a6lTeGVk2VU5CsBx5IBm08NDZDGc1/nXPOTVrtWmOVq5yC5MXklUtezjnnplLKh0grp2f7Z6YiEOeccyOYrqktSV80s49I+jHDfBtmduUwhzmXHsON0qsABcn2ZCRe1dfBMXMozGkmOFggqs/Q355jzylZ8i1QrDPybRHWXCTYlyHsFVEmrgAt1hnZ/SKqm0+xziCweJC9rEFoIMg294OM/MEsRKK5o4N8PqTtwYDsikZ+ta4N6992RF3FML90Vb1dLqXGP0PilBvtieSfkq///1QE4pxzbnjTttWWmT2aLJ5hZl8qfU/Sh4EHqxmYc865xHQtSEosJx5ssdR7h9nmXM0F9fUQhmA2OHFUZChQ3Cw3ac5LGKKWZqK2Jkxi8yXtHFjWS6bOaGroQ+qlu6uR4v4sQV9A4+aQsC+kd45RrDdUEASQ2ydUEBYa9fsDolz8XpCHTA9ku41Mb0ihXjRvLYBBlGsmu79I7pEnKXZ31/qWOTdpo9WRXAO8G1giqXRO3xZgd7UDc845F5u2qS3g18BWYA7wtyXbu4DV1QzKOedciela2W5mG4ANxMOjOJdOGmyBhUXxnB1maH4nL/3BMfzOFU/zPxfcTQB0RQE9liFvIf2EPHnweJ7oWkRXoY7st9o47XMHKLQ1kNlbRPu7OWb7CwBYoTDy9Sch5V0DXFrMkJ7tzjnnaskLEuecc5OhlD++jlqQJNPsfiuZ2dC52lNpZ8KQcE4HUWcHfXMa2PWaOnrO6aGzYz9dvVnq74UtN57EHz18GVYs+U20KD6+vg41NkB/nmMW7EG9/WS278T680T9/fHUuCNd3zsHuqmU8o/bqAWJmRUlHS8pZ2b9UxWUc865mGx6t9oasB54KGkCfKjRu4/+65xzU2S6ttoq8ULyCoj7kDjnnJtK0/2JZGD0X0mNZtYz1v7OVYsyGdTQQNDWijXUke9sZdtZjRxYZLDgIFGhl2PurqduS0jbizuxrh0Ud+8lioap6wAs3w9dXfHKvv1xc1wrrUsZ5rfX60ZcDaQ9tRWMtYOk10t6FliTrC+T9PdVj8w551zMxvGqgTELEuCLwFuBXQBm9iRwYTWDcs45l7DBCvdyXrVQVj8SM9soHVbZM3yuwLnJCEKCXBZIepMnvdUHBl3c9v5zqL9iB5fNf57O7H5e6p3NDx85m1lPhRxzRxE9tYaoPw8WUUiOLTsVNUL6y7lUmO6pLWCjpPMBk5SV9HHgubEOklQv6TeSnpT0jKSBupYlkh6RtE7SdyX59L3OOTeaGZDa+mPgA8ACYDNwRrI+lj7gEjNblhxzuaTzgM8BXzCzk4A9wLUTCdw5544W0z61ZWY7gXH3bDczAw4kq9nkZcAlxMPTA9wGfBr4ynjP72YWZXMEHe1Ex3VSrAvZ8bpGll39NDfO+ynzM3Fa9Yx7zqD9y7N4/NnFRLv3Eh04wCmsBAXxf8ZsSO/1cQUwTDt9b6Hl0iLlH8XR5iP5MqOEb2Z/OtbJkyFWHgVOAm4h7o+y18wGhlPdRPyk45xzbjjToGf7aKmtVcSFQD1wFrA2eZ0BlFWvYWZFMzsDWAicA5xWbmCSrpO0StKqPH3lHuacczNPyutIRpuP5DYASe8H3jDwFCHpq8Avx3MRM9sr6X7iuU3aJWWS8y0krncZ7pgVwAqAVnWkvDx2zrkqSvlfwHKa/84CWhmcXrc52TYqSXOBfFKINABvJq5ovx94B3A78Xzwd04gbpdSQX394ERQClA2AxIKw3jCqfp6or37DqvDCBob6b7wNF6+yjh36QvMyXWz4YVT2fjpU/jYv0PxQDdYxKmZ1RCIYrEYj8w7UIdhFWi66/UhLqXE9E5tDbgJeFzSNyXdBjwG/K8yjpsH3C9pNbASuNfM/hW4HviopHXAbODrEwvdOeeOEhVMbUm6XNLzSReMG0bZ7/clmaSzxzpnOa22viHpJ8C5SZjXm9m2Mo5bDZw5zPb1xPUlzjnnxlLByvakAdQtxBmiTcBKSXeZ2bND9msBPgw8Us55y50h8RzgjcmyAT8u8zh3tJAIW1qI+vritJMCFERYsYhyOchm0OxZYMb7HnqEixq2klX8QLyzWORdq49l3v/pYP+fh+x5pY8lrAYOH0LB8lWeEkfyFJdLp8p9LM8B1iX/oUfS7cBVwLND9vsfxFURnyjnpOUM2ngTccn0bPL6U0nlpLacc85VQuVSWwuAjSXrR3TBkHQWsMjM/q3c8Mp5InkbcIYlvb2SepLHgT8v9yLOOecmbpxzts+RtKpkfUXSCnbs60gBcDPw3vFcsNzUVjuDrbbaxnMBN8MFIWFbK9FJC+n5qy6uXriSjswB6pXnxOwuioiuKMe2Qjsb8x3sKzRy65svYsXGLYcNlDgniOdPK9Zy8ERPa7k0Gn//kJ1mNlIF+WZgUcn60C4YLcBrgAeSgXqPBe6SdKWZlRZOhymnIPlr4lZb9xO3RLsQGLGm3znnXGVVsPnvSuBkSUuIC5CrGRyyCjPbB8w5dF3pAeDjoxUiUF6rre8kJ/udZFNZrbacc85VSIUKEjMrSPogcA8QArea2TOSPgusMrO7JnLeMQsSSb8H/HzgApLaJf2umf3LRC7ophkJ5XIEJxwHr+ymuGs3CkN63n4W26/pZfmrHqEts4Nf7hG7P7GIO586gejAgThNNDAQ4hEpo41HXIaoOPzAiZOM3dNVbiaoZIdEM7sbuHvItk+NsO/F5ZyznA6Jf5k87gyceC/wl+Wc3DnnXAVM17G2SgxX2JRbSe+cc24yalhAlKucJ5JVkm6WdGLyupl4VGDnnHNVpnG+aqGcJ4sPAZ8Evpus30t5MyS6aU6ZDMpkCObO4eW3z6V3zhwKswqEzXlm/zSk8zv1/GrtGWh/N0QRmf4tFAfqR6D29RO1vr5zlZLyj3I5rba68ea+zjlXM2kf/becVlunAB8HFpfub2aXVC8s55xzh4xz5uipVk5q63vAV4GvcfgYem6mCkIynXMpzp/NgeOa2XNqSPYA1O8S0d4s9bszzH54B/byZop9fYMppCAsL500XLPcSjb99Wa/biaZBlPtllOQFMzsK1WPxDnn3PBmQEHyY0l/AvwIBidPN7PdIx/inHOuUmbCE8ny5GvpuPQGnFD5cFy1KZuDQBDZkfN7JOmlA79/NlsuL3LF6U/zq81LyN3TTue3VmP9eayQBwUULToyfTSZARcrmYrytJabaVL+kS6n1daSqQjEOefc8NL+RDJih0RJ/61k+Z1D3vOJrZxzbiqMZ3iUGhU4o/Vsv7pk+cYh711ehVicc84NJ+UFyWipLY2wPNy6q5XSEXZLm9AqGLbOwgp5guZmJFEsFg+NuqtMlmDxQvJfzdMWbKT3O8fz0p/lmNe1BoUhUaFQcpIR6kIkgro6rBjFdSkDcQAKhEUWfy0919Dvxes3nDuMSH9qa7SCxEZYHm7dOedctaT8L+5oBckySfuJC8SGZJlkvb7qkTnnnANAKX9SH7EgMbNwKgNxE1T6ATtsuYiyOYKGkjI/m4EgxHp6sIF9JRSGKJdFhSLZD9ZTXLOOOdoeN/EFrFiEIESBDqWqrFiEYZoAR729Q+KL02DJqQ59HfN7cc7FDJTyIVLKGUZ+QiQtknS/pGclPSPpw8n2Dkn3SlqbfJ1VrRicc25GSHlle9UKEqAAfMzMlgLnAR+QtJR4JOH7zOxk4D58ZGHnnBuVrPxXLVRtpkMz2wpsTZa7JD0HLACuAi5OdrsNeAC4vlpxzHilLZ0Up54UhiibQS3NcLAXtTRjrU1QjOhdPIuX35rhLRc+wSc7H6QtyNFjeV4pii/uuJSX37c4Pp+VzKGugCCXxQqFuDd80spLYZaoPz/YOsxTU85VR8p/taZkylxJi4EzgUeAzqSQAdgGdE5FDM45Ny3NkNF/J0VSM/AD4CNmtl8lfR3MzKThb5Gk64DrAOpprHaYzjmXXkdzQSIpS1yIfNvMfphs3i5pnpltlTQP2DHcsWa2AlgB0KqOlN/GKSQRtrWy9T+9mr3L8lzy2uc4rXkrV7Wspr6kTN5dzHJv91I2981iX97oyvfTlu3lwfUtHP9148VPhby3542Hp6PUC6wZXD80ZW6RqLd42HbL92P56powZAwAABPNSURBVH6rzrnp0SGxmq22BHwdeM7Mbi556y4GRxReDtxZrRicc25GMCv/VQPVfCK5APjPwFOSnki2/TlwE3CHpGuBDcC7qhiDc85Ne2l/Iqlmq61fMfKYXJdW67rOOTej1LB/SLmmpNWWmzhlMug1p9B1UgsHOwJ65ovsGXvo+Mc88771PFvDLJt75vBzXYoVo8O6jquuDjU3xSuR0d0dcXK0hqg/TzTcJFTefNe5VNIk5oybCl6QOOdcyh21qS3nnHMVYKQ+W+AFSUopm4t7py+azws3Zjj12A1s2tdG75Y2Ov65jcZV6yke6EZhiBXyh6dQk7lJ7GAv6uvDIhtMeSkYY+RE51za+BOJc865yfGCxDnn3ERNhw6JXpCkQTLXhxUKEIT0/ofX8fJVEbdd8jXu3HMW+Y8uo+/BbcxlG3MhTls1NKDXLSVY+zLRgSieH6RUMvDiEVmskabJdc6lUw07GpbLCxLnnEs5fyJxzjk3OV6QuDFFRcwEQUhm0Xya1u/jVTcbf/2n5wOQ0W+JSkZNViZL1NsHK5+iGISD84EMNZAyG2i1lUyTO9wUueNSOgeKc67q/InEOefcxBlQTHdJ4gWJc86lXNqfSKo5Z7tzzrlKqOAw8pIul/S8pHWSbhjm/Y9KelbSakn3STp+rHN6QVJLQYiyOTKLFtL1rnPh3nmceedLvHBNBz3HtxH19hH19MTNgjX4o7JCyTzpI9WPAEEuS9DYSNBQT1BXF8/lHobxuTTSwMxl8PoR56aUrPzXqOeRQuAW4ApgKXCNpKVDdnscONvMTge+D/zNWPF5QeKcc2lm43yN7hxgnZmtN7N+4HbgqsMuZ3a/mfUkqw8DC8c6qdeROOdcisU928eVBZgjaVXJ+opk6nKABcDGkvc2AeeOcq5rgZ+MdUEvSKZSEKJshqC5CSKjcNpx7DuxgX0nBPTPirCvLeTRu/awpPAYli8cSltZX9+ELhf19kJvbyW/A+dcLYxvnNWdZnb2ZC8p6T8BZwMXjbWvFyTOOZdy43wiGc1mYFHJ+sJk2+HXky4D/gK4yMzG/J+s15E451yaVbaOZCVwsqQlknLA1cBdpTtIOhP4B+BKM9tRToj+RFJJ0qH5PpTLEbS2wtxZvPCe2Zx4/gZuOeEOQsG2Yh0hxj1dr+XuLa/GftPJCf/SR/CLJ4iSuUTKETQ1xWkvBVi+v8rfnHOuNio3aKOZFSR9ELgHCIFbzewZSZ8FVpnZXcDngWbge4r/Fr1sZleOdl4vSJxzLuUUVa7JvZndDdw9ZNunSpYvG+85vSBxzrk0M1DKJzX1gsQ559Iu5Z2AvSAZamBk25FGuE3qQRQM1oeggKChnsKrl7B2eR3/39mPs3z2L3mqbyHrejtZf+cb6b55IX9y31uwfCHumQ5gRhPraWL94dco80MT9fSk/gPmnKuAlP+aV63VlqRbJe2Q9HTJtg5J90pam3ydVa3rO+fcTCGzsl+1UM3mv98ELh+y7QbgPjM7GbgvWXfOOTeaCg7aWA1VS22Z2S8kLR6y+Srg4mT5NuAB4PpqxTBhw6W1Bpr2AspmUCZD0N4GxSLRsbPZdXobXceJ1jXwzPdO55OPNiZprALH5x+BqDjOzqll8LSWczOfMd6e7VNuqutIOs1sa7K8DegcaUdJ1wHXAdTTOAWhOedc+ojapazKVbPKdjMzaeRBj5NBxlYAtKoj3XfROeeqyQuSw2yXNM/MtkqaB5TV/X7KqLQlVtK7XAFBLgvZLHbq8Rw8tpFXzszwznc8yIl1z3Nibge/7jmZW587n/r7W5h3x/MUd+2mONk50SH1Hx7n3BRJ+d+CqR5r6y5gebK8HLhziq/vnHPTi4GKVvarFqrZ/Pc7wL8Dp0raJOla4CbgzZLWApcl684550ZzFLfaumaEty6t1jUnLAhRIMI5s7HWZqKWevaf3ELmYMTuV2XoX9bNgjl7MTtAoC4Kj85n5TtO5ZGXmrDCIjDjOJ4CaXIprQEpf4x1zk2l2hUQ5fKe7c45l2aGFyTOOecmyfuROOecmwzvR5JWEgpDCEOUyaD6OmhsoNjeyIHFTWw/DzpO2kvP+ll0/LyR4it1ND+3m2j9y5xgmykWi0c+bqb8h+2cm6ZS/rfl6C1InHNuOjCgghNbVYMXJM45l2reait9kl7jYccs1NpC1NJA37HNdC3MUqyD/lahIpz0z11kNh9gbt8uoq4DUCxSjOzQ/CPOOTdlvCBxzjk3YQYU091sywsS55xLtSQTkmJHTUESNDailmZ6lx3HK2fk4PV7WdS+lygqcrA/z84ts2h+PsfsZwo0/XYX0YZNFPr7h5+XpHQ55Y+czrkZIOV/Z46agsQ556Ylb7XlnHNu0vyJxDnn3KR4QVIlI82rDiiTjb/W16FMBjIZus9bwiunZ7AzuujdI467tRnWFwl37qF9307a2QxhGDfz7c9DVBz+upZMoJzyH6xzbqbwfiTOOecmw4DIW20555ybDH8iqRCJoK4OggA1NaFcln3nLWLHO3p532t/xX9seZIA2FBopSPsoV5F9kY5Qoy8hdy08W1sefgETvpMHlvzHFYsHjkJVblNe1P+Q3XOzTAp/5szfQoS55w7GplhxRHqbFPCCxLnnEs770cyeaec3sO//nQVW4sH2VJooNtyvNDfyZfXtHPMt1u4/9Nnct/62XGpXTqUwGGPg9s5ge2jTzSW8sdH59xRKuV/m6ZFQeKcc0ctM2+15ZxzbpL8iWTyfru6kbcvPBvlcgQN9Vi+gBrqWaRXIL+Z4t59ZZ9LdXUoDFFdHRSLhzohRn19KJMhOtiLAmGFwpHHZpLblRxjkcUdF4MQZTOHb0tamamhAcIA6zlI1NMzZnxh5zEwZxZs3Epx//7hv4dsjgNXnknTD38Tbyj3QyYRNDdjpy0m3LaHwuatgx0vk6mHD33fQRjH09GO6uuJdu7CCoX4fYkD7zyX9l+8SLS/C8sXwKIktTiBD7xEOLtjzHukTGbw/g4IwkP3G7P4ZxGI8NhOyGYobtqKFfIlJwkG058DsQ6cY4opkxn2cza4Q/I9SSiTJZzfifX0UnzllZEPqasjaG0ddZ+BawctLdjx82DtBqLu7nEGX8aApck+QWMjWrII7d5HtGcvhGH8cx6Y12e89z753QpmtVPYum1chwb19US9vUeeMpsjaGrAFs4j6OqmsGFj+bE0Ng7evyoN5Gr+ROKcc27i0t+zvSZT/Um6XNLzktZJuqEWMTjn3LQwMPpvua8amPInEkkhcAvwZmATsFLSXWb27FTH4pxz00LKJ7aSTfEjk6TXA582s7cm6zcCmNlfj3RMqzrsXF06mMcvFlEYxrnXtlbs4EHsYC8EAUFHO7ZvP1aMUBhgvX2YGZYvENTXER08CGYomwOL4nqR/jiHrkBxPUgQEPX2Df7wFAzm4QfyuqV59mS7spm4viDJ+Sqbwwp5grq6Q9cgKsbXUIDl+4Ekb9vXd3jOHuLvsbWZ4q7dR9yToKWFwrITyW3cBWFI4aWNcUwD5xgpV5vcQ2AwPz/Qo18BQS57eA55oF6lvx/r6xu8fn19/D0NyW8HTU0Es9pBorjjFay/P743YRjXVYz0eQtCgob6wTqqYnHwvpdcEzj8Xo1mpIE9U5ImUCaDFYsEDQ3x5w0GP0MDSn+myT2ygfq8kp/HcPVbpfVf8bmG/zwEjY1YX198fFLHZJEdfu2xvo9CIf5ch+Fhn5NDSuuyFKBgcBQJG/hf9NDrjfGzClpaIJ+Pr93QQHTgQDxgq0WHf7ZHq4sZeH/gd7au7tD9D+fOxroOxD+jujqKB7oPrwsd6ZzAI3Yf+223hn1znFqD2XZe5q1l739v/juPmtnZlbh2uWpRR7IAKK3J2gScO3QnSdcB1wHU0zg1kTnnXNp4z/aJM7MVwAqIn0hqHI5zztWOp7aGXHACqS1JXcDzUxPhhM0BdtY6iDJMhzg9xsrwGCtjIjEeb2ZzK3FxST9NYijXTjO7vBLXLlctCpIM8FvgUmAzsBJ4t5k9M8oxq6Y65zde0yFGmB5xeoyV4TFWxnSIsdamPLVlZgVJHwTuAULg1tEKEeecc+lWkzoSM7sbuLsW13bOOVdZNemQOAErah1AGaZDjDA94vQYK8NjrIzpEGNNTXkdiXPOuZllujyROOecS6nUFyRpHZdL0kuSnpL0hKRVybYOSfdKWpt8nTXFMd0qaYekp0u2DRuTYn+X3NfVks6qYYyflrQ5uZdPSHpbyXs3JjE+L6n87r2Ti3GRpPslPSvpGUkfTran5l6OEmPa7mW9pN9IejKJ8zPJ9iWSHkni+a6kXLK9Lllfl7y/uIYxflPSiyX38oxke01+d1LNzFL7Im7V9QJwApADngSW1jquJLaXgDlDtv0NcEOyfAPwuSmO6ULgLODpsWIC3gb8BBBwHvBIDWP8NPDxYfZdmvzM64AlyWchnIIY5wFnJcstxM3Vl6bpXo4SY9rupYDmZDkLPJLcozuAq5PtXwXenyz/CfDVZPlq4Ls1jPGbwDuG2b8mvztpfqX9ieQcYJ2ZrTezfuB24KoaxzSaq4DbkuXbgN+dyoub2S+AoQNzjRTTVcC3LPYw0C5pXo1iHMlVwO1m1mdmLwLriD8TVWVmW83ssWS5C3iOeGif1NzLUWIcSa3upZnZgWQ1m7wMuAT4frJ96L0cuMffBy6VVJExqyYQ40hq8ruTZmkvSIYbl2u0X5apZMDPJD2ajAsG0GlmW5PlbUBnbUI7zEgxpe3efjBJE9xakhKseYxJauVM4v+lpvJeDokRUnYvJYWSngB2APcSPw3tNbOB0SlLYzkUZ/L+PmD2VMdoZgP38q+Se/kFSXVDYxwm/qNS2guSNHuDmZ0FXAF8QNKFpW9a/AycqiZxaYwp8RXgROAMYCvwt7UNJyapGfgB8BEzO2y6yrTcy2FiTN29NLOimZ0BLCR+CjqtxiEdYWiMkl4D3Egc6+8AHcD1NQwx1dJekGwGFpWsL0y21ZyZbU6+7gB+RPwLsn3gETf5uqN2ER4yUkypubdmtj35RY6Af2Qw5VKzGCVlif9Af9vMfphsTtW9HC7GNN7LAWa2F7gfeD1xOmigQ3RpLIfiTN5vA3bVIMbLk/ShmVkf8A1SdC/TJu0FyUrg5KSFR4648u2uGseEpCZJLQPLwFuAp4ljW57sthy4szYRHmakmO4C/kvSAuU8YF9J2mZKDckv/x7xvYQ4xquTljxLgJOB30xBPAK+DjxnZjeXvJWaezlSjCm8l3MltSfLDcQT2j1H/Mf6HcluQ+/lwD1+B/Dz5OlvqmNcU/KfBhHX4ZTey1T87qRGrWv7x3oRt5D4LXFe9S9qHU8S0wnELWCeBJ4ZiIs4l3sfsBb4v0DHFMf1HeJ0Rp44b3vtSDERtzi5JbmvTwFn1zDGf0piWE38SzqvZP+/SGJ8HrhiimJ8A3HaajXwRPJ6W5ru5Sgxpu1eng48nsTzNPCpZPsJxAXZOuB7QF2yvT5ZX5e8f0INY/x5ci+fBv4Pgy27avK7k+aX92x3zjk3KWlPbTnnnEs5L0icc85NihckzjnnJsULEuecc5PiBYlzzrlJ8YLEVZSkYjJS6jPJaKofk1S1z5mkxSoZSXiC5/jzIeu/nlxUo15rsaR3V+v8ztWCFySu0g6a2Rlm9mrijl1XAH9Z45gOKelNXeqwgsTMzq9iCIsBL0jcjOIFiasai4ePuY54EEElA+N9XtLKZCC8PxrYV9L1iud3eVLSTcm2MyQ9nOz7Iw3O//G6ZL8ngQ+UnGPY80u6WNIvJd0FPFsaY3KthuQp6tvJtgMlxz0o6U5J6yXdJOk9iueueErSicl+cyX9ILnuSkkXJNsv0uBcFo8noyHcBLwx2fZnY8T8C0n/pnj+kK9W88nOuUmpdY9If82sF3BgmG17iUfKvQ7478m2OmAV8dwYVwC/BhqT9wZ6jK8GLkqWPwt8sWT7hcny50nmNhnl/BcD3cCScmIeWE+O20s890cd8XhKn0ne+3BJPP9MPIgnwHHEw5YA/Bi4IFluBjLJOf+15FqjxdxL3AM8JB4194i5MfzlrzS8hnvMd65a3gKcLmlgjKU24jGfLgO+YWY9AGa2W1Ib0G5mDyb73gZ8LxkTqd3ieU0gHhLkijHO3w/8xuJ5OMZrpSXjKEl6AfhZsv0p4E3J8mXAUg1Om9GqeFTeh4CbkyedH5rZJh05tcZYMa9Prv0d4mFRvj/0BM7VmhckrqoknQAUiUfKFfAhM7tnyD6VmvZ1pPNfTPxEMhF9JctRyXrE4O9PAJxnZr1Djr1J0r8Rj4H10Ajf52gxDx2/yMczcqnkOVdXNZLmEk+j+r/NzIB7gPcrHv4cSacoHj35XuAPJTUm2zvMbB+wR9Ibk9P9Z+BBi4f53ivpDcn295RccqTzjyU/cMwE/Qz40MCKBuf2PtHMnjKzzxGPZH0a0EU8NW45MZ+jeOTrAPgD4FeTiNG5qvEnEldpDYpnmssCBeLU08Aw518jbrX0WDI09yvA75rZT5M/vqsk9QN3E7ekWg58NSlg1gN/mJznD4FbJRmDqaYRz19GzCuA1ZIeM7P3jLn3kf4UuEXSauLfqV8Afwx8RNKbiJ9eniGe5zsCiklDgW8CXxol5pXA/wZOIh52/UcTiM25qvPRf51LoSS19XEze3utY3FuLJ7acs45Nyn+ROKcc25S/InEOefcpHhB4pxzblK8IHHOOTcpXpA455ybFC9InHPOTYoXJM455ybl/wG2SGZbPnFFOwAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "fig = display.plot_alignment(alignment)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## WaveFlow: vocoder model\n", - "Generated spectrogram is converted to raw audio using a pretrained waveflow model." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from paddlespeech.t2s.models.waveflow import ConditionalWaveFlow" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "data:\n", - " batch_size: 8\n", - " clip_frames: 65\n", - " fmax: 8000\n", - " fmin: 0\n", - " hop_length: 256\n", - " n_fft: 1024\n", - " n_mels: 80\n", - " sample_rate: 22050\n", - " valid_size: 16\n", - " win_length: 1024\n", - "model:\n", - " channels: 128\n", - " kernel_size: [3, 3]\n", - " n_flows: 8\n", - " n_group: 16\n", - " n_layers: 8\n", - " sigma: 1.0\n", - " upsample_factors: [16, 16]\n", - "training:\n", - " lr: 0.0002\n", - " max_iteration: 3000000\n", - " save_interval: 10000\n", - " valid_interval: 1000\n" - ] - } - ], - "source": [ - "from examples.waveflow import config as waveflow_config\n", - "vocoder_config = waveflow_config.get_cfg_defaults()\n", - "print(vocoder_config)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[checkpoint] Rank 0: loaded model from ../../pretrained/waveflow/waveflow_ljspeech_ckpt_0.3/step-2000000.pdparams\n" - ] - } - ], - "source": [ - "vocoder = ConditionalWaveFlow.from_pretrained(\n", - " vocoder_config, \n", - " \"../../pretrained/waveflow/waveflow_ljspeech_ckpt_0.3/step-2000000\")\n", - "layer_tools.recursively_remove_weight_norm(vocoder)\n", - "vocoder.eval()" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "time: 9.412613868713379s\n" - ] - } - ], - "source": [ - "audio = vocoder.infer(paddle.transpose(outputs[\"mel_outputs_postnet\"], [0, 2, 1]))\n", - "wav = audio[0].numpy()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " " - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ipd.Audio(wav, rate=22050)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/paddlespeech/t2s/exps/tacotron2/synthesize.py b/paddlespeech/t2s/exps/tacotron2/synthesize.py deleted file mode 100644 index c73c32d25be432033ca47b4118f9cf669a89fbce..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/synthesize.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -from pathlib import Path - -import numpy as np -import paddle -from matplotlib import pyplot as plt - -from paddlespeech.t2s.exps.tacotron2.config import get_cfg_defaults -from paddlespeech.t2s.frontend import EnglishCharacter -from paddlespeech.t2s.models.tacotron2 import Tacotron2 -from paddlespeech.t2s.utils import display - - -def main(config, args): - if args.ngpu == 0: - paddle.set_device("cpu") - elif args.ngpu > 0: - paddle.set_device("gpu") - else: - print("ngpu should >= 0 !") - - # model - frontend = EnglishCharacter() - model = Tacotron2.from_pretrained(config, args.checkpoint_path) - model.eval() - - # inputs - input_path = Path(args.input).expanduser() - sentences = [] - with open(input_path, "rt") as f: - for line in f: - line_list = line.strip().split() - utt_id = line_list[0] - sentence = " ".join(line_list[1:]) - sentences.append((utt_id, sentence)) - - if args.output is None: - output_dir = input_path.parent / "synthesis" - else: - output_dir = Path(args.output).expanduser() - output_dir.mkdir(exist_ok=True) - - for i, sentence in enumerate(sentences): - sentence = paddle.to_tensor(frontend(sentence)).unsqueeze(0) - outputs = model.infer(sentence) - mel_output = outputs["mel_outputs_postnet"][0].numpy().T - alignment = outputs["alignments"][0].numpy().T - - np.save(str(output_dir / f"sentence_{i}"), mel_output) - display.plot_alignment(alignment) - plt.savefig(str(output_dir / f"sentence_{i}.png")) - if args.verbose: - print("spectrogram saved at {}".format(output_dir / - f"sentence_{i}.npy")) - - -if __name__ == "__main__": - config = get_cfg_defaults() - - parser = argparse.ArgumentParser( - description="generate mel spectrogram with TransformerTTS.") - parser.add_argument( - "--config", - type=str, - metavar="FILE", - help="extra config to overwrite the default config") - parser.add_argument( - "--checkpoint_path", type=str, help="path of the checkpoint to load.") - parser.add_argument("--input", type=str, help="path of the text sentences") - parser.add_argument("--output", type=str, help="path to save outputs") - parser.add_argument( - "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") - parser.add_argument( - "--opts", - nargs=argparse.REMAINDER, - help="options to overwrite --config file and the default config, passing in KEY VALUE pairs" - ) - parser.add_argument( - "-v", "--verbose", action="store_true", help="print msg") - - args = parser.parse_args() - if args.config: - config.merge_from_file(args.config) - if args.opts: - config.merge_from_list(args.opts) - config.freeze() - print(config) - print(args) - - main(config, args) diff --git a/paddlespeech/t2s/exps/tacotron2/train.py b/paddlespeech/t2s/exps/tacotron2/train.py deleted file mode 100644 index 8198348f324588573161308d862c24f7be9873b3..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/tacotron2/train.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time -from collections import defaultdict - -import numpy as np -import paddle -from paddle import distributed as dist -from paddle.io import DataLoader -from paddle.io import DistributedBatchSampler - -from paddlespeech.t2s.data import dataset -from paddlespeech.t2s.exps.tacotron2.config import get_cfg_defaults -from paddlespeech.t2s.exps.tacotron2.ljspeech import LJSpeech -from paddlespeech.t2s.exps.tacotron2.ljspeech import LJSpeechCollector -from paddlespeech.t2s.models.tacotron2 import Tacotron2 -from paddlespeech.t2s.models.tacotron2 import Tacotron2Loss -from paddlespeech.t2s.training.cli import default_argument_parser -from paddlespeech.t2s.training.experiment import ExperimentBase -from paddlespeech.t2s.utils import display -from paddlespeech.t2s.utils import mp_tools - - -class Experiment(ExperimentBase): - def compute_losses(self, inputs, outputs): - texts, mel_targets, plens, slens = inputs - - mel_outputs = outputs["mel_output"] - mel_outputs_postnet = outputs["mel_outputs_postnet"] - attention_weight = outputs["alignments"] - if self.config.model.use_stop_token: - stop_logits = outputs["stop_logits"] - else: - stop_logits = None - - losses = self.criterion(mel_outputs, mel_outputs_postnet, mel_targets, - attention_weight, slens, plens, stop_logits) - return losses - - def train_batch(self): - start = time.time() - batch = self.read_batch() - data_loader_time = time.time() - start - - self.optimizer.clear_grad() - self.model.train() - texts, mels, text_lens, output_lens = batch - outputs = self.model(texts, text_lens, mels, output_lens) - losses = self.compute_losses(batch, outputs) - loss = losses["loss"] - loss.backward() - self.optimizer.step() - iteration_time = time.time() - start - - losses_np = {k: float(v) for k, v in losses.items()} - # logging - msg = "Rank: {}, ".format(dist.get_rank()) - msg += "step: {}, ".format(self.iteration) - msg += "time: {:>.3f}s/{:>.3f}s, ".format(data_loader_time, - iteration_time) - msg += ', '.join('{}: {:>.6f}'.format(k, v) - for k, v in losses_np.items()) - self.logger.info(msg) - - if dist.get_rank() == 0: - for k, v in losses_np.items(): - self.visualizer.add_scalar(f"train_loss/{k}", v, self.iteration) - - @mp_tools.rank_zero_only - @paddle.no_grad() - def valid(self): - valid_losses = defaultdict(list) - for i, batch in enumerate(self.valid_loader): - texts, mels, text_lens, output_lens = batch - outputs = self.model(texts, text_lens, mels, output_lens) - losses = self.compute_losses(batch, outputs) - for k, v in losses.items(): - valid_losses[k].append(float(v)) - - attention_weights = outputs["alignments"] - self.visualizer.add_figure( - f"valid_sentence_{i}_alignments", - display.plot_alignment(attention_weights[0].numpy().T), - self.iteration) - self.visualizer.add_figure( - f"valid_sentence_{i}_target_spectrogram", - display.plot_spectrogram(mels[0].numpy().T), self.iteration) - self.visualizer.add_figure( - f"valid_sentence_{i}_predicted_spectrogram", - display.plot_spectrogram(outputs['mel_outputs_postnet'][0] - .numpy().T), self.iteration) - - # write visual log - valid_losses = {k: np.mean(v) for k, v in valid_losses.items()} - - # logging - msg = "Valid: " - msg += "step: {}, ".format(self.iteration) - msg += ', '.join('{}: {:>.6f}'.format(k, v) - for k, v in valid_losses.items()) - self.logger.info(msg) - - for k, v in valid_losses.items(): - self.visualizer.add_scalar(f"valid/{k}", v, self.iteration) - - def setup_model(self): - config = self.config - model = Tacotron2( - vocab_size=config.model.vocab_size, - d_mels=config.data.n_mels, - d_encoder=config.model.d_encoder, - encoder_conv_layers=config.model.encoder_conv_layers, - encoder_kernel_size=config.model.encoder_kernel_size, - d_prenet=config.model.d_prenet, - d_attention_rnn=config.model.d_attention_rnn, - d_decoder_rnn=config.model.d_decoder_rnn, - attention_filters=config.model.attention_filters, - attention_kernel_size=config.model.attention_kernel_size, - d_attention=config.model.d_attention, - d_postnet=config.model.d_postnet, - postnet_kernel_size=config.model.postnet_kernel_size, - postnet_conv_layers=config.model.postnet_conv_layers, - reduction_factor=config.model.reduction_factor, - p_encoder_dropout=config.model.p_encoder_dropout, - p_prenet_dropout=config.model.p_prenet_dropout, - p_attention_dropout=config.model.p_attention_dropout, - p_decoder_dropout=config.model.p_decoder_dropout, - p_postnet_dropout=config.model.p_postnet_dropout, - use_stop_token=config.model.use_stop_token) - - if self.parallel: - model = paddle.DataParallel(model) - - grad_clip = paddle.nn.ClipGradByGlobalNorm( - config.training.grad_clip_thresh) - optimizer = paddle.optimizer.Adam( - learning_rate=config.training.lr, - parameters=model.parameters(), - weight_decay=paddle.regularizer.L2Decay( - config.training.weight_decay), - grad_clip=grad_clip) - criterion = Tacotron2Loss( - use_stop_token_loss=config.model.use_stop_token, - use_guided_attention_loss=config.model.use_guided_attention_loss, - sigma=config.model.guided_attention_loss_sigma) - self.model = model - self.optimizer = optimizer - self.criterion = criterion - - def setup_dataloader(self): - args = self.args - config = self.config - ljspeech_dataset = LJSpeech(args.data) - - valid_set, train_set = dataset.split(ljspeech_dataset, - config.data.valid_size) - batch_fn = LJSpeechCollector(padding_idx=config.data.padding_idx) - - if not self.parallel: - self.train_loader = DataLoader( - train_set, - batch_size=config.data.batch_size, - shuffle=True, - drop_last=True, - collate_fn=batch_fn) - else: - sampler = DistributedBatchSampler( - train_set, - batch_size=config.data.batch_size, - shuffle=True, - drop_last=True) - self.train_loader = DataLoader( - train_set, batch_sampler=sampler, collate_fn=batch_fn) - - self.valid_loader = DataLoader( - valid_set, - batch_size=config.data.batch_size, - shuffle=False, - drop_last=False, - collate_fn=batch_fn) - - -def main_sp(config, args): - exp = Experiment(config, args) - exp.setup() - exp.resume_or_load() - exp.run() - - -def main(config, args): - if args.ngpu > 1: - dist.spawn(main_sp, args=(config, args), nprocs=args.ngpu) - else: - main_sp(config, args) - - -if __name__ == "__main__": - config = get_cfg_defaults() - parser = default_argument_parser() - args = parser.parse_args() - if args.config: - config.merge_from_file(args.config) - if args.opts: - config.merge_from_list(args.opts) - config.freeze() - print(config) - print(args) - - main(config, args) diff --git a/paddlespeech/t2s/exps/transformer_tts/normalize.py b/paddlespeech/t2s/exps/transformer_tts/normalize.py index 4bb77c7911a9c2cd43de83ef40c07fadc38d34f4..87e975b88ffb1b27c63885dfbe7fdb3c4cf5b718 100644 --- a/paddlespeech/t2s/exps/transformer_tts/normalize.py +++ b/paddlespeech/t2s/exps/transformer_tts/normalize.py @@ -130,6 +130,9 @@ def main(): "speech_lengths": item['speech_lengths'], "speech": str(speech_path), } + # add spk_emb for voice cloning + if "spk_emb" in item: + record["spk_emb"] = str(item["spk_emb"]) output_metadata.append(record) output_metadata.sort(key=itemgetter('utt_id')) output_metadata_path = Path(args.dumpdir) / "metadata.jsonl" diff --git a/paddlespeech/t2s/exps/fastspeech2/voice_cloning.py b/paddlespeech/t2s/exps/voice_cloning.py similarity index 57% rename from paddlespeech/t2s/exps/fastspeech2/voice_cloning.py rename to paddlespeech/t2s/exps/voice_cloning.py index 9fbd496418199d05b6319ab335f1a5437bb961d2..d6733a9487704decb838e6d5f0efb49de4cda272 100644 --- a/paddlespeech/t2s/exps/fastspeech2/voice_cloning.py +++ b/paddlespeech/t2s/exps/voice_cloning.py @@ -21,17 +21,43 @@ import soundfile as sf import yaml from yacs.config import CfgNode +from paddlespeech.s2t.utils.dynamic_import import dynamic_import from paddlespeech.t2s.frontend.zh_frontend import Frontend -from paddlespeech.t2s.models.fastspeech2 import FastSpeech2 -from paddlespeech.t2s.models.fastspeech2 import FastSpeech2Inference -from paddlespeech.t2s.models.parallel_wavegan import PWGGenerator -from paddlespeech.t2s.models.parallel_wavegan import PWGInference from paddlespeech.t2s.modules.normalizer import ZScore from paddlespeech.vector.exps.ge2e.audio_processor import SpeakerVerificationPreprocessor from paddlespeech.vector.models.lstm_speaker_encoder import LSTMSpeakerEncoder +model_alias = { + # acoustic model + "fastspeech2": + "paddlespeech.t2s.models.fastspeech2:FastSpeech2", + "fastspeech2_inference": + "paddlespeech.t2s.models.fastspeech2:FastSpeech2Inference", + "tacotron2": + "paddlespeech.t2s.models.new_tacotron2:Tacotron2", + "tacotron2_inference": + "paddlespeech.t2s.models.new_tacotron2:Tacotron2Inference", + # voc + "pwgan": + "paddlespeech.t2s.models.parallel_wavegan:PWGGenerator", + "pwgan_inference": + "paddlespeech.t2s.models.parallel_wavegan:PWGInference", +} + + +def voice_cloning(args): + # Init body. + with open(args.am_config) as f: + am_config = CfgNode(yaml.safe_load(f)) + with open(args.voc_config) as f: + voc_config = CfgNode(yaml.safe_load(f)) + + print("========Args========") + print(yaml.safe_dump(vars(args))) + print("========Config========") + print(am_config) + print(voc_config) -def voice_cloning(args, fastspeech2_config, pwg_config): # speaker encoder p = SpeakerVerificationPreprocessor( sampling_rate=16000, @@ -57,40 +83,52 @@ def voice_cloning(args, fastspeech2_config, pwg_config): phn_id = [line.strip().split() for line in f.readlines()] vocab_size = len(phn_id) print("vocab_size:", vocab_size) - odim = fastspeech2_config.n_mels - model = FastSpeech2( - idim=vocab_size, odim=odim, **fastspeech2_config["model"]) - model.set_state_dict( - paddle.load(args.fastspeech2_checkpoint)["main_params"]) - model.eval() - - vocoder = PWGGenerator(**pwg_config["generator_params"]) - vocoder.set_state_dict(paddle.load(args.pwg_checkpoint)["generator_params"]) - vocoder.remove_weight_norm() - vocoder.eval() - print("model done!") + # acoustic model + odim = am_config.n_mels + # model: {model_name}_{dataset} + am_name = args.am[:args.am.rindex('_')] + am_dataset = args.am[args.am.rindex('_') + 1:] + + am_class = dynamic_import(am_name, model_alias) + am_inference_class = dynamic_import(am_name + '_inference', model_alias) + + if am_name == 'fastspeech2': + am = am_class( + idim=vocab_size, odim=odim, spk_num=None, **am_config["model"]) + elif am_name == 'tacotron2': + am = am_class(idim=vocab_size, odim=odim, **am_config["model"]) + + am.set_state_dict(paddle.load(args.am_ckpt)["main_params"]) + am.eval() + am_mu, am_std = np.load(args.am_stat) + am_mu = paddle.to_tensor(am_mu) + am_std = paddle.to_tensor(am_std) + am_normalizer = ZScore(am_mu, am_std) + am_inference = am_inference_class(am_normalizer, am) + am_inference.eval() + print("acoustic model done!") + + # vocoder + # model: {model_name}_{dataset} + voc_name = args.voc[:args.voc.rindex('_')] + voc_class = dynamic_import(voc_name, model_alias) + voc_inference_class = dynamic_import(voc_name + '_inference', model_alias) + voc = voc_class(**voc_config["generator_params"]) + voc.set_state_dict(paddle.load(args.voc_ckpt)["generator_params"]) + voc.remove_weight_norm() + voc.eval() + voc_mu, voc_std = np.load(args.voc_stat) + voc_mu = paddle.to_tensor(voc_mu) + voc_std = paddle.to_tensor(voc_std) + voc_normalizer = ZScore(voc_mu, voc_std) + voc_inference = voc_inference_class(voc_normalizer, voc) + voc_inference.eval() + print("voc done!") frontend = Frontend(phone_vocab_path=args.phones_dict) print("frontend done!") - stat = np.load(args.fastspeech2_stat) - mu, std = stat - mu = paddle.to_tensor(mu) - std = paddle.to_tensor(std) - fastspeech2_normalizer = ZScore(mu, std) - - stat = np.load(args.pwg_stat) - mu, std = stat - mu = paddle.to_tensor(mu) - std = paddle.to_tensor(std) - pwg_normalizer = ZScore(mu, std) - - fastspeech2_inference = FastSpeech2Inference(fastspeech2_normalizer, model) - fastspeech2_inference.eval() - pwg_inference = PWGInference(pwg_normalizer, vocoder) - pwg_inference.eval() - output_dir = Path(args.output_dir) output_dir.mkdir(parents=True, exist_ok=True) @@ -112,24 +150,23 @@ def voice_cloning(args, fastspeech2_config, pwg_config): # print("spk_emb shape: ", spk_emb.shape) with paddle.no_grad(): - wav = pwg_inference( - fastspeech2_inference(phone_ids, spk_emb=spk_emb)) + wav = voc_inference(am_inference(phone_ids, spk_emb=spk_emb)) sf.write( str(output_dir / (utt_id + ".wav")), wav.numpy(), - samplerate=fastspeech2_config.fs) + samplerate=am_config.fs) print(f"{utt_id} done!") # Randomly generate numbers of 0 ~ 0.2, 256 is the dim of spk_emb random_spk_emb = np.random.rand(256) * 0.2 random_spk_emb = paddle.to_tensor(random_spk_emb) utt_id = "random_spk_emb" with paddle.no_grad(): - wav = pwg_inference(fastspeech2_inference(phone_ids, spk_emb=spk_emb)) + wav = voc_inference(am_inference(phone_ids, spk_emb=spk_emb)) sf.write( str(output_dir / (utt_id + ".wav")), wav.numpy(), - samplerate=fastspeech2_config.fs) + samplerate=am_config.fs) print(f"{utt_id} done!") @@ -137,32 +174,53 @@ def main(): # parse args and config and redirect to train_sp parser = argparse.ArgumentParser(description="") parser.add_argument( - "--fastspeech2-config", type=str, help="fastspeech2 config file.") - parser.add_argument( - "--fastspeech2-checkpoint", + '--am', type=str, - help="fastspeech2 checkpoint to load.") + default='fastspeech2_csmsc', + choices=['fastspeech2_aishell3', 'tacotron2_aishell3'], + help='Choose acoustic model type of tts task.') parser.add_argument( - "--fastspeech2-stat", + '--am_config', type=str, - help="mean and standard deviation used to normalize spectrogram when training fastspeech2." - ) + default=None, + help='Config of acoustic model. Use deault config when it is None.') parser.add_argument( - "--pwg-config", type=str, help="parallel wavegan config file.") - parser.add_argument( - "--pwg-checkpoint", + '--am_ckpt', type=str, - help="parallel wavegan generator parameters to load.") + default=None, + help='Checkpoint file of acoustic model.') parser.add_argument( - "--pwg-stat", + "--am_stat", type=str, - help="mean and standard deviation used to normalize spectrogram when training parallel wavegan." + default=None, + help="mean and standard deviation used to normalize spectrogram when training acoustic model." ) parser.add_argument( "--phones-dict", type=str, default="phone_id_map.txt", help="phone vocabulary file.") + # vocoder + parser.add_argument( + '--voc', + type=str, + default='pwgan_csmsc', + choices=['pwgan_aishell3'], + help='Choose vocoder type of tts task.') + + parser.add_argument( + '--voc_config', + type=str, + default=None, + help='Config of voc. Use deault config when it is None.') + parser.add_argument( + '--voc_ckpt', type=str, default=None, help='Checkpoint file of voc.') + parser.add_argument( + "--voc_stat", + type=str, + default=None, + help="mean and standard deviation used to normalize spectrogram when training voc." + ) parser.add_argument( "--text", type=str, @@ -190,18 +248,7 @@ def main(): else: print("ngpu should >= 0 !") - with open(args.fastspeech2_config) as f: - fastspeech2_config = CfgNode(yaml.safe_load(f)) - with open(args.pwg_config) as f: - pwg_config = CfgNode(yaml.safe_load(f)) - - print("========Args========") - print(yaml.safe_dump(vars(args))) - print("========Config========") - print(fastspeech2_config) - print(pwg_config) - - voice_cloning(args, fastspeech2_config, pwg_config) + voice_cloning(args) if __name__ == "__main__": diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/__init__.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/__init__.py deleted file mode 100644 index abf198b97e6e818e1fbe59006f98492640bcee54..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/aishell3.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/aishell3.py deleted file mode 100644 index da95582d0872ab4942fa267657034db78dee0949..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/aishell3.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import pickle -from pathlib import Path - -import numpy as np -from paddle.io import Dataset - -from paddlespeech.t2s.data import batch_spec -from paddlespeech.t2s.data import batch_text_id -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.preprocess_transcription import _phones -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.preprocess_transcription import _tones -from paddlespeech.t2s.frontend import Vocab - -voc_phones = Vocab(sorted(list(_phones))) -print("vocab_phones:\n", voc_phones) -voc_tones = Vocab(sorted(list(_tones))) -print("vocab_tones:\n", voc_tones) - - -class AiShell3(Dataset): - """Processed AiShell3 dataset.""" - - def __init__(self, root): - super().__init__() - self.root = Path(root).expanduser() - self.embed_dir = self.root / "embed" - self.mel_dir = self.root / "mel" - - with open(self.root / "metadata.pickle", 'rb') as f: - self.records = pickle.load(f) - - def __getitem__(self, index): - metadatum = self.records[index] - sentence_id = metadatum["sentence_id"] - speaker_id = sentence_id[:7] - phones = metadatum["phones"] - tones = metadatum["tones"] - phones = np.array( - [voc_phones.lookup(item) for item in phones], dtype=np.int64) - tones = np.array( - [voc_tones.lookup(item) for item in tones], dtype=np.int64) - mel = np.load(str(self.mel_dir / speaker_id / (sentence_id + ".npy"))) - embed = np.load( - str(self.embed_dir / speaker_id / (sentence_id + ".npy"))) - return phones, tones, mel, embed - - def __len__(self): - return len(self.records) - - -def collate_aishell3_examples(examples): - phones, tones, mel, embed = list(zip(*examples)) - - text_lengths = np.array([item.shape[0] for item in phones], dtype=np.int64) - spec_lengths = np.array([item.shape[1] for item in mel], dtype=np.int64) - T_dec = np.max(spec_lengths) - stop_tokens = ( - np.arange(T_dec) >= np.expand_dims(spec_lengths, -1)).astype(np.float32) - phones, _ = batch_text_id(phones) - tones, _ = batch_text_id(tones) - mel, _ = batch_spec(mel) - mel = np.transpose(mel, (0, 2, 1)) - embed = np.stack(embed) - # 7 fields - # (B, T), (B, T), (B, T, C), (B, C), (B,), (B,), (B, T) - return phones, tones, mel, embed, text_lengths, spec_lengths, stop_tokens - - -if __name__ == "__main__": - dataset = AiShell3("~/datasets/aishell3/train") - example = dataset[0] - - examples = [dataset[i] for i in range(10)] - batch = collate_aishell3_examples(examples) - - for field in batch: - print(field.shape, field.dtype) diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/chinese_g2p.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/chinese_g2p.py deleted file mode 100644 index 12de3bb76ee255f1de49e72b405cea3119d00021..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/chinese_g2p.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import List -from typing import Tuple - -from pypinyin import lazy_pinyin -from pypinyin import Style - -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.preprocess_transcription import split_syllable - - -def convert_to_pinyin(text: str) -> List[str]: - """convert text into list of syllables, other characters that are not chinese, thus - cannot be converted to pinyin are splited. - """ - syllables = lazy_pinyin( - text, style=Style.TONE3, neutral_tone_with_five=True) - return syllables - - -def convert_sentence(text: str) -> List[Tuple[str]]: - """convert a sentence into two list: phones and tones""" - syllables = convert_to_pinyin(text) - phones = [] - tones = [] - for syllable in syllables: - p, t = split_syllable(syllable) - phones.extend(p) - tones.extend(t) - - return phones, tones diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/config.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/config.py deleted file mode 100644 index 8d8c9c4e1b70022328818b9f60e541f6286420ff..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/config.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from yacs.config import CfgNode as CN - -_C = CN() -_C.data = CN( - dict( - batch_size=32, # batch size - valid_size=64, # the first N examples are reserved for validation - sample_rate=22050, # Hz, sample rate - n_fft=1024, # fft frame size - win_length=1024, # window size - hop_length=256, # hop size between ajacent frame - fmax=8000, # Hz, max frequency when converting to mel - fmin=0, # Hz, min frequency when converting to mel - d_mels=80, # mel bands - padding_idx=0, # text embedding's padding index - )) - -_C.model = CN( - dict( - vocab_size=70, - n_tones=10, - reduction_factor=1, # reduction factor - d_encoder=512, # embedding & encoder's internal size - encoder_conv_layers=3, # number of conv layer in tacotron2 encoder - encoder_kernel_size=5, # kernel size of conv layers in tacotron2 encoder - d_prenet=256, # hidden size of decoder prenet - # hidden size of the first rnn layer in tacotron2 decoder - d_attention_rnn=1024, - # hidden size of the second rnn layer in tacotron2 decoder - d_decoder_rnn=1024, - d_attention=128, # hidden size of decoder location linear layer - attention_filters=32, # number of filter in decoder location conv layer - attention_kernel_size=31, # kernel size of decoder location conv layer - d_postnet=512, # hidden size of decoder postnet - postnet_kernel_size=5, # kernel size of conv layers in postnet - postnet_conv_layers=5, # number of conv layer in decoder postnet - p_encoder_dropout=0.5, # droput probability in encoder - p_prenet_dropout=0.5, # droput probability in decoder prenet - - # droput probability of first rnn layer in decoder - p_attention_dropout=0.1, - # droput probability of second rnn layer in decoder - p_decoder_dropout=0.1, - p_postnet_dropout=0.5, # droput probability in decoder postnet - guided_attention_loss_sigma=0.2, - d_global_condition=256, - - # whether to use a classifier to predict stop probability - use_stop_token=False, - # whether to use guided attention loss in training - use_guided_attention_loss=True, )) - -_C.training = CN( - dict( - lr=1e-3, # learning rate - weight_decay=1e-6, # the coeff of weight decay - grad_clip_thresh=1.0, # the clip norm of grad clip. - valid_interval=1000, # validation - save_interval=1000, # checkpoint - max_iteration=500000, # max iteration to train - )) - - -def get_cfg_defaults(): - """Get a yacs CfgNode object with default values for my_project.""" - # Return a clone so that the defaults will not be altered - # This is for the "local variable" use pattern - return _C.clone() diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/extract_mel.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/extract_mel.py deleted file mode 100644 index d12466f6fc1239e73c1cb18d1c65505f6988ccaf..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/extract_mel.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import multiprocessing as mp -from functools import partial -from pathlib import Path - -import numpy as np -import tqdm - -from paddlespeech.t2s.audio import AudioProcessor -from paddlespeech.t2s.audio.spec_normalizer import LogMagnitude -from paddlespeech.t2s.audio.spec_normalizer import NormalizerBase -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.config import get_cfg_defaults - - -def extract_mel(fname: Path, - input_dir: Path, - output_dir: Path, - p: AudioProcessor, - n: NormalizerBase): - relative_path = fname.relative_to(input_dir) - out_path = (output_dir / relative_path).with_suffix(".npy") - out_path.parent.mkdir(parents=True, exist_ok=True) - wav = p.read_wav(fname) - mel = p.mel_spectrogram(wav) - mel = n.transform(mel) - np.save(out_path, mel) - - -def extract_mel_multispeaker(config, input_dir, output_dir, extension=".wav"): - input_dir = Path(input_dir).expanduser() - fnames = list(input_dir.rglob(f"*{extension}")) - output_dir = Path(output_dir).expanduser() - output_dir.mkdir(parents=True, exist_ok=True) - - p = AudioProcessor(config.sample_rate, config.n_fft, config.win_length, - config.hop_length, config.d_mels, config.fmin, - config.fmax) - n = LogMagnitude(1e-5) - - func = partial( - extract_mel, input_dir=input_dir, output_dir=output_dir, p=p, n=n) - - with mp.Pool(16) as pool: - list( - tqdm.tqdm( - pool.imap(func, fnames), total=len(fnames), unit="utterance")) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Extract mel spectrogram from processed wav in AiShell3 training dataset." - ) - parser.add_argument( - "--config", - type=str, - help="yaml config file to overwrite the default config") - parser.add_argument( - "--input", - type=str, - default="~/datasets/aishell3/train/normalized_wav", - help="path of the processed wav folder") - parser.add_argument( - "--output", - type=str, - default="~/datasets/aishell3/train/mel", - help="path of the folder to save mel spectrograms") - parser.add_argument( - "--opts", - nargs=argparse.REMAINDER, - help="options to overwrite --config file and the default config, passing in KEY VALUE pairs" - ) - default_config = get_cfg_defaults() - - args = parser.parse_args() - if args.config: - default_config.merge_from_file(args.config) - if args.opts: - default_config.merge_from_list(args.opts) - default_config.freeze() - audio_config = default_config.data - - extract_mel_multispeaker(audio_config, args.input, args.output) diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/lexicon.txt b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/lexicon.txt deleted file mode 100644 index cc56b55d83b267536ace2b44d03d40c78ee8e0ac..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/lexicon.txt +++ /dev/null @@ -1,4150 +0,0 @@ -zhi1 zh iii1 -zhi2 zh iii2 -zhi3 zh iii3 -zhi4 zh iii4 -zhi5 zh iii5 -chi1 ch iii1 -chi2 ch iii2 -chi3 ch iii3 -chi4 ch iii4 -chi5 ch iii5 -shi1 sh iii1 -shi2 sh iii2 -shi3 sh iii3 -shi4 sh iii4 -shi5 sh iii5 -ri1 r iii1 -ri2 r iii2 -ri3 r iii3 -ri4 r iii4 -ri5 r iii5 -zi1 z ii1 -zi2 z ii2 -zi3 z ii3 -zi4 z ii4 -zi5 z ii5 -ci1 c ii1 -ci2 c ii2 -ci3 c ii3 -ci4 c ii4 -ci5 c ii5 -si1 s ii1 -si2 s ii2 -si3 s ii3 -si4 s ii4 -si5 s ii5 -a1 a1 -a2 a2 -a3 a3 -a4 a4 -a5 a5 -ba1 b a1 -ba2 b a2 -ba3 b a3 -ba4 b a4 -ba5 b a5 -pa1 p a1 -pa2 p a2 -pa3 p a3 -pa4 p a4 -pa5 p a5 -ma1 m a1 -ma2 m a2 -ma3 m a3 -ma4 m a4 -ma5 m a5 -fa1 f a1 -fa2 f a2 -fa3 f a3 -fa4 f a4 -fa5 f a5 -da1 d a1 -da2 d a2 -da3 d a3 -da4 d a4 -da5 d a5 -ta1 t a1 -ta2 t a2 -ta3 t a3 -ta4 t a4 -ta5 t a5 -na1 n a1 -na2 n a2 -na3 n a3 -na4 n a4 -na5 n a5 -la1 l a1 -la2 l a2 -la3 l a3 -la4 l a4 -la5 l a5 -ga1 g a1 -ga2 g a2 -ga3 g a3 -ga4 g a4 -ga5 g a5 -ka1 k a1 -ka2 k a2 -ka3 k a3 -ka4 k a4 -ka5 k a5 -ha1 h a1 -ha2 h a2 -ha3 h a3 -ha4 h a4 -ha5 h a5 -zha1 zh a1 -zha2 zh a2 -zha3 zh a3 -zha4 zh a4 -zha5 zh a5 -cha1 ch a1 -cha2 ch a2 -cha3 ch a3 -cha4 ch a4 -cha5 ch a5 -sha1 sh a1 -sha2 sh a2 -sha3 sh a3 -sha4 sh a4 -sha5 sh a5 -za1 z a1 -za2 z a2 -za3 z a3 -za4 z a4 -za5 z a5 -ca1 c a1 -ca2 c a2 -ca3 c a3 -ca4 c a4 -ca5 c a5 -sa1 s a1 -sa2 s a2 -sa3 s a3 -sa4 s a4 -sa5 s a5 -o1 o1 -o2 o2 -o3 o3 -o4 o4 -o5 o5 -bo1 b uo1 -bo2 b uo2 -bo3 b uo3 -bo4 b uo4 -bo5 b uo5 -po1 p uo1 -po2 p uo2 -po3 p uo3 -po4 p uo4 -po5 p uo5 -mo1 m uo1 -mo2 m uo2 -mo3 m uo3 -mo4 m uo4 -mo5 m uo5 -fo1 f uo1 -fo2 f uo2 -fo3 f uo3 -fo4 f uo4 -fo5 f uo5 -lo1 l o1 -lo2 l o2 -lo3 l o3 -lo4 l o4 -lo5 l o5 -e1 e1 -e2 e2 -e3 e3 -e4 e4 -e5 e5 -me1 m e1 -me2 m e2 -me3 m e3 -me4 m e4 -me5 m e5 -de1 d e1 -de2 d e2 -de3 d e3 -de4 d e4 -de5 d e5 -te1 t e1 -te2 t e2 -te3 t e3 -te4 t e4 -te5 t e5 -ne1 n e1 -ne2 n e2 -ne3 n e3 -ne4 n e4 -ne5 n e5 -le1 l e1 -le2 l e2 -le3 l e3 -le4 l e4 -le5 l e5 -ge1 g e1 -ge2 g e2 -ge3 g e3 -ge4 g e4 -ge5 g e5 -ke1 k e1 -ke2 k e2 -ke3 k e3 -ke4 k e4 -ke5 k e5 -he1 h e1 -he2 h e2 -he3 h e3 -he4 h e4 -he5 h e5 -zhe1 zh e1 -zhe2 zh e2 -zhe3 zh e3 -zhe4 zh e4 -zhe5 zh e5 -che1 ch e1 -che2 ch e2 -che3 ch e3 -che4 ch e4 -che5 ch e5 -she1 sh e1 -she2 sh e2 -she3 sh e3 -she4 sh e4 -she5 sh e5 -re1 r e1 -re2 r e2 -re3 r e3 -re4 r e4 -re5 r e5 -ze1 z e1 -ze2 z e2 -ze3 z e3 -ze4 z e4 -ze5 z e5 -ce1 c e1 -ce2 c e2 -ce3 c e3 -ce4 c e4 -ce5 c e5 -se1 s e1 -se2 s e2 -se3 s e3 -se4 s e4 -se5 s e5 -ea1 ea1 -ea2 ea2 -ea3 ea3 -ea4 ea4 -ea5 ea5 -ai1 ai1 -ai2 ai2 -ai3 ai3 -ai4 ai4 -ai5 ai5 -bai1 b ai1 -bai2 b ai2 -bai3 b ai3 -bai4 b ai4 -bai5 b ai5 -pai1 p ai1 -pai2 p ai2 -pai3 p ai3 -pai4 p ai4 -pai5 p ai5 -mai1 m ai1 -mai2 m ai2 -mai3 m ai3 -mai4 m ai4 -mai5 m ai5 -dai1 d ai1 -dai2 d ai2 -dai3 d ai3 -dai4 d ai4 -dai5 d ai5 -tai1 t ai1 -tai2 t ai2 -tai3 t ai3 -tai4 t ai4 -tai5 t ai5 -nai1 n ai1 -nai2 n ai2 -nai3 n ai3 -nai4 n ai4 -nai5 n ai5 -lai1 l ai1 -lai2 l ai2 -lai3 l ai3 -lai4 l ai4 -lai5 l ai5 -gai1 g ai1 -gai2 g ai2 -gai3 g ai3 -gai4 g ai4 -gai5 g ai5 -kai1 k ai1 -kai2 k ai2 -kai3 k ai3 -kai4 k ai4 -kai5 k ai5 -hai1 h ai1 -hai2 h ai2 -hai3 h ai3 -hai4 h ai4 -hai5 h ai5 -zhai1 zh ai1 -zhai2 zh ai2 -zhai3 zh ai3 -zhai4 zh ai4 -zhai5 zh ai5 -chai1 ch ai1 -chai2 ch ai2 -chai3 ch ai3 -chai4 ch ai4 -chai5 ch ai5 -shai1 sh ai1 -shai2 sh ai2 -shai3 sh ai3 -shai4 sh ai4 -shai5 sh ai5 -zai1 z ai1 -zai2 z ai2 -zai3 z ai3 -zai4 z ai4 -zai5 z ai5 -cai1 c ai1 -cai2 c ai2 -cai3 c ai3 -cai4 c ai4 -cai5 c ai5 -sai1 s ai1 -sai2 s ai2 -sai3 s ai3 -sai4 s ai4 -sai5 s ai5 -ei1 ei1 -ei2 ei2 -ei3 ei3 -ei4 ei4 -ei5 ei5 -bei1 b ei1 -bei2 b ei2 -bei3 b ei3 -bei4 b ei4 -bei5 b ei5 -pei1 p ei1 -pei2 p ei2 -pei3 p ei3 -pei4 p ei4 -pei5 p ei5 -mei1 m ei1 -mei2 m ei2 -mei3 m ei3 -mei4 m ei4 -mei5 m ei5 -fei1 f ei1 -fei2 f ei2 -fei3 f ei3 -fei4 f ei4 -fei5 f ei5 -dei1 d ei1 -dei2 d ei2 -dei3 d ei3 -dei4 d ei4 -dei5 d ei5 -tei1 t ei1 -tei2 t ei2 -tei3 t ei3 -tei4 t ei4 -tei5 t ei5 -nei1 n ei1 -nei2 n ei2 -nei3 n ei3 -nei4 n ei4 -nei5 n ei5 -lei1 l ei1 -lei2 l ei2 -lei3 l ei3 -lei4 l ei4 -lei5 l ei5 -gei1 g ei1 -gei2 g ei2 -gei3 g ei3 -gei4 g ei4 -gei5 g ei5 -kei1 k ei1 -kei2 k ei2 -kei3 k ei3 -kei4 k ei4 -kei5 k ei5 -hei1 h ei1 -hei2 h ei2 -hei3 h ei3 -hei4 h ei4 -hei5 h ei5 -zhei1 zh ei1 -zhei2 zh ei2 -zhei3 zh ei3 -zhei4 zh ei4 -zhei5 zh ei5 -shei1 sh ei1 -shei2 sh ei2 -shei3 sh ei3 -shei4 sh ei4 -shei5 sh ei5 -zei1 z ei1 -zei2 z ei2 -zei3 z ei3 -zei4 z ei4 -zei5 z ei5 -ao1 au1 -ao2 au2 -ao3 au3 -ao4 au4 -ao5 au5 -bao1 b au1 -bao2 b au2 -bao3 b au3 -bao4 b au4 -bao5 b au5 -pao1 p au1 -pao2 p au2 -pao3 p au3 -pao4 p au4 -pao5 p au5 -mao1 m au1 -mao2 m au2 -mao3 m au3 -mao4 m au4 -mao5 m au5 -dao1 d au1 -dao2 d au2 -dao3 d au3 -dao4 d au4 -dao5 d au5 -tao1 t au1 -tao2 t au2 -tao3 t au3 -tao4 t au4 -tao5 t au5 -nao1 n au1 -nao2 n au2 -nao3 n au3 -nao4 n au4 -nao5 n au5 -lao1 l au1 -lao2 l au2 -lao3 l au3 -lao4 l au4 -lao5 l au5 -gao1 g au1 -gao2 g au2 -gao3 g au3 -gao4 g au4 -gao5 g au5 -kao1 k au1 -kao2 k au2 -kao3 k au3 -kao4 k au4 -kao5 k au5 -hao1 h au1 -hao2 h au2 -hao3 h au3 -hao4 h au4 -hao5 h au5 -zhao1 zh au1 -zhao2 zh au2 -zhao3 zh au3 -zhao4 zh au4 -zhao5 zh au5 -chao1 ch au1 -chao2 ch au2 -chao3 ch au3 -chao4 ch au4 -chao5 ch au5 -shao1 sh au1 -shao2 sh au2 -shao3 sh au3 -shao4 sh au4 -shao5 sh au5 -rao1 r au1 -rao2 r au2 -rao3 r au3 -rao4 r au4 -rao5 r au5 -zao1 z au1 -zao2 z au2 -zao3 z au3 -zao4 z au4 -zao5 z au5 -cao1 c au1 -cao2 c au2 -cao3 c au3 -cao4 c au4 -cao5 c au5 -sao1 s au1 -sao2 s au2 -sao3 s au3 -sao4 s au4 -sao5 s au5 -ou1 ou1 -ou2 ou2 -ou3 ou3 -ou4 ou4 -ou5 ou5 -pou1 p ou1 -pou2 p ou2 -pou3 p ou3 -pou4 p ou4 -pou5 p ou5 -mou1 m ou1 -mou2 m ou2 -mou3 m ou3 -mou4 m ou4 -mou5 m ou5 -fou1 f ou1 -fou2 f ou2 -fou3 f ou3 -fou4 f ou4 -fou5 f ou5 -dou1 d ou1 -dou2 d ou2 -dou3 d ou3 -dou4 d ou4 -dou5 d ou5 -tou1 t ou1 -tou2 t ou2 -tou3 t ou3 -tou4 t ou4 -tou5 t ou5 -nou1 n ou1 -nou2 n ou2 -nou3 n ou3 -nou4 n ou4 -nou5 n ou5 -lou1 l ou1 -lou2 l ou2 -lou3 l ou3 -lou4 l ou4 -lou5 l ou5 -gou1 g ou1 -gou2 g ou2 -gou3 g ou3 -gou4 g ou4 -gou5 g ou5 -kou1 k ou1 -kou2 k ou2 -kou3 k ou3 -kou4 k ou4 -kou5 k ou5 -hou1 h ou1 -hou2 h ou2 -hou3 h ou3 -hou4 h ou4 -hou5 h ou5 -zhou1 zh ou1 -zhou2 zh ou2 -zhou3 zh ou3 -zhou4 zh ou4 -zhou5 zh ou5 -chou1 ch ou1 -chou2 ch ou2 -chou3 ch ou3 -chou4 ch ou4 -chou5 ch ou5 -shou1 sh ou1 -shou2 sh ou2 -shou3 sh ou3 -shou4 sh ou4 -shou5 sh ou5 -rou1 r ou1 -rou2 r ou2 -rou3 r ou3 -rou4 r ou4 -rou5 r ou5 -zou1 z ou1 -zou2 z ou2 -zou3 z ou3 -zou4 z ou4 -zou5 z ou5 -cou1 c ou1 -cou2 c ou2 -cou3 c ou3 -cou4 c ou4 -cou5 c ou5 -sou1 s ou1 -sou2 s ou2 -sou3 s ou3 -sou4 s ou4 -sou5 s ou5 -an1 an1 -an2 an2 -an3 an3 -an4 an4 -an5 an5 -ban1 b an1 -ban2 b an2 -ban3 b an3 -ban4 b an4 -ban5 b an5 -pan1 p an1 -pan2 p an2 -pan3 p an3 -pan4 p an4 -pan5 p an5 -man1 m an1 -man2 m an2 -man3 m an3 -man4 m an4 -man5 m an5 -fan1 f an1 -fan2 f an2 -fan3 f an3 -fan4 f an4 -fan5 f an5 -dan1 d an1 -dan2 d an2 -dan3 d an3 -dan4 d an4 -dan5 d an5 -tan1 t an1 -tan2 t an2 -tan3 t an3 -tan4 t an4 -tan5 t an5 -nan1 n an1 -nan2 n an2 -nan3 n an3 -nan4 n an4 -nan5 n an5 -lan1 l an1 -lan2 l an2 -lan3 l an3 -lan4 l an4 -lan5 l an5 -gan1 g an1 -gan2 g an2 -gan3 g an3 -gan4 g an4 -gan5 g an5 -kan1 k an1 -kan2 k an2 -kan3 k an3 -kan4 k an4 -kan5 k an5 -han1 h an1 -han2 h an2 -han3 h an3 -han4 h an4 -han5 h an5 -zhan1 zh an1 -zhan2 zh an2 -zhan3 zh an3 -zhan4 zh an4 -zhan5 zh an5 -chan1 ch an1 -chan2 ch an2 -chan3 ch an3 -chan4 ch an4 -chan5 ch an5 -shan1 sh an1 -shan2 sh an2 -shan3 sh an3 -shan4 sh an4 -shan5 sh an5 -ran1 r an1 -ran2 r an2 -ran3 r an3 -ran4 r an4 -ran5 r an5 -zan1 z an1 -zan2 z an2 -zan3 z an3 -zan4 z an4 -zan5 z an5 -can1 c an1 -can2 c an2 -can3 c an3 -can4 c an4 -can5 c an5 -san1 s an1 -san2 s an2 -san3 s an3 -san4 s an4 -san5 s an5 -en1 en1 -en2 en2 -en3 en3 -en4 en4 -en5 en5 -ben1 b en1 -ben2 b en2 -ben3 b en3 -ben4 b en4 -ben5 b en5 -pen1 p en1 -pen2 p en2 -pen3 p en3 -pen4 p en4 -pen5 p en5 -men1 m en1 -men2 m en2 -men3 m en3 -men4 m en4 -men5 m en5 -fen1 f en1 -fen2 f en2 -fen3 f en3 -fen4 f en4 -fen5 f en5 -den1 d en1 -den2 d en2 -den3 d en3 -den4 d en4 -den5 d en5 -nen1 n en1 -nen2 n en2 -nen3 n en3 -nen4 n en4 -nen5 n en5 -gen1 g en1 -gen2 g en2 -gen3 g en3 -gen4 g en4 -gen5 g en5 -ken1 k en1 -ken2 k en2 -ken3 k en3 -ken4 k en4 -ken5 k en5 -hen1 h en1 -hen2 h en2 -hen3 h en3 -hen4 h en4 -hen5 h en5 -zhen1 zh en1 -zhen2 zh en2 -zhen3 zh en3 -zhen4 zh en4 -zhen5 zh en5 -chen1 ch en1 -chen2 ch en2 -chen3 ch en3 -chen4 ch en4 -chen5 ch en5 -shen1 sh en1 -shen2 sh en2 -shen3 sh en3 -shen4 sh en4 -shen5 sh en5 -ren1 r en1 -ren2 r en2 -ren3 r en3 -ren4 r en4 -ren5 r en5 -zen1 z en1 -zen2 z en2 -zen3 z en3 -zen4 z en4 -zen5 z en5 -cen1 c en1 -cen2 c en2 -cen3 c en3 -cen4 c en4 -cen5 c en5 -sen1 s en1 -sen2 s en2 -sen3 s en3 -sen4 s en4 -sen5 s en5 -ang1 ang1 -ang2 ang2 -ang3 ang3 -ang4 ang4 -ang5 ang5 -bang1 b ang1 -bang2 b ang2 -bang3 b ang3 -bang4 b ang4 -bang5 b ang5 -pang1 p ang1 -pang2 p ang2 -pang3 p ang3 -pang4 p ang4 -pang5 p ang5 -mang1 m ang1 -mang2 m ang2 -mang3 m ang3 -mang4 m ang4 -mang5 m ang5 -fang1 f ang1 -fang2 f ang2 -fang3 f ang3 -fang4 f ang4 -fang5 f ang5 -dang1 d ang1 -dang2 d ang2 -dang3 d ang3 -dang4 d ang4 -dang5 d ang5 -tang1 t ang1 -tang2 t ang2 -tang3 t ang3 -tang4 t ang4 -tang5 t ang5 -nang1 n ang1 -nang2 n ang2 -nang3 n ang3 -nang4 n ang4 -nang5 n ang5 -lang1 l ang1 -lang2 l ang2 -lang3 l ang3 -lang4 l ang4 -lang5 l ang5 -gang1 g ang1 -gang2 g ang2 -gang3 g ang3 -gang4 g ang4 -gang5 g ang5 -kang1 k ang1 -kang2 k ang2 -kang3 k ang3 -kang4 k ang4 -kang5 k ang5 -hang1 h ang1 -hang2 h ang2 -hang3 h ang3 -hang4 h ang4 -hang5 h ang5 -zhang1 zh ang1 -zhang2 zh ang2 -zhang3 zh ang3 -zhang4 zh ang4 -zhang5 zh ang5 -chang1 ch ang1 -chang2 ch ang2 -chang3 ch ang3 -chang4 ch ang4 -chang5 ch ang5 -shang1 sh ang1 -shang2 sh ang2 -shang3 sh ang3 -shang4 sh ang4 -shang5 sh ang5 -rang1 r ang1 -rang2 r ang2 -rang3 r ang3 -rang4 r ang4 -rang5 r ang5 -zang1 z ang1 -zang2 z ang2 -zang3 z ang3 -zang4 z ang4 -zang5 z ang5 -cang1 c ang1 -cang2 c ang2 -cang3 c ang3 -cang4 c ang4 -cang5 c ang5 -sang1 s ang1 -sang2 s ang2 -sang3 s ang3 -sang4 s ang4 -sang5 s ang5 -eng1 eng1 -eng2 eng2 -eng3 eng3 -eng4 eng4 -eng5 eng5 -beng1 b eng1 -beng2 b eng2 -beng3 b eng3 -beng4 b eng4 -beng5 b eng5 -peng1 p eng1 -peng2 p eng2 -peng3 p eng3 -peng4 p eng4 -peng5 p eng5 -meng1 m eng1 -meng2 m eng2 -meng3 m eng3 -meng4 m eng4 -meng5 m eng5 -feng1 f eng1 -feng2 f eng2 -feng3 f eng3 -feng4 f eng4 -feng5 f eng5 -deng1 d eng1 -deng2 d eng2 -deng3 d eng3 -deng4 d eng4 -deng5 d eng5 -teng1 t eng1 -teng2 t eng2 -teng3 t eng3 -teng4 t eng4 -teng5 t eng5 -neng1 n eng1 -neng2 n eng2 -neng3 n eng3 -neng4 n eng4 -neng5 n eng5 -leng1 l eng1 -leng2 l eng2 -leng3 l eng3 -leng4 l eng4 -leng5 l eng5 -geng1 g eng1 -geng2 g eng2 -geng3 g eng3 -geng4 g eng4 -geng5 g eng5 -keng1 k eng1 -keng2 k eng2 -keng3 k eng3 -keng4 k eng4 -keng5 k eng5 -heng1 h eng1 -heng2 h eng2 -heng3 h eng3 -heng4 h eng4 -heng5 h eng5 -zheng1 zh eng1 -zheng2 zh eng2 -zheng3 zh eng3 -zheng4 zh eng4 -zheng5 zh eng5 -cheng1 ch eng1 -cheng2 ch eng2 -cheng3 ch eng3 -cheng4 ch eng4 -cheng5 ch eng5 -sheng1 sh eng1 -sheng2 sh eng2 -sheng3 sh eng3 -sheng4 sh eng4 -sheng5 sh eng5 -reng1 r eng1 -reng2 r eng2 -reng3 r eng3 -reng4 r eng4 -reng5 r eng5 -zeng1 z eng1 -zeng2 z eng2 -zeng3 z eng3 -zeng4 z eng4 -zeng5 z eng5 -ceng1 c eng1 -ceng2 c eng2 -ceng3 c eng3 -ceng4 c eng4 -ceng5 c eng5 -seng1 s eng1 -seng2 s eng2 -seng3 s eng3 -seng4 s eng4 -seng5 s eng5 -er1 er1 -er2 er2 -er3 er3 -er4 er4 -er5 er5 -yi1 y i1 -yi2 y i2 -yi3 y i3 -yi4 y i4 -yi5 y i5 -bi1 b i1 -bi2 b i2 -bi3 b i3 -bi4 b i4 -bi5 b i5 -pi1 p i1 -pi2 p i2 -pi3 p i3 -pi4 p i4 -pi5 p i5 -mi1 m i1 -mi2 m i2 -mi3 m i3 -mi4 m i4 -mi5 m i5 -di1 d i1 -di2 d i2 -di3 d i3 -di4 d i4 -di5 d i5 -ti1 t i1 -ti2 t i2 -ti3 t i3 -ti4 t i4 -ti5 t i5 -ni1 n i1 -ni2 n i2 -ni3 n i3 -ni4 n i4 -ni5 n i5 -li1 l i1 -li2 l i2 -li3 l i3 -li4 l i4 -li5 l i5 -ji1 j i1 -ji2 j i2 -ji3 j i3 -ji4 j i4 -ji5 j i5 -qi1 q i1 -qi2 q i2 -qi3 q i3 -qi4 q i4 -qi5 q i5 -xi1 x i1 -xi2 x i2 -xi3 x i3 -xi4 x i4 -xi5 x i5 -ya1 y ia1 -ya2 y ia2 -ya3 y ia3 -ya4 y ia4 -ya5 y ia5 -dia1 d ia1 -dia2 d ia2 -dia3 d ia3 -dia4 d ia4 -dia5 d ia5 -lia1 l ia1 -lia2 l ia2 -lia3 l ia3 -lia4 l ia4 -lia5 l ia5 -jia1 j ia1 -jia2 j ia2 -jia3 j ia3 -jia4 j ia4 -jia5 j ia5 -qia1 q ia1 -qia2 q ia2 -qia3 q ia3 -qia4 q ia4 -qia5 q ia5 -xia1 x ia1 -xia2 x ia2 -xia3 x ia3 -xia4 x ia4 -xia5 x ia5 -yo1 y io1 -yo2 y io2 -yo3 y io3 -yo4 y io4 -yo5 y io5 -ye1 y ie1 -ye2 y ie2 -ye3 y ie3 -ye4 y ie4 -ye5 y ie5 -bie1 b ie1 -bie2 b ie2 -bie3 b ie3 -bie4 b ie4 -bie5 b ie5 -pie1 p ie1 -pie2 p ie2 -pie3 p ie3 -pie4 p ie4 -pie5 p ie5 -mie1 m ie1 -mie2 m ie2 -mie3 m ie3 -mie4 m ie4 -mie5 m ie5 -die1 d ie1 -die2 d ie2 -die3 d ie3 -die4 d ie4 -die5 d ie5 -tie1 t ie1 -tie2 t ie2 -tie3 t ie3 -tie4 t ie4 -tie5 t ie5 -nie1 n ie1 -nie2 n ie2 -nie3 n ie3 -nie4 n ie4 -nie5 n ie5 -lie1 l ie1 -lie2 l ie2 -lie3 l ie3 -lie4 l ie4 -lie5 l ie5 -jie1 j ie1 -jie2 j ie2 -jie3 j ie3 -jie4 j ie4 -jie5 j ie5 -qie1 q ie1 -qie2 q ie2 -qie3 q ie3 -qie4 q ie4 -qie5 q ie5 -xie1 x ie1 -xie2 x ie2 -xie3 x ie3 -xie4 x ie4 -xie5 x ie5 -yai1 y ai1 -yai2 y ai2 -yai3 y ai3 -yai4 y ai4 -yai5 y ai5 -yao1 y au1 -yao2 y au2 -yao3 y au3 -yao4 y au4 -yao5 y au5 -biao1 b iau1 -biao2 b iau2 -biao3 b iau3 -biao4 b iau4 -biao5 b iau5 -piao1 p iau1 -piao2 p iau2 -piao3 p iau3 -piao4 p iau4 -piao5 p iau5 -miao1 m iau1 -miao2 m iau2 -miao3 m iau3 -miao4 m iau4 -miao5 m iau5 -fiao1 f iau1 -fiao2 f iau2 -fiao3 f iau3 -fiao4 f iau4 -fiao5 f iau5 -diao1 d iau1 -diao2 d iau2 -diao3 d iau3 -diao4 d iau4 -diao5 d iau5 -tiao1 t iau1 -tiao2 t iau2 -tiao3 t iau3 -tiao4 t iau4 -tiao5 t iau5 -niao1 n iau1 -niao2 n iau2 -niao3 n iau3 -niao4 n iau4 -niao5 n iau5 -liao1 l iau1 -liao2 l iau2 -liao3 l iau3 -liao4 l iau4 -liao5 l iau5 -jiao1 j iau1 -jiao2 j iau2 -jiao3 j iau3 -jiao4 j iau4 -jiao5 j iau5 -qiao1 q iau1 -qiao2 q iau2 -qiao3 q iau3 -qiao4 q iau4 -qiao5 q iau5 -xiao1 x iau1 -xiao2 x iau2 -xiao3 x iau3 -xiao4 x iau4 -xiao5 x iau5 -you1 y iou1 -you2 y iou2 -you3 y iou3 -you4 y iou4 -you5 y iou5 -miu1 m iou1 -miu2 m iou2 -miu3 m iou3 -miu4 m iou4 -miu5 m iou5 -diu1 d iou1 -diu2 d iou2 -diu3 d iou3 -diu4 d iou4 -diu5 d iou5 -niu1 n iou1 -niu2 n iou2 -niu3 n iou3 -niu4 n iou4 -niu5 n iou5 -liu1 l iou1 -liu2 l iou2 -liu3 l iou3 -liu4 l iou4 -liu5 l iou5 -jiu1 j iou1 -jiu2 j iou2 -jiu3 j iou3 -jiu4 j iou4 -jiu5 j iou5 -qiu1 q iou1 -qiu2 q iou2 -qiu3 q iou3 -qiu4 q iou4 -qiu5 q iou5 -xiu1 xiou1 -xiu2 xiou2 -xiu3 xiou3 -xiu4 xiou4 -xiu5 xiou5 -yan1 y ian1 -yan2 y ian2 -yan3 y ian3 -yan4 y ian4 -yan5 y ian5 -bian1 b ian1 -bian2 b ian2 -bian3 b ian3 -bian4 b ian4 -bian5 b ian5 -pian1 p ian1 -pian2 p ian2 -pian3 p ian3 -pian4 p ian4 -pian5 p ian5 -mian1 m ian1 -mian2 m ian2 -mian3 m ian3 -mian4 m ian4 -mian5 m ian5 -dian1 d ian1 -dian2 d ian2 -dian3 d ian3 -dian4 d ian4 -dian5 d ian5 -tian1 t ian1 -tian2 t ian2 -tian3 t ian3 -tian4 t ian4 -tian5 t ian5 -nian1 n ian1 -nian2 n ian2 -nian3 n ian3 -nian4 n ian4 -nian5 n ian5 -lian1 l ian1 -lian2 l ian2 -lian3 l ian3 -lian4 l ian4 -lian5 l ian5 -jian1 j ian1 -jian2 j ian2 -jian3 j ian3 -jian4 j ian4 -jian5 j ian5 -qian1 q ian1 -qian2 q ian2 -qian3 q ian3 -qian4 q ian4 -qian5 q ian5 -xian1 x ian1 -xian2 x ian2 -xian3 x ian3 -xian4 x ian4 -xian5 x ian5 -yin1 y in1 -yin2 y in2 -yin3 y in3 -yin4 y in4 -yin5 y in5 -bin1 b in1 -bin2 b in2 -bin3 b in3 -bin4 b in4 -bin5 b in5 -pin1 p in1 -pin2 p in2 -pin3 p in3 -pin4 p in4 -pin5 p in5 -min1 m in1 -min2 m in2 -min3 m in3 -min4 m in4 -min5 m in5 -din1 d in1 -din2 d in2 -din3 d in3 -din4 d in4 -din5 d in5 -nin1 n in1 -nin2 n in2 -nin3 n in3 -nin4 n in4 -nin5 n in5 -lin1 l in1 -lin2 l in2 -lin3 l in3 -lin4 l in4 -lin5 l in5 -jin1 j in1 -jin2 j in2 -jin3 j in3 -jin4 j in4 -jin5 j in5 -qin1 q in1 -qin2 q in2 -qin3 q in3 -qin4 q in4 -qin5 q in5 -xin1 x in1 -xin2 x in2 -xin3 x in3 -xin4 x in4 -xin5 x in5 -yang1 y iang1 -yang2 y iang2 -yang3 y iang3 -yang4 y iang4 -yang5 y iang5 -biang1 b iang1 -biang2 b iang2 -biang3 b iang3 -biang4 b iang4 -biang5 b iang5 -niang1 n iang1 -niang2 n iang2 -niang3 n iang3 -niang4 n iang4 -niang5 n iang5 -liang1 l iang1 -liang2 l iang2 -liang3 l iang3 -liang4 l iang4 -liang5 l iang5 -jiang1 j iang1 -jiang2 j iang2 -jiang3 j iang3 -jiang4 j iang4 -jiang5 j iang5 -qiang1 q iang1 -qiang2 q iang2 -qiang3 q iang3 -qiang4 q iang4 -qiang5 q iang5 -xiang1 x iang1 -xiang2 x iang2 -xiang3 x iang3 -xiang4 x iang4 -xiang5 x iang5 -ying1 y ing1 -ying2 y ing2 -ying3 y ing3 -ying4 y ing4 -ying5 y ing5 -bing1 b ing1 -bing2 b ing2 -bing3 b ing3 -bing4 b ing4 -bing5 b ing5 -ping1 p ing1 -ping2 p ing2 -ping3 p ing3 -ping4 p ing4 -ping5 p ing5 -ming1 m ing1 -ming2 m ing2 -ming3 m ing3 -ming4 m ing4 -ming5 m ing5 -ding1 d ing1 -ding2 d ing2 -ding3 d ing3 -ding4 d ing4 -ding5 d ing5 -ting1 t ing1 -ting2 t ing2 -ting3 t ing3 -ting4 t ing4 -ting5 t ing5 -ning1 n ing1 -ning2 n ing2 -ning3 n ing3 -ning4 n ing4 -ning5 n ing5 -ling1 l ing1 -ling2 l ing2 -ling3 l ing3 -ling4 l ing4 -ling5 l ing5 -jing1 j ing1 -jing2 j ing2 -jing3 j ing3 -jing4 j ing4 -jing5 j ing5 -qing1 q ing1 -qing2 q ing2 -qing3 q ing3 -qing4 q ing4 -qing5 q ing5 -xing1 x ing1 -xing2 x ing2 -xing3 x ing3 -xing4 x ing4 -xing5 x ing5 -wu1 w u1 -wu2 w u2 -wu3 w u3 -wu4 w u4 -wu5 w u5 -bu1 b u1 -bu2 b u2 -bu3 b u3 -bu4 b u4 -bu5 b u5 -pu1 p u1 -pu2 p u2 -pu3 p u3 -pu4 p u4 -pu5 p u5 -mu1 m u1 -mu2 m u2 -mu3 m u3 -mu4 m u4 -mu5 m u5 -fu1 f u1 -fu2 f u2 -fu3 f u3 -fu4 f u4 -fu5 f u5 -du1 d u1 -du2 d u2 -du3 d u3 -du4 d u4 -du5 d u5 -tu1 t u1 -tu2 t u2 -tu3 t u3 -tu4 t u4 -tu5 t u5 -nu1 n u1 -nu2 n u2 -nu3 n u3 -nu4 n u4 -nu5 n u5 -lu1 l u1 -lu2 l u2 -lu3 l u3 -lu4 l u4 -lu5 l u5 -gu1 g u1 -gu2 g u2 -gu3 g u3 -gu4 g u4 -gu5 g u5 -ku1 k u1 -ku2 k u2 -ku3 k u3 -ku4 k u4 -ku5 k u5 -hu1 h u1 -hu2 h u2 -hu3 h u3 -hu4 h u4 -hu5 h u5 -zhu1 zh u1 -zhu2 zh u2 -zhu3 zh u3 -zhu4 zh u4 -zhu5 zh u5 -chu1 ch u1 -chu2 ch u2 -chu3 ch u3 -chu4 ch u4 -chu5 ch u5 -shu1 sh u1 -shu2 sh u2 -shu3 sh u3 -shu4 sh u4 -shu5 sh u5 -ru1 r u1 -ru2 r u2 -ru3 r u3 -ru4 r u4 -ru5 r u5 -zu1 z u1 -zu2 z u2 -zu3 z u3 -zu4 z u4 -zu5 z u5 -cu1 c u1 -cu2 c u2 -cu3 c u3 -cu4 c u4 -cu5 c u5 -su1 s u1 -su2 s u2 -su3 s u3 -su4 s u4 -su5 s u5 -wa1 w ua1 -wa2 w ua2 -wa3 w ua3 -wa4 w ua4 -wa5 w ua5 -gua1 g ua1 -gua2 g ua2 -gua3 g ua3 -gua4 g ua4 -gua5 g ua5 -kua1 k ua1 -kua2 k ua2 -kua3 k ua3 -kua4 k ua4 -kua5 k ua5 -hua1 h ua1 -hua2 h ua2 -hua3 h ua3 -hua4 h ua4 -hua5 h ua5 -zhua1 zh ua1 -zhua2 zh ua2 -zhua3 zh ua3 -zhua4 zh ua4 -zhua5 zh ua5 -chua1 ch ua1 -chua2 ch ua2 -chua3 ch ua3 -chua4 ch ua4 -chua5 ch ua5 -shua1 sh ua1 -shua2 sh ua2 -shua3 sh ua3 -shua4 sh ua4 -shua5 sh ua5 -wo1 w uo1 -wo2 w uo2 -wo3 w uo3 -wo4 w uo4 -wo5 w uo5 -duo1 d uo1 -duo2 d uo2 -duo3 d uo3 -duo4 d uo4 -duo5 d uo5 -tuo1 t uo1 -tuo2 t uo2 -tuo3 t uo3 -tuo4 t uo4 -tuo5 t uo5 -nuo1 n uo1 -nuo2 n uo2 -nuo3 n uo3 -nuo4 n uo4 -nuo5 n uo5 -luo1 l uo1 -luo2 l uo2 -luo3 l uo3 -luo4 l uo4 -luo5 l uo5 -guo1 g uo1 -guo2 g uo2 -guo3 g uo3 -guo4 g uo4 -guo5 g uo5 -kuo1 k uo1 -kuo2 k uo2 -kuo3 k uo3 -kuo4 k uo4 -kuo5 k uo5 -huo1 h uo1 -huo2 h uo2 -huo3 h uo3 -huo4 h uo4 -huo5 h uo5 -zhuo1 zh uo1 -zhuo2 zh uo2 -zhuo3 zh uo3 -zhuo4 zh uo4 -zhuo5 zh uo5 -chuo1 ch uo1 -chuo2 ch uo2 -chuo3 ch uo3 -chuo4 ch uo4 -chuo5 ch uo5 -shuo1 sh uo1 -shuo2 sh uo2 -shuo3 sh uo3 -shuo4 sh uo4 -shuo5 sh uo5 -ruo1 r uo1 -ruo2 r uo2 -ruo3 r uo3 -ruo4 r uo4 -ruo5 r uo5 -zuo1 z uo1 -zuo2 z uo2 -zuo3 z uo3 -zuo4 z uo4 -zuo5 z uo5 -cuo1 c uo1 -cuo2 c uo2 -cuo3 c uo3 -cuo4 c uo4 -cuo5 c uo5 -suo1 s uo1 -suo2 s uo2 -suo3 s uo3 -suo4 s uo4 -suo5 s uo5 -wai1 w uai1 -wai2 w uai2 -wai3 w uai3 -wai4 w uai4 -wai5 w uai5 -guai1 g uai1 -guai2 g uai2 -guai3 g uai3 -guai4 g uai4 -guai5 g uai5 -kuai1 k uai1 -kuai2 k uai2 -kuai3 k uai3 -kuai4 k uai4 -kuai5 k uai5 -huai1 h uai1 -huai2 h uai2 -huai3 h uai3 -huai4 h uai4 -huai5 h uai5 -zhuai1 zh uai1 -zhuai2 zh uai2 -zhuai3 zh uai3 -zhuai4 zh uai4 -zhuai5 zh uai5 -chuai1 ch uai1 -chuai2 ch uai2 -chuai3 ch uai3 -chuai4 ch uai4 -chuai5 ch uai5 -shuai1 sh uai1 -shuai2 sh uai2 -shuai3 sh uai3 -shuai4 sh uai4 -shuai5 sh uai5 -wei1 w uei1 -wei2 w uei2 -wei3 w uei3 -wei4 w uei4 -wei5 w uei5 -dui1 d uei1 -dui2 d uei2 -dui3 d uei3 -dui4 d uei4 -dui5 d uei5 -tui1 t uei1 -tui2 t uei2 -tui3 t uei3 -tui4 t uei4 -tui5 t uei5 -gui1 g uei1 -gui2 g uei2 -gui3 g uei3 -gui4 g uei4 -gui5 g uei5 -kui1 k uei1 -kui2 k uei2 -kui3 k uei3 -kui4 k uei4 -kui5 k uei5 -hui1 h uei1 -hui2 h uei2 -hui3 h uei3 -hui4 h uei4 -hui5 h uei5 -zhui1 zh uei1 -zhui2 zh uei2 -zhui3 zh uei3 -zhui4 zh uei4 -zhui5 zh uei5 -chui1 ch uei1 -chui2 ch uei2 -chui3 ch uei3 -chui4 ch uei4 -chui5 ch uei5 -shui1 sh uei1 -shui2 sh uei2 -shui3 sh uei3 -shui4 sh uei4 -shui5 sh uei5 -rui1 r uei1 -rui2 r uei2 -rui3 r uei3 -rui4 r uei4 -rui5 r uei5 -zui1 z uei1 -zui2 z uei2 -zui3 z uei3 -zui4 z uei4 -zui5 z uei5 -cui1 c uei1 -cui2 c uei2 -cui3 c uei3 -cui4 c uei4 -cui5 c uei5 -sui1 s uei1 -sui2 s uei2 -sui3 s uei3 -sui4 s uei4 -sui5 s uei5 -wan1 w uan1 -wan2 w uan2 -wan3 w uan3 -wan4 w uan4 -wan5 w uan5 -duan1 d uan1 -duan2 d uan2 -duan3 d uan3 -duan4 d uan4 -duan5 d uan5 -tuan1 t uan1 -tuan2 t uan2 -tuan3 t uan3 -tuan4 t uan4 -tuan5 t uan5 -nuan1 n uan1 -nuan2 n uan2 -nuan3 n uan3 -nuan4 n uan4 -nuan5 n uan5 -luan1 l uan1 -luan2 l uan2 -luan3 l uan3 -luan4 l uan4 -luan5 l uan5 -guan1 g uan1 -guan2 g uan2 -guan3 g uan3 -guan4 g uan4 -guan5 g uan5 -kuan1 k uan1 -kuan2 k uan2 -kuan3 k uan3 -kuan4 k uan4 -kuan5 k uan5 -huan1 h uan1 -huan2 h uan2 -huan3 h uan3 -huan4 h uan4 -huan5 h uan5 -zhuan1 zh uan1 -zhuan2 zh uan2 -zhuan3 zh uan3 -zhuan4 zh uan4 -zhuan5 zh uan5 -chuan1 ch uan1 -chuan2 ch uan2 -chuan3 ch uan3 -chuan4 ch uan4 -chuan5 ch uan5 -shuan1 sh uan1 -shuan2 sh uan2 -shuan3 sh uan3 -shuan4 sh uan4 -shuan5 sh uan5 -ruan1 r uan1 -ruan2 r uan2 -ruan3 r uan3 -ruan4 r uan4 -ruan5 r uan5 -zuan1 z uan1 -zuan2 z uan2 -zuan3 z uan3 -zuan4 z uan4 -zuan5 z uan5 -cuan1 c uan1 -cuan2 c uan2 -cuan3 c uan3 -cuan4 c uan4 -cuan5 c uan5 -suan1 s uan1 -suan2 s uan2 -suan3 s uan3 -suan4 s uan4 -suan5 s uan5 -wen1 w uen1 -wen2 w uen2 -wen3 w uen3 -wen4 w uen4 -wen5 w uen5 -dun1 d uen1 -dun2 d uen2 -dun3 d uen3 -dun4 d uen4 -dun5 d uen5 -tun1 t uen1 -tun2 t uen2 -tun3 t uen3 -tun4 t uen4 -tun5 t uen5 -nun1 n uen1 -nun2 n uen2 -nun3 n uen3 -nun4 n uen4 -nun5 n uen5 -lun1 l uen1 -lun2 l uen2 -lun3 l uen3 -lun4 l uen4 -lun5 l uen5 -gun1 g uen1 -gun2 g uen2 -gun3 g uen3 -gun4 g uen4 -gun5 g uen5 -kun1 k uen1 -kun2 k uen2 -kun3 k uen3 -kun4 k uen4 -kun5 k uen5 -hun1 h uen1 -hun2 h uen2 -hun3 h uen3 -hun4 h uen4 -hun5 h uen5 -zhun1 zh uen1 -zhun2 zh uen2 -zhun3 zh uen3 -zhun4 zh uen4 -zhun5 zh uen5 -chun1 ch uen1 -chun2 ch uen2 -chun3 ch uen3 -chun4 ch uen4 -chun5 ch uen5 -shun1 sh uen1 -shun2 sh uen2 -shun3 sh uen3 -shun4 sh uen4 -shun5 sh uen5 -run1 r uen1 -run2 r uen2 -run3 r uen3 -run4 r uen4 -run5 r uen5 -zun1 z uen1 -zun2 z uen2 -zun3 z uen3 -zun4 z uen4 -zun5 z uen5 -cun1 c uen1 -cun2 c uen2 -cun3 c uen3 -cun4 c uen4 -cun5 c uen5 -sun1 s uen1 -sun2 s uen2 -sun3 s uen3 -sun4 s uen4 -sun5 s uen5 -wang1 w uang1 -wang2 w uang2 -wang3 w uang3 -wang4 w uang4 -wang5 w uang5 -guang1 g uang1 -guang2 g uang2 -guang3 g uang3 -guang4 g uang4 -guang5 g uang5 -kuang1 k uang1 -kuang2 k uang2 -kuang3 k uang3 -kuang4 k uang4 -kuang5 k uang5 -huang1 h uang1 -huang2 h uang2 -huang3 h uang3 -huang4 h uang4 -huang5 h uang5 -zhuang1 zh uang1 -zhuang2 zh uang2 -zhuang3 zh uang3 -zhuang4 zh uang4 -zhuang5 zh uang5 -chuang1 ch uang1 -chuang2 ch uang2 -chuang3 ch uang3 -chuang4 ch uang4 -chuang5 ch uang5 -shuang1 sh uang1 -shuang2 sh uang2 -shuang3 sh uang3 -shuang4 sh uang4 -shuang5 sh uang5 -weng1 w ung1 -weng2 w ung2 -weng3 w ung3 -weng4 w ung4 -weng5 w ung5 -dong1 d ung1 -dong2 d ung2 -dong3 d ung3 -dong4 d ung4 -dong5 d ung5 -tong1 t ung1 -tong2 t ung2 -tong3 t ung3 -tong4 t ung4 -tong5 t ung5 -nong1 n ung1 -nong2 n ung2 -nong3 n ung3 -nong4 n ung4 -nong5 n ung5 -long1 l ung1 -long2 l ung2 -long3 l ung3 -long4 l ung4 -long5 l ung5 -gong1 g ung1 -gong2 g ung2 -gong3 g ung3 -gong4 g ung4 -gong5 g ung5 -kong1 k ung1 -kong2 k ung2 -kong3 k ung3 -kong4 k ung4 -kong5 k ung5 -hong1 h ung1 -hong2 h ung2 -hong3 h ung3 -hong4 h ung4 -hong5 h ung5 -zhong1 zh ung1 -zhong2 zh ung2 -zhong3 zh ung3 -zhong4 zh ung4 -zhong5 zh ung5 -chong1 ch ung1 -chong2 ch ung2 -chong3 ch ung3 -chong4 ch ung4 -chong5 ch ung5 -rong1 r ung1 -rong2 r ung2 -rong3 r ung3 -rong4 r ung4 -rong5 r ung5 -zong1 z ung1 -zong2 z ung2 -zong3 z ung3 -zong4 z ung4 -zong5 z ung5 -cong1 c ung1 -cong2 c ung2 -cong3 c ung3 -cong4 c ung4 -cong5 c ung5 -song1 s ung1 -song2 s ung2 -song3 s ung3 -song4 s ung4 -song5 s ung5 -yu1 y v1 -yu2 y v2 -yu3 y v3 -yu4 y v4 -yu5 y v5 -nv1 n v1 -nv2 n v2 -nv3 n v3 -nv4 n v4 -nv5 n v5 -lv1 l v1 -lv2 l v2 -lv3 l v3 -lv4 l v4 -lv5 l v5 -ju1 j v1 -ju2 j v2 -ju3 j v3 -ju4 j v4 -ju5 j v5 -qu1 q v1 -qu2 q v2 -qu3 q v3 -qu4 q v4 -qu5 q v5 -xu1 x v1 -xu2 x v2 -xu3 x v3 -xu4 x v4 -xu5 x v5 -yue1 y ve1 -yue2 y ve2 -yue3 y ve3 -yue4 y ve4 -yue5 y ve5 -nue1 n ve1 -nue2 n ve2 -nue3 n ve3 -nue4 n ve4 -nue5 n ve5 -nve1 n ve1 -nve2 n ve2 -nve3 n ve3 -nve4 n ve4 -nve5 n ve5 -lue1 l ve1 -lue2 l ve2 -lue3 l ve3 -lue4 l ve4 -lue5 l ve5 -lve1 l ve1 -lve2 l ve2 -lve3 l ve3 -lve4 l ve4 -lve5 l ve5 -jue1 j ve1 -jue2 j ve2 -jue3 j ve3 -jue4 j ve4 -jue5 j ve5 -que1 q ve1 -que2 q ve2 -que3 q ve3 -que4 q ve4 -que5 q ve5 -xue1 x ve1 -xue2 x ve2 -xue3 x ve3 -xue4 x ve4 -xue5 x ve5 -yuan1 y van1 -yuan2 y van2 -yuan3 y van3 -yuan4 y van4 -yuan5 y van5 -juan1 j van1 -juan2 j van2 -juan3 j van3 -juan4 j van4 -juan5 j van5 -quan1 q van1 -quan2 q van2 -quan3 q van3 -quan4 q van4 -quan5 q van5 -xuan1 x van1 -xuan2 x van2 -xuan3 x van3 -xuan4 x van4 -xuan5 x van5 -yun1 y vn1 -yun2 y vn2 -yun3 y vn3 -yun4 y vn4 -yun5 y vn5 -jun1 j vn1 -jun2 j vn2 -jun3 j vn3 -jun4 j vn4 -jun5 j vn5 -qun1 q vn1 -qun2 q vn2 -qun3 q vn3 -qun4 q vn4 -qun5 q vn5 -xun1 x vn1 -xun2 x vn2 -xun3 x vn3 -xun4 x vn4 -xun5 x vn5 -yong1 y vng1 -yong2 y vng2 -yong3 y vng3 -yong4 y vng4 -yong5 y vng5 -jiong1 j vng1 -jiong2 j vng2 -jiong3 j vng3 -jiong4 j vng4 -jiong5 j vng5 -qiong1 q vng1 -qiong2 q vng2 -qiong3 q vng3 -qiong4 q vng4 -qiong5 q vng5 -xiong1 x vng1 -xiong2 x vng2 -xiong3 x vng3 -xiong4 x vng4 -xiong5 x vng5 -zhir1 zh iii1 &r -zhir2 zh iii2 &r -zhir3 zh iii3 &r -zhir4 zh iii4 &r -zhir5 zh iii5 &r -chir1 ch iii1 &r -chir2 ch iii2 &r -chir3 ch iii3 &r -chir4 ch iii4 &r -chir5 ch iii5 &r -shir1 sh iii1 &r -shir2 sh iii2 &r -shir3 sh iii3 &r -shir4 sh iii4 &r -shir5 sh iii5 &r -rir1 r iii1 &r -rir2 r iii2 &r -rir3 r iii3 &r -rir4 r iii4 &r -rir5 r iii5 &r -zir1 z ii1 &r -zir2 z ii2 &r -zir3 z ii3 &r -zir4 z ii4 &r -zir5 z ii5 &r -cir1 c ii1 &r -cir2 c ii2 &r -cir3 c ii3 &r -cir4 c ii4 &r -cir5 c ii5 &r -sir1 s ii1 &r -sir2 s ii2 &r -sir3 s ii3 &r -sir4 s ii4 &r -sir5 s ii5 &r -ar1 a1 &r -ar2 a2 &r -ar3 a3 &r -ar4 a4 &r -ar5 a5 &r -bar1 b a1 &r -bar2 b a2 &r -bar3 b a3 &r -bar4 b a4 &r -bar5 b a5 &r -par1 p a1 &r -par2 p a2 &r -par3 p a3 &r -par4 p a4 &r -par5 p a5 &r -mar1 m a1 &r -mar2 m a2 &r -mar3 m a3 &r -mar4 m a4 &r -mar5 m a5 &r -far1 f a1 &r -far2 f a2 &r -far3 f a3 &r -far4 f a4 &r -far5 f a5 &r -dar1 d a1 &r -dar2 d a2 &r -dar3 d a3 &r -dar4 d a4 &r -dar5 d a5 &r -tar1 t a1 &r -tar2 t a2 &r -tar3 t a3 &r -tar4 t a4 &r -tar5 t a5 &r -nar1 n a1 &r -nar2 n a2 &r -nar3 n a3 &r -nar4 n a4 &r -nar5 n a5 &r -lar1 l a1 &r -lar2 l a2 &r -lar3 l a3 &r -lar4 l a4 &r -lar5 l a5 &r -gar1 g a1 &r -gar2 g a2 &r -gar3 g a3 &r -gar4 g a4 &r -gar5 g a5 &r -kar1 k a1 &r -kar2 k a2 &r -kar3 k a3 &r -kar4 k a4 &r -kar5 k a5 &r -har1 h a1 &r -har2 h a2 &r -har3 h a3 &r -har4 h a4 &r -har5 h a5 &r -zhar1 zh a1 &r -zhar2 zh a2 &r -zhar3 zh a3 &r -zhar4 zh a4 &r -zhar5 zh a5 &r -char1 ch a1 &r -char2 ch a2 &r -char3 ch a3 &r -char4 ch a4 &r -char5 ch a5 &r -shar1 sh a1 &r -shar2 sh a2 &r -shar3 sh a3 &r -shar4 sh a4 &r -shar5 sh a5 &r -zar1 z a1 &r -zar2 z a2 &r -zar3 z a3 &r -zar4 z a4 &r -zar5 z a5 &r -car1 c a1 &r -car2 c a2 &r -car3 c a3 &r -car4 c a4 &r -car5 c a5 &r -sar1 s a1 &r -sar2 s a2 &r -sar3 s a3 &r -sar4 s a4 &r -sar5 s a5 &r -or1 o1 &r -or2 o2 &r -or3 o3 &r -or4 o4 &r -or5 o5 &r -bor1 b uo1 &r -bor2 b uo2 &r -bor3 b uo3 &r -bor4 b uo4 &r -bor5 b uo5 &r -por1 p uo1 &r -por2 p uo2 &r -por3 p uo3 &r -por4 p uo4 &r -por5 p uo5 &r -mor1 m uo1 &r -mor2 m uo2 &r -mor3 m uo3 &r -mor4 m uo4 &r -mor5 m uo5 &r -for1 f uo1 &r -for2 f uo2 &r -for3 f uo3 &r -for4 f uo4 &r -for5 f uo5 &r -lor1 l o1 &r -lor2 l o2 &r -lor3 l o3 &r -lor4 l o4 &r -lor5 l o5 &r -mer1 m e1 &r -mer2 m e2 &r -mer3 m e3 &r -mer4 m e4 &r -mer5 m e5 &r -der1 d e1 &r -der2 d e2 &r -der3 d e3 &r -der4 d e4 &r -der5 d e5 &r -ter1 t e1 &r -ter2 t e2 &r -ter3 t e3 &r -ter4 t e4 &r -ter5 t e5 &r -ner1 n e1 &r -ner2 n e2 &r -ner3 n e3 &r -ner4 n e4 &r -ner5 n e5 &r -ler1 l e1 &r -ler2 l e2 &r -ler3 l e3 &r -ler4 l e4 &r -ler5 l e5 &r -ger1 g e1 &r -ger2 g e2 &r -ger3 g e3 &r -ger4 g e4 &r -ger5 g e5 &r -ker1 k e1 &r -ker2 k e2 &r -ker3 k e3 &r -ker4 k e4 &r -ker5 k e5 &r -her1 h e1 &r -her2 h e2 &r -her3 h e3 &r -her4 h e4 &r -her5 h e5 &r -zher1 zh e1 &r -zher2 zh e2 &r -zher3 zh e3 &r -zher4 zh e4 &r -zher5 zh e5 &r -cher1 ch e1 &r -cher2 ch e2 &r -cher3 ch e3 &r -cher4 ch e4 &r -cher5 ch e5 &r -sher1 sh e1 &r -sher2 sh e2 &r -sher3 sh e3 &r -sher4 sh e4 &r -sher5 sh e5 &r -rer1 r e1 &r -rer2 r e2 &r -rer3 r e3 &r -rer4 r e4 &r -rer5 r e5 &r -zer1 z e1 &r -zer2 z e2 &r -zer3 z e3 &r -zer4 z e4 &r -zer5 z e5 &r -cer1 c e1 &r -cer2 c e2 &r -cer3 c e3 &r -cer4 c e4 &r -cer5 c e5 &r -ser1 s e1 &r -ser2 s e2 &r -ser3 s e3 &r -ser4 s e4 &r -ser5 s e5 &r -air1 ai1 &r -air2 ai2 &r -air3 ai3 &r -air4 ai4 &r -air5 ai5 &r -bair1 b ai1 &r -bair2 b ai2 &r -bair3 b ai3 &r -bair4 b ai4 &r -bair5 b ai5 &r -pair1 p ai1 &r -pair2 p ai2 &r -pair3 p ai3 &r -pair4 p ai4 &r -pair5 p ai5 &r -mair1 m ai1 &r -mair2 m ai2 &r -mair3 m ai3 &r -mair4 m ai4 &r -mair5 m ai5 &r -dair1 d ai1 &r -dair2 d ai2 &r -dair3 d ai3 &r -dair4 d ai4 &r -dair5 d ai5 &r -tair1 t ai1 &r -tair2 t ai2 &r -tair3 t ai3 &r -tair4 t ai4 &r -tair5 t ai5 &r -nair1 n ai1 &r -nair2 n ai2 &r -nair3 n ai3 &r -nair4 n ai4 &r -nair5 n ai5 &r -lair1 l ai1 &r -lair2 l ai2 &r -lair3 l ai3 &r -lair4 l ai4 &r -lair5 l ai5 &r -gair1 g ai1 &r -gair2 g ai2 &r -gair3 g ai3 &r -gair4 g ai4 &r -gair5 g ai5 &r -kair1 k ai1 &r -kair2 k ai2 &r -kair3 k ai3 &r -kair4 k ai4 &r -kair5 k ai5 &r -hair1 h ai1 &r -hair2 h ai2 &r -hair3 h ai3 &r -hair4 h ai4 &r -hair5 h ai5 &r -zhair1 zh ai1 &r -zhair2 zh ai2 &r -zhair3 zh ai3 &r -zhair4 zh ai4 &r -zhair5 zh ai5 &r -chair1 ch ai1 &r -chair2 ch ai2 &r -chair3 ch ai3 &r -chair4 ch ai4 &r -chair5 ch ai5 &r -shair1 sh ai1 &r -shair2 sh ai2 &r -shair3 sh ai3 &r -shair4 sh ai4 &r -shair5 sh ai5 &r -zair1 z ai1 &r -zair2 z ai2 &r -zair3 z ai3 &r -zair4 z ai4 &r -zair5 z ai5 &r -cair1 c ai1 &r -cair2 c ai2 &r -cair3 c ai3 &r -cair4 c ai4 &r -cair5 c ai5 &r -sair1 s ai1 &r -sair2 s ai2 &r -sair3 s ai3 &r -sair4 s ai4 &r -sair5 s ai5 &r -beir1 b ei1 &r -beir2 b ei2 &r -beir3 b ei3 &r -beir4 b ei4 &r -beir5 b ei5 &r -peir1 p ei1 &r -peir2 p ei2 &r -peir3 p ei3 &r -peir4 p ei4 &r -peir5 p ei5 &r -meir1 m ei1 &r -meir2 m ei2 &r -meir3 m ei3 &r -meir4 m ei4 &r -meir5 m ei5 &r -feir1 f ei1 &r -feir2 f ei2 &r -feir3 f ei3 &r -feir4 f ei4 &r -feir5 f ei5 &r -deir1 d ei1 &r -deir2 d ei2 &r -deir3 d ei3 &r -deir4 d ei4 &r -deir5 d ei5 &r -teir1 t ei1 &r -teir2 t ei2 &r -teir3 t ei3 &r -teir4 t ei4 &r -teir5 t ei5 &r -neir1 n ei1 &r -neir2 n ei2 &r -neir3 n ei3 &r -neir4 n ei4 &r -neir5 n ei5 &r -leir1 l ei1 &r -leir2 l ei2 &r -leir3 l ei3 &r -leir4 l ei4 &r -leir5 l ei5 &r -geir1 g ei1 &r -geir2 g ei2 &r -geir3 g ei3 &r -geir4 g ei4 &r -geir5 g ei5 &r -keir1 k ei1 &r -keir2 k ei2 &r -keir3 k ei3 &r -keir4 k ei4 &r -keir5 k ei5 &r -heir1 h ei1 &r -heir2 h ei2 &r -heir3 h ei3 &r -heir4 h ei4 &r -heir5 h ei5 &r -zheir1 zh ei1 &r -zheir2 zh ei2 &r -zheir3 zh ei3 &r -zheir4 zh ei4 &r -zheir5 zh ei5 &r -sheir1 sh ei1 &r -sheir2 sh ei2 &r -sheir3 sh ei3 &r -sheir4 sh ei4 &r -sheir5 sh ei5 &r -zeir1 z ei1 &r -zeir2 z ei2 &r -zeir3 z ei3 &r -zeir4 z ei4 &r -zeir5 z ei5 &r -aor1 au1 &r -aor2 au2 &r -aor3 au3 &r -aor4 au4 &r -aor5 au5 &r -baor1 b au1 &r -baor2 b au2 &r -baor3 b au3 &r -baor4 b au4 &r -baor5 b au5 &r -paor1 p au1 &r -paor2 p au2 &r -paor3 p au3 &r -paor4 p au4 &r -paor5 p au5 &r -maor1 m au1 &r -maor2 m au2 &r -maor3 m au3 &r -maor4 m au4 &r -maor5 m au5 &r -daor1 d au1 &r -daor2 d au2 &r -daor3 d au3 &r -daor4 d au4 &r -daor5 d au5 &r -taor1 t au1 &r -taor2 t au2 &r -taor3 t au3 &r -taor4 t au4 &r -taor5 t au5 &r -naor1 n au1 &r -naor2 n au2 &r -naor3 n au3 &r -naor4 n au4 &r -naor5 n au5 &r -laor1 l au1 &r -laor2 l au2 &r -laor3 l au3 &r -laor4 l au4 &r -laor5 l au5 &r -gaor1 g au1 &r -gaor2 g au2 &r -gaor3 g au3 &r -gaor4 g au4 &r -gaor5 g au5 &r -kaor1 k au1 &r -kaor2 k au2 &r -kaor3 k au3 &r -kaor4 k au4 &r -kaor5 k au5 &r -haor1 h au1 &r -haor2 h au2 &r -haor3 h au3 &r -haor4 h au4 &r -haor5 h au5 &r -zhaor1 zh au1 &r -zhaor2 zh au2 &r -zhaor3 zh au3 &r -zhaor4 zh au4 &r -zhaor5 zh au5 &r -chaor1 ch au1 &r -chaor2 ch au2 &r -chaor3 ch au3 &r -chaor4 ch au4 &r -chaor5 ch au5 &r -shaor1 sh au1 &r -shaor2 sh au2 &r -shaor3 sh au3 &r -shaor4 sh au4 &r -shaor5 sh au5 &r -raor1 r au1 &r -raor2 r au2 &r -raor3 r au3 &r -raor4 r au4 &r -raor5 r au5 &r -zaor1 z au1 &r -zaor2 z au2 &r -zaor3 z au3 &r -zaor4 z au4 &r -zaor5 z au5 &r -caor1 c au1 &r -caor2 c au2 &r -caor3 c au3 &r -caor4 c au4 &r -caor5 c au5 &r -saor1 s au1 &r -saor2 s au2 &r -saor3 s au3 &r -saor4 s au4 &r -saor5 s au5 &r -our1 ou1 &r -our2 ou2 &r -our3 ou3 &r -our4 ou4 &r -our5 ou5 &r -pour1 p ou1 &r -pour2 p ou2 &r -pour3 p ou3 &r -pour4 p ou4 &r -pour5 p ou5 &r -mour1 m ou1 &r -mour2 m ou2 &r -mour3 m ou3 &r -mour4 m ou4 &r -mour5 m ou5 &r -four1 f ou1 &r -four2 f ou2 &r -four3 f ou3 &r -four4 f ou4 &r -four5 f ou5 &r -dour1 d ou1 &r -dour2 d ou2 &r -dour3 d ou3 &r -dour4 d ou4 &r -dour5 d ou5 &r -tour1 t ou1 &r -tour2 t ou2 &r -tour3 t ou3 &r -tour4 t ou4 &r -tour5 t ou5 &r -nour1 n ou1 &r -nour2 n ou2 &r -nour3 n ou3 &r -nour4 n ou4 &r -nour5 n ou5 &r -lour1 l ou1 &r -lour2 l ou2 &r -lour3 l ou3 &r -lour4 l ou4 &r -lour5 l ou5 &r -gour1 g ou1 &r -gour2 g ou2 &r -gour3 g ou3 &r -gour4 g ou4 &r -gour5 g ou5 &r -kour1 k ou1 &r -kour2 k ou2 &r -kour3 k ou3 &r -kour4 k ou4 &r -kour5 k ou5 &r -hour1 h ou1 &r -hour2 h ou2 &r -hour3 h ou3 &r -hour4 h ou4 &r -hour5 h ou5 &r -zhour1 zh ou1 &r -zhour2 zh ou2 &r -zhour3 zh ou3 &r -zhour4 zh ou4 &r -zhour5 zh ou5 &r -chour1 ch ou1 &r -chour2 ch ou2 &r -chour3 ch ou3 &r -chour4 ch ou4 &r -chour5 ch ou5 &r -shour1 sh ou1 &r -shour2 sh ou2 &r -shour3 sh ou3 &r -shour4 sh ou4 &r -shour5 sh ou5 &r -rour1 r ou1 &r -rour2 r ou2 &r -rour3 r ou3 &r -rour4 r ou4 &r -rour5 r ou5 &r -zour1 z ou1 &r -zour2 z ou2 &r -zour3 z ou3 &r -zour4 z ou4 &r -zour5 z ou5 &r -cour1 c ou1 &r -cour2 c ou2 &r -cour3 c ou3 &r -cour4 c ou4 &r -cour5 c ou5 &r -sour1 s ou1 &r -sour2 s ou2 &r -sour3 s ou3 &r -sour4 s ou4 &r -sour5 s ou5 &r -anr1 an1 &r -anr2 an2 &r -anr3 an3 &r -anr4 an4 &r -anr5 an5 &r -banr1 b an1 &r -banr2 b an2 &r -banr3 b an3 &r -banr4 b an4 &r -banr5 b an5 &r -panr1 p an1 &r -panr2 p an2 &r -panr3 p an3 &r -panr4 p an4 &r -panr5 p an5 &r -manr1 m an1 &r -manr2 m an2 &r -manr3 m an3 &r -manr4 m an4 &r -manr5 m an5 &r -fanr1 f an1 &r -fanr2 f an2 &r -fanr3 f an3 &r -fanr4 f an4 &r -fanr5 f an5 &r -danr1 d an1 &r -danr2 d an2 &r -danr3 d an3 &r -danr4 d an4 &r -danr5 d an5 &r -tanr1 t an1 &r -tanr2 t an2 &r -tanr3 t an3 &r -tanr4 t an4 &r -tanr5 t an5 &r -nanr1 n an1 &r -nanr2 n an2 &r -nanr3 n an3 &r -nanr4 n an4 &r -nanr5 n an5 &r -lanr1 l an1 &r -lanr2 l an2 &r -lanr3 l an3 &r -lanr4 l an4 &r -lanr5 l an5 &r -ganr1 g an1 &r -ganr2 g an2 &r -ganr3 g an3 &r -ganr4 g an4 &r -ganr5 g an5 &r -kanr1 k an1 &r -kanr2 k an2 &r -kanr3 k an3 &r -kanr4 k an4 &r -kanr5 k an5 &r -hanr1 h an1 &r -hanr2 h an2 &r -hanr3 h an3 &r -hanr4 h an4 &r -hanr5 h an5 &r -zhanr1 zh an1 &r -zhanr2 zh an2 &r -zhanr3 zh an3 &r -zhanr4 zh an4 &r -zhanr5 zh an5 &r -chanr1 ch an1 &r -chanr2 ch an2 &r -chanr3 ch an3 &r -chanr4 ch an4 &r -chanr5 ch an5 &r -shanr1 sh an1 &r -shanr2 sh an2 &r -shanr3 sh an3 &r -shanr4 sh an4 &r -shanr5 sh an5 &r -ranr1 r an1 &r -ranr2 r an2 &r -ranr3 r an3 &r -ranr4 r an4 &r -ranr5 r an5 &r -zanr1 z an1 &r -zanr2 z an2 &r -zanr3 z an3 &r -zanr4 z an4 &r -zanr5 z an5 &r -canr1 c an1 &r -canr2 c an2 &r -canr3 c an3 &r -canr4 c an4 &r -canr5 c an5 &r -sanr1 s an1 &r -sanr2 s an2 &r -sanr3 s an3 &r -sanr4 s an4 &r -sanr5 s an5 &r -benr1 b en1 &r -benr2 b en2 &r -benr3 b en3 &r -benr4 b en4 &r -benr5 b en5 &r -penr1 p en1 &r -penr2 p en2 &r -penr3 p en3 &r -penr4 p en4 &r -penr5 p en5 &r -menr1 m en1 &r -menr2 m en2 &r -menr3 m en3 &r -menr4 m en4 &r -menr5 m en5 &r -fenr1 f en1 &r -fenr2 f en2 &r -fenr3 f en3 &r -fenr4 f en4 &r -fenr5 f en5 &r -denr1 d en1 &r -denr2 d en2 &r -denr3 d en3 &r -denr4 d en4 &r -denr5 d en5 &r -nenr1 n en1 &r -nenr2 n en2 &r -nenr3 n en3 &r -nenr4 n en4 &r -nenr5 n en5 &r -genr1 g en1 &r -genr2 g en2 &r -genr3 g en3 &r -genr4 g en4 &r -genr5 g en5 &r -kenr1 k en1 &r -kenr2 k en2 &r -kenr3 k en3 &r -kenr4 k en4 &r -kenr5 k en5 &r -henr1 h en1 &r -henr2 h en2 &r -henr3 h en3 &r -henr4 h en4 &r -henr5 h en5 &r -zhenr1 zh en1 &r -zhenr2 zh en2 &r -zhenr3 zh en3 &r -zhenr4 zh en4 &r -zhenr5 zh en5 &r -chenr1 ch en1 &r -chenr2 ch en2 &r -chenr3 ch en3 &r -chenr4 ch en4 &r -chenr5 ch en5 &r -shenr1 sh en1 &r -shenr2 sh en2 &r -shenr3 sh en3 &r -shenr4 sh en4 &r -shenr5 sh en5 &r -renr1 r en1 &r -renr2 r en2 &r -renr3 r en3 &r -renr4 r en4 &r -renr5 r en5 &r -zenr1 z en1 &r -zenr2 z en2 &r -zenr3 z en3 &r -zenr4 z en4 &r -zenr5 z en5 &r -cenr1 c en1 &r -cenr2 c en2 &r -cenr3 c en3 &r -cenr4 c en4 &r -cenr5 c en5 &r -senr1 s en1 &r -senr2 s en2 &r -senr3 s en3 &r -senr4 s en4 &r -senr5 s en5 &r -angr1 ang1 &r -angr2 ang2 &r -angr3 ang3 &r -angr4 ang4 &r -angr5 ang5 &r -bangr1 b ang1 &r -bangr2 b ang2 &r -bangr3 b ang3 &r -bangr4 b ang4 &r -bangr5 b ang5 &r -pangr1 p ang1 &r -pangr2 p ang2 &r -pangr3 p ang3 &r -pangr4 p ang4 &r -pangr5 p ang5 &r -mangr1 m ang1 &r -mangr2 m ang2 &r -mangr3 m ang3 &r -mangr4 m ang4 &r -mangr5 m ang5 &r -fangr1 f ang1 &r -fangr2 f ang2 &r -fangr3 f ang3 &r -fangr4 f ang4 &r -fangr5 f ang5 &r -dangr1 d ang1 &r -dangr2 d ang2 &r -dangr3 d ang3 &r -dangr4 d ang4 &r -dangr5 d ang5 &r -tangr1 t ang1 &r -tangr2 t ang2 &r -tangr3 t ang3 &r -tangr4 t ang4 &r -tangr5 t ang5 &r -nangr1 n ang1 &r -nangr2 n ang2 &r -nangr3 n ang3 &r -nangr4 n ang4 &r -nangr5 n ang5 &r -langr1 l ang1 &r -langr2 l ang2 &r -langr3 l ang3 &r -langr4 l ang4 &r -langr5 l ang5 &r -gangr1 g ang1 &r -gangr2 g ang2 &r -gangr3 g ang3 &r -gangr4 g ang4 &r -gangr5 g ang5 &r -kangr1 k ang1 &r -kangr2 k ang2 &r -kangr3 k ang3 &r -kangr4 k ang4 &r -kangr5 k ang5 &r -hangr1 h ang1 &r -hangr2 h ang2 &r -hangr3 h ang3 &r -hangr4 h ang4 &r -hangr5 h ang5 &r -zhangr1 zh ang1 &r -zhangr2 zh ang2 &r -zhangr3 zh ang3 &r -zhangr4 zh ang4 &r -zhangr5 zh ang5 &r -changr1 ch ang1 &r -changr2 ch ang2 &r -changr3 ch ang3 &r -changr4 ch ang4 &r -changr5 ch ang5 &r -shangr1 sh ang1 &r -shangr2 sh ang2 &r -shangr3 sh ang3 &r -shangr4 sh ang4 &r -shangr5 sh ang5 &r -rangr1 r ang1 &r -rangr2 r ang2 &r -rangr3 r ang3 &r -rangr4 r ang4 &r -rangr5 r ang5 &r -zangr1 z ang1 &r -zangr2 z ang2 &r -zangr3 z ang3 &r -zangr4 z ang4 &r -zangr5 z ang5 &r -cangr1 c ang1 &r -cangr2 c ang2 &r -cangr3 c ang3 &r -cangr4 c ang4 &r -cangr5 c ang5 &r -sangr1 s ang1 &r -sangr2 s ang2 &r -sangr3 s ang3 &r -sangr4 s ang4 &r -sangr5 s ang5 &r -bengr1 b eng1 &r -bengr2 b eng2 &r -bengr3 b eng3 &r -bengr4 b eng4 &r -bengr5 b eng5 &r -pengr1 p eng1 &r -pengr2 p eng2 &r -pengr3 p eng3 &r -pengr4 p eng4 &r -pengr5 p eng5 &r -mengr1 m eng1 &r -mengr2 m eng2 &r -mengr3 m eng3 &r -mengr4 m eng4 &r -mengr5 m eng5 &r -fengr1 f eng1 &r -fengr2 f eng2 &r -fengr3 f eng3 &r -fengr4 f eng4 &r -fengr5 f eng5 &r -dengr1 d eng1 &r -dengr2 d eng2 &r -dengr3 d eng3 &r -dengr4 d eng4 &r -dengr5 d eng5 &r -tengr1 t eng1 &r -tengr2 t eng2 &r -tengr3 t eng3 &r -tengr4 t eng4 &r -tengr5 t eng5 &r -nengr1 n eng1 &r -nengr2 n eng2 &r -nengr3 n eng3 &r -nengr4 n eng4 &r -nengr5 n eng5 &r -lengr1 l eng1 &r -lengr2 l eng2 &r -lengr3 l eng3 &r -lengr4 l eng4 &r -lengr5 l eng5 &r -gengr1 g eng1 &r -gengr2 g eng2 &r -gengr3 g eng3 &r -gengr4 g eng4 &r -gengr5 g eng5 &r -kengr1 k eng1 &r -kengr2 k eng2 &r -kengr3 k eng3 &r -kengr4 k eng4 &r -kengr5 k eng5 &r -hengr1 h eng1 &r -hengr2 h eng2 &r -hengr3 h eng3 &r -hengr4 h eng4 &r -hengr5 h eng5 &r -zhengr1 zh eng1 &r -zhengr2 zh eng2 &r -zhengr3 zh eng3 &r -zhengr4 zh eng4 &r -zhengr5 zh eng5 &r -chengr1 ch eng1 &r -chengr2 ch eng2 &r -chengr3 ch eng3 &r -chengr4 ch eng4 &r -chengr5 ch eng5 &r -shengr1 sh eng1 &r -shengr2 sh eng2 &r -shengr3 sh eng3 &r -shengr4 sh eng4 &r -shengr5 sh eng5 &r -rengr1 r eng1 &r -rengr2 r eng2 &r -rengr3 r eng3 &r -rengr4 r eng4 &r -rengr5 r eng5 &r -zengr1 z eng1 &r -zengr2 z eng2 &r -zengr3 z eng3 &r -zengr4 z eng4 &r -zengr5 z eng5 &r -cengr1 c eng1 &r -cengr2 c eng2 &r -cengr3 c eng3 &r -cengr4 c eng4 &r -cengr5 c eng5 &r -sengr1 s eng1 &r -sengr2 s eng2 &r -sengr3 s eng3 &r -sengr4 s eng4 &r -sengr5 s eng5 &r -yir1 y i1 &r -yir2 y i2 &r -yir3 y i3 &r -yir4 y i4 &r -yir5 y i5 &r -bir1 b i1 &r -bir2 b i2 &r -bir3 b i3 &r -bir4 b i4 &r -bir5 b i5 &r -pir1 p i1 &r -pir2 p i2 &r -pir3 p i3 &r -pir4 p i4 &r -pir5 p i5 &r -mir1 m i1 &r -mir2 m i2 &r -mir3 m i3 &r -mir4 m i4 &r -mir5 m i5 &r -dir1 d i1 &r -dir2 d i2 &r -dir3 d i3 &r -dir4 d i4 &r -dir5 d i5 &r -tir1 t i1 &r -tir2 t i2 &r -tir3 t i3 &r -tir4 t i4 &r -tir5 t i5 &r -nir1 n i1 &r -nir2 n i2 &r -nir3 n i3 &r -nir4 n i4 &r -nir5 n i5 &r -lir1 l i1 &r -lir2 l i2 &r -lir3 l i3 &r -lir4 l i4 &r -lir5 l i5 &r -jir1 j i1 &r -jir2 j i2 &r -jir3 j i3 &r -jir4 j i4 &r -jir5 j i5 &r -qir1 q i1 &r -qir2 q i2 &r -qir3 q i3 &r -qir4 q i4 &r -qir5 q i5 &r -xir1 x i1 &r -xir2 x i2 &r -xir3 x i3 &r -xir4 x i4 &r -xir5 x i5 &r -yar1 y ia1 &r -yar2 y ia2 &r -yar3 y ia3 &r -yar4 y ia4 &r -yar5 y ia5 &r -diar1 d ia1 &r -diar2 d ia2 &r -diar3 d ia3 &r -diar4 d ia4 &r -diar5 d ia5 &r -liar1 l ia1 &r -liar2 l ia2 &r -liar3 l ia3 &r -liar4 l ia4 &r -liar5 l ia5 &r -jiar1 j ia1 &r -jiar2 j ia2 &r -jiar3 j ia3 &r -jiar4 j ia4 &r -jiar5 j ia5 &r -qiar1 q ia1 &r -qiar2 q ia2 &r -qiar3 q ia3 &r -qiar4 q ia4 &r -qiar5 q ia5 &r -xiar1 x ia1 &r -xiar2 x ia2 &r -xiar3 x ia3 &r -xiar4 x ia4 &r -xiar5 x ia5 &r -yor1 y io1 &r -yor2 y io2 &r -yor3 y io3 &r -yor4 y io4 &r -yor5 y io5 &r -yer1 y ie1 &r -yer2 y ie2 &r -yer3 y ie3 &r -yer4 y ie4 &r -yer5 y ie5 &r -bier1 b ie1 &r -bier2 b ie2 &r -bier3 b ie3 &r -bier4 b ie4 &r -bier5 b ie5 &r -pier1 p ie1 &r -pier2 p ie2 &r -pier3 p ie3 &r -pier4 p ie4 &r -pier5 p ie5 &r -mier1 m ie1 &r -mier2 m ie2 &r -mier3 m ie3 &r -mier4 m ie4 &r -mier5 m ie5 &r -dier1 d ie1 &r -dier2 d ie2 &r -dier3 d ie3 &r -dier4 d ie4 &r -dier5 d ie5 &r -tier1 t ie1 &r -tier2 t ie2 &r -tier3 t ie3 &r -tier4 t ie4 &r -tier5 t ie5 &r -nier1 n ie1 &r -nier2 n ie2 &r -nier3 n ie3 &r -nier4 n ie4 &r -nier5 n ie5 &r -lier1 l ie1 &r -lier2 l ie2 &r -lier3 l ie3 &r -lier4 l ie4 &r -lier5 l ie5 &r -jier1 j ie1 &r -jier2 j ie2 &r -jier3 j ie3 &r -jier4 j ie4 &r -jier5 j ie5 &r -qier1 q ie1 &r -qier2 q ie2 &r -qier3 q ie3 &r -qier4 q ie4 &r -qier5 q ie5 &r -xier1 x ie1 &r -xier2 x ie2 &r -xier3 x ie3 &r -xier4 x ie4 &r -xier5 x ie5 &r -yair1 y ai1 &r -yair2 y ai2 &r -yair3 y ai3 &r -yair4 y ai4 &r -yair5 y ai5 &r -yaor1 y au1 &r -yaor2 y au2 &r -yaor3 y au3 &r -yaor4 y au4 &r -yaor5 y au5 &r -biaor1 b iau1 &r -biaor2 b iau2 &r -biaor3 b iau3 &r -biaor4 b iau4 &r -biaor5 b iau5 &r -piaor1 p iau1 &r -piaor2 p iau2 &r -piaor3 p iau3 &r -piaor4 p iau4 &r -piaor5 p iau5 &r -miaor1 m iau1 &r -miaor2 m iau2 &r -miaor3 m iau3 &r -miaor4 m iau4 &r -miaor5 m iau5 &r -fiaor1 f iau1 &r -fiaor2 f iau2 &r -fiaor3 f iau3 &r -fiaor4 f iau4 &r -fiaor5 f iau5 &r -diaor1 d iau1 &r -diaor2 d iau2 &r -diaor3 d iau3 &r -diaor4 d iau4 &r -diaor5 d iau5 &r -tiaor1 t iau1 &r -tiaor2 t iau2 &r -tiaor3 t iau3 &r -tiaor4 t iau4 &r -tiaor5 t iau5 &r -niaor1 n iau1 &r -niaor2 n iau2 &r -niaor3 n iau3 &r -niaor4 n iau4 &r -niaor5 n iau5 &r -liaor1 l iau1 &r -liaor2 l iau2 &r -liaor3 l iau3 &r -liaor4 l iau4 &r -liaor5 l iau5 &r -jiaor1 j iau1 &r -jiaor2 j iau2 &r -jiaor3 j iau3 &r -jiaor4 j iau4 &r -jiaor5 j iau5 &r -qiaor1 q iau1 &r -qiaor2 q iau2 &r -qiaor3 q iau3 &r -qiaor4 q iau4 &r -qiaor5 q iau5 &r -xiaor1 x iau1 &r -xiaor2 x iau2 &r -xiaor3 x iau3 &r -xiaor4 x iau4 &r -xiaor5 x iau5 &r -your1 y iou1 &r -your2 y iou2 &r -your3 y iou3 &r -your4 y iou4 &r -your5 y iou5 &r -miur1 m iou1 &r -miur2 m iou2 &r -miur3 m iou3 &r -miur4 m iou4 &r -miur5 m iou5 &r -diur1 d iou1 &r -diur2 d iou2 &r -diur3 d iou3 &r -diur4 d iou4 &r -diur5 d iou5 &r -niur1 n iou1 &r -niur2 n iou2 &r -niur3 n iou3 &r -niur4 n iou4 &r -niur5 n iou5 &r -liur1 l iou1 &r -liur2 l iou2 &r -liur3 l iou3 &r -liur4 l iou4 &r -liur5 l iou5 &r -jiur1 j iou1 &r -jiur2 j iou2 &r -jiur3 j iou3 &r -jiur4 j iou4 &r -jiur5 j iou5 &r -qiur1 q iou1 &r -qiur2 q iou2 &r -qiur3 q iou3 &r -qiur4 q iou4 &r -qiur5 q iou5 &r -xiur1 xiou1 &r -xiur2 xiou2 &r -xiur3 xiou3 &r -xiur4 xiou4 &r -xiur5 xiou5 &r -yanr1 y ian1 &r -yanr2 y ian2 &r -yanr3 y ian3 &r -yanr4 y ian4 &r -yanr5 y ian5 &r -bianr1 b ian1 &r -bianr2 b ian2 &r -bianr3 b ian3 &r -bianr4 b ian4 &r -bianr5 b ian5 &r -pianr1 p ian1 &r -pianr2 p ian2 &r -pianr3 p ian3 &r -pianr4 p ian4 &r -pianr5 p ian5 &r -mianr1 m ian1 &r -mianr2 m ian2 &r -mianr3 m ian3 &r -mianr4 m ian4 &r -mianr5 m ian5 &r -dianr1 d ian1 &r -dianr2 d ian2 &r -dianr3 d ian3 &r -dianr4 d ian4 &r -dianr5 d ian5 &r -tianr1 t ian1 &r -tianr2 t ian2 &r -tianr3 t ian3 &r -tianr4 t ian4 &r -tianr5 t ian5 &r -nianr1 n ian1 &r -nianr2 n ian2 &r -nianr3 n ian3 &r -nianr4 n ian4 &r -nianr5 n ian5 &r -lianr1 l ian1 &r -lianr2 l ian2 &r -lianr3 l ian3 &r -lianr4 l ian4 &r -lianr5 l ian5 &r -jianr1 j ian1 &r -jianr2 j ian2 &r -jianr3 j ian3 &r -jianr4 j ian4 &r -jianr5 j ian5 &r -qianr1 q ian1 &r -qianr2 q ian2 &r -qianr3 q ian3 &r -qianr4 q ian4 &r -qianr5 q ian5 &r -xianr1 x ian1 &r -xianr2 x ian2 &r -xianr3 x ian3 &r -xianr4 x ian4 &r -xianr5 x ian5 &r -yinr1 y in1 &r -yinr2 y in2 &r -yinr3 y in3 &r -yinr4 y in4 &r -yinr5 y in5 &r -binr1 b in1 &r -binr2 b in2 &r -binr3 b in3 &r -binr4 b in4 &r -binr5 b in5 &r -pinr1 p in1 &r -pinr2 p in2 &r -pinr3 p in3 &r -pinr4 p in4 &r -pinr5 p in5 &r -minr1 m in1 &r -minr2 m in2 &r -minr3 m in3 &r -minr4 m in4 &r -minr5 m in5 &r -dinr1 d in1 &r -dinr2 d in2 &r -dinr3 d in3 &r -dinr4 d in4 &r -dinr5 d in5 &r -ninr1 n in1 &r -ninr2 n in2 &r -ninr3 n in3 &r -ninr4 n in4 &r -ninr5 n in5 &r -linr1 l in1 &r -linr2 l in2 &r -linr3 l in3 &r -linr4 l in4 &r -linr5 l in5 &r -jinr1 j in1 &r -jinr2 j in2 &r -jinr3 j in3 &r -jinr4 j in4 &r -jinr5 j in5 &r -qinr1 q in1 &r -qinr2 q in2 &r -qinr3 q in3 &r -qinr4 q in4 &r -qinr5 q in5 &r -xinr1 x in1 &r -xinr2 x in2 &r -xinr3 x in3 &r -xinr4 x in4 &r -xinr5 x in5 &r -yangr1 y iang1 &r -yangr2 y iang2 &r -yangr3 y iang3 &r -yangr4 y iang4 &r -yangr5 y iang5 &r -biangr1 b iang1 &r -biangr2 b iang2 &r -biangr3 b iang3 &r -biangr4 b iang4 &r -biangr5 b iang5 &r -niangr1 n iang1 &r -niangr2 n iang2 &r -niangr3 n iang3 &r -niangr4 n iang4 &r -niangr5 n iang5 &r -liangr1 l iang1 &r -liangr2 l iang2 &r -liangr3 l iang3 &r -liangr4 l iang4 &r -liangr5 l iang5 &r -jiangr1 j iang1 &r -jiangr2 j iang2 &r -jiangr3 j iang3 &r -jiangr4 j iang4 &r -jiangr5 j iang5 &r -qiangr1 q iang1 &r -qiangr2 q iang2 &r -qiangr3 q iang3 &r -qiangr4 q iang4 &r -qiangr5 q iang5 &r -xiangr1 x iang1 &r -xiangr2 x iang2 &r -xiangr3 x iang3 &r -xiangr4 x iang4 &r -xiangr5 x iang5 &r -yingr1 y ing1 &r -yingr2 y ing2 &r -yingr3 y ing3 &r -yingr4 y ing4 &r -yingr5 y ing5 &r -bingr1 b ing1 &r -bingr2 b ing2 &r -bingr3 b ing3 &r -bingr4 b ing4 &r -bingr5 b ing5 &r -pingr1 p ing1 &r -pingr2 p ing2 &r -pingr3 p ing3 &r -pingr4 p ing4 &r -pingr5 p ing5 &r -mingr1 m ing1 &r -mingr2 m ing2 &r -mingr3 m ing3 &r -mingr4 m ing4 &r -mingr5 m ing5 &r -dingr1 d ing1 &r -dingr2 d ing2 &r -dingr3 d ing3 &r -dingr4 d ing4 &r -dingr5 d ing5 &r -tingr1 t ing1 &r -tingr2 t ing2 &r -tingr3 t ing3 &r -tingr4 t ing4 &r -tingr5 t ing5 &r -ningr1 n ing1 &r -ningr2 n ing2 &r -ningr3 n ing3 &r -ningr4 n ing4 &r -ningr5 n ing5 &r -lingr1 l ing1 &r -lingr2 l ing2 &r -lingr3 l ing3 &r -lingr4 l ing4 &r -lingr5 l ing5 &r -jingr1 j ing1 &r -jingr2 j ing2 &r -jingr3 j ing3 &r -jingr4 j ing4 &r -jingr5 j ing5 &r -qingr1 q ing1 &r -qingr2 q ing2 &r -qingr3 q ing3 &r -qingr4 q ing4 &r -qingr5 q ing5 &r -xingr1 x ing1 &r -xingr2 x ing2 &r -xingr3 x ing3 &r -xingr4 x ing4 &r -xingr5 x ing5 &r -wur1 w u1 &r -wur2 w u2 &r -wur3 w u3 &r -wur4 w u4 &r -wur5 w u5 &r -bur1 b u1 &r -bur2 b u2 &r -bur3 b u3 &r -bur4 b u4 &r -bur5 b u5 &r -pur1 p u1 &r -pur2 p u2 &r -pur3 p u3 &r -pur4 p u4 &r -pur5 p u5 &r -mur1 m u1 &r -mur2 m u2 &r -mur3 m u3 &r -mur4 m u4 &r -mur5 m u5 &r -fur1 f u1 &r -fur2 f u2 &r -fur3 f u3 &r -fur4 f u4 &r -fur5 f u5 &r -dur1 d u1 &r -dur2 d u2 &r -dur3 d u3 &r -dur4 d u4 &r -dur5 d u5 &r -tur1 t u1 &r -tur2 t u2 &r -tur3 t u3 &r -tur4 t u4 &r -tur5 t u5 &r -nur1 n u1 &r -nur2 n u2 &r -nur3 n u3 &r -nur4 n u4 &r -nur5 n u5 &r -lur1 l u1 &r -lur2 l u2 &r -lur3 l u3 &r -lur4 l u4 &r -lur5 l u5 &r -gur1 g u1 &r -gur2 g u2 &r -gur3 g u3 &r -gur4 g u4 &r -gur5 g u5 &r -kur1 k u1 &r -kur2 k u2 &r -kur3 k u3 &r -kur4 k u4 &r -kur5 k u5 &r -hur1 h u1 &r -hur2 h u2 &r -hur3 h u3 &r -hur4 h u4 &r -hur5 h u5 &r -zhur1 zh u1 &r -zhur2 zh u2 &r -zhur3 zh u3 &r -zhur4 zh u4 &r -zhur5 zh u5 &r -chur1 ch u1 &r -chur2 ch u2 &r -chur3 ch u3 &r -chur4 ch u4 &r -chur5 ch u5 &r -shur1 sh u1 &r -shur2 sh u2 &r -shur3 sh u3 &r -shur4 sh u4 &r -shur5 sh u5 &r -rur1 r u1 &r -rur2 r u2 &r -rur3 r u3 &r -rur4 r u4 &r -rur5 r u5 &r -zur1 z u1 &r -zur2 z u2 &r -zur3 z u3 &r -zur4 z u4 &r -zur5 z u5 &r -cur1 c u1 &r -cur2 c u2 &r -cur3 c u3 &r -cur4 c u4 &r -cur5 c u5 &r -sur1 s u1 &r -sur2 s u2 &r -sur3 s u3 &r -sur4 s u4 &r -sur5 s u5 &r -war1 w ua1 &r -war2 w ua2 &r -war3 w ua3 &r -war4 w ua4 &r -war5 w ua5 &r -guar1 g ua1 &r -guar2 g ua2 &r -guar3 g ua3 &r -guar4 g ua4 &r -guar5 g ua5 &r -kuar1 k ua1 &r -kuar2 k ua2 &r -kuar3 k ua3 &r -kuar4 k ua4 &r -kuar5 k ua5 &r -huar1 h ua1 &r -huar2 h ua2 &r -huar3 h ua3 &r -huar4 h ua4 &r -huar5 h ua5 &r -zhuar1 zh ua1 &r -zhuar2 zh ua2 &r -zhuar3 zh ua3 &r -zhuar4 zh ua4 &r -zhuar5 zh ua5 &r -chuar1 ch ua1 &r -chuar2 ch ua2 &r -chuar3 ch ua3 &r -chuar4 ch ua4 &r -chuar5 ch ua5 &r -shuar1 sh ua1 &r -shuar2 sh ua2 &r -shuar3 sh ua3 &r -shuar4 sh ua4 &r -shuar5 sh ua5 &r -wor1 w uo1 &r -wor2 w uo2 &r -wor3 w uo3 &r -wor4 w uo4 &r -wor5 w uo5 &r -duor1 d uo1 &r -duor2 d uo2 &r -duor3 d uo3 &r -duor4 d uo4 &r -duor5 d uo5 &r -tuor1 t uo1 &r -tuor2 t uo2 &r -tuor3 t uo3 &r -tuor4 t uo4 &r -tuor5 t uo5 &r -nuor1 n uo1 &r -nuor2 n uo2 &r -nuor3 n uo3 &r -nuor4 n uo4 &r -nuor5 n uo5 &r -luor1 l uo1 &r -luor2 l uo2 &r -luor3 l uo3 &r -luor4 l uo4 &r -luor5 l uo5 &r -guor1 g uo1 &r -guor2 g uo2 &r -guor3 g uo3 &r -guor4 g uo4 &r -guor5 g uo5 &r -kuor1 k uo1 &r -kuor2 k uo2 &r -kuor3 k uo3 &r -kuor4 k uo4 &r -kuor5 k uo5 &r -huor1 h uo1 &r -huor2 h uo2 &r -huor3 h uo3 &r -huor4 h uo4 &r -huor5 h uo5 &r -zhuor1 zh uo1 &r -zhuor2 zh uo2 &r -zhuor3 zh uo3 &r -zhuor4 zh uo4 &r -zhuor5 zh uo5 &r -chuor1 ch uo1 &r -chuor2 ch uo2 &r -chuor3 ch uo3 &r -chuor4 ch uo4 &r -chuor5 ch uo5 &r -shuor1 sh uo1 &r -shuor2 sh uo2 &r -shuor3 sh uo3 &r -shuor4 sh uo4 &r -shuor5 sh uo5 &r -ruor1 r uo1 &r -ruor2 r uo2 &r -ruor3 r uo3 &r -ruor4 r uo4 &r -ruor5 r uo5 &r -zuor1 z uo1 &r -zuor2 z uo2 &r -zuor3 z uo3 &r -zuor4 z uo4 &r -zuor5 z uo5 &r -cuor1 c uo1 &r -cuor2 c uo2 &r -cuor3 c uo3 &r -cuor4 c uo4 &r -cuor5 c uo5 &r -suor1 s uo1 &r -suor2 s uo2 &r -suor3 s uo3 &r -suor4 s uo4 &r -suor5 s uo5 &r -wair1 w uai1 &r -wair2 w uai2 &r -wair3 w uai3 &r -wair4 w uai4 &r -wair5 w uai5 &r -guair1 g uai1 &r -guair2 g uai2 &r -guair3 g uai3 &r -guair4 g uai4 &r -guair5 g uai5 &r -kuair1 k uai1 &r -kuair2 k uai2 &r -kuair3 k uai3 &r -kuair4 k uai4 &r -kuair5 k uai5 &r -huair1 h uai1 &r -huair2 h uai2 &r -huair3 h uai3 &r -huair4 h uai4 &r -huair5 h uai5 &r -zhuair1 zh uai1 &r -zhuair2 zh uai2 &r -zhuair3 zh uai3 &r -zhuair4 zh uai4 &r -zhuair5 zh uai5 &r -chuair1 ch uai1 &r -chuair2 ch uai2 &r -chuair3 ch uai3 &r -chuair4 ch uai4 &r -chuair5 ch uai5 &r -shuair1 sh uai1 &r -shuair2 sh uai2 &r -shuair3 sh uai3 &r -shuair4 sh uai4 &r -shuair5 sh uai5 &r -weir1 w uei1 &r -weir2 w uei2 &r -weir3 w uei3 &r -weir4 w uei4 &r -weir5 w uei5 &r -duir1 d uei1 &r -duir2 d uei2 &r -duir3 d uei3 &r -duir4 d uei4 &r -duir5 d uei5 &r -tuir1 t uei1 &r -tuir2 t uei2 &r -tuir3 t uei3 &r -tuir4 t uei4 &r -tuir5 t uei5 &r -guir1 g uei1 &r -guir2 g uei2 &r -guir3 g uei3 &r -guir4 g uei4 &r -guir5 g uei5 &r -kuir1 k uei1 &r -kuir2 k uei2 &r -kuir3 k uei3 &r -kuir4 k uei4 &r -kuir5 k uei5 &r -huir1 h uei1 &r -huir2 h uei2 &r -huir3 h uei3 &r -huir4 h uei4 &r -huir5 h uei5 &r -zhuir1 zh uei1 &r -zhuir2 zh uei2 &r -zhuir3 zh uei3 &r -zhuir4 zh uei4 &r -zhuir5 zh uei5 &r -chuir1 ch uei1 &r -chuir2 ch uei2 &r -chuir3 ch uei3 &r -chuir4 ch uei4 &r -chuir5 ch uei5 &r -shuir1 sh uei1 &r -shuir2 sh uei2 &r -shuir3 sh uei3 &r -shuir4 sh uei4 &r -shuir5 sh uei5 &r -ruir1 r uei1 &r -ruir2 r uei2 &r -ruir3 r uei3 &r -ruir4 r uei4 &r -ruir5 r uei5 &r -zuir1 z uei1 &r -zuir2 z uei2 &r -zuir3 z uei3 &r -zuir4 z uei4 &r -zuir5 z uei5 &r -cuir1 c uei1 &r -cuir2 c uei2 &r -cuir3 c uei3 &r -cuir4 c uei4 &r -cuir5 c uei5 &r -suir1 s uei1 &r -suir2 s uei2 &r -suir3 s uei3 &r -suir4 s uei4 &r -suir5 s uei5 &r -wanr1 w uan1 &r -wanr2 w uan2 &r -wanr3 w uan3 &r -wanr4 w uan4 &r -wanr5 w uan5 &r -duanr1 d uan1 &r -duanr2 d uan2 &r -duanr3 d uan3 &r -duanr4 d uan4 &r -duanr5 d uan5 &r -tuanr1 t uan1 &r -tuanr2 t uan2 &r -tuanr3 t uan3 &r -tuanr4 t uan4 &r -tuanr5 t uan5 &r -nuanr1 n uan1 &r -nuanr2 n uan2 &r -nuanr3 n uan3 &r -nuanr4 n uan4 &r -nuanr5 n uan5 &r -luanr1 l uan1 &r -luanr2 l uan2 &r -luanr3 l uan3 &r -luanr4 l uan4 &r -luanr5 l uan5 &r -guanr1 g uan1 &r -guanr2 g uan2 &r -guanr3 g uan3 &r -guanr4 g uan4 &r -guanr5 g uan5 &r -kuanr1 k uan1 &r -kuanr2 k uan2 &r -kuanr3 k uan3 &r -kuanr4 k uan4 &r -kuanr5 k uan5 &r -huanr1 h uan1 &r -huanr2 h uan2 &r -huanr3 h uan3 &r -huanr4 h uan4 &r -huanr5 h uan5 &r -zhuanr1 zh uan1 &r -zhuanr2 zh uan2 &r -zhuanr3 zh uan3 &r -zhuanr4 zh uan4 &r -zhuanr5 zh uan5 &r -chuanr1 ch uan1 &r -chuanr2 ch uan2 &r -chuanr3 ch uan3 &r -chuanr4 ch uan4 &r -chuanr5 ch uan5 &r -shuanr1 sh uan1 &r -shuanr2 sh uan2 &r -shuanr3 sh uan3 &r -shuanr4 sh uan4 &r -shuanr5 sh uan5 &r -ruanr1 r uan1 &r -ruanr2 r uan2 &r -ruanr3 r uan3 &r -ruanr4 r uan4 &r -ruanr5 r uan5 &r -zuanr1 z uan1 &r -zuanr2 z uan2 &r -zuanr3 z uan3 &r -zuanr4 z uan4 &r -zuanr5 z uan5 &r -cuanr1 c uan1 &r -cuanr2 c uan2 &r -cuanr3 c uan3 &r -cuanr4 c uan4 &r -cuanr5 c uan5 &r -suanr1 s uan1 &r -suanr2 s uan2 &r -suanr3 s uan3 &r -suanr4 s uan4 &r -suanr5 s uan5 &r -wenr1 w uen1 &r -wenr2 w uen2 &r -wenr3 w uen3 &r -wenr4 w uen4 &r -wenr5 w uen5 &r -dunr1 d uen1 &r -dunr2 d uen2 &r -dunr3 d uen3 &r -dunr4 d uen4 &r -dunr5 d uen5 &r -tunr1 t uen1 &r -tunr2 t uen2 &r -tunr3 t uen3 &r -tunr4 t uen4 &r -tunr5 t uen5 &r -nunr1 n uen1 &r -nunr2 n uen2 &r -nunr3 n uen3 &r -nunr4 n uen4 &r -nunr5 n uen5 &r -lunr1 l uen1 &r -lunr2 l uen2 &r -lunr3 l uen3 &r -lunr4 l uen4 &r -lunr5 l uen5 &r -gunr1 g uen1 &r -gunr2 g uen2 &r -gunr3 g uen3 &r -gunr4 g uen4 &r -gunr5 g uen5 &r -kunr1 k uen1 &r -kunr2 k uen2 &r -kunr3 k uen3 &r -kunr4 k uen4 &r -kunr5 k uen5 &r -hunr1 h uen1 &r -hunr2 h uen2 &r -hunr3 h uen3 &r -hunr4 h uen4 &r -hunr5 h uen5 &r -zhunr1 zh uen1 &r -zhunr2 zh uen2 &r -zhunr3 zh uen3 &r -zhunr4 zh uen4 &r -zhunr5 zh uen5 &r -chunr1 ch uen1 &r -chunr2 ch uen2 &r -chunr3 ch uen3 &r -chunr4 ch uen4 &r -chunr5 ch uen5 &r -shunr1 sh uen1 &r -shunr2 sh uen2 &r -shunr3 sh uen3 &r -shunr4 sh uen4 &r -shunr5 sh uen5 &r -runr1 r uen1 &r -runr2 r uen2 &r -runr3 r uen3 &r -runr4 r uen4 &r -runr5 r uen5 &r -zunr1 z uen1 &r -zunr2 z uen2 &r -zunr3 z uen3 &r -zunr4 z uen4 &r -zunr5 z uen5 &r -cunr1 c uen1 &r -cunr2 c uen2 &r -cunr3 c uen3 &r -cunr4 c uen4 &r -cunr5 c uen5 &r -sunr1 s uen1 &r -sunr2 s uen2 &r -sunr3 s uen3 &r -sunr4 s uen4 &r -sunr5 s uen5 &r -wangr1 w uang1 &r -wangr2 w uang2 &r -wangr3 w uang3 &r -wangr4 w uang4 &r -wangr5 w uang5 &r -guangr1 g uang1 &r -guangr2 g uang2 &r -guangr3 g uang3 &r -guangr4 g uang4 &r -guangr5 g uang5 &r -kuangr1 k uang1 &r -kuangr2 k uang2 &r -kuangr3 k uang3 &r -kuangr4 k uang4 &r -kuangr5 k uang5 &r -huangr1 h uang1 &r -huangr2 h uang2 &r -huangr3 h uang3 &r -huangr4 h uang4 &r -huangr5 h uang5 &r -zhuangr1 zh uang1 &r -zhuangr2 zh uang2 &r -zhuangr3 zh uang3 &r -zhuangr4 zh uang4 &r -zhuangr5 zh uang5 &r -chuangr1 ch uang1 &r -chuangr2 ch uang2 &r -chuangr3 ch uang3 &r -chuangr4 ch uang4 &r -chuangr5 ch uang5 &r -shuangr1 sh uang1 &r -shuangr2 sh uang2 &r -shuangr3 sh uang3 &r -shuangr4 sh uang4 &r -shuangr5 sh uang5 &r -wengr1 w ung1 &r -wengr2 w ung2 &r -wengr3 w ung3 &r -wengr4 w ung4 &r -wengr5 w ung5 &r -dongr1 d ung1 &r -dongr2 d ung2 &r -dongr3 d ung3 &r -dongr4 d ung4 &r -dongr5 d ung5 &r -tongr1 t ung1 &r -tongr2 t ung2 &r -tongr3 t ung3 &r -tongr4 t ung4 &r -tongr5 t ung5 &r -nongr1 n ung1 &r -nongr2 n ung2 &r -nongr3 n ung3 &r -nongr4 n ung4 &r -nongr5 n ung5 &r -longr1 l ung1 &r -longr2 l ung2 &r -longr3 l ung3 &r -longr4 l ung4 &r -longr5 l ung5 &r -gongr1 g ung1 &r -gongr2 g ung2 &r -gongr3 g ung3 &r -gongr4 g ung4 &r -gongr5 g ung5 &r -kongr1 k ung1 &r -kongr2 k ung2 &r -kongr3 k ung3 &r -kongr4 k ung4 &r -kongr5 k ung5 &r -hongr1 h ung1 &r -hongr2 h ung2 &r -hongr3 h ung3 &r -hongr4 h ung4 &r -hongr5 h ung5 &r -zhongr1 zh ung1 &r -zhongr2 zh ung2 &r -zhongr3 zh ung3 &r -zhongr4 zh ung4 &r -zhongr5 zh ung5 &r -chongr1 ch ung1 &r -chongr2 ch ung2 &r -chongr3 ch ung3 &r -chongr4 ch ung4 &r -chongr5 ch ung5 &r -rongr1 r ung1 &r -rongr2 r ung2 &r -rongr3 r ung3 &r -rongr4 r ung4 &r -rongr5 r ung5 &r -zongr1 z ung1 &r -zongr2 z ung2 &r -zongr3 z ung3 &r -zongr4 z ung4 &r -zongr5 z ung5 &r -congr1 c ung1 &r -congr2 c ung2 &r -congr3 c ung3 &r -congr4 c ung4 &r -congr5 c ung5 &r -songr1 s ung1 &r -songr2 s ung2 &r -songr3 s ung3 &r -songr4 s ung4 &r -songr5 s ung5 &r -yur1 y v1 &r -yur2 y v2 &r -yur3 y v3 &r -yur4 y v4 &r -yur5 y v5 &r -nvr1 n v1 &r -nvr2 n v2 &r -nvr3 n v3 &r -nvr4 n v4 &r -nvr5 n v5 &r -lvr1 l v1 &r -lvr2 l v2 &r -lvr3 l v3 &r -lvr4 l v4 &r -lvr5 l v5 &r -jur1 j v1 &r -jur2 j v2 &r -jur3 j v3 &r -jur4 j v4 &r -jur5 j v5 &r -qur1 q v1 &r -qur2 q v2 &r -qur3 q v3 &r -qur4 q v4 &r -qur5 q v5 &r -xur1 x v1 &r -xur2 x v2 &r -xur3 x v3 &r -xur4 x v4 &r -xur5 x v5 &r -yuer1 y ve1 &r -yuer2 y ve2 &r -yuer3 y ve3 &r -yuer4 y ve4 &r -yuer5 y ve5 &r -nuer1 n ve1 &r -nuer2 n ve2 &r -nuer3 n ve3 &r -nuer4 n ve4 &r -nuer5 n ve5 &r -nver1 n ve1 &r -nver2 n ve2 &r -nver3 n ve3 &r -nver4 n ve4 &r -nver5 n ve5 &r -luer1 l ve1 &r -luer2 l ve2 &r -luer3 l ve3 &r -luer4 l ve4 &r -luer5 l ve5 &r -lver1 l ve1 &r -lver2 l ve2 &r -lver3 l ve3 &r -lver4 l ve4 &r -lver5 l ve5 &r -juer1 j ve1 &r -juer2 j ve2 &r -juer3 j ve3 &r -juer4 j ve4 &r -juer5 j ve5 &r -quer1 q ve1 &r -quer2 q ve2 &r -quer3 q ve3 &r -quer4 q ve4 &r -quer5 q ve5 &r -xuer1 x ve1 &r -xuer2 x ve2 &r -xuer3 x ve3 &r -xuer4 x ve4 &r -xuer5 x ve5 &r -yuanr1 y van1 &r -yuanr2 y van2 &r -yuanr3 y van3 &r -yuanr4 y van4 &r -yuanr5 y van5 &r -juanr1 j van1 &r -juanr2 j van2 &r -juanr3 j van3 &r -juanr4 j van4 &r -juanr5 j van5 &r -quanr1 q van1 &r -quanr2 q van2 &r -quanr3 q van3 &r -quanr4 q van4 &r -quanr5 q van5 &r -xuanr1 x van1 &r -xuanr2 x van2 &r -xuanr3 x van3 &r -xuanr4 x van4 &r -xuanr5 x van5 &r -yunr1 y vn1 &r -yunr2 y vn2 &r -yunr3 y vn3 &r -yunr4 y vn4 &r -yunr5 y vn5 &r -junr1 j vn1 &r -junr2 j vn2 &r -junr3 j vn3 &r -junr4 j vn4 &r -junr5 j vn5 &r -qunr1 q vn1 &r -qunr2 q vn2 &r -qunr3 q vn3 &r -qunr4 q vn4 &r -qunr5 q vn5 &r -xunr1 x vn1 &r -xunr2 x vn2 &r -xunr3 x vn3 &r -xunr4 x vn4 &r -xunr5 x vn5 &r -yongr1 y vng1 &r -yongr2 y vng2 &r -yongr3 y vng3 &r -yongr4 y vng4 &r -yongr5 y vng5 &r -jiongr1 j vng1 &r -jiongr2 j vng2 &r -jiongr3 j vng3 &r -jiongr4 j vng4 &r -jiongr5 j vng5 &r -qiongr1 q vng1 &r -qiongr2 q vng2 &r -qiongr3 q vng3 &r -qiongr4 q vng4 &r -qiongr5 q vng5 &r -xiongr1 x vng1 &r -xiongr2 x vng2 &r -xiongr3 x vng3 &r -xiongr4 x vng4 &r -xiongr5 x vng5 &r diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/preprocess_transcription.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/preprocess_transcription.py deleted file mode 100644 index ce117d420eda7eb736df26962912455524974671..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/preprocess_transcription.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import pickle -import re -from pathlib import Path - -import tqdm -import yaml - -zh_pattern = re.compile("[\u4e00-\u9fa5]") - -_tones = {'', '', '', '0', '1', '2', '3', '4', '5'} - -_pauses = {'%', '$'} - -_initials = { - 'b', - 'p', - 'm', - 'f', - 'd', - 't', - 'n', - 'l', - 'g', - 'k', - 'h', - 'j', - 'q', - 'x', - 'zh', - 'ch', - 'sh', - 'r', - 'z', - 'c', - 's', -} - -_finals = { - 'ii', - 'iii', - 'a', - 'o', - 'e', - 'ea', - 'ai', - 'ei', - 'ao', - 'ou', - 'an', - 'en', - 'ang', - 'eng', - 'er', - 'i', - 'ia', - 'io', - 'ie', - 'iai', - 'iao', - 'iou', - 'ian', - 'ien', - 'iang', - 'ieng', - 'u', - 'ua', - 'uo', - 'uai', - 'uei', - 'uan', - 'uen', - 'uang', - 'ueng', - 'v', - 've', - 'van', - 'ven', - 'veng', -} - -_ernized_symbol = {'&r'} - -_specials = {'', '', '', ''} - -_phones = _initials | _finals | _ernized_symbol | _specials | _pauses - - -def is_zh(word): - global zh_pattern - match = zh_pattern.search(word) - return match is not None - - -def ernized(syllable): - return syllable[:2] != "er" and syllable[-2] == 'r' - - -def convert(syllable): - # expansion of o -> uo - syllable = re.sub(r"([bpmf])o$", r"\1uo", syllable) - # syllable = syllable.replace("bo", "buo").replace("po", "puo").replace("mo", "muo").replace("fo", "fuo") - # expansion for iong, ong - syllable = syllable.replace("iong", "veng").replace("ong", "ueng") - - # expansion for ing, in - syllable = syllable.replace("ing", "ieng").replace("in", "ien") - - # expansion for un, ui, iu - syllable = syllable.replace("un", "uen").replace("ui", - "uei").replace("iu", "iou") - - # rule for variants of i - syllable = syllable.replace("zi", "zii").replace("ci", "cii").replace("si", "sii")\ - .replace("zhi", "zhiii").replace("chi", "chiii").replace("shi", "shiii")\ - .replace("ri", "riii") - - # rule for y preceding i, u - syllable = syllable.replace("yi", "i").replace("yu", "v").replace("y", "i") - - # rule for w - syllable = syllable.replace("wu", "u").replace("w", "u") - - # rule for v following j, q, x - syllable = syllable.replace("ju", "jv").replace("qu", - "qv").replace("xu", "xv") - - return syllable - - -def split_syllable(syllable: str): - """Split a syllable in pinyin into a list of phones and a list of tones. - Initials have no tone, represented by '0', while finals have tones from - '1,2,3,4,5'. - - e.g. - - zhang -> ['zh', 'ang'], ['0', '1'] - """ - if syllable in _pauses: - # syllable, tone - return [syllable], ['0'] - - tone = syllable[-1] - syllable = convert(syllable[:-1]) - - phones = [] - tones = [] - - global _initials - if syllable[:2] in _initials: - phones.append(syllable[:2]) - tones.append('0') - phones.append(syllable[2:]) - tones.append(tone) - elif syllable[0] in _initials: - phones.append(syllable[0]) - tones.append('0') - phones.append(syllable[1:]) - tones.append(tone) - else: - phones.append(syllable) - tones.append(tone) - return phones, tones - - -def load_aishell3_transcription(line: str): - sentence_id, pinyin, text = line.strip().split("|") - syllables = pinyin.strip().split() - - results = [] - - for syllable in syllables: - if syllable in _pauses: - results.append(syllable) - elif not ernized(syllable): - results.append(syllable) - else: - results.append(syllable[:-2] + syllable[-1]) - results.append('&r5') - - phones = [] - tones = [] - for syllable in results: - p, t = split_syllable(syllable) - phones.extend(p) - tones.extend(t) - for p in phones: - assert p in _phones, p - return { - "sentence_id": sentence_id, - "text": text, - "syllables": results, - "phones": phones, - "tones": tones - } - - -def process_aishell3(dataset_root, output_dir): - dataset_root = Path(dataset_root).expanduser() - output_dir = Path(output_dir).expanduser() - output_dir.mkdir(parents=True, exist_ok=True) - - prosody_label_path = dataset_root / "label_train-set.txt" - with open(prosody_label_path, 'rt') as f: - lines = [line.strip() for line in f] - - records = lines[5:] - - processed_records = [] - for record in tqdm.tqdm(records): - new_record = load_aishell3_transcription(record) - processed_records.append(new_record) - print(new_record) - - with open(output_dir / "metadata.pickle", 'wb') as f: - pickle.dump(processed_records, f) - - with open(output_dir / "metadata.yaml", 'wt', encoding="utf-8") as f: - yaml.safe_dump( - processed_records, f, default_flow_style=None, allow_unicode=True) - - print("metadata done!") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Preprocess transcription of AiShell3 and save them in a compact file(yaml and pickle)." - ) - parser.add_argument( - "--input", - type=str, - default="~/datasets/aishell3/train", - help="path of the training dataset,(contains a label_train-set.txt).") - parser.add_argument( - "--output", - type=str, - help="the directory to save the processed transcription." - "If not provided, it would be the same as the input.") - args = parser.parse_args() - if args.output is None: - args.output = args.input - - process_aishell3(args.input, args.output) diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/process_wav.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/process_wav.py deleted file mode 100644 index 56d8e4c3a8ff09f47a534695ba2e5a613376fba3..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/process_wav.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -from functools import partial -from multiprocessing import Pool -from pathlib import Path - -import librosa -import numpy as np -import soundfile as sf -from praatio import textgrid -from tqdm import tqdm - - -def get_valid_part(fpath): - f = textgrid.openTextgrid(fpath, includeEmptyIntervals=True) - - start = 0 - phone_entry_list = f.tierDict['phones'].entryList - first_entry = phone_entry_list[0] - if first_entry.label == "sil": - start = first_entry.end - - last_entry = phone_entry_list[-1] - if last_entry.label == "sp": - end = last_entry.start - else: - end = last_entry.end - return start, end - - -def process_utterance(fpath, source_dir, target_dir, alignment_dir): - rel_path = fpath.relative_to(source_dir) - opath = target_dir / rel_path - apath = (alignment_dir / rel_path).with_suffix(".TextGrid") - opath.parent.mkdir(parents=True, exist_ok=True) - - start, end = get_valid_part(apath) - wav, _ = librosa.load(fpath, sr=22050, offset=start, duration=end - start) - normalized_wav = wav / np.max(wav) * 0.999 - sf.write(opath, normalized_wav, samplerate=22050, subtype='PCM_16') - # print(f"{fpath} => {opath}") - - -def preprocess_aishell3(source_dir, target_dir, alignment_dir): - source_dir = Path(source_dir).expanduser() - target_dir = Path(target_dir).expanduser() - alignment_dir = Path(alignment_dir).expanduser() - - wav_paths = list(source_dir.rglob("*.wav")) - print(f"there are {len(wav_paths)} audio files in total") - fx = partial( - process_utterance, - source_dir=source_dir, - target_dir=target_dir, - alignment_dir=alignment_dir) - with Pool(16) as p: - list( - tqdm(p.imap(fx, wav_paths), total=len(wav_paths), unit="utterance")) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Process audio in AiShell3, trim silence according to the alignment " - "files generated by MFA, and normalize volume by peak.") - parser.add_argument( - "--input", - type=str, - default="~/datasets/aishell3/train/wav", - help="path of the original audio folder in aishell3.") - parser.add_argument( - "--output", - type=str, - default="~/datasets/aishell3/train/normalized_wav", - help="path of the folder to save the processed audio files.") - parser.add_argument( - "--alignment", - type=str, - default="~/datasets/aishell3/train/alignment", - help="path of the alignment files.") - args = parser.parse_args() - - preprocess_aishell3(args.input, args.output, args.alignment) diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/train.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/train.py deleted file mode 100644 index ea5f12da7bab17ff1bafc8a27df17955b3cb54af..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/train.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import time -from collections import defaultdict -from pathlib import Path - -import numpy as np -import paddle -from matplotlib import pyplot as plt -from paddle import distributed as dist -from paddle.io import DataLoader -from paddle.io import DistributedBatchSampler - -from paddlespeech.t2s.data import dataset -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.aishell3 import AiShell3 -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.aishell3 import collate_aishell3_examples -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.config import get_cfg_defaults -from paddlespeech.t2s.models.tacotron2 import Tacotron2 -from paddlespeech.t2s.models.tacotron2 import Tacotron2Loss -from paddlespeech.t2s.training.cli import default_argument_parser -from paddlespeech.t2s.training.experiment import ExperimentBase -from paddlespeech.t2s.utils import display -from paddlespeech.t2s.utils import mp_tools - - -class Experiment(ExperimentBase): - def compute_losses(self, inputs, outputs): - texts, tones, mel_targets, utterance_embeds, text_lens, output_lens, stop_tokens = inputs - - mel_outputs = outputs["mel_output"] - mel_outputs_postnet = outputs["mel_outputs_postnet"] - alignments = outputs["alignments"] - - losses = self.criterion(mel_outputs, mel_outputs_postnet, mel_targets, - alignments, output_lens, text_lens) - return losses - - def train_batch(self): - start = time.time() - batch = self.read_batch() - data_loader_time = time.time() - start - - self.optimizer.clear_grad() - self.model.train() - texts, tones, mels, utterance_embeds, text_lens, output_lens, stop_tokens = batch - outputs = self.model( - texts, - text_lens, - mels, - output_lens, - tones=tones, - global_condition=utterance_embeds) - losses = self.compute_losses(batch, outputs) - loss = losses["loss"] - loss.backward() - self.optimizer.step() - iteration_time = time.time() - start - - losses_np = {k: float(v) for k, v in losses.items()} - # logging - msg = "Rank: {}, ".format(dist.get_rank()) - msg += "step: {}, ".format(self.iteration) - msg += "time: {:>.3f}s/{:>.3f}s, ".format(data_loader_time, - iteration_time) - msg += ', '.join('{}: {:>.6f}'.format(k, v) - for k, v in losses_np.items()) - self.logger.info(msg) - - if dist.get_rank() == 0: - for key, value in losses_np.items(): - self.visualizer.add_scalar(f"train_loss/{key}", value, - self.iteration) - - @mp_tools.rank_zero_only - @paddle.no_grad() - def valid(self): - valid_losses = defaultdict(list) - for i, batch in enumerate(self.valid_loader): - texts, tones, mels, utterance_embeds, text_lens, output_lens, stop_tokens = batch - outputs = self.model( - texts, - text_lens, - mels, - output_lens, - tones=tones, - global_condition=utterance_embeds) - losses = self.compute_losses(batch, outputs) - for key, value in losses.items(): - valid_losses[key].append(float(value)) - - attention_weights = outputs["alignments"] - self.visualizer.add_figure( - f"valid_sentence_{i}_alignments", - display.plot_alignment(attention_weights[0].numpy().T), - self.iteration) - self.visualizer.add_figure( - f"valid_sentence_{i}_target_spectrogram", - display.plot_spectrogram(mels[0].numpy().T), self.iteration) - mel_pred = outputs['mel_outputs_postnet'] - self.visualizer.add_figure( - f"valid_sentence_{i}_predicted_spectrogram", - display.plot_spectrogram(mel_pred[0].numpy().T), self.iteration) - - # write visual log - valid_losses = {k: np.mean(v) for k, v in valid_losses.items()} - - # logging - msg = "Valid: " - msg += "step: {}, ".format(self.iteration) - msg += ', '.join('{}: {:>.6f}'.format(k, v) - for k, v in valid_losses.items()) - self.logger.info(msg) - - for key, value in valid_losses.items(): - self.visualizer.add_scalar(f"valid/{key}", value, self.iteration) - - @mp_tools.rank_zero_only - @paddle.no_grad() - def eval(self): - """Evaluation of Tacotron2 in autoregressive manner.""" - self.model.eval() - mel_dir = Path(self.output_dir / ("eval_{}".format(self.iteration))) - mel_dir.mkdir(parents=True, exist_ok=True) - for i, batch in enumerate(self.test_loader): - texts, tones, mels, utterance_embeds, *_ = batch - outputs = self.model.infer( - texts, tones=tones, global_condition=utterance_embeds) - - display.plot_alignment(outputs["alignments"][0].numpy().T) - plt.savefig(mel_dir / f"sentence_{i}.png") - plt.close() - np.save(mel_dir / f"sentence_{i}", - outputs["mel_outputs_postnet"][0].numpy().T) - print(f"sentence_{i}") - - def setup_model(self): - config = self.config - model = Tacotron2( - vocab_size=config.model.vocab_size, - n_tones=config.model.n_tones, - d_mels=config.data.d_mels, - d_encoder=config.model.d_encoder, - encoder_conv_layers=config.model.encoder_conv_layers, - encoder_kernel_size=config.model.encoder_kernel_size, - d_prenet=config.model.d_prenet, - d_attention_rnn=config.model.d_attention_rnn, - d_decoder_rnn=config.model.d_decoder_rnn, - attention_filters=config.model.attention_filters, - attention_kernel_size=config.model.attention_kernel_size, - d_attention=config.model.d_attention, - d_postnet=config.model.d_postnet, - postnet_kernel_size=config.model.postnet_kernel_size, - postnet_conv_layers=config.model.postnet_conv_layers, - reduction_factor=config.model.reduction_factor, - p_encoder_dropout=config.model.p_encoder_dropout, - p_prenet_dropout=config.model.p_prenet_dropout, - p_attention_dropout=config.model.p_attention_dropout, - p_decoder_dropout=config.model.p_decoder_dropout, - p_postnet_dropout=config.model.p_postnet_dropout, - d_global_condition=config.model.d_global_condition, - use_stop_token=config.model.use_stop_token, ) - - if self.parallel: - model = paddle.DataParallel(model) - - grad_clip = paddle.nn.ClipGradByGlobalNorm( - config.training.grad_clip_thresh) - optimizer = paddle.optimizer.Adam( - learning_rate=config.training.lr, - parameters=model.parameters(), - weight_decay=paddle.regularizer.L2Decay( - config.training.weight_decay), - grad_clip=grad_clip) - criterion = Tacotron2Loss( - use_stop_token_loss=config.model.use_stop_token, - use_guided_attention_loss=config.model.use_guided_attention_loss, - sigma=config.model.guided_attention_loss_sigma) - self.model = model - self.optimizer = optimizer - self.criterion = criterion - - def setup_dataloader(self): - args = self.args - config = self.config - aishell3_dataset = AiShell3(args.data) - - valid_set, train_set = dataset.split(aishell3_dataset, - config.data.valid_size) - batch_fn = collate_aishell3_examples - - if not self.parallel: - self.train_loader = DataLoader( - train_set, - batch_size=config.data.batch_size, - shuffle=True, - drop_last=True, - collate_fn=batch_fn) - else: - sampler = DistributedBatchSampler( - train_set, - batch_size=config.data.batch_size, - shuffle=True, - drop_last=True) - self.train_loader = DataLoader( - train_set, batch_sampler=sampler, collate_fn=batch_fn) - - self.valid_loader = DataLoader( - valid_set, - batch_size=config.data.batch_size, - shuffle=False, - drop_last=False, - collate_fn=batch_fn) - - self.test_loader = DataLoader( - valid_set, - batch_size=1, - shuffle=False, - drop_last=False, - collate_fn=batch_fn) - - -def main_sp(config, args): - exp = Experiment(config, args) - exp.setup() - exp.resume_or_load() - if not args.test: - exp.run() - else: - exp.eval() - - -def main(config, args): - if args.ngpu > 1: - dist.spawn(main_sp, args=(config, args), nprocs=args.ngpu) - else: - main_sp(config, args) - - -if __name__ == "__main__": - config = get_cfg_defaults() - parser = default_argument_parser() - parser.add_argument("--test", action="store_true") - args = parser.parse_args() - if args.config: - config.merge_from_file(args.config) - if args.opts: - config.merge_from_list(args.opts) - config.freeze() - print(config) - print(args) - - main(config, args) diff --git a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/voice_cloning.py b/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/voice_cloning.py deleted file mode 100644 index 4e6b8d36276c65e134221a0c2b5cf4b3ff61a469..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/exps/voice_cloning/tacotron2_ge2e/voice_cloning.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import os -from pathlib import Path - -import numpy as np -import paddle -import soundfile as sf -from matplotlib import pyplot as plt - -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.aishell3 import voc_phones -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.aishell3 import voc_tones -from paddlespeech.t2s.exps.voice_cloning.tacotron2_ge2e.chinese_g2p import convert_sentence -from paddlespeech.t2s.models.tacotron2 import Tacotron2 -from paddlespeech.t2s.models.waveflow import ConditionalWaveFlow -from paddlespeech.t2s.utils import display -from paddlespeech.vector.exps.ge2e.audio_processor import SpeakerVerificationPreprocessor -from paddlespeech.vector.models.lstm_speaker_encoder import LSTMSpeakerEncoder - - -def voice_cloning(args): - # speaker encoder - p = SpeakerVerificationPreprocessor( - sampling_rate=16000, - audio_norm_target_dBFS=-30, - vad_window_length=30, - vad_moving_average_width=8, - vad_max_silence_length=6, - mel_window_length=25, - mel_window_step=10, - n_mels=40, - partial_n_frames=160, - min_pad_coverage=0.75, - partial_overlap_ratio=0.5) - print("Audio Processor Done!") - - speaker_encoder = LSTMSpeakerEncoder( - n_mels=40, num_layers=3, hidden_size=256, output_size=256) - speaker_encoder.set_state_dict(paddle.load(args.ge2e_params_path)) - speaker_encoder.eval() - print("GE2E Done!") - - synthesizer = Tacotron2( - vocab_size=68, - n_tones=10, - d_mels=80, - d_encoder=512, - encoder_conv_layers=3, - encoder_kernel_size=5, - d_prenet=256, - d_attention_rnn=1024, - d_decoder_rnn=1024, - attention_filters=32, - attention_kernel_size=31, - d_attention=128, - d_postnet=512, - postnet_kernel_size=5, - postnet_conv_layers=5, - reduction_factor=1, - p_encoder_dropout=0.5, - p_prenet_dropout=0.5, - p_attention_dropout=0.1, - p_decoder_dropout=0.1, - p_postnet_dropout=0.5, - d_global_condition=256, - use_stop_token=False, ) - synthesizer.set_state_dict(paddle.load(args.tacotron2_params_path)) - synthesizer.eval() - print("Tacotron2 Done!") - - # vocoder - vocoder = ConditionalWaveFlow( - upsample_factors=[16, 16], - n_flows=8, - n_layers=8, - n_group=16, - channels=128, - n_mels=80, - kernel_size=[3, 3]) - vocoder.set_state_dict(paddle.load(args.waveflow_params_path)) - vocoder.eval() - print("WaveFlow Done!") - - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - input_dir = Path(args.input_dir) - - # 因为 AISHELL-3 数据集中使用 % 和 $ 表示韵律词和韵律短语的边界,它们大约对应着较短和较长的停顿,在文本中可以使用 % 和 $ 来调节韵律。 - # 值得的注意的是,句子的有效字符集仅包含汉字和 %, $, 因此输入的句子只能包含这些字符。 - sentence = "每当你觉得%想要批评什么人的时候$你切要记着%这个世界上的人%并非都具备你禀有的条件$" - phones, tones = convert_sentence(sentence) - phones = np.array( - [voc_phones.lookup(item) for item in phones], dtype=np.int64) - tones = np.array([voc_tones.lookup(item) for item in tones], dtype=np.int64) - phones = paddle.to_tensor(phones).unsqueeze(0) - tones = paddle.to_tensor(tones).unsqueeze(0) - - for name in os.listdir(input_dir): - utt_id = name.split(".")[0] - ref_audio_path = input_dir / name - mel_sequences = p.extract_mel_partials(p.preprocess_wav(ref_audio_path)) - print("mel_sequences: ", mel_sequences.shape) - with paddle.no_grad(): - embed = speaker_encoder.embed_utterance( - paddle.to_tensor(mel_sequences)) - print("embed shape: ", embed.shape) - utterance_embeds = paddle.unsqueeze(embed, 0) - outputs = synthesizer.infer( - phones, tones=tones, global_condition=utterance_embeds) - mel_input = paddle.transpose(outputs["mel_outputs_postnet"], [0, 2, 1]) - alignment = outputs["alignments"][0].numpy().T - display.plot_alignment(alignment) - plt.savefig(str(output_dir / (utt_id + ".png"))) - - with paddle.no_grad(): - wav = vocoder.infer(mel_input) - wav = wav.numpy()[0] - sf.write(str(output_dir / (utt_id + ".wav")), wav, samplerate=22050) - - -def main(): - # parse args and config and redirect to train_sp - parser = argparse.ArgumentParser(description="") - parser.add_argument( - "--ge2e_params_path", type=str, help="ge2e params path.") - parser.add_argument( - "--tacotron2_params_path", type=str, help="tacotron2 params path.") - parser.add_argument( - "--waveflow_params_path", type=str, help="waveflow params path.") - - parser.add_argument( - "--ngpu", type=int, default=1, help="if ngpu=0, use cpu.") - - parser.add_argument( - "--input-dir", - type=str, - help="input dir of *.wav, the sample rate will be resample to 16k.") - parser.add_argument("--output-dir", type=str, help="output dir.") - - args = parser.parse_args() - - if args.ngpu == 0: - paddle.set_device("cpu") - elif args.ngpu > 0: - paddle.set_device("gpu") - else: - print("ngpu should >= 0 !") - - voice_cloning(args) - - -if __name__ == "__main__": - main() diff --git a/paddlespeech/t2s/exps/tacotron2/__init__.py b/paddlespeech/t2s/exps/wavernn/__init__.py similarity index 100% rename from paddlespeech/t2s/exps/tacotron2/__init__.py rename to paddlespeech/t2s/exps/wavernn/__init__.py diff --git a/paddlespeech/t2s/exps/wavernn/synthesize.py b/paddlespeech/t2s/exps/wavernn/synthesize.py new file mode 100644 index 0000000000000000000000000000000000000000..61723e039ee4d64a1debf37d05176670a3b09667 --- /dev/null +++ b/paddlespeech/t2s/exps/wavernn/synthesize.py @@ -0,0 +1,108 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +from pathlib import Path + +import jsonlines +import numpy as np +import paddle +import soundfile as sf +import yaml +from paddle import distributed as dist +from timer import timer +from yacs.config import CfgNode + +from paddlespeech.t2s.datasets.data_table import DataTable +from paddlespeech.t2s.models.wavernn import WaveRNN + + +def main(): + parser = argparse.ArgumentParser(description="Synthesize with WaveRNN.") + + parser.add_argument("--config", type=str, help="GANVocoder config file.") + parser.add_argument("--checkpoint", type=str, help="snapshot to load.") + parser.add_argument("--test-metadata", type=str, help="dev data.") + parser.add_argument("--output-dir", type=str, help="output dir.") + parser.add_argument( + "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") + + args = parser.parse_args() + + with open(args.config) as f: + config = CfgNode(yaml.safe_load(f)) + + print("========Args========") + print(yaml.safe_dump(vars(args))) + print("========Config========") + print(config) + print( + f"master see the word size: {dist.get_world_size()}, from pid: {os.getpid()}" + ) + + if args.ngpu == 0: + paddle.set_device("cpu") + elif args.ngpu > 0: + paddle.set_device("gpu") + else: + print("ngpu should >= 0 !") + + model = WaveRNN( + hop_length=config.n_shift, sample_rate=config.fs, **config["model"]) + state_dict = paddle.load(args.checkpoint) + model.set_state_dict(state_dict["main_params"]) + + model.eval() + + with jsonlines.open(args.test_metadata, 'r') as reader: + metadata = list(reader) + test_dataset = DataTable( + metadata, + fields=['utt_id', 'feats'], + converters={ + 'utt_id': None, + 'feats': np.load, + }) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + N = 0 + T = 0 + for example in test_dataset: + utt_id = example['utt_id'] + mel = example['feats'] + mel = paddle.to_tensor(mel) # (T, C) + with timer() as t: + with paddle.no_grad(): + wav = model.generate( + c=mel, + batched=config.inference.gen_batched, + target=config.inference.target, + overlap=config.inference.overlap, + mu_law=config.mu_law, + gen_display=True) + wav = wav.numpy() + N += wav.size + T += t.elapse + speed = wav.size / t.elapse + rtf = config.fs / speed + print( + f"{utt_id}, mel: {mel.shape}, wave: {wav.shape}, time: {t.elapse}s, Hz: {speed}, RTF: {rtf}." + ) + sf.write(str(output_dir / (utt_id + ".wav")), wav, samplerate=config.fs) + print(f"generation speed: {N / T}Hz, RTF: {config.fs / (N / T) }") + + +if __name__ == "__main__": + main() diff --git a/paddlespeech/t2s/exps/wavernn/train.py b/paddlespeech/t2s/exps/wavernn/train.py new file mode 100644 index 0000000000000000000000000000000000000000..aec745f76307866b9112b1db8a0dfc8a4e4088e7 --- /dev/null +++ b/paddlespeech/t2s/exps/wavernn/train.py @@ -0,0 +1,212 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import os +import shutil +from pathlib import Path + +import jsonlines +import numpy as np +import paddle +import yaml +from paddle import DataParallel +from paddle import distributed as dist +from paddle.io import DataLoader +from paddle.io import DistributedBatchSampler +from paddle.optimizer import Adam +from yacs.config import CfgNode + +from paddlespeech.t2s.datasets.data_table import DataTable +from paddlespeech.t2s.datasets.vocoder_batch_fn import WaveRNNClip +from paddlespeech.t2s.models.wavernn import WaveRNN +from paddlespeech.t2s.models.wavernn import WaveRNNEvaluator +from paddlespeech.t2s.models.wavernn import WaveRNNUpdater +from paddlespeech.t2s.modules.losses import discretized_mix_logistic_loss +from paddlespeech.t2s.training.extensions.snapshot import Snapshot +from paddlespeech.t2s.training.extensions.visualizer import VisualDL +from paddlespeech.t2s.training.seeding import seed_everything +from paddlespeech.t2s.training.trainer import Trainer + + +def train_sp(args, config): + # decides device type and whether to run in parallel + # setup running environment correctly + world_size = paddle.distributed.get_world_size() + if (not paddle.is_compiled_with_cuda()) or args.ngpu == 0: + paddle.set_device("cpu") + else: + paddle.set_device("gpu") + if world_size > 1: + paddle.distributed.init_parallel_env() + + # set the random seed, it is a must for multiprocess training + seed_everything(config.seed) + + print( + f"rank: {dist.get_rank()}, pid: {os.getpid()}, parent_pid: {os.getppid()}", + ) + + # construct dataset for training and validation + with jsonlines.open(args.train_metadata, 'r') as reader: + train_metadata = list(reader) + train_dataset = DataTable( + data=train_metadata, + fields=["wave", "feats"], + converters={ + "wave": np.load, + "feats": np.load, + }, ) + + with jsonlines.open(args.dev_metadata, 'r') as reader: + dev_metadata = list(reader) + dev_dataset = DataTable( + data=dev_metadata, + fields=["wave", "feats"], + converters={ + "wave": np.load, + "feats": np.load, + }, ) + + batch_fn = WaveRNNClip( + mode=config.model.mode, + aux_context_window=config.model.aux_context_window, + hop_size=config.n_shift, + batch_max_steps=config.batch_max_steps, + bits=config.model.bits) + + # collate function and dataloader + train_sampler = DistributedBatchSampler( + train_dataset, + batch_size=config.batch_size, + shuffle=True, + drop_last=True) + dev_sampler = DistributedBatchSampler( + dev_dataset, + batch_size=config.batch_size, + shuffle=False, + drop_last=False) + print("samplers done!") + + train_dataloader = DataLoader( + train_dataset, + batch_sampler=train_sampler, + collate_fn=batch_fn, + num_workers=config.num_workers) + + dev_dataloader = DataLoader( + dev_dataset, + collate_fn=batch_fn, + batch_sampler=dev_sampler, + num_workers=config.num_workers) + + valid_generate_loader = DataLoader(dev_dataset, batch_size=1) + + print("dataloaders done!") + + model = WaveRNN( + hop_length=config.n_shift, sample_rate=config.fs, **config["model"]) + if world_size > 1: + model = DataParallel(model) + print("model done!") + + if config.model.mode == 'RAW': + criterion = paddle.nn.CrossEntropyLoss(axis=1) + elif config.model.mode == 'MOL': + criterion = discretized_mix_logistic_loss + else: + criterion = None + RuntimeError('Unknown model mode value - ', config.model.mode) + print("criterions done!") + clip = paddle.nn.ClipGradByGlobalNorm(config.grad_clip) + optimizer = Adam( + parameters=model.parameters(), + learning_rate=config.learning_rate, + grad_clip=clip) + + print("optimizer done!") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + if dist.get_rank() == 0: + config_name = args.config.split("/")[-1] + # copy conf to output_dir + shutil.copyfile(args.config, output_dir / config_name) + + updater = WaveRNNUpdater( + model=model, + optimizer=optimizer, + criterion=criterion, + dataloader=train_dataloader, + output_dir=output_dir, + mode=config.model.mode) + + evaluator = WaveRNNEvaluator( + model=model, + dataloader=dev_dataloader, + criterion=criterion, + output_dir=output_dir, + valid_generate_loader=valid_generate_loader, + config=config) + + trainer = Trainer( + updater, + stop_trigger=(config.train_max_steps, "iteration"), + out=output_dir) + + if dist.get_rank() == 0: + trainer.extend( + evaluator, trigger=(config.eval_interval_steps, 'iteration')) + trainer.extend(VisualDL(output_dir), trigger=(1, 'iteration')) + trainer.extend( + Snapshot(max_size=config.num_snapshots), + trigger=(config.save_interval_steps, 'iteration')) + + print("Trainer Done!") + trainer.run() + + +def main(): + # parse args and config and redirect to train_sp + + parser = argparse.ArgumentParser(description="Train a HiFiGAN model.") + parser.add_argument( + "--config", type=str, help="config file to overwrite default config.") + parser.add_argument("--train-metadata", type=str, help="training data.") + parser.add_argument("--dev-metadata", type=str, help="dev data.") + parser.add_argument("--output-dir", type=str, help="output dir.") + parser.add_argument( + "--ngpu", type=int, default=1, help="if ngpu == 0, use cpu.") + + args = parser.parse_args() + + with open(args.config, 'rt') as f: + config = CfgNode(yaml.safe_load(f)) + + print("========Args========") + print(yaml.safe_dump(vars(args))) + print("========Config========") + print(config) + print( + f"master see the word size: {dist.get_world_size()}, from pid: {os.getpid()}" + ) + + # dispatch + if args.ngpu > 1: + dist.spawn(train_sp, (args, config), nprocs=args.ngpu) + else: + train_sp(args, config) + + +if __name__ == "__main__": + main() diff --git a/paddlespeech/t2s/models/__init__.py b/paddlespeech/t2s/models/__init__.py index 65227374ed7550a219665aa559611747a3bc7f8c..3b90a414c493f55de215efe4d72cf107a81fa57b 100644 --- a/paddlespeech/t2s/models/__init__.py +++ b/paddlespeech/t2s/models/__init__.py @@ -17,6 +17,6 @@ from .melgan import * from .new_tacotron2 import * from .parallel_wavegan import * from .speedyspeech import * -from .tacotron2 import * from .transformer_tts import * from .waveflow import * +from .wavernn import * diff --git a/paddlespeech/t2s/models/new_tacotron2/tacotron2.py b/paddlespeech/t2s/models/new_tacotron2/tacotron2.py index 6a6d107356c43fd0ba1e7568287e9cfd4e9e0a03..da71077fbc01b512aa9308237e43649a33b9d865 100644 --- a/paddlespeech/t2s/models/new_tacotron2/tacotron2.py +++ b/paddlespeech/t2s/models/new_tacotron2/tacotron2.py @@ -432,6 +432,7 @@ class Tacotron2(nn.Layer): # inference h = self.enc.inference(x) + if self.spk_num is not None: sid_emb = self.sid_emb(spk_id.reshape([-1])) h = h + sid_emb @@ -478,7 +479,7 @@ class Tacotron2(nn.Layer): elif self.spk_embed_integration_type == "concat": # concat hidden states with spk embeds spk_emb = F.normalize(spk_emb).unsqueeze(1).expand( - -1, paddle.shape(hs)[1], -1) + shape=[-1, paddle.shape(hs)[1], -1]) hs = paddle.concat([hs, spk_emb], axis=-1) else: raise NotImplementedError("support only add or concat.") diff --git a/paddlespeech/t2s/models/tacotron2.py b/paddlespeech/t2s/models/tacotron2.py deleted file mode 100644 index 01ea4f7d235aa76eba3bc18d029e2e07e8ccbe5c..0000000000000000000000000000000000000000 --- a/paddlespeech/t2s/models/tacotron2.py +++ /dev/null @@ -1,1074 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import math - -import paddle -from paddle import nn -from paddle.fluid.layers import sequence_mask -from paddle.nn import functional as F -from paddle.nn import initializer as I -from tqdm import trange - -from paddlespeech.t2s.modules.conv import Conv1dBatchNorm -from paddlespeech.t2s.modules.losses import guided_attention_loss -from paddlespeech.t2s.utils import checkpoint - -__all__ = ["Tacotron2", "Tacotron2Loss"] - - -class LocationSensitiveAttention(nn.Layer): - """Location Sensitive Attention module. - - Reference: `Attention-Based Models for Speech Recognition `_ - - Parameters - ----------- - d_query: int - The feature size of query. - d_key : int - The feature size of key. - d_attention : int - The feature size of dimension. - location_filters : int - Filter size of attention convolution. - location_kernel_size : int - Kernel size of attention convolution. - """ - - def __init__(self, - d_query: int, - d_key: int, - d_attention: int, - location_filters: int, - location_kernel_size: int): - super().__init__() - - self.query_layer = nn.Linear(d_query, d_attention, bias_attr=False) - self.key_layer = nn.Linear(d_key, d_attention, bias_attr=False) - self.value = nn.Linear(d_attention, 1, bias_attr=False) - - # Location Layer - self.location_conv = nn.Conv1D( - 2, - location_filters, - kernel_size=location_kernel_size, - padding=int((location_kernel_size - 1) / 2), - bias_attr=False, - data_format='NLC') - self.location_layer = nn.Linear( - location_filters, d_attention, bias_attr=False) - - def forward(self, - query, - processed_key, - value, - attention_weights_cat, - mask=None): - """Compute context vector and attention weights. - - Parameters - ----------- - query : Tensor [shape=(batch_size, d_query)] - The queries. - processed_key : Tensor [shape=(batch_size, time_steps_k, d_attention)] - The keys after linear layer. - value : Tensor [shape=(batch_size, time_steps_k, d_key)] - The values. - attention_weights_cat : Tensor [shape=(batch_size, time_step_k, 2)] - Attention weights concat. - mask : Tensor, optional - The mask. Shape should be (batch_size, times_steps_k, 1). - Defaults to None. - - Returns - ---------- - attention_context : Tensor [shape=(batch_size, d_attention)] - The context vector. - attention_weights : Tensor [shape=(batch_size, time_steps_k)] - The attention weights. - """ - - processed_query = self.query_layer(paddle.unsqueeze(query, axis=[1])) - processed_attention_weights = self.location_layer( - self.location_conv(attention_weights_cat)) - # (B, T_enc, 1) - alignment = self.value( - paddle.tanh(processed_attention_weights + processed_key + - processed_query)) - - if mask is not None: - alignment = alignment + (1.0 - mask) * -1e9 - - attention_weights = F.softmax(alignment, axis=1) - attention_context = paddle.matmul( - attention_weights, value, transpose_x=True) - - attention_weights = paddle.squeeze(attention_weights, axis=-1) - attention_context = paddle.squeeze(attention_context, axis=1) - - return attention_context, attention_weights - - -class DecoderPreNet(nn.Layer): - """Decoder prenet module for Tacotron2. - - Parameters - ---------- - d_input: int - The input feature size. - - d_hidden: int - The hidden size. - - d_output: int - The output feature size. - - dropout_rate: float - The droput probability. - - """ - - def __init__(self, - d_input: int, - d_hidden: int, - d_output: int, - dropout_rate: float): - super().__init__() - - self.dropout_rate = dropout_rate - self.linear1 = nn.Linear(d_input, d_hidden, bias_attr=False) - self.linear2 = nn.Linear(d_hidden, d_output, bias_attr=False) - - def forward(self, x): - """Calculate forward propagation. - - Parameters - ---------- - x: Tensor [shape=(B, T_mel, C)] - Batch of the sequences of padded mel spectrogram. - - Returns - ------- - output: Tensor [shape=(B, T_mel, C)] - Batch of the sequences of padded hidden state. - - """ - - x = F.dropout(F.relu(self.linear1(x)), self.dropout_rate, training=True) - output = F.dropout( - F.relu(self.linear2(x)), self.dropout_rate, training=True) - return output - - -class DecoderPostNet(nn.Layer): - """Decoder postnet module for Tacotron2. - - Parameters - ---------- - d_mels: int - The number of mel bands. - - d_hidden: int - The hidden size of postnet. - - kernel_size: int - The kernel size of the conv layer in postnet. - - num_layers: int - The number of conv layers in postnet. - - dropout: float - The droput probability. - - """ - - def __init__(self, - d_mels: int, - d_hidden: int, - kernel_size: int, - num_layers: int, - dropout: float): - super().__init__() - self.dropout = dropout - self.num_layers = num_layers - - padding = int((kernel_size - 1) / 2) - - self.conv_batchnorms = nn.LayerList() - k = math.sqrt(1.0 / (d_mels * kernel_size)) - self.conv_batchnorms.append( - Conv1dBatchNorm( - d_mels, - d_hidden, - kernel_size=kernel_size, - padding=padding, - bias_attr=I.Uniform(-k, k), - data_format='NLC')) - - k = math.sqrt(1.0 / (d_hidden * kernel_size)) - self.conv_batchnorms.extend([ - Conv1dBatchNorm( - d_hidden, - d_hidden, - kernel_size=kernel_size, - padding=padding, - bias_attr=I.Uniform(-k, k), - data_format='NLC') for i in range(1, num_layers - 1) - ]) - - self.conv_batchnorms.append( - Conv1dBatchNorm( - d_hidden, - d_mels, - kernel_size=kernel_size, - padding=padding, - bias_attr=I.Uniform(-k, k), - data_format='NLC')) - - def forward(self, x): - """Calculate forward propagation. - - Parameters - ---------- - x: Tensor [shape=(B, T_mel, C)] - Output sequence of features from decoder. - - Returns - ------- - output: Tensor [shape=(B, T_mel, C)] - Output sequence of features after postnet. - - """ - - for i in range(len(self.conv_batchnorms) - 1): - x = F.dropout( - F.tanh(self.conv_batchnorms[i](x)), - self.dropout, - training=self.training) - output = F.dropout( - self.conv_batchnorms[self.num_layers - 1](x), - self.dropout, - training=self.training) - return output - - -class Tacotron2Encoder(nn.Layer): - """Tacotron2 encoder module for Tacotron2. - - Parameters - ---------- - d_hidden: int - The hidden size in encoder module. - - conv_layers: int - The number of conv layers. - - kernel_size: int - The kernel size of conv layers. - - p_dropout: float - The droput probability. - """ - - def __init__(self, - d_hidden: int, - conv_layers: int, - kernel_size: int, - p_dropout: float): - super().__init__() - - k = math.sqrt(1.0 / (d_hidden * kernel_size)) - self.conv_batchnorms = nn.LayerList([ - Conv1dBatchNorm( - d_hidden, - d_hidden, - kernel_size, - stride=1, - padding=int((kernel_size - 1) / 2), - bias_attr=I.Uniform(-k, k), - data_format='NLC') for i in range(conv_layers) - ]) - self.p_dropout = p_dropout - - self.hidden_size = int(d_hidden / 2) - self.lstm = nn.LSTM( - d_hidden, self.hidden_size, direction="bidirectional") - - def forward(self, x, input_lens=None): - """Calculate forward propagation of tacotron2 encoder. - - Parameters - ---------- - x: Tensor [shape=(B, T, C)] - Input embeddings. - - text_lens: Tensor [shape=(B,)], optional - Batch of lengths of each text input batch. Defaults to None. - - Returns - ------- - output : Tensor [shape=(B, T, C)] - Batch of the sequences of padded hidden states. - - """ - for conv_batchnorm in self.conv_batchnorms: - x = F.dropout( - F.relu(conv_batchnorm(x)), - self.p_dropout, - training=self.training) - - output, _ = self.lstm(inputs=x, sequence_length=input_lens) - return output - - -class Tacotron2Decoder(nn.Layer): - """Tacotron2 decoder module for Tacotron2. - - Parameters - ---------- - d_mels: int - The number of mel bands. - - reduction_factor: int - The reduction factor of tacotron. - - d_encoder: int - The hidden size of encoder. - - d_prenet: int - The hidden size in decoder prenet. - - d_attention_rnn: int - The attention rnn layer hidden size. - - d_decoder_rnn: int - The decoder rnn layer hidden size. - - d_attention: int - The hidden size of the linear layer in location sensitive attention. - - attention_filters: int - The filter size of the conv layer in location sensitive attention. - - attention_kernel_size: int - The kernel size of the conv layer in location sensitive attention. - - p_prenet_dropout: float - The droput probability in decoder prenet. - - p_attention_dropout: float - The droput probability in location sensitive attention. - - p_decoder_dropout: float - The droput probability in decoder. - - use_stop_token: bool - Whether to use a binary classifier for stop token prediction. - Defaults to False - """ - - def __init__(self, - d_mels: int, - reduction_factor: int, - d_encoder: int, - d_prenet: int, - d_attention_rnn: int, - d_decoder_rnn: int, - d_attention: int, - attention_filters: int, - attention_kernel_size: int, - p_prenet_dropout: float, - p_attention_dropout: float, - p_decoder_dropout: float, - use_stop_token: bool=False): - super().__init__() - self.d_mels = d_mels - self.reduction_factor = reduction_factor - self.d_encoder = d_encoder - self.d_attention_rnn = d_attention_rnn - self.d_decoder_rnn = d_decoder_rnn - self.p_attention_dropout = p_attention_dropout - self.p_decoder_dropout = p_decoder_dropout - - self.prenet = DecoderPreNet( - d_mels * reduction_factor, - d_prenet, - d_prenet, - dropout_rate=p_prenet_dropout) - - # attention_rnn takes attention's context vector has an - # auxiliary input - self.attention_rnn = nn.LSTMCell(d_prenet + d_encoder, d_attention_rnn) - - self.attention_layer = LocationSensitiveAttention( - d_attention_rnn, d_encoder, d_attention, attention_filters, - attention_kernel_size) - - # decoder_rnn takes prenet's output and attention_rnn's input - # as input - self.decoder_rnn = nn.LSTMCell(d_attention_rnn + d_encoder, - d_decoder_rnn) - self.linear_projection = nn.Linear(d_decoder_rnn + d_encoder, - d_mels * reduction_factor) - - self.use_stop_token = use_stop_token - if use_stop_token: - self.stop_layer = nn.Linear(d_decoder_rnn + d_encoder, 1) - - # states - temporary attributes - self.attention_hidden = None - self.attention_cell = None - - self.decoder_hidden = None - self.decoder_cell = None - - self.attention_weights = None - self.attention_weights_cum = None - self.attention_context = None - - self.key = None - self.mask = None - self.processed_key = None - - def _initialize_decoder_states(self, key): - """init states be used in decoder - """ - batch_size, encoder_steps, _ = key.shape - - self.attention_hidden = paddle.zeros( - shape=[batch_size, self.d_attention_rnn], dtype=key.dtype) - self.attention_cell = paddle.zeros( - shape=[batch_size, self.d_attention_rnn], dtype=key.dtype) - - self.decoder_hidden = paddle.zeros( - shape=[batch_size, self.d_decoder_rnn], dtype=key.dtype) - self.decoder_cell = paddle.zeros( - shape=[batch_size, self.d_decoder_rnn], dtype=key.dtype) - - self.attention_weights = paddle.zeros( - shape=[batch_size, encoder_steps], dtype=key.dtype) - self.attention_weights_cum = paddle.zeros( - shape=[batch_size, encoder_steps], dtype=key.dtype) - self.attention_context = paddle.zeros( - shape=[batch_size, self.d_encoder], dtype=key.dtype) - - self.key = key # [B, T, C] - # pre-compute projected keys to improve efficiency - self.processed_key = self.attention_layer.key_layer(key) # [B, T, C] - - def _decode(self, query): - """decode one time step - """ - cell_input = paddle.concat([query, self.attention_context], axis=-1) - - # The first lstm layer (or spec encoder lstm) - _, (self.attention_hidden, self.attention_cell) = self.attention_rnn( - cell_input, (self.attention_hidden, self.attention_cell)) - self.attention_hidden = F.dropout( - self.attention_hidden, - self.p_attention_dropout, - training=self.training) - - # Loaction sensitive attention - attention_weights_cat = paddle.stack( - [self.attention_weights, self.attention_weights_cum], axis=-1) - self.attention_context, self.attention_weights = self.attention_layer( - self.attention_hidden, self.processed_key, self.key, - attention_weights_cat, self.mask) - self.attention_weights_cum += self.attention_weights - - # The second lstm layer (or spec decoder lstm) - decoder_input = paddle.concat( - [self.attention_hidden, self.attention_context], axis=-1) - _, (self.decoder_hidden, self.decoder_cell) = self.decoder_rnn( - decoder_input, (self.decoder_hidden, self.decoder_cell)) - self.decoder_hidden = F.dropout( - self.decoder_hidden, - p=self.p_decoder_dropout, - training=self.training) - - # decode output one step - decoder_hidden_attention_context = paddle.concat( - [self.decoder_hidden, self.attention_context], axis=-1) - decoder_output = self.linear_projection( - decoder_hidden_attention_context) - if self.use_stop_token: - stop_logit = self.stop_layer(decoder_hidden_attention_context) - return decoder_output, self.attention_weights, stop_logit - return decoder_output, self.attention_weights - - def forward(self, keys, querys, mask): - """Calculate forward propagation of tacotron2 decoder. - - Parameters - ---------- - keys: Tensor[shape=(B, T_key, C)] - Batch of the sequences of padded output from encoder. - - querys: Tensor[shape(B, T_query, C)] - Batch of the sequences of padded mel spectrogram. - - mask: Tensor - Mask generated with text length. Shape should be (B, T_key, 1). - - Returns - ------- - mel_output: Tensor [shape=(B, T_query, C)] - Output sequence of features. - - alignments: Tensor [shape=(B, T_query, T_key)] - Attention weights. - """ - self._initialize_decoder_states(keys) - self.mask = mask - - querys = paddle.reshape( - querys, - [querys.shape[0], querys.shape[1] // self.reduction_factor, -1]) - start_step = paddle.zeros( - shape=[querys.shape[0], 1, querys.shape[-1]], dtype=querys.dtype) - querys = paddle.concat([start_step, querys], axis=1) - - querys = self.prenet(querys) - - mel_outputs, alignments = [], [] - stop_logits = [] - # Ignore the last time step - while len(mel_outputs) < querys.shape[1] - 1: - query = querys[:, len(mel_outputs), :] - if self.use_stop_token: - mel_output, attention_weights, stop_logit = self._decode(query) - else: - mel_output, attention_weights = self._decode(query) - mel_outputs.append(mel_output) - alignments.append(attention_weights) - if self.use_stop_token: - stop_logits.append(stop_logit) - - alignments = paddle.stack(alignments, axis=1) - mel_outputs = paddle.stack(mel_outputs, axis=1) - if self.use_stop_token: - stop_logits = paddle.concat(stop_logits, axis=1) - return mel_outputs, alignments, stop_logits - return mel_outputs, alignments - - def infer(self, key, max_decoder_steps=1000): - """Calculate forward propagation of tacotron2 decoder. - - Parameters - ---------- - keys: Tensor [shape=(B, T_key, C)] - Batch of the sequences of padded output from encoder. - - max_decoder_steps: int, optional - Number of max step when synthesize. Defaults to 1000. - - Returns - ------- - mel_output: Tensor [shape=(B, T_mel, C)] - Output sequence of features. - - alignments: Tensor [shape=(B, T_mel, T_key)] - Attention weights. - - """ - self._initialize_decoder_states(key) - self.mask = None # mask is not needed for single instance inference - encoder_steps = key.shape[1] - - # [B, C] - start_step = paddle.zeros( - shape=[key.shape[0], self.d_mels * self.reduction_factor], - dtype=key.dtype) - query = start_step # [B, C] - first_hit_end = None - - mel_outputs, alignments = [], [] - stop_logits = [] - for i in trange(max_decoder_steps): - query = self.prenet(query) - if self.use_stop_token: - mel_output, alignment, stop_logit = self._decode(query) - else: - mel_output, alignment = self._decode(query) - - mel_outputs.append(mel_output) - alignments.append(alignment) # (B=1, T) - if self.use_stop_token: - stop_logits.append(stop_logit) - - if self.use_stop_token: - if F.sigmoid(stop_logit) > 0.5: - print("hit stop condition!") - break - else: - if int(paddle.argmax(alignment[0])) == encoder_steps - 1: - if first_hit_end is None: - first_hit_end = i - elif i > (first_hit_end + 20): - print("content exhausted!") - break - if len(mel_outputs) == max_decoder_steps: - print("Warning! Reached max decoder steps!!!") - break - - query = mel_output - - alignments = paddle.stack(alignments, axis=1) - mel_outputs = paddle.stack(mel_outputs, axis=1) - if self.use_stop_token: - stop_logits = paddle.concat(stop_logits, axis=1) - return mel_outputs, alignments, stop_logits - return mel_outputs, alignments - - -class Tacotron2(nn.Layer): - """Tacotron2 model for end-to-end text-to-speech (E2E-TTS). - - This is a model of Spectrogram prediction network in Tacotron2 described - in `Natural TTS Synthesis by Conditioning WaveNet on Mel Spectrogram - Predictions `_, - which converts the sequence of characters - into the sequence of mel spectrogram. - - Parameters - ---------- - vocab_size : int - Vocabulary size of phons of the model. - - n_tones: int - Vocabulary size of tones of the model. Defaults to None. If provided, - the model has an extra tone embedding. - - d_mels: int - Number of mel bands. - - d_encoder: int - Hidden size in encoder module. - - encoder_conv_layers: int - Number of conv layers in encoder. - - encoder_kernel_size: int - Kernel size of conv layers in encoder. - - d_prenet: int - Hidden size in decoder prenet. - - d_attention_rnn: int - Attention rnn layer hidden size in decoder. - - d_decoder_rnn: int - Decoder rnn layer hidden size in decoder. - - attention_filters: int - Filter size of the conv layer in location sensitive attention. - - attention_kernel_size: int - Kernel size of the conv layer in location sensitive attention. - - d_attention: int - Hidden size of the linear layer in location sensitive attention. - - d_postnet: int - Hidden size of postnet. - - postnet_kernel_size: int - Kernel size of the conv layer in postnet. - - postnet_conv_layers: int - Number of conv layers in postnet. - - reduction_factor: int - Reduction factor of tacotron2. - - p_encoder_dropout: float - Droput probability in encoder. - - p_prenet_dropout: float - Droput probability in decoder prenet. - - p_attention_dropout: float - Droput probability in location sensitive attention. - - p_decoder_dropout: float - Droput probability in decoder. - - p_postnet_dropout: float - Droput probability in postnet. - - d_global_condition: int - Feature size of global condition. Defaults to None. If provided, The - model assumes a global condition that is concatenated to the encoder - outputs. - - """ - - def __init__(self, - vocab_size, - n_tones=None, - d_mels: int=80, - d_encoder: int=512, - encoder_conv_layers: int=3, - encoder_kernel_size: int=5, - d_prenet: int=256, - d_attention_rnn: int=1024, - d_decoder_rnn: int=1024, - attention_filters: int=32, - attention_kernel_size: int=31, - d_attention: int=128, - d_postnet: int=512, - postnet_kernel_size: int=5, - postnet_conv_layers: int=5, - reduction_factor: int=1, - p_encoder_dropout: float=0.5, - p_prenet_dropout: float=0.5, - p_attention_dropout: float=0.1, - p_decoder_dropout: float=0.1, - p_postnet_dropout: float=0.5, - d_global_condition=None, - use_stop_token=False): - super().__init__() - - std = math.sqrt(2.0 / (vocab_size + d_encoder)) - val = math.sqrt(3.0) * std # uniform bounds for std - self.embedding = nn.Embedding( - vocab_size, d_encoder, weight_attr=I.Uniform(-val, val)) - if n_tones: - self.embedding_tones = nn.Embedding( - n_tones, - d_encoder, - padding_idx=0, - weight_attr=I.Uniform(-0.1 * val, 0.1 * val)) - self.toned = n_tones is not None - - self.encoder = Tacotron2Encoder(d_encoder, encoder_conv_layers, - encoder_kernel_size, p_encoder_dropout) - - # input augmentation scheme: concat global condition to the encoder output - if d_global_condition is not None: - d_encoder += d_global_condition - self.decoder = Tacotron2Decoder( - d_mels, - reduction_factor, - d_encoder, - d_prenet, - d_attention_rnn, - d_decoder_rnn, - d_attention, - attention_filters, - attention_kernel_size, - p_prenet_dropout, - p_attention_dropout, - p_decoder_dropout, - use_stop_token=use_stop_token) - self.postnet = DecoderPostNet( - d_mels=d_mels * reduction_factor, - d_hidden=d_postnet, - kernel_size=postnet_kernel_size, - num_layers=postnet_conv_layers, - dropout=p_postnet_dropout) - - def forward(self, - text_inputs, - text_lens, - mels, - output_lens=None, - tones=None, - global_condition=None): - """Calculate forward propagation of tacotron2. - - Parameters - ---------- - text_inputs: Tensor [shape=(B, T_text)] - Batch of the sequencees of padded character ids. - - text_lens: Tensor [shape=(B,)] - Batch of lengths of each text input batch. - - mels: Tensor [shape(B, T_mel, C)] - Batch of the sequences of padded mel spectrogram. - - output_lens: Tensor [shape=(B,)], optional - Batch of lengths of each mels batch. Defaults to None. - - tones: Tensor [shape=(B, T_text)] - Batch of sequences of padded tone ids. - - global_condition: Tensor [shape(B, C)] - Batch of global conditions. Defaults to None. If the - `d_global_condition` of the model is not None, this input should be - provided. - - use_stop_token: bool - Whether to include a binary classifier to predict the stop token. - Defaults to False. - - Returns - ------- - outputs : Dict[str, Tensor] - - mel_output: output sequence of features (B, T_mel, C); - - mel_outputs_postnet: output sequence of features after postnet (B, T_mel, C); - - alignments: attention weights (B, T_mel, T_text); - - stop_logits: output sequence of stop logits (B, T_mel) - """ - # input of embedding must be int64 - text_inputs = paddle.cast(text_inputs, 'int64') - embedded_inputs = self.embedding(text_inputs) - if self.toned: - embedded_inputs += self.embedding_tones(tones) - - encoder_outputs = self.encoder(embedded_inputs, text_lens) - - if global_condition is not None: - global_condition = global_condition.unsqueeze(1) - global_condition = paddle.expand(global_condition, - [-1, encoder_outputs.shape[1], -1]) - encoder_outputs = paddle.concat([encoder_outputs, global_condition], - -1) - - # [B, T_enc, 1] - mask = sequence_mask( - text_lens, dtype=encoder_outputs.dtype).unsqueeze(-1) - if self.decoder.use_stop_token: - mel_outputs, alignments, stop_logits = self.decoder( - encoder_outputs, mels, mask=mask) - else: - mel_outputs, alignments = self.decoder( - encoder_outputs, mels, mask=mask) - mel_outputs_postnet = self.postnet(mel_outputs) - mel_outputs_postnet = mel_outputs + mel_outputs_postnet - - if output_lens is not None: - # [B, T_dec, 1] - mask = sequence_mask(output_lens).unsqueeze(-1) - mel_outputs = mel_outputs * mask # [B, T, C] - mel_outputs_postnet = mel_outputs_postnet * mask # [B, T, C] - outputs = { - "mel_output": mel_outputs, - "mel_outputs_postnet": mel_outputs_postnet, - "alignments": alignments - } - if self.decoder.use_stop_token: - outputs["stop_logits"] = stop_logits - - return outputs - - @paddle.no_grad() - def infer(self, - text_inputs, - max_decoder_steps=1000, - tones=None, - global_condition=None): - """Generate the mel sepctrogram of features given the sequences of character ids. - - Parameters - ---------- - text_inputs: Tensor [shape=(B, T_text)] - Batch of the sequencees of padded character ids. - - max_decoder_steps: int, optional - Number of max step when synthesize. Defaults to 1000. - - Returns - ------- - outputs : Dict[str, Tensor] - - mel_output: output sequence of sepctrogram (B, T_mel, C); - - mel_outputs_postnet: output sequence of sepctrogram after postnet (B, T_mel, C); - - stop_logits: output sequence of stop logits (B, T_mel); - - alignments: attention weights (B, T_mel, T_text). This key is only - present when `use_stop_token` is True. - """ - # input of embedding must be int64 - text_inputs = paddle.cast(text_inputs, 'int64') - embedded_inputs = self.embedding(text_inputs) - if self.toned: - embedded_inputs += self.embedding_tones(tones) - encoder_outputs = self.encoder(embedded_inputs) - - if global_condition is not None: - global_condition = global_condition.unsqueeze(1) - global_condition = paddle.expand(global_condition, - [-1, encoder_outputs.shape[1], -1]) - encoder_outputs = paddle.concat([encoder_outputs, global_condition], - -1) - if self.decoder.use_stop_token: - mel_outputs, alignments, stop_logits = self.decoder.infer( - encoder_outputs, max_decoder_steps=max_decoder_steps) - else: - mel_outputs, alignments = self.decoder.infer( - encoder_outputs, max_decoder_steps=max_decoder_steps) - - mel_outputs_postnet = self.postnet(mel_outputs) - mel_outputs_postnet = mel_outputs + mel_outputs_postnet - - outputs = { - "mel_output": mel_outputs, - "mel_outputs_postnet": mel_outputs_postnet, - "alignments": alignments - } - if self.decoder.use_stop_token: - outputs["stop_logits"] = stop_logits - - return outputs - - @classmethod - def from_pretrained(cls, config, checkpoint_path): - """Build a Tacotron2 model from a pretrained model. - - Parameters - ---------- - config: yacs.config.CfgNode - model configs - - checkpoint_path: Path or str - the path of pretrained model checkpoint, without extension name - - Returns - ------- - ConditionalWaveFlow - The model built from pretrained result. - """ - model = cls(vocab_size=config.model.vocab_size, - n_tones=config.model.n_tones, - d_mels=config.data.n_mels, - d_encoder=config.model.d_encoder, - encoder_conv_layers=config.model.encoder_conv_layers, - encoder_kernel_size=config.model.encoder_kernel_size, - d_prenet=config.model.d_prenet, - d_attention_rnn=config.model.d_attention_rnn, - d_decoder_rnn=config.model.d_decoder_rnn, - attention_filters=config.model.attention_filters, - attention_kernel_size=config.model.attention_kernel_size, - d_attention=config.model.d_attention, - d_postnet=config.model.d_postnet, - postnet_kernel_size=config.model.postnet_kernel_size, - postnet_conv_layers=config.model.postnet_conv_layers, - reduction_factor=config.model.reduction_factor, - p_encoder_dropout=config.model.p_encoder_dropout, - p_prenet_dropout=config.model.p_prenet_dropout, - p_attention_dropout=config.model.p_attention_dropout, - p_decoder_dropout=config.model.p_decoder_dropout, - p_postnet_dropout=config.model.p_postnet_dropout, - d_global_condition=config.model.d_global_condition, - use_stop_token=config.model.use_stop_token) - checkpoint.load_parameters(model, checkpoint_path=checkpoint_path) - return model - - -class Tacotron2Loss(nn.Layer): - """ Tacotron2 Loss module - """ - - def __init__(self, - use_stop_token_loss=True, - use_guided_attention_loss=False, - sigma=0.2): - """Tacotron 2 Criterion. - - Args: - use_stop_token_loss (bool, optional): Whether to use a loss for stop token prediction. Defaults to True. - use_guided_attention_loss (bool, optional): Whether to use a loss for attention weights. Defaults to False. - sigma (float, optional): Hyper-parameter sigma for guided attention loss. Defaults to 0.2. - """ - super().__init__() - self.spec_criterion = nn.MSELoss() - self.use_stop_token_loss = use_stop_token_loss - self.use_guided_attention_loss = use_guided_attention_loss - self.attn_criterion = guided_attention_loss - self.stop_criterion = nn.BCEWithLogitsLoss() - self.sigma = sigma - - def forward(self, - mel_outputs, - mel_outputs_postnet, - mel_targets, - attention_weights=None, - slens=None, - plens=None, - stop_logits=None): - """Calculate tacotron2 loss. - - Parameters - ---------- - mel_outputs: Tensor [shape=(B, T_mel, C)] - Output mel spectrogram sequence. - - mel_outputs_postnet: Tensor [shape(B, T_mel, C)] - Output mel spectrogram sequence after postnet. - - mel_targets: Tensor [shape=(B, T_mel, C)] - Target mel spectrogram sequence. - - attention_weights: Tensor [shape=(B, T_mel, T_enc)] - Attention weights. This should be provided when - `use_guided_attention_loss` is True. - - slens: Tensor [shape=(B,)] - Number of frames of mel spectrograms. This should be provided when - `use_guided_attention_loss` is True. - - plens: Tensor [shape=(B, )] - Number of text or phone ids of each utterance. This should be - provided when `use_guided_attention_loss` is True. - - stop_logits: Tensor [shape=(B, T_mel)] - Stop logits of each mel spectrogram frame. This should be provided - when `use_stop_token_loss` is True. - - Returns - ------- - losses : Dict[str, Tensor] - - loss: the sum of the other three losses; - - mel_loss: MSE loss compute by mel_targets and mel_outputs; - - post_mel_loss: MSE loss compute by mel_targets and mel_outputs_postnet; - - guided_attn_loss: Guided attention loss for attention weights; - - stop_loss: Binary cross entropy loss for stop token prediction. - """ - mel_loss = self.spec_criterion(mel_outputs, mel_targets) - post_mel_loss = self.spec_criterion(mel_outputs_postnet, mel_targets) - total_loss = mel_loss + post_mel_loss - if self.use_guided_attention_loss: - gal_loss = self.attn_criterion(attention_weights, slens, plens, - self.sigma) - total_loss += gal_loss - if self.use_stop_token_loss: - T_dec = mel_targets.shape[1] - stop_labels = F.one_hot(slens - 1, num_classes=T_dec) - stop_token_loss = self.stop_criterion(stop_logits, stop_labels) - total_loss += stop_token_loss - - losses = { - "loss": total_loss, - "mel_loss": mel_loss, - "post_mel_loss": post_mel_loss - } - if self.use_guided_attention_loss: - losses["guided_attn_loss"] = gal_loss - if self.use_stop_token_loss: - losses["stop_loss"] = stop_token_loss - return losses diff --git a/paddlespeech/t2s/exps/voice_cloning/__init__.py b/paddlespeech/t2s/models/wavernn/__init__.py similarity index 91% rename from paddlespeech/t2s/exps/voice_cloning/__init__.py rename to paddlespeech/t2s/models/wavernn/__init__.py index abf198b97e6e818e1fbe59006f98492640bcee54..80ffd0688bc4339a5e6d34d5a5220133dbc69a9b 100644 --- a/paddlespeech/t2s/exps/voice_cloning/__init__.py +++ b/paddlespeech/t2s/models/wavernn/__init__.py @@ -11,3 +11,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .wavernn import * +from .wavernn_updater import * diff --git a/paddlespeech/t2s/models/wavernn/wavernn.py b/paddlespeech/t2s/models/wavernn/wavernn.py new file mode 100644 index 0000000000000000000000000000000000000000..fcf39a482825133ca1669da5882700f36ddb5cf7 --- /dev/null +++ b/paddlespeech/t2s/models/wavernn/wavernn.py @@ -0,0 +1,627 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import time +from typing import List + +import numpy as np +import paddle +from paddle import nn +from paddle.nn import functional as F + +from paddlespeech.t2s.audio.codec import decode_mu_law +from paddlespeech.t2s.modules.losses import sample_from_discretized_mix_logistic +from paddlespeech.t2s.modules.nets_utils import initialize +from paddlespeech.t2s.modules.upsample import Stretch2D + + +class ResBlock(nn.Layer): + def __init__(self, dims): + super().__init__() + self.conv1 = nn.Conv1D(dims, dims, kernel_size=1, bias_attr=False) + self.conv2 = nn.Conv1D(dims, dims, kernel_size=1, bias_attr=False) + self.batch_norm1 = nn.BatchNorm1D(dims) + self.batch_norm2 = nn.BatchNorm1D(dims) + + def forward(self, x): + ''' + conv -> bn -> relu -> conv -> bn + residual connection + ''' + residual = x + x = self.conv1(x) + x = self.batch_norm1(x) + x = F.relu(x) + x = self.conv2(x) + x = self.batch_norm2(x) + return x + residual + + +class MelResNet(nn.Layer): + def __init__(self, + res_blocks: int=10, + compute_dims: int=128, + res_out_dims: int=128, + aux_channels: int=80, + aux_context_window: int=0): + super().__init__() + k_size = aux_context_window * 2 + 1 + # pay attention here, the dim reduces aux_context_window * 2 + self.conv_in = nn.Conv1D( + aux_channels, compute_dims, kernel_size=k_size, bias_attr=False) + self.batch_norm = nn.BatchNorm1D(compute_dims) + self.layers = nn.LayerList() + for _ in range(res_blocks): + self.layers.append(ResBlock(compute_dims)) + self.conv_out = nn.Conv1D(compute_dims, res_out_dims, kernel_size=1) + + def forward(self, x): + ''' + Parameters + ---------- + x : Tensor + Input tensor (B, in_dims, T). + Returns + ---------- + Tensor + Output tensor (B, res_out_dims, T). + ''' + + x = self.conv_in(x) + x = self.batch_norm(x) + x = F.relu(x) + for f in self.layers: + x = f(x) + x = self.conv_out(x) + return x + + +class UpsampleNetwork(nn.Layer): + def __init__(self, + aux_channels: int=80, + upsample_scales: List[int]=[4, 5, 3, 5], + compute_dims: int=128, + res_blocks: int=10, + res_out_dims: int=128, + aux_context_window: int=2): + super().__init__() + # total_scale is the total Up sampling multiple + total_scale = np.prod(upsample_scales) + # TODO pad*total_scale is numpy.int64 + self.indent = int(aux_context_window * total_scale) + self.resnet = MelResNet( + res_blocks=res_blocks, + aux_channels=aux_channels, + compute_dims=compute_dims, + res_out_dims=res_out_dims, + aux_context_window=aux_context_window) + self.resnet_stretch = Stretch2D(total_scale, 1) + self.up_layers = nn.LayerList() + for scale in upsample_scales: + k_size = (1, scale * 2 + 1) + padding = (0, scale) + stretch = Stretch2D(scale, 1) + + conv = nn.Conv2D( + 1, 1, kernel_size=k_size, padding=padding, bias_attr=False) + weight_ = paddle.full_like(conv.weight, 1. / k_size[1]) + conv.weight.set_value(weight_) + self.up_layers.append(stretch) + self.up_layers.append(conv) + + def forward(self, m): + ''' + Parameters + ---------- + c : Tensor + Input tensor (B, C_aux, T). + Returns + ---------- + Tensor + Output tensor (B, (T - 2 * pad) * prob(upsample_scales), C_aux). + Tensor + Output tensor (B, (T - 2 * pad) * prob(upsample_scales), res_out_dims). + ''' + # aux: [B, C_aux, T] + # -> [B, res_out_dims, T - 2 * aux_context_window] + # -> [B, 1, res_out_dims, T - 2 * aux_context_window] + aux = self.resnet(m).unsqueeze(1) + # aux: [B, 1, res_out_dims, T - 2 * aux_context_window] + # -> [B, 1, res_out_dims, (T - 2 * pad) * prob(upsample_scales)] + aux = self.resnet_stretch(aux) + # aux: [B, 1, res_out_dims, T * prob(upsample_scales)] + # -> [B, res_out_dims, T * prob(upsample_scales)] + aux = aux.squeeze(1) + # m: [B, C_aux, T] -> [B, 1, C_aux, T] + m = m.unsqueeze(1) + for f in self.up_layers: + m = f(m) + # m: [B, 1, C_aux, T*prob(upsample_scales)] + # -> [B, C_aux, T * prob(upsample_scales)] + # -> [B, C_aux, (T - 2 * pad) * prob(upsample_scales)] + m = m.squeeze(1)[:, :, self.indent:-self.indent] + # m: [B, (T - 2 * pad) * prob(upsample_scales), C_aux] + # aux: [B, (T - 2 * pad) * prob(upsample_scales), res_out_dims] + return m.transpose([0, 2, 1]), aux.transpose([0, 2, 1]) + + +class WaveRNN(nn.Layer): + def __init__( + self, + rnn_dims: int=512, + fc_dims: int=512, + bits: int=9, + aux_context_window: int=2, + upsample_scales: List[int]=[4, 5, 3, 5], + aux_channels: int=80, + compute_dims: int=128, + res_out_dims: int=128, + res_blocks: int=10, + hop_length: int=300, + sample_rate: int=24000, + mode='RAW', + init_type: str="xavier_uniform", ): + ''' + Parameters + ---------- + rnn_dims : int, optional + Hidden dims of RNN Layers. + fc_dims : int, optional + Dims of FC Layers. + bits : int, optional + bit depth of signal. + aux_context_window : int, optional + The context window size of the first convolution applied to the + auxiliary input, by default 2 + upsample_scales : List[int], optional + Upsample scales of the upsample network. + aux_channels : int, optional + Auxiliary channel of the residual blocks. + compute_dims : int, optional + Dims of Conv1D in MelResNet. + res_out_dims : int, optional + Dims of output in MelResNet. + res_blocks : int, optional + Number of residual blocks. + mode : str, optional + Output mode of the WaveRNN vocoder. `MOL` for Mixture of Logistic Distribution, + and `RAW` for quantized bits as the model's output. + init_type : str + How to initialize parameters. + ''' + super().__init__() + self.mode = mode + self.aux_context_window = aux_context_window + if self.mode == 'RAW': + self.n_classes = 2**bits + elif self.mode == 'MOL': + self.n_classes = 10 * 3 + else: + RuntimeError('Unknown model mode value - ', self.mode) + + # List of rnns to call 'flatten_parameters()' on + self._to_flatten = [] + + self.rnn_dims = rnn_dims + self.aux_dims = res_out_dims // 4 + self.hop_length = hop_length + self.sample_rate = sample_rate + + # initialize parameters + initialize(self, init_type) + + self.upsample = UpsampleNetwork( + aux_channels=aux_channels, + upsample_scales=upsample_scales, + compute_dims=compute_dims, + res_blocks=res_blocks, + res_out_dims=res_out_dims, + aux_context_window=aux_context_window) + self.I = nn.Linear(aux_channels + self.aux_dims + 1, rnn_dims) + + self.rnn1 = nn.GRU(rnn_dims, rnn_dims) + self.rnn2 = nn.GRU(rnn_dims + self.aux_dims, rnn_dims) + + self._to_flatten += [self.rnn1, self.rnn2] + + self.fc1 = nn.Linear(rnn_dims + self.aux_dims, fc_dims) + self.fc2 = nn.Linear(fc_dims + self.aux_dims, fc_dims) + self.fc3 = nn.Linear(fc_dims, self.n_classes) + + # Avoid fragmentation of RNN parameters and associated warning + self._flatten_parameters() + + nn.initializer.set_global_initializer(None) + + def forward(self, x, c): + ''' + Parameters + ---------- + x : Tensor + wav sequence, [B, T] + c : Tensor + mel spectrogram [B, C_aux, T'] + + T = (T' - 2 * aux_context_window ) * hop_length + Returns + ---------- + Tensor + [B, T, n_classes] + ''' + # Although we `_flatten_parameters()` on init, when using DataParallel + # the model gets replicated, making it no longer guaranteed that the + # weights are contiguous in GPU memory. Hence, we must call it again + self._flatten_parameters() + + bsize = paddle.shape(x)[0] + h1 = paddle.zeros([1, bsize, self.rnn_dims]) + h2 = paddle.zeros([1, bsize, self.rnn_dims]) + # c: [B, T, C_aux] + # aux: [B, T, res_out_dims] + c, aux = self.upsample(c) + + aux_idx = [self.aux_dims * i for i in range(5)] + a1 = aux[:, :, aux_idx[0]:aux_idx[1]] + a2 = aux[:, :, aux_idx[1]:aux_idx[2]] + a3 = aux[:, :, aux_idx[2]:aux_idx[3]] + a4 = aux[:, :, aux_idx[3]:aux_idx[4]] + + x = paddle.concat([x.unsqueeze(-1), c, a1], axis=2) + x = self.I(x) + res = x + x, _ = self.rnn1(x, h1) + + x = x + res + res = x + x = paddle.concat([x, a2], axis=2) + x, _ = self.rnn2(x, h2) + + x = x + res + x = paddle.concat([x, a3], axis=2) + x = F.relu(self.fc1(x)) + + x = paddle.concat([x, a4], axis=2) + x = F.relu(self.fc2(x)) + + return self.fc3(x) + + @paddle.no_grad() + def generate(self, + c, + batched: bool=True, + target: int=12000, + overlap: int=600, + mu_law: bool=True, + gen_display: bool=False): + """ + Parameters + ---------- + c : Tensor + input mels, (T', C_aux) + batched : bool + generate in batch or not + target : int + target number of samples to be generated in each batch entry + overlap : int + number of samples for crossfading between batches + mu_law : bool + use mu law or not + Returns + ---------- + wav sequence + Output (T' * prod(upsample_scales), out_channels, C_out). + """ + + self.eval() + + mu_law = mu_law if self.mode == 'RAW' else False + + output = [] + start = time.time() + + # pseudo batch + # (T, C_aux) -> (1, C_aux, T) + c = paddle.transpose(c, [1, 0]).unsqueeze(0) + T = paddle.shape(c)[-1] + wave_len = T * self.hop_length + # TODO remove two transpose op by modifying function pad_tensor + c = self.pad_tensor( + c.transpose([0, 2, 1]), pad=self.aux_context_window, + side='both').transpose([0, 2, 1]) + + c, aux = self.upsample(c) + + if batched: + # (num_folds, target + 2 * overlap, features) + c = self.fold_with_overlap(c, target, overlap) + aux = self.fold_with_overlap(aux, target, overlap) + + # for dygraph to static graph, if use seq_len of `b_size, seq_len, _ = paddle.shape(c)` in for + # will not get TensorArray + # see https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/04_dygraph_to_static/case_analysis_cn.html#list-lodtensorarray + # b_size, seq_len, _ = paddle.shape(c) + b_size = paddle.shape(c)[0] + seq_len = paddle.shape(c)[1] + + h1 = paddle.zeros([b_size, self.rnn_dims]) + h2 = paddle.zeros([b_size, self.rnn_dims]) + x = paddle.zeros([b_size, 1]) + + d = self.aux_dims + aux_split = [aux[:, :, d * i:d * (i + 1)] for i in range(4)] + + for i in range(seq_len): + m_t = c[:, i, :] + # for dygraph to static graph + # a1_t, a2_t, a3_t, a4_t = (a[:, i, :] for a in aux_split) + a1_t = aux_split[0][:, i, :] + a2_t = aux_split[1][:, i, :] + a3_t = aux_split[2][:, i, :] + a4_t = aux_split[3][:, i, :] + x = paddle.concat([x, m_t, a1_t], axis=1) + x = self.I(x) + # use GRUCell here + h1, _ = self.rnn1[0].cell(x, h1) + x = x + h1 + inp = paddle.concat([x, a2_t], axis=1) + # use GRUCell here + h2, _ = self.rnn2[0].cell(inp, h2) + + x = x + h2 + x = paddle.concat([x, a3_t], axis=1) + x = F.relu(self.fc1(x)) + + x = paddle.concat([x, a4_t], axis=1) + x = F.relu(self.fc2(x)) + + logits = self.fc3(x) + + if self.mode == 'MOL': + sample = sample_from_discretized_mix_logistic( + logits.unsqueeze(0).transpose([0, 2, 1])) + output.append(sample.reshape([-1])) + x = sample.transpose([1, 0, 2]) + + elif self.mode == 'RAW': + posterior = F.softmax(logits, axis=1) + distrib = paddle.distribution.Categorical(posterior) + # corresponding operate [np.floor((fx + 1) / 2 * mu + 0.5)] in enocde_mu_law + # distrib.sample([1])[0].cast('float32'): [0, 2**bits-1] + # sample: [-1, 1] + sample = 2 * distrib.sample([1])[0].cast('float32') / ( + self.n_classes - 1.) - 1. + output.append(sample) + x = sample.unsqueeze(-1) + else: + raise RuntimeError('Unknown model mode value - ', self.mode) + + if gen_display: + if i % 1000 == 0: + self.gen_display(i, int(seq_len), int(b_size), start) + + output = paddle.stack(output).transpose([1, 0]) + + if mu_law: + output = decode_mu_law(output, self.n_classes, False) + + if batched: + output = self.xfade_and_unfold(output, target, overlap) + else: + output = output[0] + + # Fade-out at the end to avoid signal cutting out suddenly + fade_out = paddle.linspace(1, 0, 10 * self.hop_length) + output = output[:wave_len] + output[-10 * self.hop_length:] *= fade_out + + self.train() + + # 增加 C_out 维度 + return output.unsqueeze(-1) + + def _flatten_parameters(self): + [m.flatten_parameters() for m in self._to_flatten] + + def pad_tensor(self, x, pad, side='both'): + ''' + Parameters + ---------- + x : Tensor + mel, [1, n_frames, 80] + pad : int + side : str + 'both', 'before' or 'after' + Returns + ---------- + Tensor + ''' + b, t, _ = paddle.shape(x) + # for dygraph to static graph + c = x.shape[-1] + total = t + 2 * pad if side == 'both' else t + pad + padded = paddle.zeros([b, total, c]) + if side == 'before' or side == 'both': + padded[:, pad:pad + t, :] = x + elif side == 'after': + padded[:, :t, :] = x + return padded + + def fold_with_overlap(self, x, target, overlap): + ''' + Fold the tensor with overlap for quick batched inference. + Overlap will be used for crossfading in xfade_and_unfold() + + Parameters + ---------- + x : Tensor + Upsampled conditioning features. mels or aux + shape=(1, T, features) + mels: [1, T, 80] + aux: [1, T, 128] + target : int + Target timesteps for each index of batch + overlap : int + Timesteps for both xfade and rnn warmup + overlap = hop_length * 2 + + Returns + ---------- + Tensor + shape=(num_folds, target + 2 * overlap, features) + num_flods = (time_seq - overlap) // (target + overlap) + mel: [num_folds, target + 2 * overlap, 80] + aux: [num_folds, target + 2 * overlap, 128] + + Details + ---------- + x = [[h1, h2, ... hn]] + + Where each h is a vector of conditioning features + + Eg: target=2, overlap=1 with x.size(1)=10 + + folded = [[h1, h2, h3, h4], + [h4, h5, h6, h7], + [h7, h8, h9, h10]] + ''' + + _, total_len, features = paddle.shape(x) + + # Calculate variables needed + num_folds = (total_len - overlap) // (target + overlap) + extended_len = num_folds * (overlap + target) + overlap + remaining = total_len - extended_len + + # Pad if some time steps poking out + if remaining != 0: + num_folds += 1 + padding = target + 2 * overlap - remaining + x = self.pad_tensor(x, padding, side='after') + + folded = paddle.zeros([num_folds, target + 2 * overlap, features]) + + # Get the values for the folded tensor + for i in range(num_folds): + start = i * (target + overlap) + end = start + target + 2 * overlap + folded[i] = x[0][start:end, :] + return folded + + def xfade_and_unfold(self, y, target: int=12000, overlap: int=600): + ''' Applies a crossfade and unfolds into a 1d array. + + Parameters + ---------- + y : Tensor + Batched sequences of audio samples + shape=(num_folds, target + 2 * overlap) + dtype=paddle.float32 + overlap : int + Timesteps for both xfade and rnn warmup + + Returns + ---------- + Tensor + audio samples in a 1d array + shape=(total_len) + dtype=paddle.float32 + + Details + ---------- + y = [[seq1], + [seq2], + [seq3]] + + Apply a gain envelope at both ends of the sequences + + y = [[seq1_in, seq1_target, seq1_out], + [seq2_in, seq2_target, seq2_out], + [seq3_in, seq3_target, seq3_out]] + + Stagger and add up the groups of samples: + + [seq1_in, seq1_target, (seq1_out + seq2_in), seq2_target, ...] + + ''' + # num_folds = (total_len - overlap) // (target + overlap) + num_folds, length = paddle.shape(y) + target = length - 2 * overlap + total_len = num_folds * (target + overlap) + overlap + + # Need some silence for the run warmup + slience_len = overlap // 2 + fade_len = overlap - slience_len + slience = paddle.zeros([slience_len], dtype=paddle.float32) + linear = paddle.ones([fade_len], dtype=paddle.float32) + + # Equal power crossfade + # fade_in increase from 0 to 1, fade_out reduces from 1 to 0 + t = paddle.linspace(-1, 1, fade_len, dtype=paddle.float32) + fade_in = paddle.sqrt(0.5 * (1 + t)) + fade_out = paddle.sqrt(0.5 * (1 - t)) + # Concat the silence to the fades + fade_out = paddle.concat([linear, fade_out]) + fade_in = paddle.concat([slience, fade_in]) + + # Apply the gain to the overlap samples + y[:, :overlap] *= fade_in + y[:, -overlap:] *= fade_out + + unfolded = paddle.zeros([total_len], dtype=paddle.float32) + + # Loop to add up all the samples + for i in range(num_folds): + start = i * (target + overlap) + end = start + target + 2 * overlap + unfolded[start:end] += y[i] + + return unfolded + + def gen_display(self, i, seq_len, b_size, start): + gen_rate = (i + 1) / (time.time() - start) * b_size / 1000 + pbar = self.progbar(i, seq_len) + msg = f'| {pbar} {i*b_size}/{seq_len*b_size} | Batch Size: {b_size} | Gen Rate: {gen_rate:.1f}kHz | ' + sys.stdout.write(f"\r{msg}") + + def progbar(self, i, n, size=16): + done = int(i * size) // n + bar = '' + for i in range(size): + bar += '█' if i <= done else '░' + return bar + + +class WaveRNNInference(nn.Layer): + def __init__(self, normalizer, wavernn): + super().__init__() + self.normalizer = normalizer + self.wavernn = wavernn + + def forward(self, + logmel, + batched: bool=True, + target: int=12000, + overlap: int=600, + mu_law: bool=True, + gen_display: bool=False): + normalized_mel = self.normalizer(logmel) + + wav = self.wavernn.generate( + normalized_mel, ) + # batched=batched, + # target=target, + # overlap=overlap, + # mu_law=mu_law, + # gen_display=gen_display) + + return wav diff --git a/paddlespeech/t2s/models/wavernn/wavernn_updater.py b/paddlespeech/t2s/models/wavernn/wavernn_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..b2756d00c8f83d9b96bdc1a36f755a6c6f50ca09 --- /dev/null +++ b/paddlespeech/t2s/models/wavernn/wavernn_updater.py @@ -0,0 +1,201 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from pathlib import Path + +import paddle +import soundfile as sf +from paddle import distributed as dist +from paddle.io import DataLoader +from paddle.nn import Layer +from paddle.optimizer import Optimizer + +from paddlespeech.t2s.training.extensions.evaluator import StandardEvaluator +from paddlespeech.t2s.training.reporter import report +from paddlespeech.t2s.training.updaters.standard_updater import StandardUpdater +logging.basicConfig( + format='%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s', + datefmt='[%Y-%m-%d %H:%M:%S]') +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def calculate_grad_norm(parameters, norm_type: str=2): + ''' + calculate grad norm of mdoel's parameters + parameters: + model's parameters + norm_type: str + Returns + ------------ + Tensor + grad_norm + ''' + + grad_list = [ + paddle.to_tensor(p.grad) for p in parameters if p.grad is not None + ] + norm_list = paddle.stack( + [paddle.norm(grad, norm_type) for grad in grad_list]) + total_norm = paddle.norm(norm_list) + return total_norm + + +# for save name in gen_valid_samples() +ITERATION = 0 + + +class WaveRNNUpdater(StandardUpdater): + def __init__(self, + model: Layer, + optimizer: Optimizer, + criterion: Layer, + dataloader: DataLoader, + init_state=None, + output_dir: Path=None, + mode='RAW'): + super().__init__(model, optimizer, dataloader, init_state=None) + + self.criterion = criterion + # self.scheduler = scheduler + + log_file = output_dir / 'worker_{}.log'.format(dist.get_rank()) + self.filehandler = logging.FileHandler(str(log_file)) + logger.addHandler(self.filehandler) + self.logger = logger + self.msg = "" + self.mode = mode + + def update_core(self, batch): + + self.msg = "Rank: {}, ".format(dist.get_rank()) + losses_dict = {} + # parse batch + self.model.train() + self.optimizer.clear_grad() + + wav, y, mel = batch + + y_hat = self.model(wav, mel) + if self.mode == 'RAW': + y_hat = y_hat.transpose([0, 2, 1]).unsqueeze(-1) + elif self.mode == 'MOL': + y_hat = paddle.cast(y, dtype='float32') + + y = y.unsqueeze(-1) + loss = self.criterion(y_hat, y) + loss.backward() + grad_norm = float( + calculate_grad_norm(self.model.parameters(), norm_type=2)) + + self.optimizer.step() + + report("train/loss", float(loss)) + report("train/grad_norm", float(grad_norm)) + + losses_dict["loss"] = float(loss) + losses_dict["grad_norm"] = float(grad_norm) + self.msg += ', '.join('{}: {:>.6f}'.format(k, v) + for k, v in losses_dict.items()) + global ITERATION + ITERATION = self.state.iteration + 1 + + +class WaveRNNEvaluator(StandardEvaluator): + def __init__(self, + model: Layer, + criterion: Layer, + dataloader: Optimizer, + output_dir: Path=None, + valid_generate_loader=None, + config=None): + super().__init__(model, dataloader) + + log_file = output_dir / 'worker_{}.log'.format(dist.get_rank()) + self.filehandler = logging.FileHandler(str(log_file)) + logger.addHandler(self.filehandler) + self.logger = logger + self.msg = "" + + self.criterion = criterion + self.valid_generate_loader = valid_generate_loader + self.config = config + self.mode = config.model.mode + + self.valid_samples_dir = output_dir / "valid_samples" + self.valid_samples_dir.mkdir(parents=True, exist_ok=True) + + def evaluate_core(self, batch): + self.msg = "Evaluate: " + losses_dict = {} + # parse batch + wav, y, mel = batch + y_hat = self.model(wav, mel) + + if self.mode == 'RAW': + y_hat = y_hat.transpose([0, 2, 1]).unsqueeze(-1) + elif self.mode == 'MOL': + y_hat = paddle.cast(y, dtype='float32') + + y = y.unsqueeze(-1) + loss = self.criterion(y_hat, y) + report("eval/loss", float(loss)) + + losses_dict["loss"] = float(loss) + + self.msg += ', '.join('{}: {:>.6f}'.format(k, v) + for k, v in losses_dict.items()) + self.logger.info(self.msg) + + def gen_valid_samples(self): + + for i, item in enumerate(self.valid_generate_loader): + if i >= self.config.generate_num: + break + print( + '\n| Generating: {}/{}'.format(i + 1, self.config.generate_num)) + + mel = item['feats'] + wav = item['wave'] + wav = wav.squeeze(0) + + origin_save_path = self.valid_samples_dir / '{}_steps_{}_target.wav'.format( + self.iteration, i) + sf.write(origin_save_path, wav.numpy(), samplerate=self.config.fs) + + if self.config.inference.gen_batched: + batch_str = 'gen_batched_target{}_overlap{}'.format( + self.config.inference.target, self.config.inference.overlap) + else: + batch_str = 'gen_not_batched' + gen_save_path = str(self.valid_samples_dir / + '{}_steps_{}_{}.wav'.format(self.iteration, i, + batch_str)) + # (1, T, C_aux) -> (T, C_aux) + mel = mel.squeeze(0) + gen_sample = self.model.generate( + mel, self.config.inference.gen_batched, + self.config.inference.target, self.config.inference.overlap, + self.config.mu_law) + sf.write( + gen_save_path, gen_sample.numpy(), samplerate=self.config.fs) + + def __call__(self, trainer=None): + summary = self.evaluate() + for k, v in summary.items(): + report(k, v) + # gen samples at then end of evaluate + self.iteration = ITERATION + if self.iteration % self.config.gen_eval_samples_interval_steps == 0: + self.gen_valid_samples() diff --git a/paddlespeech/t2s/modules/losses.py b/paddlespeech/t2s/modules/losses.py index 3cc7a93cb5f74d50a2562f61b18b9f1e1bab6bf8..618f444a11b0baf9815e61d38898070b5f5024e5 100644 --- a/paddlespeech/t2s/modules/losses.py +++ b/paddlespeech/t2s/modules/losses.py @@ -14,6 +14,7 @@ import math import librosa +import numpy as np import paddle from paddle import nn from paddle.fluid.layers import sequence_mask @@ -23,6 +24,145 @@ from scipy import signal from paddlespeech.t2s.modules.nets_utils import make_non_pad_mask +# Losses for WaveRNN +def log_sum_exp(x): + """ numerically stable log_sum_exp implementation that prevents overflow """ + # TF ordering + axis = len(x.shape) - 1 + m = paddle.max(x, axis=axis) + m2 = paddle.max(x, axis=axis, keepdim=True) + return m + paddle.log(paddle.sum(paddle.exp(x - m2), axis=axis)) + + +# It is adapted from https://github.com/r9y9/wavenet_vocoder/blob/master/wavenet_vocoder/mixture.py +def discretized_mix_logistic_loss(y_hat, + y, + num_classes=65536, + log_scale_min=None, + reduce=True): + if log_scale_min is None: + log_scale_min = float(np.log(1e-14)) + y_hat = y_hat.transpose([0, 2, 1]) + assert y_hat.dim() == 3 + assert y_hat.shape[1] % 3 == 0 + nr_mix = y_hat.shape[1] // 3 + + # (B x T x C) + y_hat = y_hat.transpose([0, 2, 1]) + + # unpack parameters. (B, T, num_mixtures) x 3 + logit_probs = y_hat[:, :, :nr_mix] + means = y_hat[:, :, nr_mix:2 * nr_mix] + log_scales = paddle.clip( + y_hat[:, :, 2 * nr_mix:3 * nr_mix], min=log_scale_min) + + # B x T x 1 -> B x T x num_mixtures + y = y.expand_as(means) + centered_y = paddle.cast(y, dtype=paddle.get_default_dtype()) - means + inv_stdv = paddle.exp(-log_scales) + plus_in = inv_stdv * (centered_y + 1. / (num_classes - 1)) + cdf_plus = F.sigmoid(plus_in) + min_in = inv_stdv * (centered_y - 1. / (num_classes - 1)) + cdf_min = F.sigmoid(min_in) + + # log probability for edge case of 0 (before scaling) + # equivalent: torch.log(F.sigmoid(plus_in)) + # softplus: log(1+ e^{-x}) + log_cdf_plus = plus_in - F.softplus(plus_in) + + # log probability for edge case of 255 (before scaling) + # equivalent: (1 - F.sigmoid(min_in)).log() + log_one_minus_cdf_min = -F.softplus(min_in) + + # probability for all other cases + cdf_delta = cdf_plus - cdf_min + + mid_in = inv_stdv * centered_y + # log probability in the center of the bin, to be used in extreme cases + # (not actually used in our code) + log_pdf_mid = mid_in - log_scales - 2. * F.softplus(mid_in) + + # TODO: cdf_delta <= 1e-5 actually can happen. How can we choose the value + # for num_classes=65536 case? 1e-7? not sure.. + inner_inner_cond = cdf_delta > 1e-5 + + inner_inner_cond = paddle.cast( + inner_inner_cond, dtype=paddle.get_default_dtype()) + + # inner_inner_out = inner_inner_cond * \ + # paddle.log(paddle.clip(cdf_delta, min=1e-12)) + \ + # (1. - inner_inner_cond) * (log_pdf_mid - np.log((num_classes - 1) / 2)) + + inner_inner_out = inner_inner_cond * paddle.log( + paddle.clip(cdf_delta, min=1e-12)) + (1. - inner_inner_cond) * ( + log_pdf_mid - np.log((num_classes - 1) / 2)) + + inner_cond = y > 0.999 + + inner_cond = paddle.cast(inner_cond, dtype=paddle.get_default_dtype()) + + inner_out = inner_cond * log_one_minus_cdf_min + (1. - inner_cond + ) * inner_inner_out + cond = y < -0.999 + cond = paddle.cast(cond, dtype=paddle.get_default_dtype()) + + log_probs = cond * log_cdf_plus + (1. - cond) * inner_out + log_probs = log_probs + F.log_softmax(logit_probs, -1) + + if reduce: + return -paddle.mean(log_sum_exp(log_probs)) + else: + return -log_sum_exp(log_probs).unsqueeze(-1) + + +def sample_from_discretized_mix_logistic(y, log_scale_min=None): + """ + Sample from discretized mixture of logistic distributions + Parameters + ---------- + y : Tensor + (B, C, T) + log_scale_min : float + Log scale minimum value + Returns + ---------- + Tensor + sample in range of [-1, 1]. + """ + if log_scale_min is None: + log_scale_min = float(np.log(1e-14)) + + assert y.shape[1] % 3 == 0 + nr_mix = y.shape[1] // 3 + + # (B, T, C) + y = y.transpose([0, 2, 1]) + logit_probs = y[:, :, :nr_mix] + + # sample mixture indicator from softmax + temp = paddle.uniform( + logit_probs.shape, dtype=logit_probs.dtype, min=1e-5, max=1.0 - 1e-5) + temp = logit_probs - paddle.log(-paddle.log(temp)) + argmax = paddle.argmax(temp, axis=-1) + + # (B, T) -> (B, T, nr_mix) + one_hot = F.one_hot(argmax, nr_mix) + one_hot = paddle.cast(one_hot, dtype=paddle.get_default_dtype()) + + # select logistic parameters + means = paddle.sum(y[:, :, nr_mix:2 * nr_mix] * one_hot, axis=-1) + log_scales = paddle.clip( + paddle.sum(y[:, :, 2 * nr_mix:3 * nr_mix] * one_hot, axis=-1), + min=log_scale_min) + # sample from logistic & clip to interval + # we don't actually round to the nearest 8bit value when sampling + u = paddle.uniform(means.shape, min=1e-5, max=1.0 - 1e-5) + x = means + paddle.exp(log_scales) * (paddle.log(u) - paddle.log(1. - u)) + x = paddle.clip(x, min=-1., max=-1.) + + return x + + # Loss for new Tacotron2 class GuidedAttentionLoss(nn.Layer): """Guided attention loss function module. diff --git a/paddlespeech/t2s/modules/tacotron2/attentions.py b/paddlespeech/t2s/modules/tacotron2/attentions.py index 710e326d6083da4aa550740d915afef0382a139e..af7a94f30a2598bf39af56a3774ac9231026fa0a 100644 --- a/paddlespeech/t2s/modules/tacotron2/attentions.py +++ b/paddlespeech/t2s/modules/tacotron2/attentions.py @@ -157,7 +157,7 @@ class AttLoc(nn.Layer): paddle.Tensor previous attention weights (B, T_max) """ - batch = len(enc_hs_pad) + batch = paddle.shape(enc_hs_pad)[0] # pre-compute all h outside the decoder loop if self.pre_compute_enc_h is None or self.han_mode: # (utt, frame, hdim) @@ -172,33 +172,30 @@ class AttLoc(nn.Layer): dec_z = dec_z.reshape([batch, self.dunits]) # initialize attention weight with uniform dist. - if att_prev is None: + if paddle.sum(att_prev) == 0: # if no bias, 0 0-pad goes 0 - att_prev = 1.0 - make_pad_mask(enc_hs_len) att_prev = att_prev / enc_hs_len.unsqueeze(-1) # att_prev: (utt, frame) -> (utt, 1, 1, frame) # -> (utt, att_conv_chans, 1, frame) - att_conv = self.loc_conv(att_prev.reshape([batch, 1, 1, self.h_length])) # att_conv: (utt, att_conv_chans, 1, frame) -> (utt, frame, att_conv_chans) att_conv = att_conv.squeeze(2).transpose([0, 2, 1]) # att_conv: (utt, frame, att_conv_chans) -> (utt, frame, att_dim) att_conv = self.mlp_att(att_conv) - - # dec_z_tiled: (utt, frame, att_dim) + # dec_z_tiled: (utt, frame, att_dim) dec_z_tiled = self.mlp_dec(dec_z).reshape([batch, 1, self.att_dim]) # dot with gvec # (utt, frame, att_dim) -> (utt, frame) - e = self.gvec( - paddle.tanh(att_conv + self.pre_compute_enc_h + - dec_z_tiled)).squeeze(2) + e = paddle.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) + e = self.gvec(e).squeeze(2) # NOTE: consider zero padding when compute w. if self.mask is None: self.mask = make_pad_mask(enc_hs_len) + e = masked_fill(e, self.mask, -float("inf")) # apply monotonic attention constraint (mainly for TTS) if last_attended_idx is not None: @@ -211,7 +208,6 @@ class AttLoc(nn.Layer): # utt x hdim c = paddle.sum( self.enc_h * w.reshape([batch, self.h_length, 1]), axis=1) - return c, w diff --git a/paddlespeech/t2s/modules/tacotron2/decoder.py b/paddlespeech/t2s/modules/tacotron2/decoder.py index fc15adfda30a5ded3481fe570a59a41b60da2bcc..3622fd7a247c3785a9c85900d97c513f30a13bd8 100644 --- a/paddlespeech/t2s/modules/tacotron2/decoder.py +++ b/paddlespeech/t2s/modules/tacotron2/decoder.py @@ -15,7 +15,6 @@ """Tacotron2 decoder related modules.""" import paddle import paddle.nn.functional as F -import six from paddle import nn from paddlespeech.t2s.modules.tacotron2.attentions import AttForwardTA @@ -59,7 +58,7 @@ class Prenet(nn.Layer): super().__init__() self.dropout_rate = dropout_rate self.prenet = nn.LayerList() - for layer in six.moves.range(n_layers): + for layer in range(n_layers): n_inputs = idim if layer == 0 else n_units self.prenet.append( nn.Sequential(nn.Linear(n_inputs, n_units), nn.ReLU())) @@ -78,7 +77,7 @@ class Prenet(nn.Layer): Batch of output tensors (B, ..., odim). """ - for i in six.moves.range(len(self.prenet)): + for i in range(len(self.prenet)): # F.dropout 引入了随机, tacotron2 的 dropout 是不能去掉的 x = F.dropout(self.prenet[i](x)) return x @@ -129,7 +128,7 @@ class Postnet(nn.Layer): """ super().__init__() self.postnet = nn.LayerList() - for layer in six.moves.range(n_layers - 1): + for layer in range(n_layers - 1): ichans = odim if layer == 0 else n_chans ochans = odim if layer == n_layers - 1 else n_chans if use_batch_norm: @@ -196,7 +195,7 @@ class Postnet(nn.Layer): Batch of padded output tensor. (B, odim, Tmax). """ - for i in six.moves.range(len(self.postnet)): + for i in range(len(self.postnet)): xs = self.postnet[i](xs) return xs @@ -360,7 +359,7 @@ class Decoder(nn.Layer): # define lstm network prenet_units = prenet_units if prenet_layers != 0 else odim self.lstm = nn.LayerList() - for layer in six.moves.range(dlayers): + for layer in range(dlayers): iunits = idim + prenet_units if layer == 0 else dunits lstm = nn.LSTMCell(iunits, dunits) if zoneout_rate > 0.0: @@ -437,47 +436,50 @@ class Decoder(nn.Layer): # initialize hidden states of decoder c_list = [self._zero_state(hs)] z_list = [self._zero_state(hs)] - for _ in six.moves.range(1, len(self.lstm)): - c_list += [self._zero_state(hs)] - z_list += [self._zero_state(hs)] + for _ in range(1, len(self.lstm)): + c_list.append(self._zero_state(hs)) + z_list.append(self._zero_state(hs)) prev_out = paddle.zeros([paddle.shape(hs)[0], self.odim]) # initialize attention - prev_att_w = None + prev_att_ws = [] + prev_att_w = paddle.zeros(paddle.shape(hlens)) + prev_att_ws.append(prev_att_w) self.att.reset() # loop for an output sequence outs, logits, att_ws = [], [], [] for y in ys.transpose([1, 0, 2]): if self.use_att_extra_inputs: - att_c, att_w = self.att(hs, hlens, z_list[0], prev_att_w, + att_c, att_w = self.att(hs, hlens, z_list[0], prev_att_ws[-1], prev_out) else: - att_c, att_w = self.att(hs, hlens, z_list[0], prev_att_w) + att_c, att_w = self.att(hs, hlens, z_list[0], prev_att_ws[-1]) prenet_out = self.prenet( prev_out) if self.prenet is not None else prev_out xs = paddle.concat([att_c, prenet_out], axis=1) # we only use the second output of LSTMCell in paddle _, next_hidden = self.lstm[0](xs, (z_list[0], c_list[0])) z_list[0], c_list[0] = next_hidden - for i in six.moves.range(1, len(self.lstm)): + for i in range(1, len(self.lstm)): # we only use the second output of LSTMCell in paddle _, next_hidden = self.lstm[i](z_list[i - 1], (z_list[i], c_list[i])) z_list[i], c_list[i] = next_hidden zcs = (paddle.concat([z_list[-1], att_c], axis=1) if self.use_concate else z_list[-1]) - outs += [ - self.feat_out(zcs).reshape([paddle.shape(hs)[0], self.odim, -1]) - ] - logits += [self.prob_out(zcs)] - att_ws += [att_w] + outs.append( + self.feat_out(zcs).reshape([paddle.shape(hs)[0], self.odim, -1 + ])) + logits.append(self.prob_out(zcs)) + att_ws.append(att_w) # teacher forcing prev_out = y - if self.cumulate_att_w and prev_att_w is not None: + if self.cumulate_att_w and paddle.sum(prev_att_w) != 0: prev_att_w = prev_att_w + att_w # Note: error when use += else: prev_att_w = att_w + prev_att_ws.append(prev_att_w) # (B, Lmax) logits = paddle.concat(logits, axis=1) # (B, odim, Lmax) @@ -552,6 +554,7 @@ class Decoder(nn.Layer): .. _`Deep Voice 3`: https://arxiv.org/abs/1710.07654 """ # setup + assert len(paddle.shape(h)) == 2 hs = h.unsqueeze(0) ilens = paddle.shape(h)[0] @@ -561,13 +564,16 @@ class Decoder(nn.Layer): # initialize hidden states of decoder c_list = [self._zero_state(hs)] z_list = [self._zero_state(hs)] - for _ in six.moves.range(1, len(self.lstm)): - c_list += [self._zero_state(hs)] - z_list += [self._zero_state(hs)] + for _ in range(1, len(self.lstm)): + c_list.append(self._zero_state(hs)) + z_list.append(self._zero_state(hs)) prev_out = paddle.zeros([1, self.odim]) # initialize attention - prev_att_w = None + prev_att_ws = [] + prev_att_w = paddle.zeros([ilens]) + prev_att_ws.append(prev_att_w) + self.att.reset() # setup for attention constraint @@ -579,6 +585,7 @@ class Decoder(nn.Layer): # loop for an output sequence idx = 0 outs, att_ws, probs = [], [], [] + prob = paddle.zeros([1]) while True: # updated index idx += self.reduction_factor @@ -589,7 +596,7 @@ class Decoder(nn.Layer): hs, ilens, z_list[0], - prev_att_w, + prev_att_ws[-1], prev_out, last_attended_idx=last_attended_idx, backward_window=backward_window, @@ -599,19 +606,20 @@ class Decoder(nn.Layer): hs, ilens, z_list[0], - prev_att_w, + prev_att_ws[-1], last_attended_idx=last_attended_idx, backward_window=backward_window, forward_window=forward_window, ) - att_ws += [att_w] + att_ws.append(att_w) prenet_out = self.prenet( prev_out) if self.prenet is not None else prev_out xs = paddle.concat([att_c, prenet_out], axis=1) # we only use the second output of LSTMCell in paddle _, next_hidden = self.lstm[0](xs, (z_list[0], c_list[0])) + z_list[0], c_list[0] = next_hidden - for i in six.moves.range(1, len(self.lstm)): + for i in range(1, len(self.lstm)): # we only use the second output of LSTMCell in paddle _, next_hidden = self.lstm[i](z_list[i - 1], (z_list[i], c_list[i])) @@ -619,38 +627,38 @@ class Decoder(nn.Layer): zcs = (paddle.concat([z_list[-1], att_c], axis=1) if self.use_concate else z_list[-1]) # [(1, odim, r), ...] - outs += [self.feat_out(zcs).reshape([1, self.odim, -1])] + outs.append(self.feat_out(zcs).reshape([1, self.odim, -1])) + + prob = F.sigmoid(self.prob_out(zcs))[0] + probs.append(prob) - # [(r), ...] - probs += [F.sigmoid(self.prob_out(zcs))[0]] if self.output_activation_fn is not None: prev_out = self.output_activation_fn( outs[-1][:, :, -1]) # (1, odim) else: prev_out = outs[-1][:, :, -1] # (1, odim) - if self.cumulate_att_w and prev_att_w is not None: + if self.cumulate_att_w and paddle.sum(prev_att_w) != 0: prev_att_w = prev_att_w + att_w # Note: error when use += else: prev_att_w = att_w + prev_att_ws.append(prev_att_w) if use_att_constraint: last_attended_idx = int(att_w.argmax()) - # check whether to finish generation - if sum(paddle.cast(probs[-1] >= threshold, - 'int64')) > 0 or idx >= maxlen: + if prob >= threshold or idx >= maxlen: # check mininum length if idx < minlen: continue - # (1, odim, L) - outs = paddle.concat(outs, axis=2) - if self.postnet is not None: - # (1, odim, L) - outs = outs + self.postnet(outs) - # (L, odim) - outs = outs.transpose([0, 2, 1]).squeeze(0) - probs = paddle.concat(probs, axis=0) - att_ws = paddle.concat(att_ws, axis=0) break + # (1, odim, L) + outs = paddle.concat(outs, axis=2) + if self.postnet is not None: + # (1, odim, L) + outs = outs + self.postnet(outs) + # (L, odim) + outs = outs.transpose([0, 2, 1]).squeeze(0) + probs = paddle.concat(probs, axis=0) + att_ws = paddle.concat(att_ws, axis=0) if self.output_activation_fn is not None: outs = self.output_activation_fn(outs) @@ -685,9 +693,9 @@ class Decoder(nn.Layer): # initialize hidden states of decoder c_list = [self._zero_state(hs)] z_list = [self._zero_state(hs)] - for _ in six.moves.range(1, len(self.lstm)): - c_list += [self._zero_state(hs)] - z_list += [self._zero_state(hs)] + for _ in range(1, len(self.lstm)): + c_list.append(self._zero_state(hs)) + z_list.append(self._zero_state(hs)) prev_out = paddle.zeros([paddle.shape(hs)[0], self.odim]) # initialize attention @@ -702,14 +710,14 @@ class Decoder(nn.Layer): prev_out) else: att_c, att_w = self.att(hs, hlens, z_list[0], prev_att_w) - att_ws += [att_w] + att_ws.append(att_w) prenet_out = self.prenet( prev_out) if self.prenet is not None else prev_out xs = paddle.concat([att_c, prenet_out], axis=1) # we only use the second output of LSTMCell in paddle _, next_hidden = self.lstm[0](xs, (z_list[0], c_list[0])) z_list[0], c_list[0] = next_hidden - for i in six.moves.range(1, len(self.lstm)): + for i in range(1, len(self.lstm)): z_list[i], c_list[i] = self.lstm[i](z_list[i - 1], (z_list[i], c_list[i])) # teacher forcing diff --git a/paddlespeech/t2s/modules/tacotron2/encoder.py b/paddlespeech/t2s/modules/tacotron2/encoder.py index b2ed30d1f1cebc666e68c8555c4d69dfd1140331..80c213a1a3f50b6e79c14ee259810a734bd8a865 100644 --- a/paddlespeech/t2s/modules/tacotron2/encoder.py +++ b/paddlespeech/t2s/modules/tacotron2/encoder.py @@ -14,7 +14,6 @@ # Modified from espnet(https://github.com/espnet/espnet) """Tacotron2 encoder related modules.""" import paddle -import six from paddle import nn @@ -88,7 +87,7 @@ class Encoder(nn.Layer): if econv_layers > 0: self.convs = nn.LayerList() - for layer in six.moves.range(econv_layers): + for layer in range(econv_layers): ichans = (embed_dim if layer == 0 and input_layer == "embed" else econv_chans) if use_batch_norm: @@ -130,6 +129,7 @@ class Encoder(nn.Layer): direction='bidirectional', bias_ih_attr=True, bias_hh_attr=True) + self.blstm.flatten_parameters() else: self.blstm = None @@ -157,7 +157,7 @@ class Encoder(nn.Layer): """ xs = self.embed(xs).transpose([0, 2, 1]) if self.convs is not None: - for i in six.moves.range(len(self.convs)): + for i in range(len(self.convs)): if self.use_residual: xs += self.convs[i](xs) else: @@ -167,7 +167,8 @@ class Encoder(nn.Layer): if not isinstance(ilens, paddle.Tensor): ilens = paddle.to_tensor(ilens) xs = xs.transpose([0, 2, 1]) - self.blstm.flatten_parameters() + # for dygraph to static graph + # self.blstm.flatten_parameters() # (B, Tmax, C) # see https://www.paddlepaddle.org.cn/documentation/docs/zh/faq/train_cn.html#paddletorch-nn-utils-rnn-pack-padded-sequencetorch-nn-utils-rnn-pad-packed-sequenceapi xs, _ = self.blstm(xs, sequence_length=ilens) @@ -191,6 +192,6 @@ class Encoder(nn.Layer): """ xs = x.unsqueeze(0) - ilens = paddle.to_tensor([x.shape[0]]) + ilens = paddle.shape(x)[0] return self.forward(xs, ilens)[0][0] diff --git a/paddlespeech/t2s/modules/transformer/repeat.py b/paddlespeech/t2s/modules/transformer/repeat.py index 0325a638235a8fb736da886e24225ed2f66e876a..f738b5569e02938bb525d44a2a327da0723323ba 100644 --- a/paddlespeech/t2s/modules/transformer/repeat.py +++ b/paddlespeech/t2s/modules/transformer/repeat.py @@ -41,4 +41,4 @@ def repeat(N, fn): MultiSequential Repeated model instance. """ - return MultiSequential(* [fn(n) for n in range(N)]) + return MultiSequential(*[fn(n) for n in range(N)]) diff --git a/paddlespeech/t2s/utils/__init__.py b/paddlespeech/t2s/utils/__init__.py index ce3a4ef60ce05e42600cf9f0c5e3ba0fb8d44f5e..520c81a26e275eb090b5816e4af584b52036aa6b 100644 --- a/paddlespeech/t2s/utils/__init__.py +++ b/paddlespeech/t2s/utils/__init__.py @@ -16,3 +16,7 @@ from . import display from . import layer_tools from . import mp_tools from . import scheduler + + +def str2bool(str): + return True if str.lower() == 'true' else False diff --git a/paddlespeech/vector/exps/ge2e/speaker_verification_dataset.py b/paddlespeech/vector/exps/ge2e/speaker_verification_dataset.py index 194eb7f28fb485e8fc61ba25fb9c9fcb61bf1802..ae6f6ad974a8cad5795bda99b55befd25559f5fa 100644 --- a/paddlespeech/vector/exps/ge2e/speaker_verification_dataset.py +++ b/paddlespeech/vector/exps/ge2e/speaker_verification_dataset.py @@ -123,9 +123,3 @@ class Collate(object): frame_clips = [self.random_crop(mel) for mel in examples] batced_clips = np.stack(frame_clips) return batced_clips - - -if __name__ == "__main__": - mydataset = MultiSpeakerMelDataset( - Path("/home/chenfeiyu/datasets/SV2TTS/encoder")) - print(mydataset.get_example_by_index(0, 10)) diff --git a/tests/benchmark/conformer/README.md b/tests/benchmark/conformer/README.md index 22e0009d4445820a9ca6a226a1978ac065d698a9..72242bad7e3436f0c79cb0495ff81a98f4ad01de 100644 --- a/tests/benchmark/conformer/README.md +++ b/tests/benchmark/conformer/README.md @@ -1,5 +1,5 @@ ### Prepare the environment -Please follow the instructions shown in [here](../../docs/source/install.md) to install the Deepspeech first. +Please follow the instructions shown in [here](../../../docs/source/install.md) to install the Deepspeech first. ### File list └── benchmark # 模型名 diff --git a/utils/compute_statistics.py b/utils/compute_statistics.py index e8021c19d0de3e8c64ad3a318d3e9c8ad2758315..5b2a5606547ac0d4780378bc0c4c349b5f309a8b 100755 --- a/utils/compute_statistics.py +++ b/utils/compute_statistics.py @@ -22,6 +22,7 @@ from sklearn.preprocessing import StandardScaler from tqdm import tqdm from paddlespeech.t2s.datasets.data_table import DataTable +from paddlespeech.t2s.utils import str2bool def main(): @@ -41,9 +42,6 @@ def main(): help="path to save statistics. if not provided, " "stats will be saved in the above root directory with name stats.npy") - def str2bool(str): - return True if str.lower() == 'true' else False - parser.add_argument( "--use-relative-path", type=str2bool,