diff --git a/README.md b/README.md index 7bb380e2d5f53d3e326c385da3842de11315183a..812c8a6c54fffe9cbbfa728255c666d033cf29ad 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ sudo apt-get install libsndfile1 ### Install PaddlePaddle -See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **1.8.0** or above. +See [install](https://www.paddlepaddle.org.cn/install/quick) for more details. This repo requires PaddlePaddle **1.8.2** or above. ### Install Parakeet @@ -74,7 +74,7 @@ Entries to the introduction, and the launch of training and synthsis for differe ## Pre-trained models and audio samples -Parakeet also releases some well-trained parameters for the example models, which can be accessed in the following tables. Each column of these tables lists resources for one model, including the url link to the pre-trained model, the dataset that the model is trained on, and synthesized audio samples based on the pre-trained model. +Parakeet also releases some well-trained parameters for the example models, which can be accessed in the following tables. Each column of these tables lists resources for one model, including the url link to the pre-trained model, the dataset that the model is trained on, and synthesized audio samples based on the pre-trained model. Click each model name to download, then you can get the compressed package which contains the pre-trained model and the `yaml` config describing how the model is trained. #### Vocoders @@ -174,29 +174,77 @@ We provide the model checkpoints of WaveFlow with 64 and 128 residual channels, #### TTS models +We also provide checkpoints for different end-to-end TTS models, and present the synthesized audio examples for some randomly chosen famous quotes. The corresponding texts are displayed as follows. + +||Text | From | +|:-:|:-- | :--: | +0|*Life was like a box of chocolates, you never know what you're gonna get.* | *Forrest Gump* | +1|*With great power there must come great responsibility.* | *Spider-Man*| +2|*To be or not to be, that’s a question.*|*Hamlet*| +3|*Death is just a part of life, something we're all destined to do.*| *Forrest Gump*| +4|*Don’t argue with the people of strong determination, because they may change the fact!*| *William Shakespeare* | + +Users have the option to use different vocoders to convert the linear/mel spectrogam to the raw audio in TTS models. Taking this into account, we are going to release the checkpoints for TTS models adapted to different vocoders, including the [Griffin-Lim](https://ieeexplore.ieee.org/document/1164317) algorithm and some neural vocoders. + +##### 1) Griffin-Lim +
+ - + + + @@ -204,8 +252,9 @@ We provide the model checkpoints of WaveFlow with 64 and 128 residual channels,
+ Deep Voice 3 + - Deep Voice 3 + Transformer TTS - Transformer TTS + FastSpeech
LJSpeech LJSpeech LJSpeech
- To be added soon + +
+ +
+ +
+ +
+ + 		- To be added soon + +
+ +
+ +
+ +
+ + + 		+ +
+ +
+ +
+ +
+ +
-Click each link to download, then you can get the compressed package which contains the pre-trained model and the `yaml` config describing how to train the model. +##### 2) Neural vocoders +under preparation ## Copyright and License diff --git a/examples/clarinet/README.md b/examples/clarinet/README.md index ca74b2d90a6b784ae3d99e96f7b992ea919fb433..cb02475d3579c2849ce35ff80c55b364ed5f10b9 100644 --- a/examples/clarinet/README.md +++ b/examples/clarinet/README.md @@ -129,7 +129,7 @@ Example script: ```bash python synthesis.py \ - --config=./configs/wavenet_single_gaussian.yaml \ + --config=./configs/clarinet_ljspeech.yaml \ --data=./LJSpeech-1.1/ \ --device=0 \ --iteration=500000 \ @@ -140,7 +140,7 @@ or ```bash python synthesis.py \ - --config=./configs/wavenet_single_gaussian.yaml \ + --config=./configs/clarinet_ljspeech.yaml \ --data=./LJSpeech-1.1/ \ --device=0 \ --checkpoint="experiment/checkpoints/step-500000" \ diff --git a/examples/deepvoice3/data.py b/examples/deepvoice3/data.py index 8ab2bd37e10a5f0e0c12875fb0387063b3e1d5b7..8da3dfbec7f4ab8c0570ea0c65a18c35dc14ec75 100644 --- a/examples/deepvoice3/data.py +++ b/examples/deepvoice3/data.py @@ -230,7 +230,7 @@ def make_data_loader(data_root, config): ref_level_db=c["ref_level_db"], max_norm=c["max_norm"], clip_norm=c["clip_norm"]) - ljspeech = CacheDataset(TransformDataset(meta, transform)) + ljspeech = TransformDataset(meta, transform) # use meta data's text length as a sort key for the sampler batch_size = config["train"]["batch_size"] diff --git a/examples/deepvoice3/utils.py b/examples/deepvoice3/utils.py index a0e8c7d3486db5dc2141e760f238a43462f2f617..20e3219f0945478d313aed8b64ac61952de9095a 100644 --- a/examples/deepvoice3/utils.py +++ b/examples/deepvoice3/utils.py @@ -60,7 +60,7 @@ def add_options(parser): def make_evaluator(config, text_sequences, output_dir, writer=None): c = config["transform"] - p_replace = c["replace_pronunciation_prob"] + p_replace = 0.0 sample_rate = c["sample_rate"] preemphasis = c["preemphasis"] win_length = c["win_length"] @@ -121,7 +121,7 @@ class Evaluator(object): en.text_to_sequence( text, p=self.p_replace), dtype=np.int64) length = len(text) - text_positions = np.arange(1, 1 + length) + text_positions = np.arange(1, 1 + length, dtype=np.int64) text = np.expand_dims(text, 0) text_positions = np.expand_dims(text_positions, 0) diff --git a/parakeet/models/waveflow/waveflow_modules.py b/parakeet/models/waveflow/waveflow_modules.py index 51f9108fba0ac0f20b3fc6766f2baba69b33b57d..31b29dcdf51d81fb4d16933966d74e838e35e5a7 100644 --- a/parakeet/models/waveflow/waveflow_modules.py +++ b/parakeet/models/waveflow/waveflow_modules.py @@ -79,7 +79,7 @@ class Conditioner(dg.Layer): stride=(1, s), param_attr=param_attr, bias_attr=bias_attr, - dtype="float32") + dtype=dtype) self.upsample_conv2d.append(conv_trans2d) for i, layer in enumerate(self.upsample_conv2d): @@ -88,12 +88,7 @@ class Conditioner(dg.Layer): def forward(self, x): x = fluid.layers.unsqueeze(x, 1) for layer in self.upsample_conv2d: - in_dtype = x.dtype - if in_dtype == fluid.core.VarDesc.VarType.FP16: - x = fluid.layers.cast(x, "float32") x = layer(x) - if in_dtype == fluid.core.VarDesc.VarType.FP16: - x = fluid.layers.cast(x, "float16") x = fluid.layers.leaky_relu(x, alpha=0.4) return fluid.layers.squeeze(x, [1]) @@ -101,12 +96,7 @@ class Conditioner(dg.Layer): def infer(self, x): x = fluid.layers.unsqueeze(x, 1) for layer in self.upsample_conv2d: - in_dtype = x.dtype - if in_dtype == fluid.core.VarDesc.VarType.FP16: - x = fluid.layers.cast(x, "float32") x = layer(x) - if in_dtype == fluid.core.VarDesc.VarType.FP16: - x = fluid.layers.cast(x, "float16") # Trim conv artifacts. time_cutoff = layer._filter_size[1] - layer._stride[1] x = fluid.layers.leaky_relu(x[:, :, :, :-time_cutoff], alpha=0.4) @@ -348,7 +338,7 @@ class WaveFlowModule(dg.Layer): mel = self.conditioner(mel) assert mel.shape[2] >= audio.shape[1] # Prune out the tail of audio/mel so that time/n_group == 0. - pruned_len = audio.shape[1] // self.n_group * self.n_group + pruned_len = int(audio.shape[1] // self.n_group * self.n_group) if audio.shape[1] > pruned_len: audio = audio[:, :pruned_len] diff --git a/parakeet/modules/weight_norm.py b/parakeet/modules/weight_norm.py index 7f68cd97b79a1f171a25344bd655225b2516e503..51732a7f13fc71a6b9056f737b5dc05cda0789dd 100644 --- a/parakeet/modules/weight_norm.py +++ b/parakeet/modules/weight_norm.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np from paddle import fluid import paddle.fluid.dygraph as dg import paddle.fluid.layers as F @@ -44,10 +43,10 @@ def norm_except(param, dim, power): if dim is None: return norm(param, dim, power) elif dim == 0: - param_matrix = F.reshape(param, (shape[0], np.prod(shape[1:]))) + param_matrix = F.reshape(param, (shape[0], -1)) return norm(param_matrix, dim=1, power=power) elif dim == -1 or dim == ndim - 1: - param_matrix = F.reshape(param, (np.prod(shape[:-1]), shape[-1])) + param_matrix = F.reshape(param, (-1, shape[-1])) return norm(param_matrix, dim=0, power=power) else: perm = list(range(ndim)) @@ -62,24 +61,26 @@ def compute_l2_normalized_weight(v, g, dim): ndim = len(shape) if dim is None: - v_normalized = v / (F.reduce_sum(F.square(v)) + 1e-12) + v_normalized = v / (F.sqrt(F.reduce_sum(F.square(v))) + 1e-12) elif dim == 0: - param_matrix = F.reshape(v, (shape[0], np.prod(shape[1:]))) + param_matrix = F.reshape(v, (shape[0], -1)) v_normalized = F.l2_normalize(param_matrix, axis=1) + v_normalized = F.reshape(v_normalized, shape) elif dim == -1 or dim == ndim - 1: - param_matrix = F.reshape(v, (np.prod(shape[:-1]), shape[-1])) + param_matrix = F.reshape(v, (-1, shape[-1])) v_normalized = F.l2_normalize(param_matrix, axis=0) + v_normalized = F.reshape(v_normalized, shape) else: perm = list(range(ndim)) perm[0] = dim perm[dim] = 0 transposed_param = F.transpose(v, perm) - param_matrix = F.reshape( - transposed_param, - (transposed_param.shape[0], np.prod(transposed_param.shape[1:]))) + transposed_shape = transposed_param.shape + param_matrix = F.reshape(transposed_param, + (transposed_param.shape[0], -1)) v_normalized = F.l2_normalize(param_matrix, axis=1) + v_normalized = F.reshape(v_normalized, transposed_shape) v_normalized = F.transpose(v_normalized, perm) - v_normalized = F.reshape(v_normalized, shape) weight = F.elementwise_mul(v_normalized, g, axis=dim) return weight @@ -87,7 +88,14 @@ def compute_l2_normalized_weight(v, g, dim): def compute_weight(v, g, dim, power): assert len(g.shape) == 1, "magnitude should be a vector" if power == 2: - return compute_l2_normalized_weight(v, g, dim) + in_dtype = v.dtype + if in_dtype == fluid.core.VarDesc.VarType.FP16: + v = F.cast(v, "float32") + g = F.cast(g, "float32") + weight = compute_l2_normalized_weight(v, g, dim) + if in_dtype == fluid.core.VarDesc.VarType.FP16: + weight = F.cast(weight, "float16") + return weight else: v_normalized = F.elementwise_div( v, (norm_except(v, dim, power) + 1e-12), axis=dim) diff --git a/setup.py b/setup.py index 062a02d78f5e87967a9c11abc42f90e2cf6acef4..7d329eac6855204632f286d7d970f4599c2fff69 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ import os import io import re -import six import sys from setuptools import setup, find_packages @@ -55,8 +54,8 @@ setup_info = dict( 'inflect', 'librosa', 'unidecode', - 'numba', - 'tqdm', + 'numba==0.47.0', + 'tqdm==4.19.8', 'matplotlib', 'tensorboardX', 'tensorboard', @@ -65,7 +64,7 @@ setup_info = dict( 'pandas', 'sox', 'soundfile', - 'llvmlite==0.31.0' if sys.version_info < (3, 6) else "llvmlite", + 'llvmlite==0.31.0', ], # Package info