Merge pull request #820 from pfZhu/zpf

modify code

ernie-sat/README.md

@@ -11,7 +11,7 @@ ERNIE-SAT中我们提出了两项创新：
 
 ### 1.安装飞桨
 
-我们的代码基于 Paddle(version>=2.0)
+本项目的代码基于 Paddle(version>=2.0)
 
 
 ### 2.预训练模型
@@ -23,7 +23,7 @@ ERNIE-SAT中我们提出了两项创新：
 
 ### 3.下载
 
-1. 我们使用parallel wavegan作为声码器(vocoder): 
+1. 本项目使用parallel wavegan作为声码器(vocoder): 
     - [pwg_aishell3_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/pwgan/pwg_aishell3_ckpt_0.5.zip)  
 
 创建download文件夹，下载上述预训练的声码器(vocoder)模型并将其解压
@@ -34,7 +34,7 @@ cd download
 unzip pwg_aishell3_ckpt_0.5.zip
 ```
 
- 2. 我们使用[FastSpeech2](https://arxiv.org/abs/2006.04558) 作为音素(phoneme)的持续时间预测器:
+ 2. 本项目使用[FastSpeech2](https://arxiv.org/abs/2006.04558) 作为音素(phoneme)的持续时间预测器:
     - [fastspeech2_conformer_baker_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_conformer_baker_ckpt_0.5.zip)  中文场景下使用 
     - [fastspeech2_nosil_ljspeech_ckpt_0.5.zip](https://paddlespeech.bj.bcebos.com/Parakeet/released_models/fastspeech2/fastspeech2_nosil_ljspeech_ckpt_0.5.zip)  英文场景下使用
 
@@ -48,7 +48,7 @@ unzip fastspeech2_nosil_ljspeech_ckpt_0.5.zip
 
 ### 4.推理
 
-我们目前开源了语音编辑、个性化语音合成、跨语言语音合成的推理代码，后续会逐步开源。
+本项目当前开源了语音编辑、个性化语音合成、跨语言语音合成的推理代码，后续会逐步开源。
 注：当前采用的声码器版本与[模型训练时版本](https://github.com/kan-bayashi/ParallelWaveGAN)在英文上存在差异，您可使用模型训练时版本作为您的声码器，模型将在后续更新中升级。
 
 我们提供特定音频文件, 以及其对应的文本、音素相关文件:

ernie-sat/prompt/dev/mfa_end

+p243_new 1.0225 1.0525 1.0925 1.1325 1.1725 1.2625 1.3625 1.4125 1.5125 1.6225 1.6625 1.7925 1.8625 2.0025 2.0925 2.1725 2.2625 2.4325 2.4725 2.5225 2.5825 2.6125 2.6425 2.7425 2.8025 2.9025 2.9525 3.0525 3.0825 3.2125 3.4525
 Prompt_003_new 0.0425 0.0925 0.1825 0.2125 0.2425 0.3225 0.3725 0.4725 0.5325 0.5625 0.6225 0.7425 0.8625 0.9725 0.9975 1.0125 1.0825 1.2625 1.3125
 p299_096 0.7525 0.7925 0.8725 0.9125 0.9425 1.0325 1.0625 1.1925 1.2625 1.3225 1.3725 1.4125 1.5125 1.5425 1.6525 1.6925 1.7325 1.7625 1.8425 1.9625 2.0225 2.1825 2.3325 2.6825
-p243_new 1.0225 1.0525 1.0925 1.1325 1.1725 1.2625 1.3625 1.4125 1.5125 1.6225 1.6625 1.7925 1.8625 2.0025 2.0925 2.1725 2.2625 2.4325 2.4725 2.5225 2.5825 2.6125 2.6425 2.7425 2.8025 2.9025 2.9525 3.0525 3.0825 3.2125 3.4525

ernie-sat/prompt/dev/mfa_start

-Prompt_003_new 0.0125 0.0425 0.0925 0.1825 0.2125 0.2425 0.3225 0.3725 0.4725 0.5325 0.5625 0.6225 0.7425 0.8625 0.9725 0.9975 1.0125 1.0825 1.2625 
 p243_new 0.0125 1.0225 1.0525 1.0925 1.1325 1.1725 1.2625 1.3625 1.4125 1.5125 1.6225 1.6625 1.7925 1.8625 2.0025 2.0925 2.1725 2.2625 2.4325 2.4725 2.5225 2.5825 2.6125 2.6425 2.7425 2.8025 2.9025 2.9525 3.0525 3.0825 3.2125
+Prompt_003_new 0.0125 0.0425 0.0925 0.1825 0.2125 0.2425 0.3225 0.3725 0.4725 0.5325 0.5625 0.6225 0.7425 0.8625 0.9725 0.9975 1.0125 1.0825 1.2625 
 p299_096 0.0125 0.7525 0.7925 0.8725 0.9125 0.9425 1.0325 1.0625 1.1925 1.2625 1.3225 1.3725 1.4125 1.5125 1.5425 1.6525 1.6925 1.7325 1.7625 1.8425 1.9625 2.0225 2.1825 2.3325

ernie-sat/prompt/dev/mfa_text

+p243_new sp F AO1 R DH AE1 T R IY1 Z AH0 N sp K AH1 V ER0 SH UH1 D N AA1 T B IY1 G IH1 V AH0 N sp
 Prompt_003_new DH IH1 S W AA1 Z N AA1 T DH AH0 SH OW1 F AO1 R M IY1 sp 
 p299_096 sp W IY1 AA1 R T R AY1 NG T UW1 AH0 S T AE1 B L IH0 SH AH0 D EY1 T sp
-p243_new sp F AO1 R DH AE1 T R IY1 Z AH0 N sp K AH1 V ER0 SH UH1 D N AA1 T B IY1 G IH1 V AH0 N sp

ernie-sat/prompt/dev/mfa_wav.scp

-Prompt_003_new ../../prompt_wav/this_was_not_the_show_for_me.wav
 p243_new ../../prompt_wav/p243_313.wav
+Prompt_003_new ../../prompt_wav/this_was_not_the_show_for_me.wav
 p299_096 ../../prompt_wav/p299_096.wav

ernie-sat/prompt/dev/text

-Prompt_003_new This was not the show for me.
 p243_new For that reason cover should not be given.
+Prompt_003_new This was not the show for me.
 p299_096 We are trying to establish a date.

ernie-sat/prompt/dev/wav.scp

+p243_new ../../prompt_wav/p243_313.wav
 Prompt_003_new ../../prompt_wav/this_was_not_the_show_for_me.wav
 p299_096 ../../prompt_wav/p299_096.wav
-p243_new ../../prompt_wav/p243_313.wav

ernie-sat/run_clone_en_to_zh.sh

-# en --> zh  的 clone
-python sedit_inference_0520.py \
+# en --> zh  的 语音合成
+# 根据Prompt_003_new对应的语音: This was not the show for me. 来合成:  '今天天气很好'
+
+python inference.py \
 --task_name cross-lingual_clone \
 --model_name paddle_checkpoint_ench \
 --uid Prompt_003_new \
 --new_str '今天天气很好' \
 --prefix ./prompt/dev/ \
---clone_prefix ./prompt/dev_aishell3/ \
---clone_uid SSB07510054 \
 --source_language english \
 --target_language chinese \
---output_name task_cross_lingual_pred.wav \
+--output_name pred_zh.wav \
 --voc pwgan_aishell3 \
 --voc_config download/pwg_aishell3_ckpt_0.5/default.yaml \
 --voc_ckpt download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \

ernie-sat/run_gen_en.sh

 # 纯英文的语音合成
-# python sedit_inference_0518.py \
-# --task_name synthesize \
-# --model_name paddle_checkpoint_en \
-# --uid p323_083 \
-# --new_str 'I enjoy my life.' \
-# --prefix ./prompt/dev/ \
-# --source_language english \
-# --target_language english \
-# --output_name pred.wav \
-# --voc pwgan_aishell3 \
-# --voc_config download/pwg_aishell3_ckpt_0.5/default.yaml \
-# --voc_ckpt download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \
-# --voc_stat download/pwg_aishell3_ckpt_0.5/feats_stats.npy \
-# --am fastspeech2_ljspeech \
-# --am_config download/fastspeech2_nosil_ljspeech_ckpt_0.5/default.yaml \
-# --am_ckpt download/fastspeech2_nosil_ljspeech_ckpt_0.5/snapshot_iter_100000.pdz \
-# --am_stat download/fastspeech2_nosil_ljspeech_ckpt_0.5/speech_stats.npy \
-# --phones_dict download/fastspeech2_nosil_ljspeech_ckpt_0.5/phone_id_map.txt
+# 根据p299_096对应的语音: This was not the show for me. 来合成:  'I enjoy my life.'
 
-
-# 纯英文的语音合成
-python sedit_inference_0520.py \
+python inference.py \
 --task_name synthesize \
 --model_name paddle_checkpoint_en \
 --uid p299_096 \
@@ -28,7 +9,7 @@ python sedit_inference_0520.py \
 --prefix ./prompt/dev/ \
 --source_language english \
 --target_language english \
---output_name task_synthesize_pred.wav \
+--output_name pred.wav \
 --voc pwgan_aishell3 \
 --voc_config download/pwg_aishell3_ckpt_0.5/default.yaml \
 --voc_ckpt download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \

ernie-sat/run_sedit_en.sh

 # 纯英文的语音编辑
-python sedit_inference_0520.py \
+# 将p243_new对应的原始语音: For that reason cover should not be given. 编辑成'for that reason cover is impossible to be given.'对应的语音
+
+python inference.py \
 --task_name edit \
 --model_name paddle_checkpoint_en \
 --uid p243_new \
@@ -7,7 +9,7 @@ python sedit_inference_0520.py \
 --prefix ./prompt/dev/ \
 --source_language english \
 --target_language english \
---output_name task_edit_pred.wav \
+--output_name pred.wav \
 --voc pwgan_aishell3 \
 --voc_config download/pwg_aishell3_ckpt_0.5/default.yaml \
 --voc_ckpt download/pwg_aishell3_ckpt_0.5/snapshot_iter_1000000.pdz \

ernie-sat/util.py → ernie-sat/utils.py

@@ -70,13 +70,15 @@ def get_voc_out(mel, target_language="chinese"):
     print("current vocoder: ", args.voc)
     with open(args.voc_config) as f:
         voc_config = CfgNode(yaml.safe_load(f))
+    # print(voc_config)
 
     voc_inference = get_voc_inference(args, voc_config)
 
     mel = paddle.to_tensor(mel)
+    # print("masked_mel: ", mel.shape)
     with paddle.no_grad():
         wav = voc_inference(mel)
-    print("shepe of wav (time x n_channels):%s"%wav.shape)   # (31800,1)
+    # print("shepe of wav (time x n_channels):%s"%wav.shape)   
     return np.squeeze(wav)
 
 # dygraph
@@ -134,6 +136,7 @@ def get_am_inference(args, am_config):
 
 def evaluate_durations(phns, target_language="chinese", fs=24000, hop_length=300):
     args = parse_args()
+    # args = parser.parse_args(args=[])
     if args.ngpu == 0:
         paddle.set_device("cpu")
     elif args.ngpu > 0:
@@ -154,6 +157,7 @@ def evaluate_durations(phns, target_language="chinese", fs=24000, hop_length=300
     # acoustic model
     am, am_inference, am_name, am_dataset,phn_id = get_am_inference(args, am_config)
     
+
     torch_phns = phns
     vocab_phones = {}
     for tone, id in phn_id:
@@ -165,17 +169,16 @@ def evaluate_durations(phns, target_language="chinese", fs=24000, hop_length=300
     ]
     phone_ids = [vocab_phones[item] for item in phonemes]
     phone_ids_new = phone_ids
-    
     phone_ids_new.append(vocab_size-1)
     phone_ids_new = paddle.to_tensor(np.array(phone_ids_new, np.int64))
     normalized_mel, d_outs, p_outs, e_outs = am.inference(phone_ids_new, spk_id=None, spk_emb=None)
     pre_d_outs = d_outs
     phoneme_durations_new = pre_d_outs * hop_length / fs
     phoneme_durations_new = phoneme_durations_new.tolist()[:-1]
-
     return phoneme_durations_new
 
 
+
 def sentence2phns(sentence, target_language="en"):
     args = parse_args()
     if target_language == 'en':