diff --git a/paddlespeech/t2s/models/melgan/melgan.py b/paddlespeech/t2s/models/melgan/melgan.py
index 8dfc05a0707279ca329876861dfb40926c859e64..32fcf65882b51808d69e1422bc4f46df07ba115a 100644
--- a/paddlespeech/t2s/models/melgan/melgan.py
+++ b/paddlespeech/t2s/models/melgan/melgan.py
@@ -93,8 +93,7 @@ class MelGANGenerator(nn.Layer):
         initialize(self, init_type)
 
         # for compatibility
-        if nonlinear_activation == "LeakyReLU":
-            nonlinear_activation = "leakyrelu"
+        nonlinear_activation = nonlinear_activation.lower()
 
         # check hyper parameters is valid
         assert channels >= np.prod(upsample_scales)
@@ -328,8 +327,7 @@ class MelGANDiscriminator(nn.Layer):
         super().__init__()
 
         # for compatibility
-        if nonlinear_activation == "LeakyReLU":
-            nonlinear_activation = "leakyrelu"
+        nonlinear_activation = nonlinear_activation.lower()
 
         # initialize parameters
         initialize(self, init_type)
@@ -479,8 +477,7 @@ class MelGANMultiScaleDiscriminator(nn.Layer):
         initialize(self, init_type)
 
         # for compatibility
-        if nonlinear_activation == "LeakyReLU":
-            nonlinear_activation = "leakyrelu"
+        nonlinear_activation = nonlinear_activation.lower()
 
         self.discriminators = nn.LayerList()
 
diff --git a/paddlespeech/t2s/models/parallel_wavegan/parallel_wavegan_updater.py b/paddlespeech/t2s/models/parallel_wavegan/parallel_wavegan_updater.py
index 8ed50bc853834254ba4739b865501003bfa0df01..79707aa4ebcade609a49f1aa6d5eae2e535398bd 100644
--- a/paddlespeech/t2s/models/parallel_wavegan/parallel_wavegan_updater.py
+++ b/paddlespeech/t2s/models/parallel_wavegan/parallel_wavegan_updater.py
@@ -10,8 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations 
-# under the License.
+# limitations under the License.
 import logging
 from pathlib import Path
 from typing import Dict
@@ -28,6 +27,7 @@ from paddlespeech.t2s.training.extensions.evaluator import StandardEvaluator
 from paddlespeech.t2s.training.reporter import report
 from paddlespeech.t2s.training.updaters.standard_updater import StandardUpdater
 from paddlespeech.t2s.training.updaters.standard_updater import UpdaterState
+
 logging.basicConfig(
     format='%(asctime)s [%(levelname)s] [%(filename)s:%(lineno)d] %(message)s',
     datefmt='[%Y-%m-%d %H:%M:%S]')
diff --git a/paddlespeech/t2s/modules/residual_stack.py b/paddlespeech/t2s/modules/residual_stack.py
index ee3ba64d1e74031a55ec510e6770dd214dafaab0..b4f95229c7e4e0d0510bf9265fc704d1d690fcb7 100644
--- a/paddlespeech/t2s/modules/residual_stack.py
+++ b/paddlespeech/t2s/modules/residual_stack.py
@@ -60,8 +60,7 @@ class ResidualStack(nn.Layer):
         """
         super().__init__()
         # for compatibility
-        if nonlinear_activation == "LeakyReLU":
-            nonlinear_activation = "leakyrelu"
+        nonlinear_activation = nonlinear_activation.lower()
 
         # defile residual stack part
         if not use_causal_conv: