diff --git a/python/paddle/fluid/dygraph/parallel.py b/python/paddle/fluid/dygraph/parallel.py
index 51e0527e4fa99fde78696fc6714cc15b1a6cbcb7..b90c6bbb4ea5010c376c74a918048d2805e68e2c 100644
--- a/python/paddle/fluid/dygraph/parallel.py
+++ b/python/paddle/fluid/dygraph/parallel.py
@@ -706,7 +706,12 @@ class DataParallel(layers.Layer):
                 if param.trainable:
                     layers_param.append((sublayer, param))
 
-        trainable_parameters = [param for _, param in layers_param]
+        trainable_parameters = list(
+            filter(
+                lambda x: not getattr(x, "no_sync", False),
+                [param for _, param in layers_param],
+            )
+        )
 
         assert len(trainable_parameters) > 0, (
             "This model does not have any parameters to train, and "