diff --git a/python/paddle/v2/parameters.py b/python/paddle/v2/parameters.py
index ea504d5104716d157add87ed3f6e31ea69e0a3f0..f2c5a4d49cfc44d3fda3b19a7e2d52fbf9b04129 100644
--- a/python/paddle/v2/parameters.py
+++ b/python/paddle/v2/parameters.py
@@ -224,7 +224,8 @@ class Parameters(object):
                 except ValueError:
                     # If no such parameter in gradient machine, then don't copy
                     pass
-            self.__gradient_machines__.append(gradient_machine)
+
+        self.__gradient_machines__.append(gradient_machine)
 
 
 def __get_parameter_in_gradient_machine__(gradient_machine, name):