diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py
index 46420063e4a33885a0a9597b677948183bd0685b..c7c4df1fcf4f70dd6f2fdc05942bead926eefe18 100644
--- a/python/paddle/fluid/dygraph/layers.py
+++ b/python/paddle/fluid/dygraph/layers.py
@@ -369,6 +369,11 @@ class Layer(core.Layer):
             return object.__getattribute__(self, name)
 
     def __setattr__(self, name, value):
+        def _remove_if_exist(*dicts):
+            for d in dicts:
+                if name in d:
+                    del d[name]
+
         if isinstance(getattr(type(self), name, None), property):
             object.__setattr__(self, name, value)
         if isinstance(value, framework.Parameter):
@@ -382,12 +387,15 @@ class Layer(core.Layer):
 
                 value.set_value(self._loaddict_holder[value.name])
 
+            _remove_if_exist(self.__dict__, self._sub_layers)
             params[name] = value
         elif isinstance(value, core.Layer):
             layers = self.__dict__.get('_sub_layers', None)
             if layers is None:
                 raise ValueError(
                     "super(YourLayer, self).__init__() should be called first")
+
+            _remove_if_exist(self.__dict__, self._parameters)
             layers[name] = value
         else:
             object.__setattr__(self, name, value)
diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py
index 5e8e30e4c205de1192f4e6b2ba8768ccac6465d7..d1a2d02747890047fcd914be09952ba459fb94e7 100644
--- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py
+++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py
@@ -37,6 +37,7 @@ class MyLayer(fluid.Layer):
 class MLP(fluid.Layer):
     def __init__(self, input_size):
         super(MLP, self).__init__()
+        self._linear1 = None
         self._linear1 = Linear(
             input_size,
             3,
@@ -71,6 +72,7 @@ class SimpleRNNCell(fluid.Layer):
         i2h_param_shape = [self.step_input_size, self.hidden_size]
         h2h_param_shape = [self.hidden_size, self.hidden_size]
         h2o_param_shape = [self.output_size, self.hidden_size]
+        self._i2h_w = None
         self._i2h_w = self.create_parameter(
             attr=self.param_attr,
             shape=i2h_param_shape,