diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index c9dcaf0e291d0ff4e5af84eefe7ed45af0b87228..99d9ce3140566927388e5b6baf4c28606a666fc0 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -774,7 +774,7 @@ class Optimizer:
                 ), "Optimizer set error, {} should in state dict".format(
                     var_name
                 )
-                var.set_value(self._accumulators_holder[var_name])
+                var.set_value(self._accumulators_holder.pop(var_name))
 
         self._accumulators[name][param.name] = var
         return var