diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py
index 59663bb81908860edf3e16e80353f76df1f369cb..6a9d504cc29bc9c21da7b0ce2a0726104ff3f680 100644
--- a/python/paddle/optimizer/optimizer.py
+++ b/python/paddle/optimizer/optimizer.py
@@ -730,10 +730,22 @@ class Optimizer:
         )
         if device is None:
             device = self._get_device_for_param(param.name)
-        with device_guard(device):
-            self.helper.set_variable_initializer(
-                var, initializer=Constant(value=float(fill_value))
+
+        if in_dygraph_mode() and (
+            device == 'cpu' or isinstance(device, core.CPUPlace)
+        ):
+            _C_ops.full_(
+                var,
+                var.shape,
+                str(float(fill_value)),
+                var.dtype,
+                core.CPUPlace(),
             )
+        else:
+            with device_guard(device):
+                self.helper.set_variable_initializer(
+                    var, initializer=Constant(value=float(fill_value))
+                )
 
         if framework._non_static_mode():
             if len(self._accumulators_holder) > 0: