diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py
index 5d9f7d1ee7dfeb823983504d26f74b994291b181..4f21ace202120031db9f331713d56f7bdacc0ba6 100644
--- a/python/paddle/fluid/optimizer.py
+++ b/python/paddle/fluid/optimizer.py
@@ -499,6 +499,9 @@ class Optimizer(object):
         optimize_ops = self.apply_optimize(
             loss, startup_program=startup_program, params_grads=params_grads)
 
+        if framework.in_dygraph_mode():
+            framework._dygraph_tracer()._clear_ops()
+
         return optimize_ops, params_grads