diff --git a/python/paddle/fluid/dygraph/tracer.py b/python/paddle/fluid/dygraph/tracer.py
index e5e715bcdcea1b7631e2e5ec283bf5ffbd38ad28..ad938188bfbb5e6343eae43bc0d611cd8c0a8506 100644
--- a/python/paddle/fluid/dygraph/tracer.py
+++ b/python/paddle/fluid/dygraph/tracer.py
@@ -25,6 +25,8 @@ __all__ = ['Tracer']
 
 def release_op(op):
     del framework._dygraph_tracer()._ops[op._trace_id].inputs
+    del framework._dygraph_tracer()._ops[op._trace_id].outputs
+    del framework._dygraph_tracer()._ops[op._trace_id].backward_refs
 
 
 class Tracer(core.Tracer):
@@ -60,6 +62,7 @@ class Tracer(core.Tracer):
                     op.previous_ops.append(var.op)
                     inps[k].append(var._ivar)
 
+        op.outputs = outputs
         outs = defaultdict(list)
         for k, vars in six.iteritems(outputs):
             if isinstance(vars, framework.Variable):