diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc
index ed94540c268e5ed990c1d92859c6a2093c052868..b6a20615783c4dd9eea3dd4a8a189ea03acb2bef 100644
--- a/paddle/framework/backward.cc
+++ b/paddle/framework/backward.cc
@@ -408,6 +408,11 @@ std::vector<std::unique_ptr<OpDescBind>> MakeBlockBackward(
 
     for (const auto& desc : op_grads) {
       for (const std::string& out_name : desc->OutputArgumentNames()) {
+        if (out_name.find("@GRAD") == std::string::npos) {
+          // Not all outputs of a backward operator is a gradient. Only gradient
+          // need to be sum. Skip variables are not gradient.
+          continue;
+        }
         dup_out_ops[out_name].emplace_back(grad_desc_idx);
       }
       ++grad_desc_idx;
diff --git a/python/paddle/v2/framework/layers.py b/python/paddle/v2/framework/layers.py
index 7e1ec10efa64d6fdaffd3b4c502b6dda3d92a29f..a5536c3573165dae590bc6e3d57c679b70957b70 100644
--- a/python/paddle/v2/framework/layers.py
+++ b/python/paddle/v2/framework/layers.py
@@ -823,7 +823,7 @@ def zeros(shape, dtype, main_program=None):
     return fill_constant(value=0.0, **locals())
 
 
-def increment(x, value=1.0, in_place=False, main_program=None):
+def increment(x, value=1.0, in_place=True, main_program=None):
     helper = LayerHelper("increment", **locals())
     if in_place:
         tmp = x