diff --git a/paddle/fluid/operators/reduce_ops/reduce_mean_op_xpu.cc b/paddle/fluid/operators/reduce_ops/reduce_mean_op_xpu.cc
index 2a79d031dcb02c97ff506152cb8ad6b010260a4e..108bba3b4522a1fee4fba769e844d08d6040bcfd 100644
--- a/paddle/fluid/operators/reduce_ops/reduce_mean_op_xpu.cc
+++ b/paddle/fluid/operators/reduce_ops/reduce_mean_op_xpu.cc
@@ -112,6 +112,7 @@ class ReduceMeanGradXPUKernel : public framework::OpKernel<T> {
         d = d + xdims.size();
       }
       reduce_numel *= xdims[d];
+      ydims.insert(ydims.begin() + d, 1);
     }
 
     float val = 1.0f / static_cast<float>(reduce_numel);