diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
index 344dfe23996fd742fd572699ae09567ed3b4a539..ba56e5e36f9851276b4986022452c7914e30dde4 100644
--- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
@@ -27,9 +27,11 @@ __global__ void CrossEntropyGrad(T* logit_grad, const int64_t* labels,
   CUDA_KERNEL_LOOP(index, n * remain) {
     int idx_n = index / remain;
     int idx_remain = index % remain;
-    int idx = idx_n * d + labels[index] * remain + idx_remain;
-    logit_grad[idx] -=
-        ignore_index == labels[index] ? static_cast<T>(0.) : static_cast<T>(1.);
+    int tmp = labels[index];
+    if (ignore_index != tmp) {
+      int idx = idx_n * d + tmp * remain + idx_remain;
+      logit_grad[idx] -= static_cast<T>(1.);
+    }
   }
 }