diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
index dbda4b9b7e03a41a9630722dfe82fbde62ee5437..80837bb3c38ec7823ce5c7066add20313e3c0d03 100644
--- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
@@ -28,9 +28,11 @@ __global__ void CrossEntropyGrad(T* logit_grad, const int64_t* labels,
        i += blockDim.x * gridDim.x) {
     int idx_n = i / remain;
     int idx_remain = i % remain;
-    int idx = idx_n * d + labels[i] * remain + idx_remain;
-    logit_grad[idx] -=
-        ignore_index == labels[i] ? static_cast<T>(0.) : static_cast<T>(1.);
+    int tmp = labels[i];
+    if (ignore_index != tmp) {
+      int idx = idx_n * d + tmp * remain + idx_remain;
+      logit_grad[idx] -= static_cast<T>(1.);
+    }
   }
 }