diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
index b81a37a68782b9e25d09ad117652c8fcead8cfde..6a9dca9fe2a6ae8f8c7859c938f83696d74791fc 100644
--- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
+++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu
@@ -73,17 +73,21 @@ __global__ void CrossEntropyHardLabel(T* loss, const T* softmax,
 
   // thread ids compute loss[ids] using softmax[idx]
   if (ids < n * d) {
-    int64_t idx = idx_n * dim * d + labels[ids] * d + idx_d;
-    if (IgnoreIndex == true) {
-      // IgnoreIndex is true
-      if (labels[ids] == ignore_idx) {
-        loss[ids] = static_cast<T>(0.0);
+    if (labels[ids] < 0) {  // label is negative
+      loss[ids] = static_cast<T>(0.0);
+    } else {  // label is positive of zero
+      int64_t idx = idx_n * dim * d + labels[ids] * d + idx_d;
+      if (IgnoreIndex == true) {
+        // IgnoreIndex is true
+        if (labels[ids] == ignore_idx) {
+          loss[ids] = static_cast<T>(0.0);
+        } else {
+          loss[ids] = -Log(softmax[idx]);
+        }
       } else {
+        // IgnoreIndex is false
         loss[ids] = -Log(softmax[idx]);
       }
-    } else {
-      // IgnoreIndex is false
-      loss[ids] = -Log(softmax[idx]);
     }
   }
 }