diff --git a/paddle/fluid/operators/cross_entropy_op.h b/paddle/fluid/operators/cross_entropy_op.h
index 89bacfc33edceb77017aad599c081710f4d4db33..309ba46cfa3b35fd4f6a4a889965b717b890a303 100644
--- a/paddle/fluid/operators/cross_entropy_op.h
+++ b/paddle/fluid/operators/cross_entropy_op.h
@@ -154,9 +154,9 @@ struct HardLabelCrossEntropyForwardFunctor {
 
   HOSTDEVICE void operator()(int64_t idx) const {
     auto label = label_[idx];
-    PADDLE_ASSERT_MSG(label >= 0 && label < feature_size_,
-                      "The label is out of the range.", label);
     if (label != ignore_index_) {
+      PADDLE_ASSERT_MSG(label >= 0 && label < feature_size_,
+                        "The label is out of the range.", label);
       auto match_x = x_[idx * feature_size_ + label];
       y_[idx] = -math::TolerableValue<T>()(real_log(match_x));
       match_x_[idx] = match_x;