diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cc b/paddle/fluid/operators/softmax_with_cross_entropy_op.cc
index 2f4702ed71d6464b056b3fb60b0c895f835bfb2a..7c024e50dd21abea4e29257ebc53f7baa7f25235 100644
--- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cc
+++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cc
@@ -111,11 +111,8 @@ class SoftmaxWithCrossEntropyOp : public framework::OperatorWithKernel {
     PADDLE_ENFORCE_EQ(
         rank, labels_dims.size(),
         "Input(logits) and Input(Label) shall have the same rank.");
-    bool check = true;
-    if ((!ctx->IsRuntime()) && (framework::product(logits_dims) <= 0 ||
-                                framework::product(labels_dims) <= 0)) {
-      check = false;
-    }
+    bool check = ctx->IsRuntime() || (framework::product(logits_dims) > 0 &&
+                                      framework::product(labels_dims) > 0);
     if (check) {
       PADDLE_ENFORCE_EQ(framework::slice_ddim(logits_dims, 0, rank - 1),
                         framework::slice_ddim(labels_dims, 0, rank - 1),