diff --git a/PaddleNLP/examples/language_model/bert/README.md b/PaddleNLP/examples/language_model/bert/README.md
index 098888345b51cf4f12366ba7e9850c49a33f449a..660f5a4768252f914eaa92928c5d7a5e41a41be9 100644
--- a/PaddleNLP/examples/language_model/bert/README.md
+++ b/PaddleNLP/examples/language_model/bert/README.md
@@ -131,7 +131,7 @@ python -u ./run_glue.py \
 | Task  | Metric                       | Result            |
 |:-----:|:----------------------------:|:-----------------:|
 | SST-2 | Accuracy                     |      0.92660      |
-| QNLI  | Accuracy                     |      0.91781      |
+| QNLI  | Accuracy                     |      0.91707      |
 | CoLA  | Mattehew's corr              |      0.59557      |
 | MRPC  | F1/Accuracy                  |  0.91667/0.88235  |
 | STS-B | Person/Spearman corr         |  0.88847/0.88350  |
diff --git a/PaddleNLP/examples/language_model/bert/run_pretrain.py b/PaddleNLP/examples/language_model/bert/run_pretrain.py
index e70ed0b7dccdfc9c5f1fbf0c803ab7bd01deec5a..0cbc8e00c1928ab9c84ae12461f71d7c3cf5bf68 100644
--- a/PaddleNLP/examples/language_model/bert/run_pretrain.py
+++ b/PaddleNLP/examples/language_model/bert/run_pretrain.py
@@ -340,13 +340,13 @@ def do_train(args):
 
         train_data_loader, _ = create_pretraining_dataset(
             data_file, args.max_predictions_per_seq, shared_file_list, args,
-            worker_init) 
+            worker_init)
 
         # TODO(guosheng): better way to process single file
-        if f_start_id + 1 == len(files): single_file = True
-            
+        single_file = True if f_start_id + 1 == len(files) else False
+
         for f_id in range(f_start_id, len(files)):
-            if not single_file:
+            if not single_file and f_id == f_start_id:
                 continue
             if paddle.distributed.get_world_size() > num_files:
                 data_file = files[(f_id * paddle.distributed.get_world_size() +