diff --git a/PaddleNLP/examples/language_model/transformer-xl/eval.py b/PaddleNLP/examples/language_model/transformer-xl/eval.py
index a6c49cf89d245d988b05b1a670e249d92457b118..cd50c2d7ef324e11fc13089234934a449be7a728 100644
--- a/PaddleNLP/examples/language_model/transformer-xl/eval.py
+++ b/PaddleNLP/examples/language_model/transformer-xl/eval.py
@@ -118,9 +118,9 @@ def do_eval(args):
 
     logger_info = ''
     if valid_loss is not None:
-        logger_info = logger_info + _logger(valid_loss)
+        logger_info = logger_info + _logger(valid_loss) + " | "
     if test_loss is not None:
-        logger_info = logger_info + _logger(test_loss)
+        logger_info = logger_info + _logger(test_loss) + " | "
     logger.info(logger_info)
 
 
diff --git a/PaddleNLP/examples/language_model/transformer-xl/train.py b/PaddleNLP/examples/language_model/transformer-xl/train.py
index 116a9352eb96b8fccd25256b9a9d6feddc7fd0e6..912972121a24e4ca87d9ee4b127e133cc19d5dcf 100644
--- a/PaddleNLP/examples/language_model/transformer-xl/train.py
+++ b/PaddleNLP/examples/language_model/transformer-xl/train.py
@@ -242,8 +242,9 @@ def do_train(args):
                 logger.info(logger_info)
 
                 if args.save_model and rank == 0:
-                    model_dir = os.path.join(args.save_model,
-                                             "step_" + str(step_idx))
+                    model_dir = os.path.join(
+                        args.save_model,
+                        "step_" + str(step_idx) + "_" + str(eval_loss))
                     if not os.path.exists(model_dir):
                         os.makedirs(model_dir)
                     paddle.save(