diff --git a/PaddleNLP/language_model/train.py b/PaddleNLP/language_model/train.py
index 10562712f807672e2016c85419517cefac0ffc50..39d6261e200776840d1ccc26fbb885637a79dde1 100644
--- a/PaddleNLP/language_model/train.py
+++ b/PaddleNLP/language_model/train.py
@@ -125,7 +125,6 @@ def main():
             res_vars = lm_model.lm_model(
                 config.hidden_size,
                 config.vocab_size,
-                config.batch_size,
                 num_layers=config.num_layers,
                 num_steps=config.num_steps,
                 init_scale=config.init_scale,
@@ -160,7 +159,6 @@ def main():
             lm_model.lm_model(
                 config.hidden_size,
                 config.vocab_size,
-                config.batch_size,
                 num_layers=config.num_layers,
                 num_steps=config.num_steps,
                 init_scale=config.init_scale,
@@ -319,7 +317,7 @@ def main():
                 print(
                     "-- Epoch:[%d]; Batch:[%d]; Time: %.5f s; ppl: %.5f, lr: %.5f"
                     % (epoch_id, batch_id, batch_time, ppl[0], lr[0]))
-            
+
             # profiler tools for benchmark
             if args.profile and batch_id == log_interval:
                 profiler.reset_profiler()
diff --git a/PaddleNLP/models/language_model/lm_model.py b/PaddleNLP/models/language_model/lm_model.py
index 2326e49f95801413946cdeee1a2910d7cc1cc279..19bab98a038b0789e8fda369b8e55fd7219bd500 100644
--- a/PaddleNLP/models/language_model/lm_model.py
+++ b/PaddleNLP/models/language_model/lm_model.py
@@ -26,7 +26,6 @@ from paddle.fluid.contrib.layers import basic_lstm
 
 def lm_model(hidden_size,
              vocab_size,
-             batch_size,
              num_layers=2,
              num_steps=20,
              init_scale=0.1,
@@ -253,7 +252,6 @@ def lm_model(hidden_size,
 
         return real_res, last_hidden, last_cell
 
-    batch_size_each = batch_size // fluid.core.get_cuda_device_count()
     x = fluid.data(name="x", shape=[None, num_steps, 1], dtype='int64')
     y = fluid.data(name="y", shape=[None, 1], dtype='int64')