update code to make test acc=0.116

python/paddle_fl/mobile/README.md

@@ -60,10 +60,10 @@ mpirun -np 2 python application.py lm_data
 ### 训练结果
 
 ```shell
-framework.py : INFO  infer results: 0.085723
+framework.py : INFO  infer results: 0.116334
 ```
 
-即：在测试集上的，测试Top1为 8.6% 
+即：在测试集上的，测试Top1为 11.6% 
 
 ## 添加自己的数据集和Trainer
 
@@ -89,7 +89,7 @@ framework.py : INFO  infer results: 0.085723
 - Step1 模型初始化
 
     1. 全局参数初始化：由编号为0的simulator来做模型初始化工作，初始化之后，它会通过UpdateGlobalParams()接口将参数传递给Scheduler；
-   
+     
     2. 个性化参数初始化
 
 - Step2 模型分发

python/paddle_fl/mobile/application.py

@@ -36,17 +36,22 @@ simulator = SimulationFramework(role_maker)
 language_model_trainer = LanguageModelTrainer()
 
 language_model_trainer.set_trainer_configs({
-    "epoch": 3,
+    "epoch": 1,
     "max_steps_in_epoch": -1,
-    "lr": 0.1,
+    "lr": 1.0,
     "batch_size": 5,
+    "max_grad_norm": 5,
+    "n_hidden": 256,
+    "num_layers": 2,
+    "init_scale": 0.1,
+    "dropout_prob": 0.0,
 })
 
 sampler = UniformSampler()
-sampler.set_sample_num(30)
+sampler.set_sample_num(10)
 sampler.set_min_ins_num(1)
 test_sampler = Test1percentSampler()
-fed_avg_optimizer = FedAvgOptimizer(learning_rate=2.0)
+fed_avg_optimizer = FedAvgOptimizer(learning_rate=1.85)
 
 simulator.set_trainer(language_model_trainer)
 simulator.set_sampler(sampler)
@@ -68,5 +73,8 @@ elif simulator.is_simulator():
     print("dates: {}".format(dates))
 
     time.sleep(10)
-    simulator.run_simulation(
-        base_path, dates, sim_num_everyday=100, do_test=True, test_skip_day=1)
+    simulator.run_simulation(base_path,
+                             dates,
+                             sim_num_everyday=100,
+                             do_test=True,
+                             test_skip_day=1)

python/paddle_fl/mobile/model/language_model.py

@@ -22,8 +22,6 @@ import paddle.fluid as fluid
 
 import paddle.fluid.layers as layers
 import paddle.fluid as fluid
-from paddle.fluid.layers.control_flow import StaticRNN as PaddingRNN
-import numpy as np
 from paddle.fluid import ParamAttr
 from paddle.fluid.contrib.layers import basic_lstm
 
@@ -31,19 +29,19 @@ from paddle.fluid.contrib.layers import basic_lstm
 class LanguageModel(ModelBase):
     def __init__(self):
         # model args
-        self.hidden_size_ = 200
-        self.vocab_size_ = 10000
+        self.seq_len_ = 10  # fixed
+        self.n_hidden_ = 256
         self.num_layers_ = 2
-        self.num_steps_ = 10  # fix
-        self.init_scale_ = 0.1
-        self.dropout_ = 0.0
-        self.rnn_model_ = 'basic_lstm'
         self.pad_symbol_ = 0
         self.unk_symbol_ = 1
+        self.vocab_size_ = 10000
+        self.init_scale_ = 0.1
+        self.max_grad_norm_ = 5
+        self.dropout_prob_ = 0.0
 
         # results
         self.correct_ = None
-        self.prediction_ = None
+        self.pred_ = None
         self.loss_ = None
 
         # private vars
@@ -53,6 +51,13 @@ class LanguageModel(ModelBase):
         self.input_name_list_ = None
         self.target_var_names_ = []
 
+    def update_params(self, config):
+        self.n_hidden_ = config.get("n_hidden", 256)
+        self.num_layers_ = config.get("num_layers", 2)
+        self.init_scale_ = config.get("init_scale", 0.1)
+        self.max_grad_norm_ = config.get("max_grad_norm", 5)
+        self.dropout_prob_ = config.get("dropout_prob", 0.0)
+
     def get_model_input_names(self):
         return self.input_name_list_
 
@@ -63,381 +68,157 @@ class LanguageModel(ModelBase):
         return self.loss_.name
 
     def get_model_metrics(self):
-        return {"correct": self.correct_.name}
+        metrics = {
+            "init_hidden": self.last_hidden_.name,
+            "init_cell": self.last_cell_.name,
+            "correct": self.correct_.name
+        }
+        return metrics
 
     def get_target_names(self):
         return self.target_var_names_
 
     def build_model(self, model_configs):
-        hidden_size = self.hidden_size_
-        init_scale = self.init_scale_
-        dropout = self.dropout_
-        num_layers = self.num_layers_
-        num_steps = self.num_steps_
-        pad_symbol = self.pad_symbol_
-        unk_symbol = self.unk_symbol_
-        vocab_size = self.vocab_size_
-        rnn_model = self.rnn_model_
-        x = fluid.data(name="x", shape=[None, num_steps], dtype='int64')
-        y = fluid.data(name="y", shape=[None, num_steps], dtype='int64')
-        x = layers.reshape(x, shape=[-1, num_steps, 1])
-        y = layers.reshape(y, shape=[-1, 1])
-        self.input_name_list_ = ['x', 'y']
-
-        init_hidden = layers.fill_constant_batch_size_like(
-            input=x,
-            shape=[-1, num_layers, hidden_size],
-            value=0,
-            dtype="float32")
-        init_cell = layers.fill_constant_batch_size_like(
-            input=x,
-            shape=[-1, num_layers, hidden_size],
-            value=0,
-            dtype="float32")
+        self.update_params(model_configs)
+        features = fluid.layers.data(name="features",
+                                     shape=[None, self.seq_len_],
+                                     dtype='int64')
+        labels = fluid.layers.data(name="labels",
+                                   shape=[None, self.seq_len_],
+                                   dtype='int64')
+        sequence_length_ph = fluid.layers.data(name="seq_len_ph",
+                                               shape=[None],
+                                               dtype='int64')
+        sequence_mask_ph = fluid.layers.data(name="seq_mask_ph",
+                                             shape=[None],
+                                             dtype='float32')
+
+        init_hidden = fluid.layers.data(
+            name="init_hidden",
+            shape=[None, self.num_layers_, self.n_hidden_],
+            dtype='float32')
+        init_cell = fluid.layers.data(
+            name="init_cell",
+            shape=[None, self.num_layers_, self.n_hidden_],
+            dtype='float32')
 
         init_hidden = layers.transpose(init_hidden, perm=[1, 0, 2])
         init_cell = layers.transpose(init_cell, perm=[1, 0, 2])
 
         init_hidden_reshape = layers.reshape(
-            init_hidden, shape=[num_layers, -1, hidden_size])
+            init_hidden, shape=[self.num_layers_, -1, self.n_hidden_])
         init_cell_reshape = layers.reshape(
-            init_cell, shape=[num_layers, -1, hidden_size])
+            init_cell, shape=[self.num_layers_, -1, self.n_hidden_])
 
-        x_emb = layers.embedding(
-            input=x,
-            size=[vocab_size, hidden_size],
+        features = layers.reshape(features, shape=[-1, self.seq_len_, 1])
+
+        # word embedding
+        inputs = layers.embedding(
+            input=features,
+            size=[self.vocab_size_, self.n_hidden_],
             dtype='float32',
             is_sparse=False,
             param_attr=fluid.ParamAttr(
                 name='embedding_para',
                 initializer=fluid.initializer.UniformInitializer(
-                    low=-init_scale, high=init_scale)))
-
-        x_emb = layers.reshape(
-            x_emb, shape=[-1, num_steps, hidden_size], inplace=True)
-        if dropout != None and dropout > 0.0:
-            x_emb = layers.dropout(
-                x_emb,
-                dropout_prob=dropout,
-                dropout_implementation='upscale_in_train')
-
-        if rnn_model == "padding":
-            rnn_out, last_hidden, last_cell = self._padding_rnn(
-                x_emb,
-                len=num_steps,
-                init_hidden=init_hidden_reshape,
-                init_cell=init_cell_reshape)
-        elif rnn_model == "static":
-            rnn_out, last_hidden, last_cell = self._encoder_static(
-                x_emb,
-                len=num_steps,
-                init_hidden=init_hidden_reshape,
-                init_cell=init_cell_reshape)
-        elif rnn_model == "cudnn":
-            x_emb = layers.transpose(x_emb, perm=[1, 0, 2])
-            rnn_out, last_hidden, last_cell = layers.lstm(
-                x_emb,
-                init_hidden_reshape,
-                init_cell_reshape,
-                num_steps,
-                hidden_size,
-                num_layers,
-                is_bidirec=False,
-                default_initializer=fluid.initializer.UniformInitializer(
-                    low=-init_scale, high=init_scale))
-            rnn_out = layers.transpose(rnn_out, perm=[1, 0, 2])
-        elif rnn_model == "basic_lstm":
-            rnn_out, last_hidden, last_cell = basic_lstm(
-                x_emb,
-                init_hidden,
-                init_cell,
-                hidden_size,
-                num_layers=num_layers,
-                batch_first=True,
-                dropout_prob=dropout,
-                param_attr=ParamAttr(
-                    initializer=fluid.initializer.UniformInitializer(
-                        low=-init_scale, high=init_scale)),
-                bias_attr=ParamAttr(
-                    initializer=fluid.initializer.Constant(0.0)),
-                forget_bias=0.0)
-        else:
-            raise Exception("type not support")
-
-        rnn_out = layers.reshape(
-            rnn_out, shape=[-1, num_steps, hidden_size], inplace=True)
-
-        softmax_weight = layers.create_parameter(
-            [hidden_size, vocab_size],
+                    low=-self.init_scale_, high=self.init_scale_)))
+
+        # LSTM
+        output, last_hidden, last_cell = self._build_rnn_graph(
+            inputs, init_hidden, init_cell, sequence_length_ph)
+
+        output = layers.reshape(output,
+                                shape=[-1, self.seq_len_, self.n_hidden_],
+                                inplace=True)
+        self.last_hidden_ = layers.reshape(
+            last_hidden, [-1, self.num_layers_, self.n_hidden_])
+        self.last_cell_ = layers.reshape(
+            last_cell, [-1, self.num_layers_, self.n_hidden_])
+
+        # softmax
+        softmax_w = layers.create_parameter(
+            [self.n_hidden_, self.vocab_size_],
             dtype="float32",
-            name="softmax_weight",
+            name="softmax_w",
             default_initializer=fluid.initializer.UniformInitializer(
-                low=-init_scale, high=init_scale))
-        softmax_bias = layers.create_parameter(
-            [vocab_size],
+                low=-self.init_scale_, high=self.init_scale_))
+        softmax_b = layers.create_parameter(
+            [self.vocab_size_],
             dtype="float32",
-            name='softmax_bias',
+            name='softmax_b',
             default_initializer=fluid.initializer.UniformInitializer(
-                low=-init_scale, high=init_scale))
+                low=-self.init_scale_, high=self.init_scale_))
 
-        projection = layers.matmul(rnn_out, softmax_weight)
-        projection = layers.elementwise_add(projection, softmax_bias)
-        projection = layers.reshape(
-            projection, shape=[-1, vocab_size], inplace=True)
+        logits = layers.matmul(output, softmax_w)
+        logits = layers.elementwise_add(logits, softmax_b)
+        logits = layers.reshape(logits,
+                                shape=[-1, self.vocab_size_],
+                                inplace=True)
 
         # correct predictions
-        labels_reshaped = fluid.layers.reshape(y, [-1])
-        pred = fluid.layers.cast(
-            fluid.layers.argmax(projection, 1), dtype="int64")
-        correct_pred = fluid.layers.cast(
-            fluid.layers.equal(pred, labels_reshaped), dtype="int64")
-        self.prediction_ = pred
-        self.target_var_names_.append(pred)
+        labels_reshaped = layers.reshape(labels, [-1])
+        pred = layers.cast(layers.argmax(logits, 1), dtype="int64")
+        correct_pred = layers.cast(layers.equal(pred, labels_reshaped),
+                                   dtype="int64")
+        self.pred_ = pred
 
         # predicting unknown is always considered wrong
-        unk_tensor = fluid.layers.fill_constant(
-            fluid.layers.shape(labels_reshaped),
-            value=unk_symbol,
-            dtype='int64')
-        pred_unk = fluid.layers.cast(
-            fluid.layers.equal(pred, unk_tensor), dtype="int64")
-        correct_unk = fluid.layers.elementwise_mul(pred_unk, correct_pred)
+        # only in paddle 1.8
+        unk_tensor = layers.fill_constant(layers.shape(labels_reshaped),
+                                          value=self.unk_symbol_,
+                                          dtype='int64')
+        pred_unk = layers.cast(layers.equal(pred, unk_tensor), dtype="int64")
+        correct_unk = layers.elementwise_mul(pred_unk, correct_pred)
 
         # predicting padding is always considered wrong
-        pad_tensor = fluid.layers.fill_constant(
-            fluid.layers.shape(labels_reshaped), value=0, dtype='int64')
-        pred_pad = fluid.layers.cast(
-            fluid.layers.equal(pred, pad_tensor), dtype="int64")
-        correct_pad = fluid.layers.elementwise_mul(pred_pad, correct_pred)
-
-        # acc
-        correct_count = fluid.layers.reduce_sum(correct_pred) \
+        pad_tensor = layers.fill_constant(layers.shape(labels_reshaped),
+                                          value=self.pad_symbol_,
+                                          dtype='int64')
+        pred_pad = layers.cast(layers.equal(pred, pad_tensor), dtype="int64")
+        correct_pad = layers.elementwise_mul(pred_pad, correct_pred)
+
+        # Reshape logits to be a 3-D tensor for sequence loss
+        logits = layers.reshape(logits, [-1, self.seq_len_, self.vocab_size_])
+
+        labels = layers.reshape(labels, [-1, self.seq_len_, 1])
+        loss = layers.softmax_with_cross_entropy(logits=logits,
+                                                 label=labels,
+                                                 soft_label=False,
+                                                 return_softmax=False)
+        sequence_mask = layers.reshape(sequence_mask_ph,
+                                       [-1, self.seq_len_, 1])
+        loss = layers.reduce_mean(layers.elementwise_mul(loss, sequence_mask))
+
+        eval_metric_ops = fluid.layers.reduce_sum(correct_pred) \
                 - fluid.layers.reduce_sum(correct_unk) \
                 - fluid.layers.reduce_sum(correct_pad)
-        self.correct_ = correct_count
-        self.target_var_names_.append(correct_count)
-
-        loss = layers.softmax_with_cross_entropy(
-            logits=projection, label=y, soft_label=False)
 
-        loss = layers.reshape(loss, shape=[-1, num_steps], inplace=True)
-        loss = layers.reduce_mean(loss, dim=[0])
-        loss = layers.reduce_sum(loss)
         self.loss_ = loss
-        self.target_var_names_.append(loss)
-
-        loss.persistable = True
-
-        # This will feed last_hidden, last_cell to init_hidden, init_cell, which
-        # can be used directly in next batch. This can avoid the fetching of
-        # last_hidden and last_cell and feeding of init_hidden and init_cell in
-        # each training step.
-        #last_hidden = layers.transpose(last_hidden, perm=[1, 0, 2])
-        #last_cell = layers.transpose(last_cell, perm=[1, 0, 2])
-        #self.input_name_list_ = ['x', 'y', 'init_hidden', 'init_cell']
+        self.correct_ = eval_metric_opsself.input_name_list_ = [
+            'features', 'labels', 'seq_len_ph', 'seq_mask_ph', 'init_hidden',
+            'init_cell'
+        ]
+        self.target_var_names_ = [
+            self.loss_, self.last_hidden_, self.last_cell_, self.correct_
+        ]
 
         self.program_ = fluid.default_main_program()
         self.startup_program_ = fluid.default_startup_program()
 
-    def _padding_rnn(input_embedding, len=3, init_hidden=None, init_cell=None):
-        weight_1_arr = []
-        weight_2_arr = []
-        bias_arr = []
-        hidden_array = []
-        cell_array = []
-        mask_array = []
-        hidden_size = self.hidden_size_
-        init_scale = self.init_scale_
-        dropout = slef.dropout_
-        num_layers = self.num_layers_
-        num_steps = self._num_steps_
-        for i in range(num_layers):
-            weight_1 = layers.create_parameter(
-                [hidden_size * 2, hidden_size * 4],
-                dtype="float32",
-                name="fc_weight1_" + str(i),
-                default_initializer=fluid.initializer.UniformInitializer(
-                    low=-init_scale, high=init_scale))
-            weight_1_arr.append(weight_1)
-            bias_1 = layers.create_parameter(
-                [hidden_size * 4],
-                dtype="float32",
-                name="fc_bias1_" + str(i),
-                default_initializer=fluid.initializer.Constant(0.0))
-            bias_arr.append(bias_1)
-
-            pre_hidden = layers.slice(
-                init_hidden, axes=[0], starts=[i], ends=[i + 1])
-            pre_cell = layers.slice(
-                init_cell, axes=[0], starts=[i], ends=[i + 1])
-            pre_hidden = layers.reshape(pre_hidden, shape=[-1, hidden_size])
-            pre_cell = layers.reshape(pre_cell, shape=[-1, hidden_size])
-            hidden_array.append(pre_hidden)
-            cell_array.append(pre_cell)
-
-        input_embedding = layers.transpose(input_embedding, perm=[1, 0, 2])
-        rnn = PaddingRNN()
-
-        with rnn.step():
-            input = rnn.step_input(input_embedding)
-            for k in range(num_layers):
-                pre_hidden = rnn.memory(init=hidden_array[k])
-                pre_cell = rnn.memory(init=cell_array[k])
-                weight_1 = weight_1_arr[k]
-                bias = bias_arr[k]
-
-                nn = layers.concat([input, pre_hidden], 1)
-                gate_input = layers.matmul(x=nn, y=weight_1)
-
-                gate_input = layers.elementwise_add(gate_input, bias)
-                i = layers.slice(
-                    gate_input, axes=[1], starts=[0], ends=[hidden_size])
-                j = layers.slice(
-                    gate_input,
-                    axes=[1],
-                    starts=[hidden_size],
-                    ends=[hidden_size * 2])
-                f = layers.slice(
-                    gate_input,
-                    axes=[1],
-                    starts=[hidden_size * 2],
-                    ends=[hidden_size * 3])
-                o = layers.slice(
-                    gate_input,
-                    axes=[1],
-                    starts=[hidden_size * 3],
-                    ends=[hidden_size * 4])
-
-                c = pre_cell * layers.sigmoid(f) + layers.sigmoid(
-                    i) * layers.tanh(j)
-                m = layers.tanh(c) * layers.sigmoid(o)
-
-                rnn.update_memory(pre_hidden, m)
-                rnn.update_memory(pre_cell, c)
-
-                rnn.step_output(m)
-                rnn.step_output(c)
-
-                input = m
-
-                if dropout != None and dropout > 0.0:
-                    input = layers.dropout(
-                        input,
-                        dropout_prob=dropout,
-                        dropout_implementation='upscale_in_train')
-
-            rnn.step_output(input)
-        rnnout = rnn()
-
-        last_hidden_array = []
-        last_cell_array = []
-        real_res = rnnout[-1]
-        for i in range(num_layers):
-            m = rnnout[i * 2]
-            c = rnnout[i * 2 + 1]
-            m.stop_gradient = True
-            c.stop_gradient = True
-            last_h = layers.slice(
-                m, axes=[0], starts=[num_steps - 1], ends=[num_steps])
-            last_hidden_array.append(last_h)
-            last_c = layers.slice(
-                c, axes=[0], starts=[num_steps - 1], ends=[num_steps])
-            last_cell_array.append(last_c)
-        real_res = layers.transpose(x=real_res, perm=[1, 0, 2])
-        last_hidden = layers.concat(last_hidden_array, 0)
-        last_cell = layers.concat(last_cell_array, 0)
-
-        return real_res, last_hidden, last_cell
-
-    def _encoder_static(input_embedding,
-                        len=3,
-                        init_hidden=None,
-                        init_cell=None):
-        weight_1_arr = []
-        weight_2_arr = []
-        bias_arr = []
-        hidden_array = []
-        cell_array = []
-        mask_array = []
-        hidden_size = self.hidden_size_
-        init_scale = self.init_scale_
-        dropout = slef.dropout_
-        num_layers = self.num_layers_
-        for i in range(num_layers):
-            weight_1 = layers.create_parameter(
-                [hidden_size * 2, hidden_size * 4],
-                dtype="float32",
-                name="fc_weight1_" + str(i),
-                default_initializer=fluid.initializer.UniformInitializer(
-                    low=-init_scale, high=init_scale))
-            weight_1_arr.append(weight_1)
-            bias_1 = layers.create_parameter(
-                [hidden_size * 4],
-                dtype="float32",
-                name="fc_bias1_" + str(i),
-                default_initializer=fluid.initializer.Constant(0.0))
-            bias_arr.append(bias_1)
-
-            pre_hidden = layers.slice(
-                init_hidden, axes=[0], starts=[i], ends=[i + 1])
-            pre_cell = layers.slice(
-                init_cell, axes=[0], starts=[i], ends=[i + 1])
-            pre_hidden = layers.reshape(
-                pre_hidden, shape=[-1, hidden_size], inplace=True)
-            pre_cell = layers.reshape(
-                pre_cell, shape=[-1, hidden_size], inplace=True)
-            hidden_array.append(pre_hidden)
-            cell_array.append(pre_cell)
-
-        res = []
-        sliced_inputs = layers.split(
-            input_embedding, num_or_sections=len, dim=1)
-
-        for index in range(len):
-            input = sliced_inputs[index]
-            input = layers.reshape(
-                input, shape=[-1, hidden_size], inplace=True)
-            for k in range(num_layers):
-                pre_hidden = hidden_array[k]
-                pre_cell = cell_array[k]
-                weight_1 = weight_1_arr[k]
-                bias = bias_arr[k]
-                nn = layers.concat([input, pre_hidden], 1)
-                gate_input = layers.matmul(x=nn, y=weight_1)
-
-                gate_input = layers.elementwise_add(gate_input, bias)
-                i, j, f, o = layers.split(
-                    gate_input, num_or_sections=4, dim=-1)
-
-                c = pre_cell * layers.sigmoid(f) + layers.sigmoid(
-                    i) * layers.tanh(j)
-                m = layers.tanh(c) * layers.sigmoid(o)
-
-                hidden_array[k] = m
-                cell_array[k] = c
-                input = m
-
-                if dropout != None and dropout > 0.0:
-                    input = layers.dropout(
-                        input,
-                        dropout_prob=dropout,
-                        dropout_implementation='upscale_in_train')
-
-            res.append(input)
-
-        last_hidden = layers.concat(hidden_array, 1)
-        last_hidden = layers.reshape(
-            last_hidden, shape=[-1, num_layers, hidden_size], inplace=True)
-        last_hidden = layers.transpose(x=last_hidden, perm=[1, 0, 2])
-
-        last_cell = layers.concat(cell_array, 1)
-        last_cell = layers.reshape(
-            last_cell, shape=[-1, num_layers, hidden_size])
-        last_cell = layers.transpose(x=last_cell, perm=[1, 0, 2])
-
-        real_res = layers.concat(res, 0)
-        real_res = layers.reshape(
-            real_res, shape=[len, -1, hidden_size], inplace=True)
-        real_res = layers.transpose(x=real_res, perm=[1, 0, 2])
-
-        return real_res, last_hidden, last_cell
+    def _build_rnn_graph(self, inputs, init_hidden, init_cell,
+                         sequence_length_ph):
+        rnn_out, last_hidden, last_cell = basic_lstm(
+            input=inputs,
+            init_hidden=init_hidden,
+            init_cell=init_cell,
+            hidden_size=self.n_hidden_,
+            num_layers=self.num_layers_,
+            batch_first=True,
+            dropout_prob=self.dropout_prob_,
+            sequence_length=sequence_length_ph,
+            param_attr=ParamAttr(
+                initializer=fluid.initializer.UniformInitializer(
+                    low=-self.init_scale_, high=self.init_scale_)),
+            bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0)),
+            forget_bias=0.0)
+        return rnn_out, last_hidden, last_cell

python/paddle_fl/mobile/reader/leaf_reddit_reader.py

@@ -108,7 +108,7 @@ def train_reader(lines):
 
             input_data, input_length = process_x(data_x, VOCAB)
             target_data = process_y(data_y, VOCAB)
-            yield [input_data] + [target_data]
+            yield [input_data] + [target_data] + [input_length] + [data_mask]
 
     return local_iter
 

python/paddle_fl/mobile/trainer/language_model_trainer.py

@@ -34,10 +34,10 @@ def train_one_user(arg_dict, trainer_config):
     max_training_steps = trainer_config["max_training_steps"]
     batch_size = trainer_config["batch_size"]
     # logging.info("training one user...")
-    main_program = fluid.Program.parse_from_string(trainer_config[
-        "main_program_desc"])
-    startup_program = fluid.Program.parse_from_string(trainer_config[
-        "startup_program_desc"])
+    main_program = fluid.Program.parse_from_string(
+        trainer_config["main_program_desc"])
+    startup_program = fluid.Program.parse_from_string(
+        trainer_config["startup_program_desc"])
     place = fluid.CPUPlace()
     exe = fluid.Executor(place)
     scope = fluid.global_scope()
@@ -46,10 +46,9 @@ def train_one_user(arg_dict, trainer_config):
         exit()
     exe.run(startup_program)
 
-    feeder = fluid.DataFeeder(
-        feed_list=trainer_config["input_names"],
-        place=place,
-        program=main_program)
+    feeder = fluid.DataFeeder(feed_list=trainer_config["input_names"],
+                              place=place,
+                              program=main_program)
     data_server_endpoints = arg_dict["data_endpoints"]
     # create data clients
     data_client = DataClient()
@@ -76,36 +75,43 @@ def train_one_user(arg_dict, trainer_config):
     epoch = trainer_config["epoch"]
     max_steps_in_epoch = trainer_config.get("max_steps_in_epoch", -1)
     metrics = trainer_config["metrics"]
-    metric_keys = metrics.keys()
-    fetch_list = [main_program.global_block().var(trainer_config["loss_name"])]
-    for key in metric_keys:
-        fetch_list.append(main_program.global_block().var(metrics[key]))
+    fetch_list = []
+    for var in trainer_config["target_names"]:
+        fetch_list.append(var)
 
-    seq_len = 10
     for ei in range(epoch):
+        fetch_res_list = []
         trained_sample_num = 0
         step = 0
-        fetch_res_list = []
-        total_loss = 0.0
-        total_correct = 0
+        num_layers = trainer_config["num_layers"]
+        hidden_size = trainer_config["n_hidden"]
+        tot_loss, tot_correct = 0, 0
+        tot_samples = 0
+        init_hidden, init_cell = generate_init_data(batch_size, num_layers,
+                                                    hidden_size)
         for data in train_reader():
+            feed_data, input_lengths = prepare_input(batch_size, data,
+                                                     init_hidden, init_cell)
             fetch_res = exe.run(main_program,
-                                feed=feeder.feed(data),
+                                feed=feeder.feed(feed_data),
                                 fetch_list=fetch_list)
+            loss, last_hidden, last_cell, correct = fetch_res
+
+            init_hidden = np.array(last_hidden)
+            init_cell = np.array(last_cell)
+            tot_loss += np.array(loss)
+            tot_correct += np.array(correct)
+            tot_samples += np.sum(input_lengths)
             step += 1
             trained_sample_num += len(data)
-            fetch_res_list.append([x[0] for x in fetch_res])
+            fetch_res_list.append([np.array(loss), np.array(correct)])
             if max_steps_in_epoch != -1 and step >= max_steps_in_epoch:
                 break
 
         if show_metric and trained_sample_num > 0:
-            loss = sum([x[0] for x in fetch_res_list]) / trained_sample_num
-            print("loss: {}, ppl: {}".format(loss, np.exp(loss)))
-            for i, key in enumerate(metric_keys):
-                if key == "correct":
-                    value = float(sum([x[i + 1] for x in fetch_res_list
-                                       ])) / trained_sample_num
-                    print("correct: {}".format(value / seq_len))
+            loss = tot_loss / step
+            acc = float(tot_correct) / tot_samples
+            print("loss: {}, acc: {}".format(loss, acc))
 
     local_updated_param_dict = {}
     # update user param
@@ -142,10 +148,10 @@ def infer_one_user(arg_dict, trainer_config):
     # run startup program, set params
     uid = arg_dict["uid"]
     batch_size = trainer_config["batch_size"]
-    startup_program = fluid.Program.parse_from_string(trainer_config[
-        "startup_program_desc"])
-    infer_program = fluid.Program.parse_from_string(trainer_config[
-        "infer_program_desc"])
+    startup_program = fluid.Program.parse_from_string(
+        trainer_config["startup_program_desc"])
+    infer_program = fluid.Program.parse_from_string(
+        trainer_config["infer_program_desc"])
     place = fluid.CPUPlace()
     exe = fluid.Executor(place)
     scope = fluid.global_scope()
@@ -169,7 +175,6 @@ def infer_one_user(arg_dict, trainer_config):
                           arg_dict["global_params"], scope)
 
     # reader
-
     date = arg_dict["date"]
     global_param_dict = arg_dict["global_params"]
     user_data = data_client.get_data_by_uid(uid, date)
@@ -179,36 +184,60 @@ def infer_one_user(arg_dict, trainer_config):
     # run infer program
     os.mkdir(arg_dict["infer_result_dir"])
     #pred_file = open(arg_dict["infer_result_dir"] + '/' + "pred_file", "w")
-    feeder = fluid.DataFeeder(
-        feed_list=trainer_config["input_names"],
-        place=place,
-        program=infer_program)
+    feeder = fluid.DataFeeder(feed_list=trainer_config["input_names"],
+                              place=place,
+                              program=infer_program)
 
     fetch_list = trainer_config["target_names"]
     #logging.info("fetch_list: {}".format(fetch_list))
     fetch_res = []
     sample_count = 0
 
-    total_loss = 0.0
-    total_correct = 0
-    iters = 0
-    steps = 0
-    seq_len = 10
+    num_layers = trainer_config["num_layers"]
+    hidden_size = trainer_config["n_hidden"]
+    tot_correct, tot_loss = 0, 0
+    tot_samples, tot_batches = 0, 0
+    init_hidden, init_cell = generate_init_data(batch_size, num_layers,
+                                                hidden_size)
     for data in infer_reader():
-        # feed_data = [x["features"] + [x["label"]] for x in data]
-        # prediction, acc_val= exe.run(infer_program,
-        pred, correct_count, loss = exe.run(infer_program,
-                                            feed=feeder.feed(data),
-                                            fetch_list=fetch_list)
-        total_loss += loss
-        total_correct += correct_count
-        steps += 1
-        sample_count += len(data)
-
-    correct = float(total_correct) / (seq_len * sample_count)
-    # logging.info("correct: {}".format(correct))
+        feed_data, input_lengths = prepare_input(batch_size, data, init_hidden,
+                                                 init_cell)
+        fetch_res = exe.run(infer_program,
+                            feed=feeder.feed(feed_data),
+                            fetch_list=fetch_list)
+        loss, last_hidden, last_cell, correct = fetch_res
+
+        cost_eval = np.array(loss)
+        init_hidden = np.array(last_hidden)
+        init_cell = np.array(last_cell)
+        correct_val = np.array(correct)
+        tot_loss += cost_eval
+        tot_correct += correct_val
+        tot_samples += np.sum(input_lengths)
+        tot_batches += 1
+
+    loss = tot_loss / tot_batches
+    acc = float(tot_correct) / tot_samples
+    logging.info("infer acc: {}".format(acc))
     with open(arg_dict["infer_result_dir"] + "/res", "w") as f:
-        f.write("%d\t%f\n" % (1, correct))
+        f.write("%d\t%f\n" % (1, acc))
+
+
+def prepare_input(batch_size, data, init_hidden, init_cell):
+    init_hidden = np.split(init_hidden, batch_size)
+    init_cell = np.split(init_cell, batch_size)
+    data = [[features] + [labels] + [seq_len_ph] + [seq_mask_ph] + [init_hidden[i]] + [init_cell[i]  ] \
+              for i, (features, labels, seq_len_ph, seq_mask_ph) in enumerate(data)]
+    input_lengths = [x[2] for x in data]
+    return data, input_lengths
+
+
+def generate_init_data(batch_size, num_layers, hidden_size):
+    init_hidden = np.zeros((batch_size, num_layers, hidden_size),
+                           dtype='float32')
+    init_cell = np.zeros((batch_size, num_layers, hidden_size),
+                         dtype='float32')
+    return init_hidden, init_cell
 
 
 def save_and_upload(arg_dict, trainer_config, dfs_upload_path):
@@ -219,7 +248,6 @@ def save_and_upload(arg_dict, trainer_config, dfs_upload_path):
 def evaluate_a_group(group):
     group_list = []
     for label, pred, _ in group:
-        # print("%s\t%s\n" % (label, pred))
         group_list.append((int(label), float(pred)))
     random.shuffle(group_list)
     labels = [x[0] for x in group_list]
@@ -236,7 +264,6 @@ class LanguageModelTrainer(TrainerBase):
     """
     LanguageModelTrainer only support training with PaddlePaddle
     """
-
     def __init__(self):
         super(LanguageModelTrainer, self).__init__()
         self.main_program_ = fluid.Program()
@@ -270,10 +297,13 @@ class LanguageModelTrainer(TrainerBase):
         """
         with fluid.program_guard(self.main_program_, self.startup_program_):
             self.input_model_ = LanguageModel()
-            model_configs = {}
+            model_configs = self.trainer_config
             self.input_model_.build_model(model_configs)
+
             optimizer = fluid.optimizer.SGD(
-                learning_rate=self.trainer_config["lr"])
+                learning_rate=self.trainer_config["lr"],
+                grad_clip=fluid.clip.GradientClipByGlobalNorm(
+                    clip_norm=self.trainer_config["max_grad_norm"]))
             optimizer.minimize(self.input_model_.get_model_loss())
 
         self.main_program_desc_ = self.main_program_.desc.serialize_to_string()
@@ -283,13 +313,16 @@ class LanguageModelTrainer(TrainerBase):
                                     self.input_model_.get_model_loss_name())
         self.update_trainer_configs(
             "input_names",
-            self.input_model_.get_model_input_names(), )
+            self.input_model_.get_model_input_names(),
+        )
         self.update_trainer_configs(
             "target_names",
-            self.input_model_.get_target_names(), )
+            self.input_model_.get_target_names(),
+        )
         self.update_trainer_configs(
             "metrics",
-            self.input_model_.get_model_metrics(), )
+            self.input_model_.get_model_metrics(),
+        )
         self.update_trainer_configs("show_metric", True)
         self.update_trainer_configs("max_training_steps", "inf")
         self.update_trainer_configs("shuffle", False)