trainer.test() (#10453)

* a draft of trainer.test()

* polish trainer.test()

* polish trainer.test()

* update code format

* update

* polish code

* polish code

* polish code

* Make trainer.test follow the rule of returning [loss, metric, metric, ..]

python/paddle/fluid/framework.py

@@ -160,6 +160,7 @@ class Variable(object):
                  persistable=None,
                  error_clip=None,
                  stop_gradient=False,
+                 is_data=False,
                  **kwargs):
         self.block = block
         self.error_clip = error_clip
@@ -238,6 +239,7 @@ class Variable(object):
         self.block.vars[name] = self
         self.op = None
         self.stop_gradient = stop_gradient
+        self.is_data = is_data
 
     def __str__(self):
         return self.to_string(True)
@@ -475,7 +477,7 @@ class Operator(object):
                 if isinstance(attrs[attr_name], Block):
                     self.desc.set_block_attr(attr_name, attrs[attr_name].desc)
                 elif isinstance(attrs[attr_name], core.BlockDesc) or \
-                   isinstance(attrs[attr_name], core.ProgramDesc):
+                        isinstance(attrs[attr_name], core.ProgramDesc):
                     self.desc.set_serialized_attr(
                         attr_name, attrs[attr_name].serialize_to_string())
                 else:
@@ -978,7 +980,8 @@ class Block(object):
                 shape=var.shape,
                 dtype=var.dtype,
                 type=var.type,
-                persistable=True)
+                persistable=True,
+                is_data=var.is_data)
         else:
             ret_var = self.create_var(
                 name=var.name,
@@ -986,7 +989,8 @@ class Block(object):
                 dtype=var.dtype,
                 type=var.type,
                 lod_level=var.lod_level,
-                persistable=True)
+                persistable=True,
+                is_data=var.is_data)
         return ret_var
 
 
@@ -1051,6 +1055,7 @@ class Program(object):
             p.sync_with_cpp()
 
         p.copy_param_info_from(self)
+        p.copy_data_info_from(self)
         return p
 
     def prune(self, targets):
@@ -1172,6 +1177,26 @@ class Program(object):
                              "program, with represent the same topology")
         self.global_block().copy_param_info_from(other.global_block())
 
+    def copy_data_info_from(self, other):
+        """
+        Copy the information of data variables from other program.
+        Args:
+            other(Program): Other program
+
+        Returns:
+            None
+        """
+        if not isinstance(other, Program):
+            raise TypeError("copy_param_info_from should be invoked with "
+                            "Program")
+
+        if len(self.blocks) != len(other.blocks):
+            raise ValueError("copy_param_info_from should be invoked with two "
+                             "program, with represent the same topology")
+        for var in other.global_block().vars.itervalues():
+            if var.is_data:
+                self.global_block().var(var.name).is_data = True
+
     def list_vars(self):
         for each_block in self.blocks:
             for each_var in each_block.vars.itervalues():

python/paddle/fluid/layers/io.py

@@ -78,8 +78,8 @@ def data(name,
         dtype=dtype,
         type=type,
         stop_gradient=stop_gradient,
-        lod_level=lod_level)
-    data_var.is_data = True
+        lod_level=lod_level,
+        is_data=True)
     return data_var
 
 

python/paddle/fluid/tests/book/high-level-api/word2vec/no_test_word2vec_new_api.py

@@ -80,8 +80,11 @@ def inference_program(is_sparse):
 
 
 def train_program(is_sparse):
-    next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
+    # The declaration of 'next_word' must be after the invoking of inference_program,
+    # or the data input order of train program would be [next_word, firstw, secondw,
+    # thirdw, forthw], which is not correct.
     predict_word = inference_program(is_sparse)
+    next_word = fluid.layers.data(name='nextw', shape=[1], dtype='int64')
     cost = fluid.layers.cross_entropy(input=predict_word, label=next_word)
     avg_cost = fluid.layers.mean(cost)
     return avg_cost
@@ -90,14 +93,17 @@ def train_program(is_sparse):
 def train(use_cuda, is_sparse, save_path):
     train_reader = paddle.batch(
         paddle.dataset.imikolov.train(word_dict, N), BATCH_SIZE)
+    test_reader = paddle.batch(
+        paddle.dataset.imikolov.test(word_dict, N), BATCH_SIZE)
 
     place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
 
     def event_handler(event):
-        print type(event)
+        # print type(event)
         if isinstance(event, fluid.EndEpochEvent):
-            avg_cost = trainer.test(reader=paddle.dataset.imikolov.test(
-                word_dict, N))
+            outs = trainer.test(reader=test_reader)
+            avg_cost = outs[0]
+            print("loss= ", avg_cost)
 
             if avg_cost < 5.0:
                 trainer.save_params(save_path)

python/paddle/fluid/trainer.py

@@ -75,11 +75,15 @@ class Trainer(object):
         self.train_program = framework.Program()
 
         with framework.program_guard(self.train_program, self.startup_program):
-            loss = program_func()
+            program_func_outs = program_func()
+            self.test_outputs = program_func_outs if isinstance(
+                program_func_outs, list) else [program_func_outs]
+            self.test_program = self.train_program.clone()
             if not isinstance(optimizer, opt_module.Optimizer):
                 raise TypeError(
                     "The optimizer should be an instance of Optimizer")
-
+            # The fisrt element of program_func_outs is loss.
+            loss = self.test_outputs[0]
             optimize_ops, params_grads = optimizer.minimize(loss)
 
         self.place = Trainer._check_and_get_place(place)
@@ -168,8 +172,17 @@ class Trainer(object):
 
         self._train_by_executor(num_epochs, event_handler, reader, feed_order)
 
-    def test(self, reader):
-        pass
+    def test(self, reader, feed_order=None):
+        """
+        Test the model on given test data
+
+        Args:
+            reader: The reader that yields test data.
+            feed_order: Feeding order of reader. None will following the defining
+                order in program
+        """
+
+        return self._test_by_executor(reader, feed_order, self.test_outputs)
 
     def save_params(self, param_path):
         # reference: save_persistables in io.py
@@ -225,22 +238,10 @@ class Trainer(object):
 
         """
         with self._prog_and_scope_guard():
-            exe = executor.Executor(self.place)
-            if feed_order is None:
-                feed_var_list = [
-                    var
-                    for var in self.train_program.global_block(
-                    ).vars.itervalues()
-                    if hasattr(var, 'is_data') and var.is_data
-                ]
-            else:
-                feed_var_list = [
-                    self.train_program.global_block().var(var_name)
-                    for var_name in feed_order
-                ]
-
+            feed_var_list = build_feed_var_list(self.train_program, feed_order)
             feeder = data_feeder.DataFeeder(
                 feed_list=feed_var_list, place=self.place)
+            exe = executor.Executor(self.place)
             for epoch_id in range(num_epochs):
                 event_handler(BeginEpochEvent(epoch_id))
                 for step_id, data in enumerate(reader()):
@@ -248,3 +249,48 @@ class Trainer(object):
                     exe.run(feed=feeder.feed(data), fetch_list=[])
                     event_handler(EndStepEvent(epoch_id, step_id))
                 event_handler(EndEpochEvent(epoch_id))
+
+    def _test_by_executor(self, reader, feed_order, fetch_list):
+        with executor.scope_guard(self.scope):
+            feed_var_list = build_feed_var_list(self.test_program, feed_order)
+            feeder = data_feeder.DataFeeder(
+                feed_list=feed_var_list, place=self.place)
+            exe = executor.Executor(self.place)
+            accumulated = len(fetch_list) * [0]
+            count = 0
+            for data in reader():
+                outs = exe.run(program=self.test_program,
+                               feed=feeder.feed(data),
+                               fetch_list=fetch_list)
+                accumulated = [x[0] + x[1][0] for x in zip(accumulated, outs)]
+                count += 1
+
+            return [x / count for x in accumulated]
+
+
+def build_feed_var_list(program, feed_order):
+    if not isinstance(program, framework.Program):
+        raise TypeError("The 'program' should be an object of Program")
+
+    if feed_order is None:
+        feed_var_list = [
+            var for var in program.global_block().vars.itervalues()
+            if var.is_data
+        ]
+    elif isinstance(feed_order, list):
+        feed_var_list = [
+            program.global_block().var(var_name) for var_name in feed_order
+        ]
+    else:
+        if not isinstance(feed_order, dict):
+            raise TypeError(
+                "The 'feed_order' should be either None, list or dict.")
+        if not sorted(feed_order.values()) == range(len(feed_order)):
+            raise ValueError(
+                "The values of 'feed_order' should be a permutation of [0, len(feed_order))"
+            )
+        sorted_pair_list = sorted(feed_order.items(), key=lambda item: item[1])
+        feed_var_list = [
+            program.global_block().var(pair[0]) for pair in sorted_pair_list
+        ]
+    return feed_var_list