Add unit test for fuse_opt_ops (#16550)

* add unit test for fuse_opt_ops
test=develop

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -118,6 +118,7 @@ endif()
 py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf SERIAL)
 py_test_modules(test_parallel_executor_fetch_feed MODULES test_parallel_executor_fetch_feed SERIAL)
 set_tests_properties(test_parallel_executor_fetch_feed PROPERTIES TIMEOUT 450)
+set_tests_properties(test_parallel_executor_seresnext PROPERTIES TIMEOUT 740)
 py_test_modules(test_parallel_executor_transformer MODULES test_parallel_executor_transformer SERIAL)
 py_test_modules(test_layers MODULES test_layers ENVS FLAGS_cudnn_deterministic=1)
 if(NOT WIN32)

python/paddle/fluid/tests/unittests/simple_nets.py

+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddle.fluid as fluid
+import numpy as np
+
+
+def simple_fc_net(use_feed=None):
+    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
+    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+
+    hidden = img
+    for _ in range(4):
+        hidden = fluid.layers.fc(
+            hidden,
+            size=200,
+            act='relu',
+            bias_attr=fluid.ParamAttr(
+                initializer=fluid.initializer.Constant(value=1.0)))
+    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
+    loss = fluid.layers.cross_entropy(input=prediction, label=label)
+    loss = fluid.layers.mean(loss)
+    return loss
+
+
+def fc_with_batchnorm(use_feed=None):
+    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
+    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
+
+    hidden = img
+    for _ in range(2):
+        hidden = fluid.layers.fc(
+            hidden,
+            size=200,
+            act='relu',
+            bias_attr=fluid.ParamAttr(
+                initializer=fluid.initializer.Constant(value=1.0)))
+
+        hidden = fluid.layers.batch_norm(input=hidden)
+
+    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
+    loss = fluid.layers.cross_entropy(input=prediction, label=label)
+    loss = fluid.layers.mean(loss)
+    return loss
+
+
+def init_data(batch_size=32, img_shape=[784], label_range=9):
+    np.random.seed(5)
+    assert isinstance(img_shape, list)
+    input_shape = [batch_size] + img_shape
+    img = np.random.random(size=input_shape).astype(np.float32)
+    label = np.array(
+        [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape(
+            (-1, 1)).astype("int64")
+    return img, label

python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from simple_nets import simple_fc_net, fc_with_batchnorm, init_data
 from parallel_executor_test_base import TestParallelExecutorBase
 import paddle.fluid as fluid
 import paddle.fluid.core as core
@@ -22,45 +22,6 @@ import unittest
 import os
 
 
-def simple_fc_net(use_feed):
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
-def fc_with_batchnorm(use_feed):
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-
-    hidden = img
-    for _ in range(2):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-
-        hidden = fluid.layers.batch_norm(input=hidden)
-
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
 class TestMNIST(TestParallelExecutorBase):
     @classmethod
     def setUpClass(cls):
@@ -75,10 +36,10 @@ class TestMNIST(TestParallelExecutorBase):
         label = np.ones(shape=[32, 1], dtype='int64')
         return img, label
 
-    def _compare_fuse_all_reduce_ops(self, model, use_cuda, random_data=True):
+    def _compare_fuse_all_reduce_ops(self, model, use_cuda):
         if use_cuda and not core.is_compiled_with_cuda():
             return
-        img, label = self._init_data(random_data)
+        img, label = init_data()
 
         def _optimizer(learning_rate=1e-6):
             optimizer = fluid.optimizer.SGD(

python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py

@@ -12,108 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from simple_nets import simple_fc_net, fc_with_batchnorm, init_data
 from parallel_executor_test_base import TestParallelExecutorBase
 import paddle.fluid as fluid
 import paddle.fluid.core as core
-import numpy as np
-import paddle
-import paddle.dataset.mnist as mnist
 import unittest
 import os
 
-MNIST_RECORDIO_FILE = "./mnist_test_pe.recordio"
-
-
-def simple_fc_net(use_feed):
-    if use_feed:
-        img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-        label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-    else:
-        reader = fluid.layers.open_files(
-            filenames=[MNIST_RECORDIO_FILE],
-            shapes=[[-1, 784], [-1, 1]],
-            lod_levels=[0, 0],
-            dtypes=['float32', 'int64'])
-        reader = fluid.layers.io.double_buffer(reader)
-        img, label = fluid.layers.read_file(reader)
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
-def fc_with_batchnorm(use_feed):
-    if use_feed:
-        img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-        label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-    else:
-        reader = fluid.layers.open_files(
-            filenames=[MNIST_RECORDIO_FILE],
-            shapes=[[-1, 784], [-1, 1]],
-            lod_levels=[0, 0],
-            dtypes=['float32', 'int64'])
-        reader = fluid.layers.io.double_buffer(reader)
-        img, label = fluid.layers.read_file(reader)
-
-    hidden = img
-    for _ in range(2):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-
-        hidden = fluid.layers.batch_norm(input=hidden)
-
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
 
 class TestMNIST(TestParallelExecutorBase):
     @classmethod
     def setUpClass(cls):
         os.environ['CPU_NUM'] = str(4)
-        # Convert mnist to recordio file
-        with fluid.program_guard(fluid.Program(), fluid.Program()):
-            reader = paddle.batch(mnist.train(), batch_size=4)
-            feeder = fluid.DataFeeder(
-                feed_list=[  # order is image and label
-                    fluid.layers.data(
-                        name='image', shape=[784]),
-                    fluid.layers.data(
-                        name='label', shape=[1], dtype='int64'),
-                ],
-                place=fluid.CPUPlace())
-            fluid.recordio_writer.convert_reader_to_recordio_file(
-                MNIST_RECORDIO_FILE, reader, feeder)
 
-    def _init_data(self, random=True):
-        np.random.seed(5)
-        if random:
-            img = np.random.random(size=[32, 784]).astype(np.float32)
-        else:
-            img = np.ones(shape=[32, 784], dtype='float32')
-        label = np.ones(shape=[32, 1], dtype='int64')
-        return img, label
-
-    def _compare_fuse_elewise_add_act_ops(self,
-                                          model,
-                                          use_cuda,
-                                          random_data=True):
+    def _compare_fuse_elewise_add_act_ops(self, model, use_cuda):
         if use_cuda and not core.is_compiled_with_cuda():
             return
-        img, label = self._init_data(random_data)
+        img, label = init_data()
 
         def _optimizer(learning_rate=1e-6):
             optimizer = fluid.optimizer.SGD(

python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py

@@ -11,78 +11,26 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from simple_nets import simple_fc_net, fc_with_batchnorm, init_data
 from parallel_executor_test_base import TestParallelExecutorBase
 import paddle.fluid as fluid
 import paddle.fluid.core as core
-import numpy as np
-import paddle
-import paddle.dataset.mnist as mnist
 import unittest
 import os
 
 
-def simple_fc_net(use_feed):
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
-def fc_with_batchnorm(use_feed):
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-
-    hidden = img
-    for _ in range(2):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='relu',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-
-        hidden = fluid.layers.batch_norm(input=hidden)
-
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
 class TestFuseAdamOps(TestParallelExecutorBase):
     @classmethod
     def setUpClass(cls):
         os.environ['CPU_NUM'] = str(4)
 
-    def _init_data(self, random=True):
-        np.random.seed(5)
-        if random:
-            img = np.random.random(size=[32, 784]).astype(np.float32)
-        else:
-            img = np.ones(shape=[32, 784], dtype='float32')
-        label = np.ones(shape=[32, 1], dtype='int64')
-        return img, label
-
     def _compare_fused_optimizer_ops(self,
                                      model,
                                      use_cuda,
-                                     random_data=True,
                                      optimizer=fluid.optimizer.Adam):
         if use_cuda and not core.is_compiled_with_cuda():
             return
-        img, label = self._init_data(random_data)
+        img, label = init_data()
         not_fuse_op_first_loss, not_fuse_op_last_loss = self.check_network_convergence(
             model,
             feed_dict={"image": img,
@@ -111,7 +59,7 @@ class TestFuseAdamOps(TestParallelExecutorBase):
 
     def test_batchnorm_fc_with_fuse_op(self):
         self._compare_fused_optimizer_ops(fc_with_batchnorm, True)
-        # self._compare_fused_optimizer_ops(fc_with_batchnorm, False)
+        self._compare_fused_optimizer_ops(fc_with_batchnorm, False)
 
 
 class TestFuseSGDOps(TestFuseAdamOps):

python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py

@@ -21,25 +21,8 @@ import os
 os.environ['FLAGS_enable_parallel_graph'] = str(1)
 import paddle.fluid.core as core
 import os
-import paddle.fluid as fluid
 from parallel_executor_test_base import TestParallelExecutorBase
-
-
-def simple_fc_net(use_feed):
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='tanh',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
+from simple_nets import simple_fc_net, init_data
 
 
 class TestMNIST(TestParallelExecutorBase):
@@ -47,19 +30,12 @@ class TestMNIST(TestParallelExecutorBase):
     def setUpClass(cls):
         os.environ['CPU_NUM'] = str(4)
 
-    def _init_data(self):
-        np.random.seed(5)
-        img = np.random.random(size=[32, 784]).astype(np.float32)
-        label = np.ones(shape=[32, 1], dtype='int64')
-        return img, label
-
     # simple_fc
     def check_simple_fc_convergence(self, use_cuda, use_reduce=False):
         if use_cuda and not core.is_compiled_with_cuda():
             return
 
-        img, label = self._init_data()
-
+        img, label = init_data()
         self.check_network_convergence(
             simple_fc_net,
             feed_dict={"image": img,
@@ -75,8 +51,7 @@ class TestMNIST(TestParallelExecutorBase):
         if use_cuda and not core.is_compiled_with_cuda():
             return
 
-        img, label = self._init_data()
-
+        img, label = init_data()
         single_first_loss, single_last_loss = self.check_network_convergence(
             method=simple_fc_net,
             seed=1,

python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py

@@ -23,9 +23,11 @@ from paddle.fluid.initializer import init_on_cpu
 from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter
 import paddle.fluid.core as core
 from parallel_executor_test_base import TestParallelExecutorBase
+from simple_nets import init_data
 import unittest
 import math
 import numpy as np
+from functools import partial
 
 # FIXME(zcd): If the neural net has dropout_op, the output of ParallelExecutor
 # and Executor is different. Because, for ParallelExecutor, the dropout_op of
@@ -187,17 +189,6 @@ class TestResnet(TestParallelExecutorBase):
         remove_dropout = False
         remove_bn = False
 
-    def _init_data(self, batch_size=2, random=True):
-        np.random.seed(5)
-        if random:
-            img = np.random.random(
-                size=[batch_size] + img_shape).astype(np.float32)
-        else:
-            img = np.ones(shape=[batch_size] + img_shape, dtype='float32')
-        label = [np.random.randint(0, 999) for _ in range(batch_size)]
-        label = np.array(label).astype(np.int64).reshape(-1, 1)
-        return img, label
-
     def _compare_reduce_and_allreduce(self,
                                       model,
                                       use_cuda,
@@ -209,7 +200,8 @@ class TestResnet(TestParallelExecutorBase):
         global remove_bn
         remove_bn = True
 
-        img, label = self._init_data(batch_size=batch_size)
+        img, label = init_data(
+            batch_size=batch_size, img_shape=img_shape, label_range=999)
         all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence(
             model,
             feed_dict={"image": img,
@@ -276,10 +268,12 @@ class TestResnet(TestParallelExecutorBase):
 
     def _check_resnet_convergence(self,
                                   model,
-                                  use_cuda=True,
-                                  use_reduce=False,
+                                  check_func_1,
+                                  check_func_2,
+                                  use_cuda,
                                   iter=20,
-                                  delta2=1e-5):
+                                  delta2=1e-5,
+                                  compare_seperately=True):
         if use_cuda and not core.is_compiled_with_cuda():
             return
 
@@ -288,31 +282,33 @@ class TestResnet(TestParallelExecutorBase):
         remove_dropout = True
         remove_bn = True
 
-        img, label = self._init_data(batch_size=batch_size)
-        single_first_loss, single_last_loss = self.check_network_convergence(
+        img, label = init_data(
+            batch_size=batch_size, img_shape=img_shape, label_range=999)
+        func_1_first_loss, func_1_last_loss = check_func_1(
             model,
             feed_dict={"image": img,
                        "label": label},
             iter=iter,
             batch_size=batch_size,
-            use_cuda=use_cuda,
-            use_reduce=use_reduce,
-            optimizer=optimizer,
-            use_parallel_executor=False)
-        parallel_first_loss, parallel_last_loss = self.check_network_convergence(
+            use_cuda=use_cuda)
+        func_2_first_loss, func_2_last_loss = check_func_2(
             model,
             feed_dict={"image": img,
                        "label": label},
             iter=iter,
             batch_size=batch_size,
-            use_cuda=use_cuda,
-            use_reduce=use_reduce,
-            optimizer=optimizer)
+            use_cuda=use_cuda)
 
+        if compare_seperately:
+            for loss in zip(func_1_first_loss, func_2_first_loss):
+                self.assertAlmostEquals(loss[0], loss[1], delta=1e-5)
+            for loss in zip(func_1_last_loss, func_2_last_loss):
+                self.assertAlmostEquals(loss[0], loss[1], delta=delta2)
+        else:
             self.assertAlmostEquals(
-            np.mean(parallel_first_loss), single_first_loss[0], delta=1e-5)
+                np.mean(func_1_first_loss), func_2_first_loss[0], delta=1e-5)
             self.assertAlmostEquals(
-            np.mean(parallel_last_loss), single_last_loss[0], delta=delta2)
+                np.mean(func_1_last_loss), func_2_last_loss[0], delta=delta2)
 
     def _compare_with_fused_all_reduce(self,
                                        model,
@@ -325,7 +321,8 @@ class TestResnet(TestParallelExecutorBase):
         global remove_bn
         remove_bn = True
 
-        img, label = self._init_data(batch_size=batch_size)
+        img, label = init_data(
+            batch_size=batch_size, img_shape=img_shape, label_range=999)
         all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence(
             model,
             feed_dict={"image": img,
@@ -350,11 +347,6 @@ class TestResnet(TestParallelExecutorBase):
         for loss in zip(all_reduce_last_loss, reduce_last_loss):
             self.assertAlmostEquals(loss[0], loss[1], delta=delta2)
 
-    def test_seresnext_with_learning_rate_decay(self):
-        self._check_resnet_convergence(model=SE_ResNeXt50Small, use_cuda=True)
-        self._check_resnet_convergence(
-            model=SE_ResNeXt50Small, use_cuda=False, iter=2, delta2=1e-3)
-
     def test_seresnext_with_reduce(self):
         self._compare_reduce_and_allreduce(
             model=SE_ResNeXt50Small, use_cuda=True, delta2=1e-2)
@@ -367,6 +359,50 @@ class TestResnet(TestParallelExecutorBase):
         self._compare_with_fused_all_reduce(
             model=SE_ResNeXt50Small, use_cuda=False, iter=2, delta2=1e-3)
 
+    def test_seresnext_with_learning_rate_decay(self):
+        check_func_1 = partial(
+            self.check_network_convergence,
+            optimizer=optimizer,
+            use_parallel_executor=True)
+        check_func_2 = partial(
+            self.check_network_convergence,
+            optimizer=optimizer,
+            use_parallel_executor=False)
+        self._check_resnet_convergence(
+            SE_ResNeXt50Small,
+            check_func_1,
+            check_func_2,
+            use_cuda=True,
+            compare_seperately=False)
+        self._check_resnet_convergence(
+            SE_ResNeXt50Small,
+            check_func_1,
+            check_func_2,
+            use_cuda=False,
+            compare_seperately=False,
+            iter=2,
+            delta2=1e-3)
+
+    def test_seresnext_with_fused_optimizer_ops(self):
+        check_func_1 = partial(
+            self.check_network_convergence, fuse_all_optimizer_ops=False)
+        check_func_2 = partial(
+            self.check_network_convergence, fuse_all_optimizer_ops=True)
+        # TODO(zcd): this test failed random, I will fix it in next PR.
+        # self._check_resnet_convergence(
+        #     SE_ResNeXt50Small,
+        #     check_func_1,
+        #     check_func_2,
+        #     use_cuda=True,
+        #     delta2=1e-3)
+        self._check_resnet_convergence(
+            SE_ResNeXt50Small,
+            check_func_1,
+            check_func_2,
+            use_cuda=False,
+            iter=2,
+            delta2=1e-3)
+
 
 if __name__ == '__main__':
     unittest.main()

python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from __future__ import print_function
-
+from simple_nets import simple_fc_net
 import paddle.fluid as fluid
 from paddle.fluid import compiler
 import paddle.fluid.core as core
@@ -24,23 +24,6 @@ import sys
 import math
 
 
-def simple_fc_net():
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='tanh',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
 class ParallelExecutorTestingDuringTraining(unittest.TestCase):
     def check_network_convergence(self, use_cuda, build_strategy=None):
         os.environ['CPU_NUM'] = str(4)

python/paddle/fluid/tests/unittests/test_pass_builder.py

@@ -14,6 +14,7 @@
 
 from __future__ import print_function
 
+from simple_nets import simple_fc_net
 import paddle.fluid as fluid
 import paddle.fluid.core as core
 from paddle.fluid import compiler
@@ -24,23 +25,6 @@ import sys
 import math
 
 
-def simple_fc_net():
-    img = fluid.layers.data(name='image', shape=[784], dtype='float32')
-    label = fluid.layers.data(name='label', shape=[1], dtype='int64')
-    hidden = img
-    for _ in range(4):
-        hidden = fluid.layers.fc(
-            hidden,
-            size=200,
-            act='tanh',
-            bias_attr=fluid.ParamAttr(
-                initializer=fluid.initializer.Constant(value=1.0)))
-    prediction = fluid.layers.fc(hidden, size=10, act='softmax')
-    loss = fluid.layers.cross_entropy(input=prediction, label=label)
-    loss = fluid.layers.mean(loss)
-    return loss
-
-
 class TestPassBuilder(unittest.TestCase):
     def check_network_convergence(self, use_cuda, build_strategy=None):
         os.environ['CPU_NUM'] = str(4)