diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
index a2f1d77460819a933e8ab536d0ebc7dd32bedf3f..da7e30ff10643278eb355c65157a02a8fae6cff3 100644
--- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
+++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py
@@ -124,8 +124,10 @@ class TestMNIST(TestParallelExecutorBase):
 
     def test_simple_fc_with_new_strategy(self):
         # use_cuda, use_reduce
-        self._compare_reduce_and_allreduce(simple_fc_net, True)
-        self._compare_reduce_and_allreduce(simple_fc_net, False)
+        # NOTE: the computation result of nccl_reduce is non-deterministic,
+        # related issue: https://github.com/NVIDIA/nccl/issues/157
+        self._compare_reduce_and_allreduce(simple_fc_net, True, 1e-5, 1e-2)
+        self._compare_reduce_and_allreduce(simple_fc_net, False, 1e-5, 1e-2)
 
     def check_simple_fc_parallel_accuracy(self, use_cuda):
         if use_cuda and not core.is_compiled_with_cuda():
@@ -179,7 +181,7 @@ class TestMNIST(TestParallelExecutorBase):
         # NOTE: the computation result of nccl_reduce is non-deterministic,
         # related issue: https://github.com/NVIDIA/nccl/issues/157
         self._compare_reduce_and_allreduce(fc_with_batchnorm, True, 1e-5, 1e-2)
-        self._compare_reduce_and_allreduce(fc_with_batchnorm, False)
+        self._compare_reduce_and_allreduce(fc_with_batchnorm, False, 1e-5, 1e-2)
 
 
 if __name__ == '__main__':