diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py
index e55f8707a9a8ac2b0d69c65b15e6593025511999..47083ca7e954c85bb42fcc88639f3e757283cbea 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py
@@ -61,9 +61,7 @@ class TestDistTransformer2x2Sync(TestDistBase):
 
     def test_transformer(self):
         download_files()
-        #Note: loss on test dataset of the first 5 batch are:
-        # 10.518872, 10.518871, 10.518868, 10.518862, 10.518855
-        self.check_with_place("dist_transformer.py", delta=1e-7)
+        self.check_with_place("dist_transformer.py", delta=1e-5)
 
 
 class TestDistTransformer2x2Async(TestDistBase):