diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py
index 2c18eff983e4c46a1e193c6fd42e8606cf0f0d8b..81411daae711138305397c5e93c9ae34a2beb265 100644
--- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py
+++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py
@@ -56,7 +56,10 @@ class TestImperativeQat(unittest.TestCase):
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = False
         self.check_export_model_accuracy = True
-        self.diff_threshold = 0.01
+        # The original model and quantized model may have different prediction.
+        # There are 32 test data and we allow at most one is different.
+        # Hence, the diff_threshold is 1 / 32 = 0.03125
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
 
     def func_qat(self):
@@ -207,7 +210,7 @@ class TestImperativeQat(unittest.TestCase):
             quant_acc = fluid.layers.accuracy(quant_out, label).numpy()
             paddle.enable_static()
             delta_value = fp32_acc - quant_acc
-            self.assertLess(delta_value, self.diff_threshold)
+            self.assertLessEqual(delta_value, self.diff_threshold)
 
     def test_qat(self):
         with _test_eager_guard():
@@ -221,7 +224,7 @@ class TestImperativeQatONNXFormat(unittest.TestCase):
         self.weight_quantize_type = 'abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = True
-        self.diff_threshold = 0.025
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
 
 
diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py
index 3770ee486499df05c37ed0033a59c2221e0b7d9f..f9fa636debc458664be85ce0001e9a460fb1fb4b 100644
--- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py
+++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py
@@ -43,7 +43,7 @@ class TestImperativeQatChannelWise(TestImperativeQat):
     def set_vars(self):
         self.weight_quantize_type = 'channel_wise_abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
-        self.diff_threshold = 0.01
+        self.diff_threshold = 0.03125
         self.onnx_format = False
         self.fuse_conv_bn = False
         print('weight_quantize_type', self.weight_quantize_type)
@@ -55,7 +55,7 @@ class TestImperativeQatChannelWiseONNXFormat(TestImperativeQat):
         self.weight_quantize_type = 'channel_wise_abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = True
-        self.diff_threshold = 0.025
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
         print('weight_quantize_type', self.weight_quantize_type)
 
diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py
index db7f15c4cef177d7cb20f491299876d125d496a8..4c491598d2124fd9903ef43b75b204857d8c2503 100644
--- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py
+++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py
@@ -43,7 +43,7 @@ class TestImperativeQatfuseBN(TestImperativeQat):
     def set_vars(self):
         self.weight_quantize_type = 'abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
-        self.diff_threshold = 0.01
+        self.diff_threshold = 0.03125
         self.onnx_format = False
         self.fuse_conv_bn = True
 
diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py
index b6cef5ca17bdcac641842573ac1c42653747a5d7..6a65382baf4b5d528cba7bbdaf831bcb769b4f37 100644
--- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py
+++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py
@@ -126,7 +126,7 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest):
                 attrs, False), 1e-5
             self.trt_param.precision = paddle_infer.PrecisionType.Half
             yield self.create_inference_config(), generate_trt_nodes_num(
-                attrs, False), 1e-5
+                attrs, False), 1e-3
 
         # for dynamic_shape
         generate_dynamic_shape(attrs)
@@ -135,7 +135,7 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest):
             attrs, True), 1e-5
         self.trt_param.precision = paddle_infer.PrecisionType.Half
         yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True), 1e-5
+            attrs, True), 1e-3
 
     def test(self):
         self.run_test()