Increase the threshold of softmax and imperative qat UT (#45819)

python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py

@@ -56,7 +56,10 @@ class TestImperativeQat(unittest.TestCase):
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = False
         self.check_export_model_accuracy = True
-        self.diff_threshold = 0.01
+        # The original model and quantized model may have different prediction.
+        # There are 32 test data and we allow at most one is different.
+        # Hence, the diff_threshold is 1 / 32 = 0.03125
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
 
     def func_qat(self):
@@ -207,7 +210,7 @@ class TestImperativeQat(unittest.TestCase):
             quant_acc = fluid.layers.accuracy(quant_out, label).numpy()
             paddle.enable_static()
             delta_value = fp32_acc - quant_acc
-            self.assertLess(delta_value, self.diff_threshold)
+            self.assertLessEqual(delta_value, self.diff_threshold)
 
     def test_qat(self):
         with _test_eager_guard():
@@ -221,7 +224,7 @@ class TestImperativeQatONNXFormat(unittest.TestCase):
         self.weight_quantize_type = 'abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = True
-        self.diff_threshold = 0.025
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
 
 

python/paddle/fluid/contrib/slim/tests/test_imperative_qat_channelwise.py

@@ -43,7 +43,7 @@ class TestImperativeQatChannelWise(TestImperativeQat):
     def set_vars(self):
         self.weight_quantize_type = 'channel_wise_abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
-        self.diff_threshold = 0.01
+        self.diff_threshold = 0.03125
         self.onnx_format = False
         self.fuse_conv_bn = False
         print('weight_quantize_type', self.weight_quantize_type)
@@ -55,7 +55,7 @@ class TestImperativeQatChannelWiseONNXFormat(TestImperativeQat):
         self.weight_quantize_type = 'channel_wise_abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
         self.onnx_format = True
-        self.diff_threshold = 0.025
+        self.diff_threshold = 0.03125
         self.fuse_conv_bn = False
         print('weight_quantize_type', self.weight_quantize_type)
 

python/paddle/fluid/contrib/slim/tests/test_imperative_qat_fuse.py

@@ -43,7 +43,7 @@ class TestImperativeQatfuseBN(TestImperativeQat):
     def set_vars(self):
         self.weight_quantize_type = 'abs_max'
         self.activation_quantize_type = 'moving_average_abs_max'
-        self.diff_threshold = 0.01
+        self.diff_threshold = 0.03125
         self.onnx_format = False
         self.fuse_conv_bn = True
 

python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_softmax.py

@@ -126,7 +126,7 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest):
                 attrs, False), 1e-5
             self.trt_param.precision = paddle_infer.PrecisionType.Half
             yield self.create_inference_config(), generate_trt_nodes_num(
-                attrs, False), 1e-5
+                attrs, False), 1e-3
 
         # for dynamic_shape
         generate_dynamic_shape(attrs)
@@ -135,7 +135,7 @@ class TrtConvertSoftmaxTest(TrtLayerAutoScanTest):
             attrs, True), 1e-5
         self.trt_param.precision = paddle_infer.PrecisionType.Half
         yield self.create_inference_config(), generate_trt_nodes_num(
-            attrs, True), 1e-5
+            attrs, True), 1e-3
 
     def test(self):
         self.run_test()