From 37e0e7a96bb9c05851bcc6d09821cff545037079 Mon Sep 17 00:00:00 2001 From: "joanna.wozna.intel" Date: Thu, 14 Nov 2019 03:02:52 +0100 Subject: [PATCH] QAT int8 accuracy little improvement (#21074) test=develop --- .../contrib/slim/quantization/quantization_mkldnn_pass.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/contrib/slim/quantization/quantization_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quantization_mkldnn_pass.py index bdfd7cdef6f..eb8c131357b 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quantization_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quantization_mkldnn_pass.py @@ -514,11 +514,11 @@ class FakeQAT2MkldnnINT8PerfPass(object): weights = np.array( self._load_param(self._scope, weight_var_name)) scales = 1.0 / np.amax( - np.abs(weights.reshape(weights.shape[0], -1)), + np.abs(weights.reshape(weights.shape[0], -1)).astype( + np.float64), axis=axis) - lod_tensor = self._convert_scale2tensor( - scales.astype(np.float64)) + lod_tensor = self._convert_scale2tensor(scales) use_unsigned_int = False self._var_quant_scales[weight_var_name] = (use_unsigned_int, lod_tensor) -- GitLab