From 672ff18868d9c45389873810af49a362c3fd552e Mon Sep 17 00:00:00 2001 From: Yanzhan Yang Date: Thu, 11 Jul 2019 12:53:52 +0800 Subject: [PATCH] fix quantification diff calculation bug on linux (#1742) --- tools/quantification/convert.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/quantification/convert.cpp b/tools/quantification/convert.cpp index 606d2f5b93..75b077c493 100644 --- a/tools/quantification/convert.cpp +++ b/tools/quantification/convert.cpp @@ -333,7 +333,7 @@ void LoadWithDumpForFloat32(const paddle_mobile::framework::VarDesc &var_desc, c float value = static_cast (memory)[g]; auto factor = (uint8_t) round((value - min_value) / (max_value - min_value) * 255); float value_quantized = min_value + (factor / 255.0) * (max_value - min_value); - diff += abs(value - value_quantized); + diff += fabs(value - value_quantized); fwrite(&value_quantized, sizeof(float), 1, out_file); } if (memory_size > 0) { -- GitLab