未验证 提交 672ff188 编写于 作者: Y Yanzhan Yang 提交者: GitHub

fix quantification diff calculation bug on linux (#1742)

上级 d9074aac
...@@ -333,7 +333,7 @@ void LoadWithDumpForFloat32(const paddle_mobile::framework::VarDesc &var_desc, c ...@@ -333,7 +333,7 @@ void LoadWithDumpForFloat32(const paddle_mobile::framework::VarDesc &var_desc, c
float value = static_cast<float *> (memory)[g]; float value = static_cast<float *> (memory)[g];
auto factor = (uint8_t) round((value - min_value) / (max_value - min_value) * 255); auto factor = (uint8_t) round((value - min_value) / (max_value - min_value) * 255);
float value_quantized = min_value + (factor / 255.0) * (max_value - min_value); float value_quantized = min_value + (factor / 255.0) * (max_value - min_value);
diff += abs(value - value_quantized); diff += fabs(value - value_quantized);
fwrite(&value_quantized, sizeof(float), 1, out_file); fwrite(&value_quantized, sizeof(float), 1, out_file);
} }
if (memory_size > 0) { if (memory_size > 0) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册