diff --git a/paddle/fluid/inference/api/mkldnn_quantizer.cc b/paddle/fluid/inference/api/mkldnn_quantizer.cc
index d0e47eea16f14076293152a5992566f92f4d8b35..654b58a2ded341a5e422f9ff9a3abb43a646b808 100644
--- a/paddle/fluid/inference/api/mkldnn_quantizer.cc
+++ b/paddle/fluid/inference/api/mkldnn_quantizer.cc
@@ -306,7 +306,7 @@ AnalysisPredictor::MkldnnQuantizer::GetKLScalingFactor(
     std::vector<int> reference_distr_P(&hist[0], &hist[i]);
     auto outliers_count =
         std::accumulate(&hist[i], &hist[precision_hist_num_bins], 0);
-    if (reference_distr_P[i - 1] == 0) {
+    if (i <= 0 || reference_distr_P[i - 1] == 0) {
       continue;
     }
     reference_distr_P[i - 1] += outliers_count;