diff --git a/paddle/fluid/operators/math/pooling.cc b/paddle/fluid/operators/math/pooling.cc
index 40cea7483f39781a46689ecd1aa21d0bf8cf2c07..fec738378a64cfdaad75f9aea8512d7ecb142136 100644
--- a/paddle/fluid/operators/math/pooling.cc
+++ b/paddle/fluid/operators/math/pooling.cc
@@ -60,19 +60,25 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
           if (adaptive) {
             hstart = AdaptStartIndex(ph, input_height, output_height);
             hend = AdaptEndIndex(ph, input_height, output_height);
-          } else {
-            hstart = ph * stride_height - padding_height;
-            hend = std::min(hstart + ksize_height, input_height);
-            hstart = std::max(hstart, 0);
           }
           for (int pw = 0; pw < output_width; ++pw) {
+            int pool_size = 1;
             if (adaptive) {
               wstart = AdaptStartIndex(pw, input_width, output_width);
               wend = AdaptEndIndex(pw, input_width, output_width);
             } else {
+              hstart = ph * stride_height - padding_height;
               wstart = pw * stride_width - padding_width;
-              wend = std::min(wstart + ksize_width, input_width);
+              hend = std::min(hstart + ksize_height,
+                              input_height + padding_height);
+              wend =
+                  std::min(wstart + ksize_width, input_width + padding_width);
+              pool_size = (hend - hstart) * (wend - wstart);
+
               wstart = std::max(wstart, 0);
+              hstart = std::max(hstart, 0);
+              hend = std::min(hend, input_height);
+              wend = std::min(wend, input_width);
             }
 
             T ele = pool_process.initial();
@@ -81,9 +87,10 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                 pool_process.compute(input_data[h * input_width + w], &ele);
               }
             }
-            int pool_size = (exclusive || adaptive)
-                                ? (hend - hstart) * (wend - wstart)
-                                : ksize_height * ksize_width;
+            if (exclusive || adaptive) {
+              pool_size = (hend - hstart) * (wend - wstart);
+            }
+
             pool_process.finalize(static_cast<T>(pool_size), &ele);
             output_data[ph * output_width + pw] = ele;
           }
@@ -137,19 +144,25 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                hstart = ph * stride_height - padding_height;
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+                pool_size = (hend - hstart) * (wend - wstart);
+
                 wstart = std::max(wstart, 0);
+                hstart = std::max(hstart, 0);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
               }
 
               T ele = pool_process.initial();
@@ -158,9 +171,9 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                   pool_process.compute(input_data[h * input_width + w], &ele);
                 }
               }
-              int pool_size = (exclusive || adaptive)
-                                  ? (hend - hstart) * (wend - wstart)
-                                  : ksize_height * ksize_width;
+              if (exclusive || adaptive) {
+                pool_size = (hend - hstart) * (wend - wstart);
+              }
               pool_process.finalize(static_cast<T>(pool_size), &ele);
               output_data[ph * output_width + pw] = ele;
             }
@@ -178,19 +191,25 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                hstart = ph * stride_height - padding_height;
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+                pool_size = (hend - hstart) * (wend - wstart);
+
                 wstart = std::max(wstart, 0);
+                hstart = std::max(hstart, 0);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
               }
               T ele = pool_process.initial();
               for (int h = hstart; h < hend; ++h) {
@@ -201,10 +220,9 @@ class Pool2dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                       &ele);
                 }
               }
-              int pool_size = (exclusive || adaptive)
-                                  ? (hend - hstart) * (wend - wstart)
-                                  : ksize_height * ksize_width;
-
+              if (exclusive || adaptive) {
+                pool_size = (hend - hstart) * (wend - wstart);
+              }
               pool_process.finalize(static_cast<T>(pool_size), &ele);
               output_data[ph * output_width * output_channels +
                           pw * output_channels + c] = ele;
@@ -262,23 +280,29 @@ class Pool2dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
           if (adaptive) {
             hstart = AdaptStartIndex(ph, input_height, output_height);
             hend = AdaptEndIndex(ph, input_height, output_height);
-          } else {
-            hstart = ph * stride_height - padding_height;
-            hend = std::min(hstart + ksize_height, input_height);
-            hstart = std::max(hstart, 0);
           }
           for (int pw = 0; pw < output_width; ++pw) {
+            int pool_size = 1;
             if (adaptive) {
               wstart = AdaptStartIndex(pw, input_width, output_width);
               wend = AdaptEndIndex(pw, input_width, output_width);
             } else {
+              hstart = ph * stride_height - padding_height;
               wstart = pw * stride_width - padding_width;
-              wend = std::min(wstart + ksize_width, input_width);
+              hend = std::min(hstart + ksize_height,
+                              input_height + padding_height);
+              wend =
+                  std::min(wstart + ksize_width, input_width + padding_width);
+              pool_size = (hend - hstart) * (wend - wstart);
+
               wstart = std::max(wstart, 0);
+              hstart = std::max(hstart, 0);
+              hend = std::min(hend, input_height);
+              wend = std::min(wend, input_width);
+            }
+            if (exclusive || adaptive) {
+              pool_size = (hend - hstart) * (wend - wstart);
             }
-            int pool_size = (exclusive || adaptive)
-                                ? (hend - hstart) * (wend - wstart)
-                                : ksize_height * ksize_width;
             float scale = 1.0 / pool_size;
             for (int h = hstart; h < hend; ++h) {
               for (int w = wstart; w < wend; ++w) {
@@ -346,23 +370,29 @@ class Pool2dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                hstart = ph * stride_height - padding_height;
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+                pool_size = (hend - hstart) * (wend - wstart);
+
                 wstart = std::max(wstart, 0);
+                hstart = std::max(hstart, 0);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
+              }
+              if (exclusive || adaptive) {
+                pool_size = (hend - hstart) * (wend - wstart);
               }
-              int pool_size = (exclusive || adaptive)
-                                  ? (hend - hstart) * (wend - wstart)
-                                  : ksize_height * ksize_width;
               float scale = 1.0 / pool_size;
               for (int h = hstart; h < hend; ++h) {
                 for (int w = wstart; w < wend; ++w) {
@@ -391,23 +421,29 @@ class Pool2dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                hstart = ph * stride_height - padding_height;
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+                pool_size = (hend - hstart) * (wend - wstart);
+
                 wstart = std::max(wstart, 0);
+                hstart = std::max(hstart, 0);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
+              }
+              if (exclusive || adaptive) {
+                pool_size = (hend - hstart) * (wend - wstart);
               }
-              int pool_size = (exclusive || adaptive)
-                                  ? (hend - hstart) * (wend - wstart)
-                                  : ksize_height * ksize_width;
               float scale = 1.0 / pool_size;
               for (int h = hstart; h < hend; ++h) {
                 for (int w = wstart; w < wend; ++w) {
@@ -672,34 +708,43 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
     int dstart, dend;
     int hstart, hend;
     int wstart, wend;
+
     for (int i = 0; i < batch_size; i++) {
       for (int c = 0; c < output_channels; ++c) {
         for (int pd = 0; pd < output_depth; ++pd) {
           if (adaptive) {
             dstart = AdaptStartIndex(pd, input_depth, output_depth);
             dend = AdaptEndIndex(pd, input_depth, output_depth);
-          } else {
-            dstart = pd * stride_depth - padding_depth;
-            dend = std::min(dstart + ksize_depth, input_depth);
-            dstart = std::max(dstart, 0);
           }
+
           for (int ph = 0; ph < output_height; ++ph) {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
+
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                dstart = pd * stride_depth - padding_depth;
+                dend =
+                    std::min(dstart + ksize_depth, input_depth + padding_depth);
+                hstart = ph * stride_height - padding_height;
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+                pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart);
+                dstart = std::max(dstart, 0);
+                hstart = std::max(hstart, 0);
                 wstart = std::max(wstart, 0);
+                dend = std::min(dend, input_depth);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
               }
               int output_idx = (pd * output_height + ph) * output_width + pw;
               T ele = pool_process.initial();
@@ -712,10 +757,9 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                   }
                 }
               }
-              int pool_size =
-                  (exclusive || adaptive)
-                      ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                      : ksize_depth * ksize_height * ksize_width;
+              if (exclusive || adaptive) {
+                pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart);
+              }
               pool_process.finalize(static_cast<T>(pool_size), &ele);
               output_data[output_idx] = ele;
             }
@@ -767,7 +811,6 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
     int dstart, dend;
     int hstart, hend;
     int wstart, wend;
-
     if (!channel_last) {
       const int input_stride = input_depth * input_height * input_width;
       const int output_stride = output_depth * output_height * output_width;
@@ -777,29 +820,40 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               dstart = AdaptStartIndex(pd, input_depth, output_depth);
               dend = AdaptEndIndex(pd, input_depth, output_depth);
-            } else {
-              dstart = pd * stride_depth - padding_depth;
-              dend = std::min(dstart + ksize_depth, input_depth);
-              dstart = std::max(dstart, 0);
             }
+
             for (int ph = 0; ph < output_height; ++ph) {
               if (adaptive) {
                 hstart = AdaptStartIndex(ph, input_height, output_height);
                 hend = AdaptEndIndex(ph, input_height, output_height);
-              } else {
-                hstart = ph * stride_height - padding_height;
-                hend = std::min(hstart + ksize_height, input_height);
-                hstart = std::max(hstart, 0);
               }
+
               for (int pw = 0; pw < output_width; ++pw) {
+                int pool_size = 1;
                 if (adaptive) {
                   wstart = AdaptStartIndex(pw, input_width, output_width);
                   wend = AdaptEndIndex(pw, input_width, output_width);
                 } else {
+                  dstart = pd * stride_depth - padding_depth;
+                  dend = std::min(dstart + ksize_depth,
+                                  input_depth + padding_depth);
+                  hstart = ph * stride_height - padding_height;
+                  hend = std::min(hstart + ksize_height,
+                                  input_height + padding_height);
                   wstart = pw * stride_width - padding_width;
-                  wend = std::min(wstart + ksize_width, input_width);
+                  wend = std::min(wstart + ksize_width,
+                                  input_width + padding_width);
+
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                  dstart = std::max(dstart, 0);
+                  hstart = std::max(hstart, 0);
                   wstart = std::max(wstart, 0);
+                  dend = std::min(dend, input_depth);
+                  hend = std::min(hend, input_height);
+                  wend = std::min(wend, input_width);
                 }
+
                 int output_idx = (pd * output_height + ph) * output_width + pw;
                 T ele = pool_process.initial();
                 for (int d = dstart; d < dend; ++d) {
@@ -811,10 +865,10 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                     }
                   }
                 }
-                int pool_size =
-                    (exclusive || adaptive)
-                        ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                        : ksize_depth * ksize_height * ksize_width;
+                if (exclusive || adaptive) {
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                }
                 pool_process.finalize(static_cast<T>(pool_size), &ele);
                 output_data[output_idx] = ele;
               }
@@ -835,28 +889,38 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               dstart = AdaptStartIndex(pd, input_depth, output_depth);
               dend = AdaptEndIndex(pd, input_depth, output_depth);
-            } else {
-              dstart = pd * stride_depth - padding_depth;
-              dend = std::min(dstart + ksize_depth, input_depth);
-              dstart = std::max(dstart, 0);
             }
+
             for (int ph = 0; ph < output_height; ++ph) {
               if (adaptive) {
                 hstart = AdaptStartIndex(ph, input_height, output_height);
                 hend = AdaptEndIndex(ph, input_height, output_height);
-              } else {
-                hstart = ph * stride_height - padding_height;
-                hend = std::min(hstart + ksize_height, input_height);
-                hstart = std::max(hstart, 0);
               }
+
               for (int pw = 0; pw < output_width; ++pw) {
+                int pool_size = 1;
                 if (adaptive) {
                   wstart = AdaptStartIndex(pw, input_width, output_width);
                   wend = AdaptEndIndex(pw, input_width, output_width);
                 } else {
+                  dstart = pd * stride_depth - padding_depth;
+                  dend = std::min(dstart + ksize_depth,
+                                  input_depth + padding_depth);
+                  hstart = ph * stride_height - padding_height;
+                  hend = std::min(hstart + ksize_height,
+                                  input_height + padding_height);
                   wstart = pw * stride_width - padding_width;
-                  wend = std::min(wstart + ksize_width, input_width);
+                  wend = std::min(wstart + ksize_width,
+                                  input_width + padding_width);
+
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                  dstart = std::max(dstart, 0);
+                  hstart = std::max(hstart, 0);
                   wstart = std::max(wstart, 0);
+                  dend = std::min(dend, input_depth);
+                  hend = std::min(hend, input_height);
+                  wend = std::min(wend, input_width);
                 }
 
                 T ele = pool_process.initial();
@@ -871,10 +935,10 @@ class Pool3dFunctor<platform::CPUDeviceContext, PoolProcess, T> {
                     }
                   }
                 }
-                int pool_size =
-                    (exclusive || adaptive)
-                        ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                        : ksize_depth * ksize_height * ksize_width;
+                if (exclusive || adaptive) {
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                }
                 pool_process.finalize(static_cast<T>(pool_size), &ele);
                 int output_idx =
                     ((pd * output_height + ph) * output_width + pw) *
@@ -943,34 +1007,42 @@ class Pool3dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
           if (adaptive) {
             dstart = AdaptStartIndex(pd, input_depth, output_depth);
             dend = AdaptEndIndex(pd, input_depth, output_depth);
-          } else {
-            dstart = pd * stride_depth - padding_depth;
-            dend = std::min(dstart + ksize_depth, input_depth);
-            dstart = std::max(dstart, 0);
           }
+
           for (int ph = 0; ph < output_height; ++ph) {
             if (adaptive) {
               hstart = AdaptStartIndex(ph, input_height, output_height);
               hend = AdaptEndIndex(ph, input_height, output_height);
-            } else {
-              hstart = ph * stride_height - padding_height;
-              hend = std::min(hstart + ksize_height, input_height);
-              hstart = std::max(hstart, 0);
             }
+
             for (int pw = 0; pw < output_width; ++pw) {
+              int pool_size = 1;
               if (adaptive) {
                 wstart = AdaptStartIndex(pw, input_width, output_width);
                 wend = AdaptEndIndex(pw, input_width, output_width);
               } else {
+                dstart = pd * stride_depth - padding_depth;
+                dend =
+                    std::min(dstart + ksize_depth, input_depth + padding_depth);
+                hstart = ph * stride_height - padding_height;
+                hend = std::min(hstart + ksize_height,
+                                input_height + padding_height);
                 wstart = pw * stride_width - padding_width;
-                wend = std::min(wstart + ksize_width, input_width);
+                wend =
+                    std::min(wstart + ksize_width, input_width + padding_width);
+
+                pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart);
+                dstart = std::max(dstart, 0);
+                hstart = std::max(hstart, 0);
                 wstart = std::max(wstart, 0);
+                dend = std::min(dend, input_depth);
+                hend = std::min(hend, input_height);
+                wend = std::min(wend, input_width);
               }
 
-              int pool_size =
-                  (exclusive || adaptive)
-                      ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                      : ksize_depth * ksize_height * ksize_width;
+              if (exclusive || adaptive) {
+                pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart);
+              }
               float scale = 1.0 / pool_size;
               for (int d = dstart; d < dend; ++d) {
                 for (int h = hstart; h < hend; ++h) {
@@ -1046,34 +1118,44 @@ class Pool3dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               dstart = AdaptStartIndex(pd, input_depth, output_depth);
               dend = AdaptEndIndex(pd, input_depth, output_depth);
-            } else {
-              dstart = pd * stride_depth - padding_depth;
-              dend = std::min(dstart + ksize_depth, input_depth);
-              dstart = std::max(dstart, 0);
             }
+
             for (int ph = 0; ph < output_height; ++ph) {
               if (adaptive) {
                 hstart = AdaptStartIndex(ph, input_height, output_height);
                 hend = AdaptEndIndex(ph, input_height, output_height);
-              } else {
-                hstart = ph * stride_height - padding_height;
-                hend = std::min(hstart + ksize_height, input_height);
-                hstart = std::max(hstart, 0);
               }
+
               for (int pw = 0; pw < output_width; ++pw) {
+                int pool_size = 1;
                 if (adaptive) {
                   wstart = AdaptStartIndex(pw, input_width, output_width);
                   wend = AdaptEndIndex(pw, input_width, output_width);
                 } else {
+                  dstart = pd * stride_depth - padding_depth;
+                  dend = std::min(dstart + ksize_depth,
+                                  input_depth + padding_depth);
+                  hstart = ph * stride_height - padding_height;
+                  hend = std::min(hstart + ksize_height,
+                                  input_height + padding_height);
                   wstart = pw * stride_width - padding_width;
-                  wend = std::min(wstart + ksize_width, input_width);
+                  wend = std::min(wstart + ksize_width,
+                                  input_width + padding_width);
+
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                  dstart = std::max(dstart, 0);
+                  hstart = std::max(hstart, 0);
                   wstart = std::max(wstart, 0);
+                  dend = std::min(dend, input_depth);
+                  hend = std::min(hend, input_height);
+                  wend = std::min(wend, input_width);
                 }
 
-                int pool_size =
-                    (exclusive || adaptive)
-                        ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                        : ksize_depth * ksize_height * ksize_width;
+                if (exclusive || adaptive) {
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                }
                 float scale = 1.0 / pool_size;
                 for (int d = dstart; d < dend; ++d) {
                   for (int h = hstart; h < hend; ++h) {
@@ -1108,34 +1190,44 @@ class Pool3dGradFunctor<platform::CPUDeviceContext, PoolProcess, T> {
             if (adaptive) {
               dstart = AdaptStartIndex(pd, input_depth, output_depth);
               dend = AdaptEndIndex(pd, input_depth, output_depth);
-            } else {
-              dstart = pd * stride_depth - padding_depth;
-              dend = std::min(dstart + ksize_depth, input_depth);
-              dstart = std::max(dstart, 0);
             }
+
             for (int ph = 0; ph < output_height; ++ph) {
               if (adaptive) {
                 hstart = AdaptStartIndex(ph, input_height, output_height);
                 hend = AdaptEndIndex(ph, input_height, output_height);
-              } else {
-                hstart = ph * stride_height - padding_height;
-                hend = std::min(hstart + ksize_height, input_height);
-                hstart = std::max(hstart, 0);
               }
+
               for (int pw = 0; pw < output_width; ++pw) {
+                int pool_size = 1;
                 if (adaptive) {
                   wstart = AdaptStartIndex(pw, input_width, output_width);
                   wend = AdaptEndIndex(pw, input_width, output_width);
                 } else {
+                  dstart = pd * stride_depth - padding_depth;
+                  dend = std::min(dstart + ksize_depth,
+                                  input_depth + padding_depth);
+                  hstart = ph * stride_height - padding_height;
+                  hend = std::min(hstart + ksize_height,
+                                  input_height + padding_height);
                   wstart = pw * stride_width - padding_width;
-                  wend = std::min(wstart + ksize_width, input_width);
+                  wend = std::min(wstart + ksize_width,
+                                  input_width + padding_width);
+
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                  dstart = std::max(dstart, 0);
+                  hstart = std::max(hstart, 0);
                   wstart = std::max(wstart, 0);
+                  dend = std::min(dend, input_depth);
+                  hend = std::min(hend, input_height);
+                  wend = std::min(wend, input_width);
                 }
 
-                int pool_size =
-                    (exclusive || adaptive)
-                        ? (dend - dstart) * (hend - hstart) * (wend - wstart)
-                        : ksize_depth * ksize_height * ksize_width;
+                if (exclusive || adaptive) {
+                  pool_size =
+                      (dend - dstart) * (hend - hstart) * (wend - wstart);
+                }
                 float scale = 1.0 / pool_size;
                 for (int d = dstart; d < dend; ++d) {
                   for (int h = hstart; h < hend; ++h) {
diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py
index a12a328b653b26dac31b11839d02e867a012c4bc..5e8828c3e91269db64298c71c7e5173db70314b9 100644
--- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py
+++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py
@@ -102,14 +102,21 @@ def avg_pool2D_forward_naive(x,
                 c_start = adaptive_start_index(j, W, ksize[1])
                 c_end = adaptive_end_index(j, W, ksize[1])
             else:
-                r_start = np.max((i * strides[0] - paddings[0], 0))
-                r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H))
-                c_start = np.max((j * strides[1] - paddings[1], 0))
-                c_end = np.min((j * strides[1] + ksize[1] - paddings[1], W))
+                r_start = i * strides[0] - paddings[0]
+                r_end = i * strides[0] + ksize[0] - paddings[0]
+                c_start = j * strides[1] - paddings[1]
+                c_end = j * strides[1] + ksize[1] - paddings[1]
+                field_size = (r_end - r_start) * (c_end - c_start)
+                r_start = np.max((r_start, 0))
+                r_end = np.min((r_end, H))
+                c_start = np.max((c_start, 0))
+                c_end = np.min((c_end, W))
+
             x_masked = x[:, :, r_start:r_end, c_start:c_end]
 
-            field_size = ((r_end - r_start) * (c_end - c_start)) \
-                if (exclusive or adaptive) else (ksize[0] * ksize[1])
+            if (exclusive or adaptive):
+                field_size = (r_end - r_start) * (c_end - c_start)
+
             if data_type == np.int8 or data_type == np.uint8:
                 out[:, :, i, j] = (np.rint(
                     np.sum(x_masked, axis=(2, 3)) /
@@ -207,22 +214,34 @@ def pool2D_forward_naive(x,
                 in_w_start = adaptive_start_index(j, W, ksize[1])
                 in_w_end = adaptive_end_index(j, W, ksize[1])
             else:
-                in_w_start = np.max((j * strides[1] - pad_w_left, 0))
-                in_w_end = np.min((j * strides[1] + ksize[1] - pad_w_left, W))
+                in_h_start = i * strides[0] - pad_h_up
+                in_w_start = j * strides[1] - pad_w_left
+                in_h_end = i * strides[0] + ksize[0] - pad_h_up
+                in_w_end = j * strides[1] + ksize[1] - pad_w_left
+
+                field_size = (in_h_end - in_h_start) * (in_w_end - in_w_start)
+                in_h_start = np.max((in_h_start, 0))
+                in_w_start = np.max((in_w_start, 0))
+                in_h_end = np.min((in_h_end, H))
+                in_w_end = np.min((in_w_end, W))
 
             if data_format == 'NCHW':
                 x_masked = x[:, :, in_h_start:in_h_end, in_w_start:in_w_end]
                 if pool_type == 'avg':
-                    field_size = ((in_h_end - in_h_start) * (in_w_end - in_w_start)) \
-                        if (exclusive or adaptive) else (ksize[0] * ksize[1])
+                    if (exclusive or adaptive):
+                        field_size = (in_h_end - in_h_start) * (
+                            in_w_end - in_w_start)
+
+#                         if (exclusive or adaptive) else (ksize[0] * ksize[1])
                     out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / field_size
                 elif pool_type == 'max':
                     out[:, :, i, j] = np.max(x_masked, axis=(2, 3))
             elif data_format == 'NHWC':
                 x_masked = x[:, in_h_start:in_h_end, in_w_start:in_w_end, :]
                 if pool_type == 'avg':
-                    field_size = ((in_h_end - in_h_start) * (in_w_end - in_w_start)) \
-                        if (exclusive or adaptive) else (ksize[0] * ksize[1])
+                    if (exclusive or adaptive):
+                        field_size = (in_h_end - in_h_start) * (
+                            in_w_end - in_w_start)
                     out[:, i, j, :] = np.sum(x_masked, axis=(1, 2)) / field_size
                 elif pool_type == 'max':
                     out[:, i, j, :] = np.max(x_masked, axis=(1, 2))
diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py
index 3d139e9b90c10e352599d506fb3ca837472348f5..eab7126c7a42235ce6ece7c6020f7049c9776d66 100644
--- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py
+++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py
@@ -116,32 +116,44 @@ def pool3D_forward_naive(x,
         if adaptive:
             d_start = adaptive_start_index(k, D, ksize[0])
             d_end = adaptive_end_index(k, D, ksize[0])
-        else:
-            d_start = np.max((k * strides[0] - pad_d_forth, 0))
-            d_end = np.min((k * strides[0] + ksize[0] - pad_d_forth, D))
 
         for i in range(H_out):
             if adaptive:
                 h_start = adaptive_start_index(i, H, ksize[1])
                 h_end = adaptive_end_index(i, H, ksize[1])
-            else:
-                h_start = np.max((i * strides[1] - pad_h_up, 0))
-                h_end = np.min((i * strides[1] + ksize[1] - pad_h_up, H))
 
             for j in range(W_out):
                 if adaptive:
                     w_start = adaptive_start_index(j, W, ksize[2])
                     w_end = adaptive_end_index(j, W, ksize[2])
                 else:
-                    w_start = np.max((j * strides[2] - pad_w_left, 0))
-                    w_end = np.min((j * strides[2] + ksize[2] - pad_w_left, W))
 
+                    d_start = k * strides[0] - pad_d_forth
+                    d_end = np.min((k * strides[0] + ksize[0] - pad_d_forth,
+                                    D + pad_d_back))
+                    h_start = i * strides[1] - pad_h_up
+                    h_end = np.min(
+                        (i * strides[1] + ksize[1] - pad_h_up, H + pad_h_down))
+                    w_start = j * strides[2] - pad_w_left
+                    w_end = np.min((j * strides[2] + ksize[2] - pad_w_left,
+                                    W + pad_w_right))
+
+                    field_size = (d_end - d_start) * (h_end - h_start) * (
+                        w_end - w_start)
+                    w_start = np.max((w_start, 0))
+                    d_start = np.max((d_start, 0))
+                    h_start = np.max((h_start, 0))
+                    w_end = np.min((w_end, W))
+                    d_end = np.min((d_end, D))
+                    h_end = np.min((h_end, H))
                 if data_format == 'NCDHW':
                     x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:
                                  w_end]
                     if pool_type == 'avg':
-                        field_size = (d_end - d_start) * (h_end - h_start) * (w_end - w_start) \
-                            if (exclusive or adaptive) else ksize[0] * ksize[1] * ksize[2]
+                        if (exclusive or adaptive):
+                            field_size = (d_end - d_start) * (
+                                h_end - h_start) * (w_end - w_start)
+
                         out[:, :, k, i, j] = np.sum(x_masked,
                                                     axis=(2, 3, 4)) / field_size
                     elif pool_type == 'max':
@@ -151,8 +163,10 @@ def pool3D_forward_naive(x,
                     x_masked = x[:, d_start:d_end, h_start:h_end, w_start:
                                  w_end, :]
                     if pool_type == 'avg':
-                        field_size = (d_end - d_start) * (h_end - h_start) * (w_end - w_start) \
-                            if (exclusive or adaptive) else ksize[0] * ksize[1] * ksize[2]
+                        if (exclusive or adaptive):
+                            field_size = (d_end - d_start) * (
+                                h_end - h_start) * (w_end - w_start)
+
                         out[:, k, i, j, :] = np.sum(x_masked,
                                                     axis=(1, 2, 3)) / field_size
                     elif pool_type == 'max':
@@ -564,7 +578,7 @@ class TestAvgInclude_AsyPadding(TestCase2):
         self.exclusive = False
 
     def init_paddings(self):
-        self.paddings = [1, 2, 1, 1, 1, 0]
+        self.paddings = [2, 2, 1, 1, 0, 0]
 
 
 @unittest.skipIf(not core.is_compiled_with_cuda(),