diff --git a/src/operators/math/im2col.cpp b/src/operators/math/im2col.cpp
index 73e554a395f4c0ca995e7a167b8a6ef989b28a4c..090ccdf24e214fc86b8a4032df228d50caa65ef9 100644
--- a/src/operators/math/im2col.cpp
+++ b/src/operators/math/im2col.cpp
@@ -73,7 +73,9 @@ class Im2ColFunctor<ColFormat::kCFO, CPU, T> {
     const int osize = col_height;
     const int isize = im_height;
     bool pad1 = padding[0] > 0;
-    bool pad2 = padding[1] > 0;
+    bool pad2 =
+        (pad1 && padding[1] &&
+         (((isize - 2 * padding[0] + filter_height) % stride[0] == 0) ? 1 : 0));
     int fill = isize % 2;
     if (stride[0] == 1 && filter_height == 3 && pad1 && pad2 &&
         dilation[0] == 1 && im_height > 2) {