diff --git a/paddle/function/neon/NeonDepthwiseConv.cpp b/paddle/function/neon/NeonDepthwiseConv.cpp
index 85bc95bb88ca606e289fb6dad4946a77faf3d5fb..199e23c4cba8a5a552f7313199d99cc83ce57d43 100644
--- a/paddle/function/neon/NeonDepthwiseConv.cpp
+++ b/paddle/function/neon/NeonDepthwiseConv.cpp
@@ -66,18 +66,16 @@ class NeonDepthwiseConvFunction : public ConvFunctionBase {
     float* inputPadding = inputData;
     int padInputHeight = inputHeight + 2 * paddingH();
     int padInputWidth = inputWidth + 2 * paddingW();
-    if (paddingH() > 0 || paddingW() > 0) {
-      int newSize = batchSize * inputChannels * padInputHeight * padInputWidth;
-      resizeBuffer<Device>(newSize);
-      inputPadding = reinterpret_cast<float*>(memory_->getBuf());
-      neon::Padding<float>::run(inputData,
+    int newSize = batchSize * (inputChannels + 1) * padInputHeight * padInputWidth;
+    resizeBuffer<Device>(newSize);
+    inputPadding = reinterpret_cast<float*>(memory_->getBuf());
+    neon::Padding<float>::run(inputData,
                                 inputPadding,
                                 batchSize * inputChannels,
                                 inputHeight,
                                 inputWidth,
                                 padInputHeight,
                                 padInputWidth);
-    }
 
     std::function<void(
         const float*, const float*, int, int, int, int, int, int, float*)>