diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp
index ffbf366fa9f4b70ee4d5b580084eaff17fb580eb..3387cae3962baa1b10885be753d9ec92284b31f8 100644
--- a/paddle/function/GemmConvOp.cpp
+++ b/paddle/function/GemmConvOp.cpp
@@ -126,6 +126,11 @@ public:
       inputData += inputChannels * inputHeight * inputWidth;
       outputData += outputChannels * outputHeight * outputWidth;
     }
+#ifdef PADDLE_MOBILE_INFERENCE
+    if (Device == DEVICE_TYPE_CPU) {
+      delete memory_;
+    }
+#endif
   }
 };
 
@@ -233,11 +238,6 @@ public:
       inputGrad += inputChannels * inputHeight * inputWidth;
       outputGrad += outputChannels * outputHeight * outputWidth;
     }
-#ifdef PADDLE_MOBILE_INFERENCE
-    if (Device == DEVICE_TYPE_CPU) {
-      delete memory_;
-    }
-#endif
   }
 };