diff --git a/paddle/math/RowBuffer.h b/paddle/math/RowBuffer.h
index dbb829c4e24a659e4a97c0a3ba4c5c78b68815d3..9ef5b89680b00981188d78cb312dc75e2c0a79ee 100644
--- a/paddle/math/RowBuffer.h
+++ b/paddle/math/RowBuffer.h
@@ -99,7 +99,11 @@ public:
   /**
    * @brief clear local buffer. It only affect auto-growth buffer.
    */
-  inline void clear() { rowStore_.clear(); }
+  inline void clear() {
+    // swap an empty vector to it to free the memory.
+    std::vector<real, AlignedAllocator<real, 32>> empty;
+    rowStore_.swap(empty);
+  }
 
   /**
    * @brief get current number of rows.