diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
index 0d3b11f78c5ab4de4d42c9d03fcf3e9a855258bb..9ce4fd07829ba5fc7b73812f40dfb7dfdf2b1bc9 100644
--- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
+++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.cc
@@ -57,6 +57,7 @@ Allocation *AutoGrowthBestFitAllocator::AllocateImpl(size_t size) {
       block_it->is_free_ = false;
     }
   } else {
+    FreeIdleChunks();
     size_t realloc_size = std::max(size, chunk_size_);
 
     try {
@@ -119,6 +120,20 @@ void AutoGrowthBestFitAllocator::FreeImpl(Allocation *allocation) {
   delete allocation;
 }
 
+void AutoGrowthBestFitAllocator::FreeIdleChunks() {
+  for (auto chunk_it = chunks_.begin(); chunk_it != chunks_.end();) {
+    auto &blocks = chunk_it->blocks_;
+    if (blocks.size() == 1 && blocks.begin()->is_free_) {
+      auto &block = *blocks.begin();
+      VLOG(2) << "Free chunk with size " << block.size_;
+      free_blocks_.erase(std::make_pair(block.size_, block.ptr_));
+      chunk_it = chunks_.erase(chunk_it);
+    } else {
+      ++chunk_it;
+    }
+  }
+}
+
 }  // namespace allocation
 }  // namespace memory
 }  // namespace paddle
diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.h b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.h
index a31dd7cf7734afc709ec9aa58c2996f3f1d06633..27257883d558e7b3a09f8aeb5264d093e7d5c480 100644
--- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.h
+++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator.h
@@ -27,7 +27,7 @@ namespace allocation {
 
 class AutoGrowthBestFitAllocator : public Allocator {
  public:
-  explicit AutoGrowthBestFitAllocator(
+  AutoGrowthBestFitAllocator(
       const std::shared_ptr<Allocator> &underlying_allocator, size_t alignment,
       size_t chunk_size = 0);
 
@@ -39,6 +39,8 @@ class AutoGrowthBestFitAllocator : public Allocator {
   void FreeImpl(Allocation *allocation) override;
 
  private:
+  void FreeIdleChunks();
+
   template <typename T>
   using List = std::list<T>;