diff --git a/paddle/fluid/platform/device_context.h b/paddle/fluid/platform/device_context.h
index bdc2670dc20f07f0aed366686526aeba1d8066cf..77443c72be8e76e655cf801a0a01ea11fb1e315b 100644
--- a/paddle/fluid/platform/device_context.h
+++ b/paddle/fluid/platform/device_context.h
@@ -218,6 +218,8 @@ class CudnnWorkspaceHandle {
     if (required_workspace_bytes <= WorkspaceSize()) {
       return;
     }
+    // reset allocation first before re-allocate to save memory
+    allocation_.reset();
     allocation_ = memory::Alloc(device_context_, required_workspace_bytes);
   }