diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt
index cf1b5c10bb5e50b23bfa6553a17af998e3bd4ab6..0a20856c0fd050b60cf45736db2a3dd3d5265d26 100644
--- a/paddle/fluid/framework/CMakeLists.txt
+++ b/paddle/fluid/framework/CMakeLists.txt
@@ -447,3 +447,5 @@ cc_library(paddle_framework DEPS ${FLUID_FRAMEWORK_MODULES})
 if(WITH_TESTING AND TEST selected_rows_test)
   set_tests_properties(selected_rows_test PROPERTIES TIMEOUT 120)
 endif()
+
+cc_test(scope_guard_test SRCS scope_guard_test.cc)
diff --git a/paddle/fluid/framework/scope_guard.h b/paddle/fluid/framework/scope_guard.h
new file mode 100644
index 0000000000000000000000000000000000000000..83387842e94ef1944fb0bbbbac5b27c27ed85d2d
--- /dev/null
+++ b/paddle/fluid/framework/scope_guard.h
@@ -0,0 +1,52 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <type_traits>
+#include <utility>
+#include "paddle/fluid/platform/macros.h"
+
+namespace paddle {
+namespace framework {
+
+template <typename ReleaseCallback>
+class ScopeGuard {
+ public:
+  explicit ScopeGuard(const ReleaseCallback &callback) : callback_(callback) {}
+
+  ~ScopeGuard() { callback_(); }
+
+ private:
+  DISABLE_COPY_AND_ASSIGN(ScopeGuard);
+
+ private:
+  ReleaseCallback callback_;
+};
+
+// Two macros are needed here.
+// See:
+// https://stackoverflow.com/questions/10379691/creating-macro-using-line-for-different-variable-names
+#define _PADDLE_CONCAT_TOKEN(x, y) x##y
+#define PADDLE_CONCAT_TOKEN(x, y) _PADDLE_CONCAT_TOKEN(x, y)
+
+#define DEFINE_PADDLE_SCOPE_GUARD(...)                                     \
+  auto PADDLE_CONCAT_TOKEN(__scope_guard_func, __LINE__) = __VA_ARGS__;    \
+  ::paddle::framework::ScopeGuard<typename std::remove_reference<decltype( \
+      PADDLE_CONCAT_TOKEN(__scope_guard_func, __LINE__))>::type>           \
+      PADDLE_CONCAT_TOKEN(__scope_guard, __LINE__)(                        \
+          PADDLE_CONCAT_TOKEN(__scope_guard_func, __LINE__))
+
+}  // namespace framework
+}  // namespace paddle
diff --git a/paddle/fluid/framework/scope_guard_test.cc b/paddle/fluid/framework/scope_guard_test.cc
new file mode 100644
index 0000000000000000000000000000000000000000..d7a7a6168a3687af95803eb584246556eb30fa0b
--- /dev/null
+++ b/paddle/fluid/framework/scope_guard_test.cc
@@ -0,0 +1,37 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/framework/scope_guard.h"
+#include "gtest/gtest.h"
+
+namespace paddle {
+namespace framework {
+
+TEST(scope_guard, scope_guard_test) {
+  int n = 10;
+  {
+    DEFINE_PADDLE_SCOPE_GUARD([&n] { ++n; });
+  }
+  EXPECT_EQ(n, 11);
+  try {
+    DEFINE_PADDLE_SCOPE_GUARD([&] { --n; });
+    DEFINE_PADDLE_SCOPE_GUARD([&] { --n; });
+    throw std::runtime_error("any exception");
+  } catch (std::runtime_error &) {
+    EXPECT_EQ(n, 9);
+  }
+}
+
+}  // namespace framework
+}  // namespace paddle