diff --git a/paddle/fluid/framework/generator.cc b/paddle/fluid/framework/generator.cc
index 737dbafb64cb21ec0a1b82151fda7d9d5d5980b2..4b64722a7abf5ab83950cd057bd08a8409314d09 100644
--- a/paddle/fluid/framework/generator.cc
+++ b/paddle/fluid/framework/generator.cc
@@ -156,17 +156,15 @@ uint64_t Generator::Random64() {
 
 std::pair<uint64_t, uint64_t> Generator::IncrementOffset(
     uint64_t increament_offset) {
-  uint64_t cur_offset = this->state_.thread_offset;
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
   std::lock_guard<std::mutex> lock(this->mu_);
-
+  uint64_t cur_offset = this->state_.thread_offset;
   this->state_.thread_offset += increament_offset;
-
+  return std::make_pair(this->state_.current_seed, cur_offset);
 #else
   PADDLE_THROW(platform::errors::PermissionDenied(
       "Increment Offset only support in CUDA place"));
 #endif
-  return std::make_pair(this->state_.current_seed, cur_offset);
 }
 
 void Generator::SetIsInitPy(bool is_init_py) {