diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc
index bd3bbf2108efc17df9d980c5a4e3b9c04a847ed8..785e5f2f757855b309716ac9df6d4d0f1b53cfdb 100644
--- a/paddle/fluid/imperative/tracer.cc
+++ b/paddle/fluid/imperative/tracer.cc
@@ -413,6 +413,7 @@ void Tracer::TraceOp(const std::string& type,
   std::map<phi::DenseTensor*, std::shared_ptr<phi::Allocation>>
       need_backup_inputs2holder;
   std::map<phi::DenseTensor*, phi::DDim> need_backup_inputs2strides;
+  std::map<phi::DenseTensor*, size_t> need_backup_inputs2offset;
   if (FLAGS_use_stride_kernel) {
     for (auto& iter : inplace_map) {
       auto inputs_iter = ins.find(iter.first);
@@ -431,6 +432,7 @@ void Tracer::TraceOp(const std::string& type,
                     ->GetMutable<phi::DenseTensor>();
             need_backup_inputs2holder[dense_tensor] = dense_tensor->Holder();
             need_backup_inputs2strides[dense_tensor] = dense_tensor->strides();
+            need_backup_inputs2offset[dense_tensor] = dense_tensor->offset();
           }
         }
       }
@@ -449,9 +451,11 @@ void Tracer::TraceOp(const std::string& type,
     for (auto& iter : need_backup_inputs2outputs) {
       iter.first->ResetHolder(need_backup_inputs2holder[iter.first]);
       iter.first->set_strides(need_backup_inputs2strides[iter.first]);
+      iter.first->set_offset(need_backup_inputs2offset[iter.first]);
       paddle::experimental::TransStrideLegacy(dev_ctx, iter.second, iter.first);
       iter.second->ResetHolder(need_backup_inputs2holder[iter.first]);
       iter.second->set_strides(need_backup_inputs2strides[iter.first]);
+      iter.second->set_offset(need_backup_inputs2offset[iter.first]);
     }
   } else {
     TraceOpImpl<egr::EagerVariable>(type,