diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h
index 81e4ca89805ab274bee378fee072f3d946408a97..c1fa94ef423f9a088ba4ae6ac6170731f6678785 100644
--- a/paddle/fluid/inference/tensorrt/engine.h
+++ b/paddle/fluid/inference/tensorrt/engine.h
@@ -852,6 +852,15 @@ class TRTEngineManager {
   size_t max_ctx_mem_size_{0};
   std::unordered_map<PredictorID, AllocationPtr> context_memorys_;
   std::unordered_map<std::string, std::unique_ptr<TensorRTEngine>> engines_;
+  // createInferBuilder loads trt kernels and take a few second
+  // But as long as one IBuilder lives, trt kernel will not be unloaded
+  // Hence, a persistent IBuilder to avoid TensorRT unload/reload kernels
+  std::unique_ptr<nvinfer1::IBuilder, std::function<void(nvinfer1::IBuilder*)>>
+      holder{createInferBuilder(&NaiveLogger::Global()), [](auto* ptr) {
+               if (ptr) {
+                 ptr->destroy();
+               }
+             }};
 };
 
 }  // namespace tensorrt