diff --git a/Dockerfile b/Dockerfile
index 9ac58f37f2893613ca9f82be08136d9da674737e..c257dbfc2987323f8ed2a24dfffa8b3c15e09399 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,7 +49,11 @@ ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
 RUN curl -s -q https://glide.sh/get | sh
 
 # Install TensorRT
-# The unnecessary files has been removed to make the library small. It only contains include and lib now.
+# following TensorRT.tar.gz is not the default official one, we do two miny changes:
+# 1. Remove the unnecessary files to make the library small. TensorRT.tar.gz only contains include and lib now,
+#    and its size is only one-third of the official one.
+# 2. Manually add ~IPluginFactory() in IPluginFactory class of NvInfer.h, otherwise, it couldn't work in paddle.
+#    See https://github.com/PaddlePaddle/Paddle/issues/10129 for details.
 RUN wget -qO- http://paddlepaddledeps.bj.bcebos.com/TensorRT-4.0.0.3.Ubuntu-16.04.4.x86_64-gnu.cuda-8.0.cudnn7.0.tar.gz | \
     tar -xz -C /usr/local && \
     cp -rf /usr/local/TensorRT/include /usr && \
diff --git a/cmake/tensorrt.cmake b/cmake/tensorrt.cmake
index 0c07d36bed65400164853b99f18ec0335341cd94..ac19b1651893f18b14c62a0986df75bed25d7e80 100644
--- a/cmake/tensorrt.cmake
+++ b/cmake/tensorrt.cmake
@@ -30,4 +30,6 @@ if(TENSORRT_FOUND)
 
     message(STATUS "Current TensorRT header is ${TENSORRT_INCLUDE_DIR}/NvInfer.h. "
         "Current TensorRT version is v${TENSORRT_MAJOR_VERSION}. ")
+    include_directories(${TENSORRT_INCLUDE_DIR})
+    list(APPEND EXTERNAL_LIBS ${TENSORRT_LIBRARY})
 endif()
diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt
index cc45bfe9b17d767be039cc0d8d83234b6994d6c1..50f635a41a99b2ae292d13afde5637a3bf4e6f8c 100644
--- a/paddle/fluid/inference/CMakeLists.txt
+++ b/paddle/fluid/inference/CMakeLists.txt
@@ -21,7 +21,8 @@ endif()
 
 if(WITH_TESTING)
   add_subdirectory(tests/book)
-  if (TENSORRT_FOUND)
-    add_subdirectory(tensorrt)
-  endif()
+endif()
+
+if (TENSORRT_FOUND)
+  add_subdirectory(tensorrt)
 endif()