diff --git a/paddle/fluid/inference/tensorrt/engine.cc b/paddle/fluid/inference/tensorrt/engine.cc index 2addff52829c8066ad0bc0c9d85fc0cf6b286dfc..cb815e00c4430b9b41da65aa10202df1e7a43857 100644 --- a/paddle/fluid/inference/tensorrt/engine.cc +++ b/paddle/fluid/inference/tensorrt/engine.cc @@ -54,6 +54,7 @@ void TensorRTEngine::Execute(int batch_size, std::vector *buffers, } else { #if IS_TRT_VERSION_GE(6000) infer_context->enqueueV2(buffers->data(), stream, nullptr); + GetEngineInfo(); #endif } SetRuntimeBatch(batch_size); @@ -237,6 +238,11 @@ void TensorRTEngine::FreezeNetwork() { #endif } +#if IS_TRT_VERSION_GE(8200) + infer_builder_config_->setProfilingVerbosity( + nvinfer1::ProfilingVerbosity::kDETAILED); +#endif + #if IS_TRT_VERSION_LT(8000) infer_engine_.reset(infer_builder_->buildEngineWithConfig( *network(), *infer_builder_config_)); @@ -253,6 +259,8 @@ void TensorRTEngine::FreezeNetwork() { infer_engine_, platform::errors::Fatal( "Build TensorRT cuda engine failed! Please recheck " "you configurations related to paddle-TensorRT.")); + + GetEngineInfo(); } nvinfer1::ITensor *TensorRTEngine::DeclareInput(const std::string &name, diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h index 9397d4e89de42360faf823c63c052ac0799e4bcd..e6f58c8c8e8f4ef6380cc862cf917678e271232e 100644 --- a/paddle/fluid/inference/tensorrt/engine.h +++ b/paddle/fluid/inference/tensorrt/engine.h @@ -321,6 +321,8 @@ class TensorRTEngine { "on the same GPU architecture;\n2. The Paddle Inference version of " "generating serialization file and doing inference are " "consistent.")); + + GetEngineInfo(); } void SetRuntimeBatch(size_t batch_size); @@ -538,6 +540,18 @@ class TensorRTEngine { } } + void GetEngineInfo() { +#if IS_TRT_VERSION_GE(8200) + std::unique_ptr infer_inspector( + infer_engine_->createEngineInspector()); + infer_inspector->setExecutionContext(context()); + VLOG(3) << infer_inspector->getEngineInformation( + nvinfer1::LayerInformationFormat::kJSON); +#else + VLOG(3) << "Inspector needs TensorRT version 8.2 and after."; +#endif + } + private: // Each ICudaEngine object is bound to a specific GPU when it is instantiated, // ensure that the thread is associated with the correct device by calling