Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wmsofts
Paddle
提交
fcd6d675
P
Paddle
项目概览
wmsofts
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
fcd6d675
编写于
1月 05, 2023
作者:
X
xiaoxiaohehe001
提交者:
GitHub
1月 05, 2023
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[Paddle Inference] Add ci flags for a persistent IBuilder. (#49538)
上级
66d516ff
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
38 addition
and
19 deletion
+38
-19
paddle/fluid/inference/tensorrt/engine.h
paddle/fluid/inference/tensorrt/engine.h
+12
-19
paddle/fluid/inference/tensorrt/helper.h
paddle/fluid/inference/tensorrt/helper.h
+11
-0
paddle/phi/core/flags.cc
paddle/phi/core/flags.cc
+13
-0
paddle/scripts/paddle_build.sh
paddle/scripts/paddle_build.sh
+2
-0
未找到文件。
paddle/fluid/inference/tensorrt/engine.h
浏览文件 @
fcd6d675
...
...
@@ -42,6 +42,8 @@ limitations under the License. */
#include "paddle/phi/core/stream.h"
#include "paddle/utils/any.h"
DECLARE_bool
(
trt_ibuilder_cache
);
namespace
paddle
{
namespace
inference
{
namespace
tensorrt
{
...
...
@@ -679,16 +681,6 @@ class TensorRTEngine {
std
::
vector
<
std
::
unique_ptr
<
nvinfer1
::
IPluginV2IOExt
>>
owned_plugin_v2ioext_
;
// TensorRT related internal members
template
<
typename
T
>
struct
Destroyer
{
void
operator
()(
T
*
x
)
{
if
(
x
)
{
x
->
destroy
();
}
}
};
template
<
typename
T
>
using
infer_ptr
=
std
::
unique_ptr
<
T
,
Destroyer
<
T
>>
;
infer_ptr
<
nvinfer1
::
IBuilder
>
infer_builder_
;
infer_ptr
<
nvinfer1
::
INetworkDefinition
>
infer_network_
;
infer_ptr
<
nvinfer1
::
ICudaEngine
>
infer_engine_
;
...
...
@@ -733,6 +725,15 @@ class TRTEngineManager {
using
AllocationPtr
=
phi
::
Allocator
::
AllocationPtr
;
public:
TRTEngineManager
()
{
// createInferBuilder loads trt kernels and take a few second
// But as long as one IBuilder lives, trt kernel will not be unloaded
// Hence, a persistent IBuilder to avoid TensorRT unload/reload kernels
if
(
FLAGS_trt_ibuilder_cache
)
{
holder_
.
reset
(
createInferBuilder
(
&
NaiveLogger
::
Global
()));
}
}
bool
Empty
()
const
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
return
engines_
.
size
()
==
0
;
...
...
@@ -855,15 +856,7 @@ class TRTEngineManager {
size_t
max_ctx_mem_size_
{
0
};
std
::
unordered_map
<
PredictorID
,
AllocationPtr
>
context_memorys_
;
std
::
unordered_map
<
std
::
string
,
std
::
unique_ptr
<
TensorRTEngine
>>
engines_
;
// createInferBuilder loads trt kernels and take a few second
// But as long as one IBuilder lives, trt kernel will not be unloaded
// Hence, a persistent IBuilder to avoid TensorRT unload/reload kernels
std
::
unique_ptr
<
nvinfer1
::
IBuilder
,
std
::
function
<
void
(
nvinfer1
::
IBuilder
*
)
>>
holder
{
createInferBuilder
(
&
NaiveLogger
::
Global
()),
[](
auto
*
ptr
)
{
if
(
ptr
)
{
ptr
->
destroy
();
}
}};
infer_ptr
<
nvinfer1
::
IBuilder
>
holder_
;
};
}
// namespace tensorrt
...
...
paddle/fluid/inference/tensorrt/helper.h
浏览文件 @
fcd6d675
...
...
@@ -95,6 +95,17 @@ static std::tuple<int, int, int> GetTrtCompileVersion() {
NV_TENSORRT_MAJOR
,
NV_TENSORRT_MINOR
,
NV_TENSORRT_PATCH
};
}
template
<
typename
T
>
struct
Destroyer
{
void
operator
()(
T
*
x
)
{
if
(
x
)
{
x
->
destroy
();
}
}
};
template
<
typename
T
>
using
infer_ptr
=
std
::
unique_ptr
<
T
,
Destroyer
<
T
>>
;
// A logger for create TensorRT infer builder.
class
NaiveLogger
:
public
nvinfer1
::
ILogger
{
public:
...
...
paddle/phi/core/flags.cc
浏览文件 @
fcd6d675
...
...
@@ -1165,3 +1165,16 @@ PADDLE_DEFINE_EXPORTED_bool(enable_cudnn_frontend, false, "");
*/
PADDLE_DEFINE_EXPORTED_int32
(
cudnn_cache_saturation_count
,
1
,
""
);
#endif // PADDLE_WITH_CUDNN_FRONTEND
/**
* CI related FLAG
* Name: trt_ibuilder_cache
* Since Version: 2.5.0
* Value Range: bool, default=false
* Example:
* Note: This FLAG is only enabled when CI is running. If True, a persistent
* IBuilder is added to avoid TensorRT unload/reload kernels.
*/
PADDLE_DEFINE_EXPORTED_bool
(
trt_ibuilder_cache
,
false
,
"Add a persistent ibuilder."
);
paddle/scripts/paddle_build.sh
浏览文件 @
fcd6d675
...
...
@@ -1403,6 +1403,7 @@ EOF
set
-x
# set trt_convert ut to run 15% cases.
export
TEST_NUM_PERCENT_CASES
=
0.15
export
FLAGS_trt_ibuilder_cache
=
1
precison_cases
=
""
bash
$PADDLE_ROOT
/tools/check_added_ut.sh
if
[
${
PRECISION_TEST
:-
OFF
}
==
"ON"
]
;
then
...
...
@@ -2547,6 +2548,7 @@ EOF
set
-x
# set trt_convert ut to run 15% cases.
export
TEST_NUM_PERCENT_CASES
=
0.15
export
FLAGS_trt_ibuilder_cache
=
1
precison_cases
=
""
bash
$PADDLE_ROOT
/tools/check_added_ut.sh
#check change of pr_unnitests and dev_unnitests
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录