From 8552a182b185e8e3b89a077250798053315581b4 Mon Sep 17 00:00:00 2001 From: Pei Yang Date: Wed, 14 Apr 2021 15:12:21 +0800 Subject: [PATCH] [Paddle-TRT] Add check for TRT runtime dynamic shape (#32155) * add check for runtime dynamic shape * add unittest * add lower bound case * adjust timeout of new ut to 120s --- .../operators/tensorrt/tensorrt_engine_op.h | 64 +++++++++++++ .../unittests/ir/inference/CMakeLists.txt | 1 + .../ir/inference/test_trt_dynamic_shape.py | 92 +++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py diff --git a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h index 1f0ae40798..1f3029d94b 100644 --- a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h +++ b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h @@ -76,6 +76,54 @@ static void RuntimeStaticShapeCheck(std::vector runtime_input_shape, model_input_shape_str, runtime_input_shape_str)); } +static void RuntimeDynamicShapeCheck( + const std::string &x, const std::vector &runtime_input_shape, + const std::vector &min_input_shape, + const std::vector &max_input_shape) { + PADDLE_ENFORCE_EQ(runtime_input_shape.size(), min_input_shape.size(), + platform::errors::InvalidArgument( + "TRT engine runtime input dims size(%d) inconsistent " + "with the dynamic shape size(%d)", + runtime_input_shape.size(), min_input_shape.size())); + auto is_input_shape_valid = [&]( + const std::vector &runtime_input_shape, + const std::vector &min_input_shape, + const std::vector &max_input_shape) -> bool { + for (size_t i = 0; i < runtime_input_shape.size(); i++) { + if (runtime_input_shape[i] <= max_input_shape[i] && + runtime_input_shape[i] >= min_input_shape[i]) { + continue; + } else { + return false; + } + } + return true; + }; + auto comma_fold = [](std::string a, int b) { + return std::move(a) + ", " + std::to_string(b); + }; + std::string runtime_input_shape_str = std::accumulate( + std::next(runtime_input_shape.begin()), runtime_input_shape.end(), + std::to_string(runtime_input_shape[0]), comma_fold); + std::string min_input_shape_str = + std::accumulate(std::next(min_input_shape.begin()), min_input_shape.end(), + std::to_string(min_input_shape[0]), comma_fold); + std::string max_input_shape_str = + std::accumulate(std::next(max_input_shape.begin()), max_input_shape.end(), + std::to_string(max_input_shape[0]), comma_fold); + PADDLE_ENFORCE_EQ(is_input_shape_valid(runtime_input_shape, min_input_shape, + max_input_shape), + true, + platform::errors::InvalidArgument( + "TRT runtime input shape of %s is invalid. Expect " + "runtime input shape to be within min/max input shape " + "configured in SetTRTDynamicShapeInfo()," + "but got runtime input shape = [%s], min input shape = " + "[%s], max input shape = [%s].", + x, runtime_input_shape_str, min_input_shape_str, + max_input_shape_str)); +} + class TensorRTEngineOp : public framework::OperatorBase { private: std::vector input_names_; @@ -272,6 +320,22 @@ class TensorRTEngineOp : public framework::OperatorBase { } } else { #if IS_TRT_VERSION_GE(6000) + std::map> min_input_shape = + engine->min_input_shape(); + std::map> max_input_shape = + engine->max_input_shape(); + PADDLE_ENFORCE_EQ( + min_input_shape.count(x), true, + platform::errors::InvalidArgument( + "Input %s not found in TRT engine min_input_shape.", x)); + PADDLE_ENFORCE_EQ( + max_input_shape.count(x), true, + platform::errors::InvalidArgument( + "Input %s not found in TRT engine max_input_shape.", x)); + auto x_min_input_shape = min_input_shape[x]; + auto x_max_input_shape = max_input_shape[x]; + RuntimeDynamicShapeCheck(x, t_shape, x_min_input_shape, + x_max_input_shape); auto *trt_context = engine->context(); trt_context->setBindingDimensions( bind_index, inference::tensorrt::Vec2TRT_Dims(t_shape, x, true)); diff --git a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt index 3ebed01777..8e4c091cd0 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/ir/inference/CMakeLists.txt @@ -33,4 +33,5 @@ set_tests_properties(test_trt_subgraph_pass PROPERTIES TIMEOUT 120) set_tests_properties(test_trt_activation_pass PROPERTIES TIMEOUT 120) set_tests_properties(test_trt_conv_pass PROPERTIES TIMEOUT 120) set_tests_properties(test_trt_multiclass_nms_op PROPERTIES TIMEOUT 200) +set_tests_properties(test_trt_dynamic_shape PROPERTIES TIMEOUT 120) endif() diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py new file mode 100644 index 0000000000..fd69a8bf6c --- /dev/null +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_dynamic_shape.py @@ -0,0 +1,92 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from inference_pass_test import InferencePassTest +import paddle.fluid as fluid +import paddle.fluid.core as core +from paddle.fluid.core import PassVersionChecker +from paddle.fluid.core import AnalysisConfig + + +class TRTDynamicShapeTest(InferencePassTest): + def setUp(self): + with fluid.program_guard(self.main_program, self.startup_program): + data = fluid.data( + name="data", shape=[-1, 3, 16, 16], dtype="float32") + out = fluid.layers.conv2d( + input=data, + num_filters=3, + filter_size=3, + groups=1, + padding=[1, 1], + bias_attr=False, + act=None) + + self.feeds = self.set_feeds() + self.enable_trt = True + self.trt_parameters = TRTDynamicShapeTest.TensorRTParam( + 1 << 30, 1, 1, AnalysisConfig.Precision.Float32, False, False) + self.dynamic_shape_params = TRTDynamicShapeTest.DynamicShapeParam({ + 'data': [1, 3, 8, 8] + }, {'data': [1, 3, 32, 32]}, {'data': [1, 3, 16, 16]}, False) + self.fetch_list = [out] + + def set_feeds(self): + return {"data": np.random.random([1, 3, 16, 16]).astype("float32"), } + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + self.check_output_with_option(use_gpu) + + +class TRTDynamicShapeOutOfBound1Test(TRTDynamicShapeTest): + def set_feeds(self): + return {"data": np.random.random([1, 3, 64, 16]).astype("float32"), } + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + with self.assertRaises(Exception): + self.check_output_with_option(use_gpu) + + +class TRTDynamicShapeOutOfBound2Test(TRTDynamicShapeTest): + def set_feeds(self): + return {"data": np.random.random([2, 3, 16, 16]).astype("float32"), } + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + with self.assertRaises(Exception): + self.check_output_with_option(use_gpu) + + +class TRTDynamicShapeOutOfBound3Test(TRTDynamicShapeTest): + def set_feeds(self): + return {"data": np.random.random([1, 3, 4, 16]).astype("float32"), } + + def test_check_output(self): + if core.is_compiled_with_cuda(): + use_gpu = True + with self.assertRaises(Exception): + self.check_output_with_option(use_gpu) + + +if __name__ == "__main__": + unittest.main() -- GitLab