diff --git a/paddle/fluid/inference/tensorrt/convert/tile_op.cc b/paddle/fluid/inference/tensorrt/convert/tile_op.cc index 9d013240d37fdb5bb94c537226099077fb59031b..ee84f61ac54fbacb1e9b4fbfbaaca541cd425093 100644 --- a/paddle/fluid/inference/tensorrt/convert/tile_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/tile_op.cc @@ -1,8 +1,11 @@ /* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -11,65 +14,117 @@ limitations under the License. */ #include "paddle/fluid/inference/tensorrt/convert/op_converter.h" -namespace paddle { -namespace framework { -class Scope; -namespace proto { -class OpDesc; -} // namespace proto -} // namespace framework -} // namespace paddle - namespace paddle { namespace inference { namespace tensorrt { -/* - * ReshapeOp - */ class TileOpConverter : public OpConverter { public: void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { #if IS_TRT_VERSION_GE(7000) - VLOG(4) << "convert a fluid tile op to tensorrt tile layer"; + VLOG(4) << "convert a tile op to tensorrt tile layer"; framework::OpDesc op_desc(op, nullptr); // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); - nvinfer1::Dims input_shape = input->getDimensions(); - std::vector repeat_times = - PADDLE_GET_CONST(std::vector, op_desc.GetAttr("repeat_times")); - - nvinfer1::Dims output_dim = input_shape; - nvinfer1::Dims output_stride; - // If input_dims.nbDims + 1 < repeat_times.size() means we - // should expand 1 on batchsize. trt doesn't support this behavior. - PADDLE_ENFORCE_GE(input_shape.nbDims + 1, - repeat_times.size(), - platform::errors::InvalidArgument( - "Can't change batchsize, please check repeat_times")); - int diff = input_shape.nbDims + 1 - repeat_times.size(); - if (diff > 0) repeat_times.insert(repeat_times.begin(), diff, 1); - - // Can't expand on batchsize - PADDLE_ENFORCE_EQ( - repeat_times[0], - 1, - platform::errors::InvalidArgument( - "Can't expand on batchsize, please check repeat_times")); - output_stride.nbDims = input_shape.nbDims; - for (int i = 0; i < input_shape.nbDims; i++) { - output_dim.d[i] = output_dim.d[i] * repeat_times[i + 1]; - output_stride.d[i] = 1; + auto inputs = op_desc.Inputs(); + auto input_shape = input->getDimensions(); + auto rank = input_shape.nbDims; + auto output_name = op_desc.Output("Out")[0]; + + if (engine_->with_dynamic_shape()) { + std::vector start(rank, 0); + std::vector stride(rank, 1); + auto start_tensor = + Add1DConstantLayer(start, output_name + "start_tensor"); + auto stride_tensor = + Add1DConstantLayer(stride, output_name + "stride_tensor"); + auto input_shape_tensor = Shape(input); + + nvinfer1::ITensor* repeat_tensor = nullptr; + int32_t repeat_rank = 0; + if (inputs.find("RepeatTimes") != inputs.end() && + op_desc.Input("RepeatTimes").size() >= 1) { + repeat_tensor = engine_->GetITensor(op_desc.Input("RepeatTimes")[0]); + repeat_rank = repeat_tensor->getDimensions().d[0]; + } else if (inputs.find("repeat_times_tensor") != inputs.end() && + op_desc.Input("repeat_times_tensor").size() >= 1) { + int32_t repeat_size = op_desc.Input("repeat_times_tensor").size(); + std::vector repeat_tensors; + for (int32_t i = 0; i < repeat_size; ++i) { + repeat_tensors.push_back( + engine_->GetITensor(op_desc.Input("repeat_times_tensor")[i])); + } + repeat_tensor = Concat(repeat_tensors); + repeat_rank = repeat_size; + } else { + std::vector repeat_times = PADDLE_GET_CONST( + std::vector, op_desc.GetAttr("repeat_times")); + repeat_tensor = + Add1DConstantLayer(repeat_times, output_name + "_shape_tensor_"); + repeat_rank = repeat_times.size(); + } + + nvinfer1::ITensor* repeat_expand_tensor; + if (rank > repeat_rank) { + auto* one_rank_tensor = + Add1DConstantLayer(std::vector(rank - repeat_rank, 1), + output_name + "_one_rank_tensor_"); + std::vector itensors; + itensors.push_back(one_rank_tensor); + itensors.push_back(repeat_tensor); + repeat_expand_tensor = Concat(itensors); + } else { + repeat_expand_tensor = repeat_tensor; + } + auto output_shape_tensor = Prod(input_shape_tensor, repeat_expand_tensor); + auto layer = TRT_ENGINE_ADD_LAYER(engine_, + Slice, + *input, + nvinfer1::Dims{}, + nvinfer1::Dims{}, + nvinfer1::Dims{}); + + layer->setInput(1, *start_tensor); + layer->setInput(2, *output_shape_tensor); + layer->setInput(3, *stride_tensor); + layer->setMode(nvinfer1::SliceMode::kWRAP); + RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); + + } else { + std::vector repeat_times = + PADDLE_GET_CONST(std::vector, op_desc.GetAttr("repeat_times")); + auto output_dim = input_shape; + auto output_stride = input_shape; + // If input_dims.nbDims + 1 < repeat_times.size() means we + // should expand 1 on batchsize. trt doesn't support this behavior. + PADDLE_ENFORCE_GE( + rank + 1, + repeat_times.size(), + platform::errors::InvalidArgument( + "Can't change batchsize, please check repeat_times")); + int32_t diff = rank + 1 - repeat_times.size(); + if (diff > 0) repeat_times.insert(repeat_times.begin(), diff, 1); + + // Can't expand on batchsize + PADDLE_ENFORCE_EQ( + repeat_times[0], + 1, + platform::errors::InvalidArgument( + "Can't expand on batchsize, please check repeat_times")); + output_stride.nbDims = rank; + for (int32_t i = 0; i < rank; i++) { + output_dim.d[i] = output_dim.d[i] * repeat_times[i + 1]; + output_stride.d[i] = 1; + } + auto layer = TRT_ENGINE_ADD_LAYER( + engine_, Slice, *input, input_shape, output_dim, output_stride); + layer->setMode(nvinfer1::SliceMode::kWRAP); + RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); } - auto* layer = TRT_ENGINE_ADD_LAYER( - engine_, Slice, *input, input_shape, output_dim, output_stride); - layer->setMode(nvinfer1::SliceMode::kWRAP); - auto output_name = op_desc.Output("Out")[0]; - RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode); #endif } }; diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 7c85b9b8a312764bba387ad33eb175f59e1a70ab..bdf343b140c6b6176ba6aad3d2f98f7547930719 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -2232,18 +2232,19 @@ struct SimpleOpTypeSetTeller : public Teller { if (op_type == "tile") { // Paddle-TRT does not support the input tensors. auto tile_inputs = desc.Inputs(); - if (tile_inputs.find("repeat_times_tensor") != tile_inputs.end()) { - if (desc.Input("repeat_times_tensor").size() >= 1) { - return false; + if (!with_dynamic_shape) { + if (tile_inputs.find("repeat_times_tensor") != tile_inputs.end()) { + if (desc.Input("repeat_times_tensor").size() >= 1) { + return false; + } } - } - if (tile_inputs.find("RepeatTimes") != tile_inputs.end()) { - if (desc.Input("RepeatTimes").size() >= 1) { - return false; + if (tile_inputs.find("RepeatTimes") != tile_inputs.end()) { + if (desc.Input("RepeatTimes").size() >= 1) { + return false; + } } + if (!desc.HasAttr("repeat_times")) return false; } - if (with_dynamic_shape) return false; - if (!with_dynamic_shape && !desc.HasAttr("repeat_times")) return false; } #endif diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py index 8898303670ebc2701018761eb3d49340ae9e3ba7..240fd65b83960686ee4b4ba8f7d8cc5b34d78c4d 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py @@ -70,7 +70,7 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): self, program_config ) -> (paddle_infer.Config, List[int], float): def generate_dynamic_shape(attrs): - self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]} + self.dynamic_shape.min_input_shape = {"input_data": [1, 2, 3, 4]} self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]} self.dynamic_shape.opt_input_shape = {"input_data": [1, 3, 64, 64]} @@ -82,10 +82,7 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): def generate_trt_nodes_num(attrs, dynamic_shape): ver = paddle_infer.get_trt_compile_version() if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 >= 7000: - if dynamic_shape: - return 0, 3 - else: - return 1, 2 + return 1, 2 else: return 0, 3 @@ -120,5 +117,187 @@ class TrtConvertTileTest(TrtLayerAutoScanTest): self.run_test(*args, **kwargs) +class TrtConvertTileTest2(TrtLayerAutoScanTest): + def is_program_valid(self, program_config: ProgramConfig) -> bool: + return True + + def sample_program_configs(self): + def generate_input1(attrs: List[Dict[str, Any]]): + return np.ones([1, 2, 3, 4]).astype(np.float32) + + dics = [{}] + dics_intput = [ + {"X": ["tile_input"], "RepeatTimes": ["repeat_times"]}, + ] + ops_config = [ + { + "op_type": "fill_constant", + "op_inputs": {}, + "op_outputs": {"Out": ["repeat_times"]}, + "op_attrs": { + "dtype": 2, + "str_value": "10", + "shape": [1], + }, + }, + { + "op_type": "tile", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["tile_out"]}, + "op_attrs": dics[0], + }, + ] + ops = self.generate_op_config(ops_config) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "tile_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) + }, + outputs=["tile_out"], + ) + + yield program_config + + def sample_predictor_configs( + self, program_config + ) -> (paddle_infer.Config, List[int], float): + def generate_dynamic_shape(attrs): + self.dynamic_shape.min_input_shape = {"tile_input": [1, 2, 3, 4]} + self.dynamic_shape.max_input_shape = {"tile_input": [4, 3, 64, 64]} + self.dynamic_shape.opt_input_shape = {"tile_input": [1, 2, 3, 4]} + + def clear_dynamic_shape(): + self.dynamic_shape.min_input_shape = {} + self.dynamic_shape.max_input_shape = {} + self.dynamic_shape.opt_input_shape = {} + + def generate_trt_nodes_num(attrs, dynamic_shape): + return 1, 2 + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + # for dynamic_shape + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), 1e-5 + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), 1e-3 + + def add_skip_trt_case(self): + pass + + def test(self): + self.add_skip_trt_case() + self.run_test() + + +class TrtConvertTileTest3(TrtLayerAutoScanTest): + def is_program_valid(self, program_config: ProgramConfig) -> bool: + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + return True + + def sample_program_configs(self): + def generate_input1(attrs: List[Dict[str, Any]]): + return np.ones([1, 2, 3, 4]).astype(np.float32) + + dics = [{}] + dics_intput = [ + { + "X": ["tile_input"], + "repeat_times_tensor": ["repeat_times1", "repeat_times2"], + }, + ] + ops_config = [ + { + "op_type": "fill_constant", + "op_inputs": {}, + "op_outputs": {"Out": ["repeat_times1"]}, + "op_attrs": { + "dtype": 2, + "str_value": "10", + "shape": [1], + }, + }, + { + "op_type": "fill_constant", + "op_inputs": {}, + "op_outputs": {"Out": ["repeat_times2"]}, + "op_attrs": { + "dtype": 2, + "str_value": "12", + "shape": [1], + }, + }, + { + "op_type": "tile", + "op_inputs": dics_intput[0], + "op_outputs": {"Out": ["tile_out"]}, + "op_attrs": dics[0], + }, + ] + ops = self.generate_op_config(ops_config) + program_config = ProgramConfig( + ops=ops, + weights={}, + inputs={ + "tile_input": TensorConfig( + data_gen=partial(generate_input1, dics) + ) + }, + outputs=["tile_out"], + ) + + yield program_config + + def sample_predictor_configs( + self, program_config + ) -> (paddle_infer.Config, List[int], float): + def generate_dynamic_shape(attrs): + self.dynamic_shape.min_input_shape = {"tile_input": [1, 2, 3, 4]} + self.dynamic_shape.max_input_shape = {"tile_input": [4, 3, 64, 64]} + self.dynamic_shape.opt_input_shape = {"tile_input": [1, 2, 3, 4]} + + def clear_dynamic_shape(): + self.dynamic_shape.min_input_shape = {} + self.dynamic_shape.max_input_shape = {} + self.dynamic_shape.opt_input_shape = {} + + def generate_trt_nodes_num(attrs, dynamic_shape): + return 1, 2 + + attrs = [ + program_config.ops[i].attrs for i in range(len(program_config.ops)) + ] + + # for dynamic_shape + generate_dynamic_shape(attrs) + self.trt_param.precision = paddle_infer.PrecisionType.Float32 + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), 1e-5 + self.trt_param.precision = paddle_infer.PrecisionType.Half + yield self.create_inference_config(), generate_trt_nodes_num( + attrs, True + ), 1e-3 + + def add_skip_trt_case(self): + pass + + def test(self): + self.add_skip_trt_case() + self.run_test() + + if __name__ == "__main__": unittest.main()