[Paddle Inference] Add trt tile converter for dynamic shape. (#50841)

* add_trt_tile

* tile_trt

paddle/fluid/inference/tensorrt/convert/tile_op.cc

 /* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
 http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -11,65 +14,117 @@ limitations under the License. */
 
 #include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
 
-namespace paddle {
-namespace framework {
-class Scope;
-namespace proto {
-class OpDesc;
-}  // namespace proto
-}  // namespace framework
-}  // namespace paddle
-
 namespace paddle {
 namespace inference {
 namespace tensorrt {
 
-/*
- * ReshapeOp
- */
 class TileOpConverter : public OpConverter {
  public:
   void operator()(const framework::proto::OpDesc& op,
                   const framework::Scope& scope,
                   bool test_mode) override {
 #if IS_TRT_VERSION_GE(7000)
-    VLOG(4) << "convert a fluid tile op to tensorrt tile layer";
+    VLOG(4) << "convert a tile op to tensorrt tile layer";
 
     framework::OpDesc op_desc(op, nullptr);
     // Declare inputs
     auto* input = engine_->GetITensor(op_desc.Input("X")[0]);
-    nvinfer1::Dims input_shape = input->getDimensions();
-    std::vector<int> repeat_times =
-        PADDLE_GET_CONST(std::vector<int>, op_desc.GetAttr("repeat_times"));
-
-    nvinfer1::Dims output_dim = input_shape;
-    nvinfer1::Dims output_stride;
-    // If input_dims.nbDims + 1 < repeat_times.size() means we
-    // should expand 1 on batchsize. trt doesn't support this behavior.
-    PADDLE_ENFORCE_GE(input_shape.nbDims + 1,
-                      repeat_times.size(),
-                      platform::errors::InvalidArgument(
-                          "Can't change batchsize, please check repeat_times"));
-    int diff = input_shape.nbDims + 1 - repeat_times.size();
-    if (diff > 0) repeat_times.insert(repeat_times.begin(), diff, 1);
-
-    // Can't expand on batchsize
-    PADDLE_ENFORCE_EQ(
-        repeat_times[0],
-        1,
-        platform::errors::InvalidArgument(
-            "Can't expand on batchsize, please check repeat_times"));
-    output_stride.nbDims = input_shape.nbDims;
-    for (int i = 0; i < input_shape.nbDims; i++) {
-      output_dim.d[i] = output_dim.d[i] * repeat_times[i + 1];
-      output_stride.d[i] = 1;
+    auto inputs = op_desc.Inputs();
+    auto input_shape = input->getDimensions();
+    auto rank = input_shape.nbDims;
+    auto output_name = op_desc.Output("Out")[0];
+
+    if (engine_->with_dynamic_shape()) {
+      std::vector<int32_t> start(rank, 0);
+      std::vector<int32_t> stride(rank, 1);
+      auto start_tensor =
+          Add1DConstantLayer(start, output_name + "start_tensor");
+      auto stride_tensor =
+          Add1DConstantLayer(stride, output_name + "stride_tensor");
+      auto input_shape_tensor = Shape(input);
+
+      nvinfer1::ITensor* repeat_tensor = nullptr;
+      int32_t repeat_rank = 0;
+      if (inputs.find("RepeatTimes") != inputs.end() &&
+          op_desc.Input("RepeatTimes").size() >= 1) {
+        repeat_tensor = engine_->GetITensor(op_desc.Input("RepeatTimes")[0]);
+        repeat_rank = repeat_tensor->getDimensions().d[0];
+      } else if (inputs.find("repeat_times_tensor") != inputs.end() &&
+                 op_desc.Input("repeat_times_tensor").size() >= 1) {
+        int32_t repeat_size = op_desc.Input("repeat_times_tensor").size();
+        std::vector<nvinfer1::ITensor*> repeat_tensors;
+        for (int32_t i = 0; i < repeat_size; ++i) {
+          repeat_tensors.push_back(
+              engine_->GetITensor(op_desc.Input("repeat_times_tensor")[i]));
+        }
+        repeat_tensor = Concat(repeat_tensors);
+        repeat_rank = repeat_size;
+      } else {
+        std::vector<int32_t> repeat_times = PADDLE_GET_CONST(
+            std::vector<int32_t>, op_desc.GetAttr("repeat_times"));
+        repeat_tensor =
+            Add1DConstantLayer(repeat_times, output_name + "_shape_tensor_");
+        repeat_rank = repeat_times.size();
+      }
+
+      nvinfer1::ITensor* repeat_expand_tensor;
+      if (rank > repeat_rank) {
+        auto* one_rank_tensor =
+            Add1DConstantLayer(std::vector<int32_t>(rank - repeat_rank, 1),
+                               output_name + "_one_rank_tensor_");
+        std::vector<nvinfer1::ITensor*> itensors;
+        itensors.push_back(one_rank_tensor);
+        itensors.push_back(repeat_tensor);
+        repeat_expand_tensor = Concat(itensors);
+      } else {
+        repeat_expand_tensor = repeat_tensor;
+      }
+      auto output_shape_tensor = Prod(input_shape_tensor, repeat_expand_tensor);
+      auto layer = TRT_ENGINE_ADD_LAYER(engine_,
+                                        Slice,
+                                        *input,
+                                        nvinfer1::Dims{},
+                                        nvinfer1::Dims{},
+                                        nvinfer1::Dims{});
+
+      layer->setInput(1, *start_tensor);
+      layer->setInput(2, *output_shape_tensor);
+      layer->setInput(3, *stride_tensor);
+      layer->setMode(nvinfer1::SliceMode::kWRAP);
+      RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode);
+
+    } else {
+      std::vector<int> repeat_times =
+          PADDLE_GET_CONST(std::vector<int>, op_desc.GetAttr("repeat_times"));
+      auto output_dim = input_shape;
+      auto output_stride = input_shape;
+      // If input_dims.nbDims + 1 < repeat_times.size() means we
+      // should expand 1 on batchsize. trt doesn't support this behavior.
+      PADDLE_ENFORCE_GE(
+          rank + 1,
+          repeat_times.size(),
+          platform::errors::InvalidArgument(
+              "Can't change batchsize, please check repeat_times"));
+      int32_t diff = rank + 1 - repeat_times.size();
+      if (diff > 0) repeat_times.insert(repeat_times.begin(), diff, 1);
+
+      // Can't expand on batchsize
+      PADDLE_ENFORCE_EQ(
+          repeat_times[0],
+          1,
+          platform::errors::InvalidArgument(
+              "Can't expand on batchsize, please check repeat_times"));
+      output_stride.nbDims = rank;
+      for (int32_t i = 0; i < rank; i++) {
+        output_dim.d[i] = output_dim.d[i] * repeat_times[i + 1];
+        output_stride.d[i] = 1;
+      }
+      auto layer = TRT_ENGINE_ADD_LAYER(
+          engine_, Slice, *input, input_shape, output_dim, output_stride);
+      layer->setMode(nvinfer1::SliceMode::kWRAP);
+      RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode);
     }
 
-    auto* layer = TRT_ENGINE_ADD_LAYER(
-        engine_, Slice, *input, input_shape, output_dim, output_stride);
-    layer->setMode(nvinfer1::SliceMode::kWRAP);
-    auto output_name = op_desc.Output("Out")[0];
-    RreplenishLayerAndOutput(layer, "tile", {output_name}, test_mode);
 #endif
   }
 };

paddle/fluid/inference/tensorrt/op_teller.cc

@@ -2232,18 +2232,19 @@ struct SimpleOpTypeSetTeller : public Teller {
     if (op_type == "tile") {
       // Paddle-TRT does not support the input tensors.
       auto tile_inputs = desc.Inputs();
-      if (tile_inputs.find("repeat_times_tensor") != tile_inputs.end()) {
-        if (desc.Input("repeat_times_tensor").size() >= 1) {
-          return false;
+      if (!with_dynamic_shape) {
+        if (tile_inputs.find("repeat_times_tensor") != tile_inputs.end()) {
+          if (desc.Input("repeat_times_tensor").size() >= 1) {
+            return false;
+          }
         }
-      }
-      if (tile_inputs.find("RepeatTimes") != tile_inputs.end()) {
-        if (desc.Input("RepeatTimes").size() >= 1) {
-          return false;
+        if (tile_inputs.find("RepeatTimes") != tile_inputs.end()) {
+          if (desc.Input("RepeatTimes").size() >= 1) {
+            return false;
+          }
         }
+        if (!desc.HasAttr("repeat_times")) return false;
       }
-      if (with_dynamic_shape) return false;
-      if (!with_dynamic_shape && !desc.HasAttr("repeat_times")) return false;
     }
 #endif
 

python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_tile.py

@@ -70,7 +70,7 @@ class TrtConvertTileTest(TrtLayerAutoScanTest):
         self, program_config
     ) -> (paddle_infer.Config, List[int], float):
         def generate_dynamic_shape(attrs):
-            self.dynamic_shape.min_input_shape = {"input_data": [1, 3, 32, 32]}
+            self.dynamic_shape.min_input_shape = {"input_data": [1, 2, 3, 4]}
             self.dynamic_shape.max_input_shape = {"input_data": [4, 3, 64, 64]}
             self.dynamic_shape.opt_input_shape = {"input_data": [1, 3, 64, 64]}
 
@@ -82,10 +82,7 @@ class TrtConvertTileTest(TrtLayerAutoScanTest):
         def generate_trt_nodes_num(attrs, dynamic_shape):
             ver = paddle_infer.get_trt_compile_version()
             if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 >= 7000:
-                if dynamic_shape:
-                    return 0, 3
-                else:
-                    return 1, 2
+                return 1, 2
             else:
                 return 0, 3
 
@@ -120,5 +117,187 @@ class TrtConvertTileTest(TrtLayerAutoScanTest):
         self.run_test(*args, **kwargs)
 
 
+class TrtConvertTileTest2(TrtLayerAutoScanTest):
+    def is_program_valid(self, program_config: ProgramConfig) -> bool:
+        return True
+
+    def sample_program_configs(self):
+        def generate_input1(attrs: List[Dict[str, Any]]):
+            return np.ones([1, 2, 3, 4]).astype(np.float32)
+
+        dics = [{}]
+        dics_intput = [
+            {"X": ["tile_input"], "RepeatTimes": ["repeat_times"]},
+        ]
+        ops_config = [
+            {
+                "op_type": "fill_constant",
+                "op_inputs": {},
+                "op_outputs": {"Out": ["repeat_times"]},
+                "op_attrs": {
+                    "dtype": 2,
+                    "str_value": "10",
+                    "shape": [1],
+                },
+            },
+            {
+                "op_type": "tile",
+                "op_inputs": dics_intput[0],
+                "op_outputs": {"Out": ["tile_out"]},
+                "op_attrs": dics[0],
+            },
+        ]
+        ops = self.generate_op_config(ops_config)
+        program_config = ProgramConfig(
+            ops=ops,
+            weights={},
+            inputs={
+                "tile_input": TensorConfig(
+                    data_gen=partial(generate_input1, dics)
+                )
+            },
+            outputs=["tile_out"],
+        )
+
+        yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            self.dynamic_shape.min_input_shape = {"tile_input": [1, 2, 3, 4]}
+            self.dynamic_shape.max_input_shape = {"tile_input": [4, 3, 64, 64]}
+            self.dynamic_shape.opt_input_shape = {"tile_input": [1, 2, 3, 4]}
+
+        def clear_dynamic_shape():
+            self.dynamic_shape.min_input_shape = {}
+            self.dynamic_shape.max_input_shape = {}
+            self.dynamic_shape.opt_input_shape = {}
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for dynamic_shape
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-5
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-3
+
+    def add_skip_trt_case(self):
+        pass
+
+    def test(self):
+        self.add_skip_trt_case()
+        self.run_test()
+
+
+class TrtConvertTileTest3(TrtLayerAutoScanTest):
+    def is_program_valid(self, program_config: ProgramConfig) -> bool:
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+        return True
+
+    def sample_program_configs(self):
+        def generate_input1(attrs: List[Dict[str, Any]]):
+            return np.ones([1, 2, 3, 4]).astype(np.float32)
+
+        dics = [{}]
+        dics_intput = [
+            {
+                "X": ["tile_input"],
+                "repeat_times_tensor": ["repeat_times1", "repeat_times2"],
+            },
+        ]
+        ops_config = [
+            {
+                "op_type": "fill_constant",
+                "op_inputs": {},
+                "op_outputs": {"Out": ["repeat_times1"]},
+                "op_attrs": {
+                    "dtype": 2,
+                    "str_value": "10",
+                    "shape": [1],
+                },
+            },
+            {
+                "op_type": "fill_constant",
+                "op_inputs": {},
+                "op_outputs": {"Out": ["repeat_times2"]},
+                "op_attrs": {
+                    "dtype": 2,
+                    "str_value": "12",
+                    "shape": [1],
+                },
+            },
+            {
+                "op_type": "tile",
+                "op_inputs": dics_intput[0],
+                "op_outputs": {"Out": ["tile_out"]},
+                "op_attrs": dics[0],
+            },
+        ]
+        ops = self.generate_op_config(ops_config)
+        program_config = ProgramConfig(
+            ops=ops,
+            weights={},
+            inputs={
+                "tile_input": TensorConfig(
+                    data_gen=partial(generate_input1, dics)
+                )
+            },
+            outputs=["tile_out"],
+        )
+
+        yield program_config
+
+    def sample_predictor_configs(
+        self, program_config
+    ) -> (paddle_infer.Config, List[int], float):
+        def generate_dynamic_shape(attrs):
+            self.dynamic_shape.min_input_shape = {"tile_input": [1, 2, 3, 4]}
+            self.dynamic_shape.max_input_shape = {"tile_input": [4, 3, 64, 64]}
+            self.dynamic_shape.opt_input_shape = {"tile_input": [1, 2, 3, 4]}
+
+        def clear_dynamic_shape():
+            self.dynamic_shape.min_input_shape = {}
+            self.dynamic_shape.max_input_shape = {}
+            self.dynamic_shape.opt_input_shape = {}
+
+        def generate_trt_nodes_num(attrs, dynamic_shape):
+            return 1, 2
+
+        attrs = [
+            program_config.ops[i].attrs for i in range(len(program_config.ops))
+        ]
+
+        # for dynamic_shape
+        generate_dynamic_shape(attrs)
+        self.trt_param.precision = paddle_infer.PrecisionType.Float32
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-5
+        self.trt_param.precision = paddle_infer.PrecisionType.Half
+        yield self.create_inference_config(), generate_trt_nodes_num(
+            attrs, True
+        ), 1e-3
+
+    def add_skip_trt_case(self):
+        pass
+
+    def test(self):
+        self.add_skip_trt_case()
+        self.run_test()
+
+
 if __name__ == "__main__":
     unittest.main()