ut_helper.h 6.5 KB
Newer Older
Y
Yan Chunwei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

/*
 * This file implements a UT framework to make the validation of transforming
 * Fluid Op to TRT Layer.
 */

#pragma once

22 23 24
#include <string>
#include <vector>

Y
Yan Chunwei 已提交
25 26 27 28 29
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/op_registry.h"
#include "paddle/fluid/inference/analysis/helper.h"
#include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
#include "paddle/fluid/inference/tensorrt/engine.h"
30
#include "paddle/fluid/inference/utils/singleton.h"
Y
Yan Chunwei 已提交
31 32 33 34 35 36 37 38 39 40 41

namespace paddle {
namespace inference {
namespace tensorrt {

/*
 * Get a random float value between [low, high]
 */
float random(float low, float high) {
  static std::random_device rd;
  static std::mt19937 mt(rd());
42
  std::uniform_real_distribution<double> dist(low, high);
Y
Yan Chunwei 已提交
43 44 45 46 47 48 49 50 51
  return dist(mt);
}

void RandomizeTensor(framework::LoDTensor* tensor, const platform::Place& place,
                     const platform::DeviceContext& ctx) {
  auto dims = tensor->dims();
  size_t num_elements = analysis::AccuDims(dims, dims.size());
  PADDLE_ENFORCE_GT(num_elements, 0);
  auto* data = tensor->mutable_data<float>(place);
52

Y
Yan Chunwei 已提交
53 54 55 56 57 58 59 60 61 62 63 64 65
  for (size_t i = 0; i < num_elements; i++) {
    *(data + i) = random(0., 1.);
  }
}

/*
 * Help to validate the correctness between Fluid Op and the corresponding TRT
 * layer.
 */
class TRTConvertValidation {
 public:
  TRTConvertValidation() = delete;

66
  TRTConvertValidation(int max_batch_size,
67
                       const std::unordered_set<std::string>& parameters,
G
gongweibao 已提交
68
                       framework::Scope& scope,  // NOLINT
N
nhzlx 已提交
69
                       int workspace_size = 1 << 10, bool if_add_batch = true)
70 71
      : parameters_(parameters),
        scope_(scope),
N
nhzlx 已提交
72 73
        if_add_batch_(if_add_batch),
        max_batch_size_(max_batch_size) {
Y
Yan Chunwei 已提交
74
    // create engine.
75
    engine_.reset(new TensorRTEngine(max_batch_size, workspace_size, &stream_));
Y
Yan Chunwei 已提交
76 77 78 79 80 81 82 83 84 85 86 87
    engine_->InitNetwork();

    PADDLE_ENFORCE_EQ(cudaStreamCreate(&stream_), 0);
  }

  // Declare a Variable as input with random initialization.
  void DeclInputVar(const std::string& name, const nvinfer1::Dims& dims) {
    DeclVar(name, dims);
    // Declare TRT inputs.
    engine_->DeclareInput(name, nvinfer1::DataType::kFLOAT, dims);
  }

88 89
  // Declare a parameter varaible in the scope.
  void DeclParamVar(const std::string& name, const nvinfer1::Dims& dims) {
90
    DeclVar(name, dims, true);
91 92
  }

Y
Yan Chunwei 已提交
93 94 95 96
  void DeclOutputVar(const std::string& name, const nvinfer1::Dims& dims) {
    DeclVar(name, dims);
  }

97
  // Declare a variable in a fluid Scope.
98 99
  void DeclVar(const std::string& name, const nvinfer1::Dims& dims,
               bool is_param = false) {
Y
Yan Chunwei 已提交
100 101 102 103
    platform::CPUPlace place;
    platform::CPUDeviceContext ctx(place);

    // Init Fluid tensor.
104
    std::vector<int> dim_vec(dims.d, dims.d + dims.nbDims);
105
    // There is no batchsize in ITensor's shape, but We should add it to
N
nhzlx 已提交
106 107 108 109
    // tensor's shape of fluid. If the variable is not parameter and the
    // if_add_batch_ flag is true, add the max batchsize to dim_vec.
    if (is_param != true && if_add_batch_ == true)
      dim_vec.insert(dim_vec.begin(), max_batch_size_);
Y
Yan Chunwei 已提交
110 111 112 113 114 115 116 117 118
    auto* x = scope_.Var(name);
    auto* x_tensor = x->GetMutable<framework::LoDTensor>();
    x_tensor->Resize(framework::make_ddim(dim_vec));
    RandomizeTensor(x_tensor, place, ctx);
  }

  void SetOp(const framework::proto::OpDesc& desc) {
    op_ = framework::OpRegistry::CreateOp(desc);

119 120
    Singleton<OpConverter>::Global().ConvertOp(
        desc, parameters_, scope_, engine_.get(), true /*test_mode*/);
Y
Yan Chunwei 已提交
121 122 123 124

    engine_->FreezeNetwork();

    // Declare outputs.
F
fengjiayi 已提交
125
    op_desc_.reset(new framework::OpDesc(desc, nullptr));
Y
Yan Chunwei 已提交
126 127 128

    // Set Inputs.
    for (const auto& input : op_desc_->InputArgumentNames()) {
129
      if (parameters_.count(input)) continue;
Y
Yan Chunwei 已提交
130 131 132
      auto* var = scope_.FindVar(input);
      PADDLE_ENFORCE(var);
      auto tensor = var->GetMutable<framework::LoDTensor>();
133

Y
Yan Chunwei 已提交
134
      engine_->SetInputFromCPU(
135
          input, static_cast<void*>(tensor->data<void>()),
Y
Yan Chunwei 已提交
136 137 138 139 140 141 142
          sizeof(float) *
              analysis::AccuDims(tensor->dims(), tensor->dims().size()));
    }
  }

  void Execute(int batch_size) {
    // Execute Fluid Op
N
nhzlx 已提交
143
    PADDLE_ENFORCE_LE(batch_size, max_batch_size_);
Y
Yan Chunwei 已提交
144 145 146
    platform::CPUPlace place;
    platform::CPUDeviceContext ctx(place);
    op_->Run(scope_, place);
147 148 149
    // Execute TRT.
    engine_->Execute(batch_size);
    cudaStreamSynchronize(*engine_->stream());
Y
Yan Chunwei 已提交
150 151

    ASSERT_FALSE(op_desc_->OutputArgumentNames().empty());
N
nhzlx 已提交
152
    const size_t output_space_size = 2000;
Y
Yan Chunwei 已提交
153 154
    for (const auto& output : op_desc_->OutputArgumentNames()) {
      std::vector<float> fluid_out;
155
      std::vector<float> trt_out(output_space_size);
N
nhzlx 已提交
156
      engine_->GetOutputInCPU(output, &trt_out[0], output_space_size);
157
      cudaStreamSynchronize(*engine_->stream());
Y
Yan Chunwei 已提交
158 159 160 161

      auto* var = scope_.FindVar(output);
      auto tensor = var->GetMutable<framework::LoDTensor>();
      framework::TensorToVector(*tensor, ctx, &fluid_out);
N
nhzlx 已提交
162 163 164

      size_t fluid_out_size = fluid_out.size();
      if (if_add_batch_ == true) {
N
nhzlx 已提交
165 166
        fluid_out_size =
            batch_size * (framework::product(tensor->dims()) / max_batch_size_);
N
nhzlx 已提交
167
      }
Y
Yan Chunwei 已提交
168 169
      // Compare two output
      ASSERT_FALSE(fluid_out.empty());
N
nhzlx 已提交
170
      for (size_t i = 0; i < fluid_out_size; i++) {
171 172
        // Loose the threshold for CI in different machine model.
        EXPECT_LT(std::abs(fluid_out[i] - trt_out[i]), 2e-5);
Y
Yan Chunwei 已提交
173 174 175 176 177 178 179 180 181 182 183
      }
    }
  }

  framework::Scope& scope() { return scope_; }

 private:
  std::unique_ptr<TensorRTEngine> engine_;
  cudaStream_t stream_;
  std::unique_ptr<framework::OperatorBase> op_;
  std::unique_ptr<framework::OpDesc> op_desc_;
184 185
  const std::unordered_set<std::string>& parameters_;
  framework::Scope& scope_;
N
nhzlx 已提交
186 187 188 189 190 191
  // The ITensor of trt does not cotain the batch size,
  // bug, in most cases, we need to set batch size for
  // fluid's tensor shape. This variable indicates
  // whether to add batch size to tensor shape of fluid.
  bool if_add_batch_;
  int max_batch_size_;
Y
Yan Chunwei 已提交
192 193 194 195 196
};

}  // namespace tensorrt
}  // namespace inference
}  // namespace paddle