paddle_inference_api.h 5.2 KB
Newer Older
Y
Yan Chunwei 已提交
1 2
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

Y
Yan Chunwei 已提交
3 4 5
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Y
Yan Chunwei 已提交
6

Y
Yan Chunwei 已提交
7
http://www.apache.org/licenses/LICENSE-2.0
Y
Yan Chunwei 已提交
8

Y
Yan Chunwei 已提交
9 10 11 12 13
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
Y
Yan Chunwei 已提交
14

15 16 17
/*
 * This file contains the definition of a simple Inference API for Paddle.
 *
18
 * ATTENTION: It requires some C++11 features, for lower version C++ or C, we
19 20 21
 * might release another API.
 */

Y
Yan Chunwei 已提交
22 23
#pragma once

24
#include <cassert>
25
#include <memory>
Y
Yan Chunwei 已提交
26 27 28 29 30
#include <string>
#include <vector>

namespace paddle {

X
Xin Pan 已提交
31 32 33 34 35
enum PaddleDType {
  FLOAT32,
  INT64,
};

36 37 38 39 40 41 42
class PaddleBuf {
 public:
  PaddleBuf() = default;
  PaddleBuf(PaddleBuf&& other);
  // Copy only available when memory is managed externally.
  explicit PaddleBuf(const PaddleBuf&);
  PaddleBuf& operator=(const PaddleBuf&);
43
  PaddleBuf& operator=(PaddleBuf&&);
44 45 46 47
  // Do not own the memory.
  PaddleBuf(void* data, size_t length)
      : data_(data), length_(length), memory_owned_{false} {}
  // Own memory.
W
Wu Yi 已提交
48
  explicit PaddleBuf(size_t length)
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
      : data_(new char[length]), length_(length), memory_owned_(true) {}
  // Resize to `length` bytes.
  void Resize(size_t length);
  // Reset to external memory.
  void Reset(void* data, size_t length);
  bool empty() const { return length_ == 0; }
  void* data() const { return data_; }
  size_t length() const { return length_; }

  ~PaddleBuf() { Free(); }

 private:
  void Free();
  void* data_{nullptr};  // pointer to the data memory.
  size_t length_{0};     // number of memory bytes.
  bool memory_owned_{true};
X
Xin Pan 已提交
65 66
};

67
struct PaddleTensor {
68
  PaddleTensor() = default;
69 70
  std::string name;  // variable name.
  std::vector<int> shape;
X
Xin Pan 已提交
71 72
  PaddleBuf data;  // blob of data.
  PaddleDType dtype;
Y
Yan Chunwei 已提交
73
  std::vector<std::vector<uint64_t>> lod;  // lod data
74 75
};

Y
Yan Chunwei 已提交
76
enum class PaddleEngineKind {
77 78 79
  kNative = 0,         // Use the native Fluid facility.
  kAnakin,             // Use Anakin for inference.
  kAutoMixedTensorRT,  // Automatically mix Fluid with TensorRT.
Y
Yan Chunwei 已提交
80 81 82 83 84
  // TODO(Superjomn) support following engines latter.
  // kTensorRT,           // Use TensorRT for inference.
  // kAutoMixedAnakin,    // Automatically mix Fluid with Anakin.
};

85
/*
Y
Yan Chunwei 已提交
86 87 88
 * A simple Inference API for Paddle. Currently this API can be used by
 * non-sequence scenerios.
 */
89
class PaddlePredictor {
W
Wu Yi 已提交
90
 public:
91 92 93
  struct Config;
  PaddlePredictor() = default;
  PaddlePredictor(const PaddlePredictor&) = delete;
94
  PaddlePredictor& operator=(const PaddlePredictor&) = delete;
Y
Yan Chunwei 已提交
95 96

  // Predict an record.
X
Xin Pan 已提交
97
  // The caller should be responsible for allocating and releasing the memory of
98 99 100
  // `inputs`. `inputs` should be available until Run returns. Caller should be
  // responsible for the output tensor's buffer, either allocated or passed from
  // outside.
101
  virtual bool Run(const std::vector<PaddleTensor>& inputs,
102 103
                   std::vector<PaddleTensor>* output_data,
                   int batch_size = -1) = 0;
104 105 106 107

  // Clone a predictor that share the model weights, the Cloned predictor should
  // be thread-safe.
  virtual std::unique_ptr<PaddlePredictor> Clone() = 0;
Y
Yan Chunwei 已提交
108 109

  // Destroy the Predictor.
110
  virtual ~PaddlePredictor() = default;
111 112 113

  // The common configs for all the predictors.
  struct Config {
Y
Yan Chunwei 已提交
114
    std::string model_dir;  // path to the model directory.
Y
Yan Chunwei 已提交
115 116 117
  };
};

Y
Yan Chunwei 已提交
118
struct NativeConfig : public PaddlePredictor::Config {
Y
Yan Chunwei 已提交
119
  // GPU related fields.
Y
Yan Chunwei 已提交
120
  bool use_gpu{false};
Y
Yan Chunwei 已提交
121 122 123
  int device{0};
  float fraction_of_gpu_memory{-1.f};  // Negative to notify initialization.

Y
Yan Chunwei 已提交
124 125 126 127
  std::string prog_file;
  std::string param_file;
};

Y
Yan Chunwei 已提交
128 129
// Configurations for Anakin engine.
struct AnakinConfig : public PaddlePredictor::Config {
C
cuichaowen 已提交
130
  enum TargetType { NVGPU = 0, X86 };
Y
Yan Chunwei 已提交
131 132 133
  int device;
  std::string model_file;
  int max_batch_size{-1};
C
cuichaowen 已提交
134
  TargetType target_type;
Y
Yan Chunwei 已提交
135 136
};

137 138 139
struct TensorRTConfig : public NativeConfig {
  // Determine whether a subgraph will be executed by TRT.
  int min_subgraph_size{1};
140 141 142 143 144 145 146 147
  // While TensorRT allows an engine optimized for a given max batch size
  // to run at any smaller size, the performance for those smaller
  // sizes may not be as well-optimized. Therefore, Max batch is best
  // equivalent to the runtime batch size.
  int max_batch_size{1};
  // For workspace_size, refer it from here:
  // https://docs.nvidia.com/deeplearning/sdk/tensorrt-developer-guide/index.html#troubleshooting
  int workspace_size{1 << 30};
148 149
};

Y
Yan Chunwei 已提交
150 151 152 153 154 155 156 157 158
// A factory to help create different predictors.
//
// FOR EXTENSION DEVELOPER:
// Different predictors are designated by config type and engine kind. Similar
// configs can be merged, but there shouldn't be a huge config containing
// different fields for more than one kind of predictors.
//
// Similarly, each engine kind should map to a unique predictor implementation.
template <typename ConfigT, PaddleEngineKind engine = PaddleEngineKind::kNative>
159
std::unique_ptr<PaddlePredictor> CreatePaddlePredictor(const ConfigT& config);
160 161 162

int PaddleDtypeSize(PaddleDType dtype);

Y
Yan Chunwei 已提交
163
}  // namespace paddle