paddle_analysis_config.h 11.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once

#include <cassert>
17
#include <map>
18 19
#include <memory>
#include <string>
20
#include <unordered_set>
21
#include <utility>
22 23
#include <vector>

24 25
/*! \file */

26 27 28 29
// Here we include some header files with relative paths, for that in deploy,
// the abstract path of this header file will be changed.
#include "paddle_api.h"           // NOLINT
#include "paddle_pass_builder.h"  // NOLINT
30 31 32
#ifdef PADDLE_WITH_MKLDNN
#include "paddle_mkldnn_quantizer_config.h"  // NOLINT
#endif
33 34 35 36

namespace paddle {

class AnalysisPredictor;
37
struct MkldnnQuantizerConfig;
38 39

// NOTE WIP, not stable yet.
40 41
struct AnalysisConfig {
  AnalysisConfig() = default;
42
  explicit AnalysisConfig(const AnalysisConfig& other);
43 44 45
  explicit AnalysisConfig(const std::string& model_dir);
  explicit AnalysisConfig(const std::string& prog_file,
                          const std::string& params_file);
N
nhzlx 已提交
46 47 48 49
  enum class Precision {
    kFloat32 = 0,
    kInt8,
  };
50

51 52
  /** Set model with a directory.
   */
53
  void SetModel(const std::string& model_dir) { model_dir_ = model_dir; }
54 55
  /** Set model with two specific pathes for program and parameters.
   */
56 57
  void SetModel(const std::string& prog_file_path,
                const std::string& params_file_path);
58 59
  /** Set program file path.
   */
60
  void SetProgFile(const std::string& x) { prog_file_ = x; }
61 62
  /** Set parameter composed file path.
   */
63
  void SetParamsFile(const std::string& x) { params_file_ = x; }
64 65 66 67 68
  /** Set opt cache dir.
   */
  void SetOptimCacheDir(const std::string& opt_cache_dir) {
    opt_cache_dir_ = opt_cache_dir;
  }
69 70
  /** Get the model directory path.
   */
71
  const std::string& model_dir() const { return model_dir_; }
72 73
  /** Get the program file path.
   */
74
  const std::string& prog_file() const { return prog_file_; }
75 76
  /** Get the composed parameters file.
   */
77 78 79
  const std::string& params_file() const { return params_file_; }

  // GPU related.
80 81 82 83 84 85

  /**
   * \brief Turn on GPU.
   * @param memory_pool_init_size_mb initial size of the GPU memory pool in MB.
   * @param device_id the GPU card to use (default is 0).
   */
86
  void EnableUseGpu(uint64_t memory_pool_init_size_mb, int device_id = 0);
87 88
  /** Turn off the GPU.
   */
89
  void DisableGpu();
90 91
  /** A bool state telling whether the GPU is turned on.
   */
92
  bool use_gpu() const { return use_gpu_; }
93 94
  /** Get the GPU device id.
   */
95
  int gpu_device_id() const { return device_id_; }
96 97
  /** Get the initial size in MB of the GPU memory pool.
   */
98
  int memory_pool_init_size_mb() const { return memory_pool_init_size_mb_; }
99 100
  /** Get the proportion of the initial memory pool size compared to the device.
   */
101
  float fraction_of_gpu_memory_for_pool() const;
102

103 104 105 106
  /** \brief Control whether to perform IR graph optimization.
   *
   * If turned off, the AnalysisConfig will act just like a NativeConfig.
   */
107
  void SwitchIrOptim(int x = true) { enable_ir_optim_ = x; }
108 109
  /** A boolean state tell whether the ir graph optimization is actived.
   */
110
  bool ir_optim() const { return enable_ir_optim_; }
111

112 113 114 115
  /** \brief INTERNAL Determine whether to use the feed and fetch operators.
   * Just for internal development, not stable yet.
   * When ZeroCopyTensor is used, this should turned off.
   */
116
  void SwitchUseFeedFetchOps(int x = true) { use_feed_fetch_ops_ = x; }
117 118
  /** A boolean state telling whether to use the feed and fetch operators.
   */
119
  bool use_feed_fetch_ops_enabled() const { return use_feed_fetch_ops_; }
120

121 122 123 124 125 126
  /** \brief Control whether to specify the inputs' names.
   *
   * The PaddleTensor type has a `name` member, assign it with the corresponding
   * variable name. This is used only when the input PaddleTensors passed to the
   * `PaddlePredictor.Run(...)` cannot follow the order in the training phase.
   */
127
  void SwitchSpecifyInputNames(bool x = true) { specify_input_name_ = x; }
128 129 130 131

  /** A boolean state tell whether the input PaddleTensor names specified should
   * be used to reorder the inputs in `PaddlePredictor.Run(...)`.
   */
132
  bool specify_input_name() const { return specify_input_name_; }
133

134 135 136 137 138 139 140 141 142 143 144 145 146
  /**
   * \brief Turn on the TensorRT engine.
   *
   * The TensorRT engine will accelerate some subgraphes in the original Fluid
   * computation graph. In some models such as TensorRT50, GoogleNet and so on,
   * it gains significant performance acceleration.
   *
   * @param workspace_size the memory size(in byte) used for TensorRT workspace.
   * @param max_batch_size the maximum batch size of this prediction task,
   * better set as small as possible, or performance loss.
   * @param min_subgrpah_size the minimum TensorRT subgraph size needed, if a
   * subgraph is less than this, it will not transfer to TensorRT engine.
   */
147
  void EnableTensorRtEngine(int workspace_size = 1 << 20,
N
nhzlx 已提交
148
                            int max_batch_size = 1, int min_subgraph_size = 3,
N
nhzlx 已提交
149
                            Precision precision = Precision::kFloat32,
150
                            bool use_static = false,
151
                            bool use_calib_mode = true);
152 153
  /** A boolean state telling whether the TensorRT engine is used.
   */
154
  bool tensorrt_engine_enabled() const { return use_tensorrt_; }
155 156 157
  /**
   *  \brief Turn on the usage of Anakin sub-graph engine.
   */
158 159
  void EnableAnakinEngine(
      int max_batch_size = 1,
160
      std::map<std::string, std::vector<int>> max_input_shape = {},
161 162 163 164
      int min_subgraph_size = 6, Precision precision = Precision::kFloat32,
      bool auto_config_layout = false,
      std::vector<std::string> passes_filter = {},
      std::vector<std::string> ops_filter = {});
165 166 167 168

  /** A boolean state indicating whether the Anakin sub-graph engine is used.
  */
  bool anakin_engine_enabled() const { return use_anakin_; }
169

Y
Yan Chunwei 已提交
170 171 172 173
  /** \brief Control whether to debug IR graph analysis phase.
   *
   * This will generate DOT files for visualizing the computation graph after
   * each analysis pass applied.
174
   */
Y
Yan Chunwei 已提交
175
  void SwitchIrDebug(int x = true);
176

M
mozga-intel 已提交
177 178 179 180 181 182 183
  /** Turn on NGRAPH.
   */
  void EnableNgraph();
  /** A boolean state telling whether to use the NGRAPH.
   */
  bool ngraph_enabled() const { return use_ngraph_; }

184 185
  /** Turn on MKLDNN.
   */
L
luotao1 已提交
186
  void EnableMKLDNN();
187 188
  /** A boolean state telling whether to use the MKLDNN.
   */
189 190
  bool mkldnn_enabled() const { return use_mkldnn_; }

191 192
  /** Set and get the number of cpu math library threads.
   */
193
  void SetCpuMathLibraryNumThreads(int cpu_math_library_num_threads);
194 195
  /** An int state telling how many threads are used in the CPU math library.
   */
196 197 198 199
  int cpu_math_library_num_threads() const {
    return cpu_math_library_num_threads_;
  }

200 201
  /** Transform the AnalysisConfig to NativeConfig.
   */
Y
Yan Chunwei 已提交
202
  NativeConfig ToNativeConfig() const;
203 204 205
  /** Specify the operator type list to use MKLDNN acceleration.
   * @param op_list the operator type list.
   */
206 207 208
  void SetMKLDNNOp(std::unordered_set<std::string> op_list) {
    mkldnn_enabled_op_types_ = op_list;
  }
209

210 211 212 213 214 215 216 217
  /** Turn on quantization.
   */
  void EnableMkldnnQuantizer();

  /** A boolean state telling whether the quantization is enabled.
  */
  bool mkldnn_quantizer_enabled() const { return use_mkldnn_quantizer_; }

218
  MkldnnQuantizerConfig* mkldnn_quantizer_config() const;
219

220 221 222 223 224 225
  /** Specify the memory buffer of program and parameter
   * @param prog_buffer the memory buffer of program.
   * @param prog_buffer_size the size of the data.
   * @param params_buffer the memory buffer of the composed parameters file.
   * @param params_buffer_size the size of the commposed parameters data.
   */
T
Tao Luo 已提交
226
  void SetModelBuffer(const char* prog_buffer, size_t prog_buffer_size,
227 228 229
                      const char* params_buffer, size_t params_buffer_size);
  /** A boolean state telling whether the model is set from the CPU memory.
   */
T
Tao Luo 已提交
230
  bool model_from_memory() const { return model_from_memory_; }
T
Tao Luo 已提交
231

Y
Yan Chunwei 已提交
232 233 234
  /** Turn on memory optimize
   * NOTE still in development, will release latter.
   */
Y
Yan Chunwei 已提交
235 236
  void EnableMemoryOptim(bool static_optim = false,
                         bool force_update_static_cache = false);
Y
Yan Chunwei 已提交
237 238
  /** Tell whether the memory optimization is activated. */
  bool enable_memory_optim() const;
239 240
  void SetInValid() const { is_valid_ = false; }
  bool is_valid() const { return is_valid_; }
Y
Yan Chunwei 已提交
241

242 243
  friend class ::paddle::AnalysisPredictor;

244 245 246
  /** NOTE just for developer, not an official API, easily to be broken.
   * Get a pass builder for customize the passes in IR analysis phase.
   */
247
  PassStrategy* pass_builder() const;
248
  void PartiallyRelease();
249 250 251 252 253 254 255

 protected:
  // Update the config.
  void Update();

  std::string SerializeInfoCache();

256
 protected:
257 258
  // Model pathes.
  std::string model_dir_;
259 260
  mutable std::string prog_file_;
  mutable std::string params_file_;
261

S
Sylwester Fraczek 已提交
262
  // GPU related.
263 264 265 266
  bool use_gpu_{false};
  int device_id_{0};
  uint64_t memory_pool_init_size_mb_{100};  // initial size is 100MB.

S
Sylwester Fraczek 已提交
267
  // TensorRT related.
268
  bool use_tensorrt_{false};
269 270
  // For workspace_size, refer it from here:
  // https://docs.nvidia.com/deeplearning/sdk/tensorrt-developer-guide/index.html#troubleshooting
271
  int tensorrt_workspace_size_;
272 273 274 275
  // While TensorRT allows an engine optimized for a given max batch size
  // to run at any smaller size, the performance for those smaller
  // sizes may not be as well-optimized. Therefore, Max batch is best
  // equivalent to the runtime batch size.
276
  int tensorrt_max_batchsize_;
277 278 279 280 281
  //  We transform the Ops that can be converted into TRT layer in the model,
  //  and aggregate these Ops into subgraphs for TRT execution.
  //  We set this variable to control the minimum number of nodes in the
  //  subgraph, 3 as default value.
  int tensorrt_min_subgraph_size_{3};
N
nhzlx 已提交
282
  Precision tensorrt_precision_mode_;
N
nhzlx 已提交
283
  bool trt_use_static_engine_;
284
  bool trt_use_calib_mode_;
285

Y
Yan Chunwei 已提交
286 287
  // memory reuse related.
  bool enable_memory_optim_{false};
Y
Yan Chunwei 已提交
288 289
  bool static_memory_optim_{false};
  bool static_memory_optim_force_update_{false};
Y
Yan Chunwei 已提交
290

M
mozga-intel 已提交
291
  bool use_ngraph_{false};
292 293 294
  bool use_mkldnn_{false};
  std::unordered_set<std::string> mkldnn_enabled_op_types_;

T
Tao Luo 已提交
295
  bool model_from_memory_{false};
296

297 298 299 300 301 302 303 304 305 306 307 308
  bool enable_ir_optim_{true};
  bool use_feed_fetch_ops_{true};
  bool ir_debug_{false};

  bool specify_input_name_{false};

  int cpu_math_library_num_threads_{1};

  // A runtime cache, shouldn't be transferred to others.
  std::string serialized_info_cache_;

  mutable std::unique_ptr<PassStrategy> pass_builder_;
309

310
  bool use_anakin_{false};
311
  int anakin_max_batchsize_;
312
  int anakin_min_subgraph_size_{6};
313
  std::map<std::string, std::vector<int>> anakin_max_input_shape_;
314 315 316 317
  Precision anakin_precision_mode_;
  bool anakin_auto_config_layout_{false};
  std::vector<std::string> anakin_passes_filter_;
  std::vector<std::string> anakin_ops_filter_;
318 319 320

  bool use_mkldnn_quantizer_{false};
  std::shared_ptr<MkldnnQuantizerConfig> mkldnn_quantizer_config_;
321 322 323 324
  // If the config is already used on a predictor, it becomes invalid.
  // Any config can only be used with one predictor.
  // Variables held by config can take up a lot of memory in some cases.
  // So we release the memory when the predictor is set up.
325 326
  mutable bool is_valid_{true};
  std::string opt_cache_dir_;
327 328 329
};

}  // namespace paddle