paddle_analysis_config.h 18.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
14 15 16 17 18 19 20 21 22 23 24

///
/// \file paddle_analysis_config.h
///
/// \brief Paddle Analysis Config API信息
///
/// \author paddle-infer@baidu.com
/// \date 2020-03-20
/// \since 1.7
///

25 26 27
#pragma once

#include <cassert>
28
#include <map>
29 30
#include <memory>
#include <string>
31
#include <unordered_set>
32
#include <utility>
33 34
#include <vector>

35 36
/*! \file */

37 38 39 40
// Here we include some header files with relative paths, for that in deploy,
// the abstract path of this header file will be changed.
#include "paddle_api.h"           // NOLINT
#include "paddle_pass_builder.h"  // NOLINT
41 42 43
#ifdef PADDLE_WITH_MKLDNN
#include "paddle_mkldnn_quantizer_config.h"  // NOLINT
#endif
44 45 46 47

namespace paddle {

class AnalysisPredictor;
48
struct MkldnnQuantizerConfig;
49

50
///
51
/// \brief configuration manager for AnalysisPredictor.
52 53
/// \since 1.7.0
///
54
/// AnalysisConfig manages configurations of AnalysisPredictor.
55 56 57 58 59
/// During inference procedure, there are many parameters(model/params path,
/// place of inference, etc.)
/// to be specified, and various optimizations(subgraph fusion, memory
/// optimazation, TensorRT engine, etc.)
/// to be done. Users can manage these settings by creating and modifying an
60 61
/// AnalysisConfig,
/// and loading it into AnalysisPredictor.
62
///
63 64
struct AnalysisConfig {
  AnalysisConfig() = default;
65
  ///
66 67
  /// \brief Construct a new AnalysisConfig from another
  /// AnalysisConfig.
68
  ///
69
  /// \param[in] other another AnalysisConfig
70
  ///
71
  explicit AnalysisConfig(const AnalysisConfig& other);
72
  ///
73
  /// \brief Construct a new AnalysisConfig from a no-combined model.
74 75 76
  ///
  /// \param[in] model_dir model directory of the no-combined model.
  ///
77
  explicit AnalysisConfig(const std::string& model_dir);
78
  ///
79
  /// \brief Construct a new AnalysisConfig from a combined model.
80 81 82 83
  ///
  /// \param[in] prog_file model file path of the combined model.
  /// \param[in] params_file params file path of the combined model.
  ///
84 85
  explicit AnalysisConfig(const std::string& prog_file,
                          const std::string& params_file);
86 87 88
  ///
  /// \brief Precision of inference in TensorRT.
  ///
N
nhzlx 已提交
89
  enum class Precision {
90 91 92
    kFloat32 = 0,  ///< fp32
    kInt8,         ///< int8
    kHalf,         ///< fp16
N
nhzlx 已提交
93
  };
94

95 96 97 98 99
  ///
  /// \brief Set the no-combined model dir path.
  ///
  /// \param model_dir model dir path.
  ///
100
  void SetModel(const std::string& model_dir) { model_dir_ = model_dir; }
101 102 103 104 105 106 107 108

  ///
  /// \brief Set the combined model with two specific pathes for program and
  /// parameters.
  ///
  /// \param prog_file_path model file path of the combined model.
  /// \param params_file_path params file path of the combined model.
  ///
109 110
  void SetModel(const std::string& prog_file_path,
                const std::string& params_file_path);
111 112 113 114 115
  ///
  /// \brief Set the model file path of a combined model.
  ///
  /// \param x model file path.
  ///
116
  void SetProgFile(const std::string& x) { prog_file_ = x; }
117 118 119 120 121
  ///
  /// \brief Set the params file path of a combined model.
  ///
  /// \param x params file path.
  ///
122
  void SetParamsFile(const std::string& x) { params_file_ = x; }
123 124 125 126 127 128

  ///
  /// \brief Set the path of optimization cache directory.
  ///
  /// \param opt_cache_dir the path of optimization cache directory.
  ///
129 130 131
  void SetOptimCacheDir(const std::string& opt_cache_dir) {
    opt_cache_dir_ = opt_cache_dir;
  }
132 133 134 135 136
  ///
  /// \brief Get the model directory path.
  ///
  /// \return const std::string& The model directory path.
  ///
137
  const std::string& model_dir() const { return model_dir_; }
138 139 140 141 142
  ///
  /// \brief Get the program file path.
  ///
  /// \return const std::string& The program file path.
  ///
143
  const std::string& prog_file() const { return prog_file_; }
144 145 146 147 148
  ///
  /// \brief Get the combined parameters file.
  ///
  /// \return const std::string& The combined parameters file.
  ///
149 150
  const std::string& params_file() const { return params_file_; }

151
  // Padding related.
152 153 154 155 156

  ///
  /// \brief Turn off FC Padding.
  ///
  ///
157
  void DisableFCPadding();
158 159 160 161 162
  ///
  /// \brief A boolean state telling whether fc padding is used.
  ///
  /// \return bool Whether fc padding is used.
  ///
163 164
  bool use_fc_padding() const { return use_fc_padding_; }

165
  // GPU related.
166

167 168 169 170 171 172
  ///
  /// \brief Turn on GPU.
  ///
  /// \param memory_pool_init_size_mb initial size of the GPU memory pool in MB.
  /// \param device_id device_id the GPU card to use (default is 0).
  ///
173
  void EnableUseGpu(uint64_t memory_pool_init_size_mb, int device_id = 0);
174 175 176 177
  ///
  /// \brief Turn off GPU.
  ///
  ///
178
  void DisableGpu();
179 180 181 182 183
  ///
  /// \brief A boolean state telling whether the GPU is turned on.
  ///
  /// \return bool Whether the GPU is turned on.
  ///
184
  bool use_gpu() const { return use_gpu_; }
185 186 187 188 189
  ///
  /// \brief Get the GPU device id.
  ///
  /// \return int The GPU device id.
  ///
190
  int gpu_device_id() const { return device_id_; }
191 192 193 194 195
  ///
  /// \brief Get the initial size in MB of the GPU memory pool.
  ///
  /// \return int The initial size in MB of the GPU memory pool.
  ///
196
  int memory_pool_init_size_mb() const { return memory_pool_init_size_mb_; }
197 198 199 200 201 202
  ///
  /// \brief Get the proportion of the initial memory pool size compared to the
  /// device.
  ///
  /// \return float The proportion of the initial memory pool size.
  ///
203
  float fraction_of_gpu_memory_for_pool() const;
204

205 206 207 208 209
  // CUDNN related.
  ///
  /// \brief Turn on CUDNN.
  ///
  ///
210
  void EnableCUDNN();
211 212 213 214 215
  ///
  /// \brief A boolean state telling whether to use CUDNN.
  ///
  /// \return bool Whether to use CUDNN.
  ///
216 217
  bool cudnn_enabled() const { return use_cudnn_; }

218 219 220 221 222 223
  ///
  /// \brief Control whether to perform IR graph optimization.
  /// If turned off, the AnalysisConfig will act just like a NativeConfig.
  ///
  /// \param x Whether the ir graph optimization is actived.
  ///
224
  void SwitchIrOptim(int x = true) { enable_ir_optim_ = x; }
225 226 227 228 229 230
  ///
  /// \brief A boolean state telling whether the ir graph optimization is
  /// actived.
  ///
  /// \return bool Whether to use ir graph optimization.
  ///
231
  bool ir_optim() const { return enable_ir_optim_; }
232

233 234 235 236 237 238 239
  ///
  /// \brief INTERNAL Determine whether to use the feed and fetch operators.
  /// Just for internal development, not stable yet.
  /// When ZeroCopyTensor is used, this should be turned off.
  ///
  /// \param x Whether to use the feed and fetch operators.
  ///
240
  void SwitchUseFeedFetchOps(int x = true) { use_feed_fetch_ops_ = x; }
241 242 243 244 245 246
  ///
  /// \brief A boolean state telling whether to use the feed and fetch
  /// operators.
  ///
  /// \return bool Whether to use the feed and fetch operators.
  ///
247
  bool use_feed_fetch_ops_enabled() const { return use_feed_fetch_ops_; }
248

249 250 251 252 253 254 255 256 257 258 259
  ///
  /// \brief Control whether to specify the inputs' names.
  /// The ZeroCopyTensor type has a name member, assign it with the
  /// corresponding
  /// variable name. This is used only when the input ZeroCopyTensors passed to
  /// the
  /// AnalysisPredictor.ZeroCopyRun() cannot follow the order in the training
  /// phase.
  ///
  /// \param x Whether to specify the inputs' names.
  ///
260
  void SwitchSpecifyInputNames(bool x = true) { specify_input_name_ = x; }
261 262 263 264 265 266 267
  ///
  /// \brief A boolean state tell whether the input ZeroCopyTensor names
  /// specified should
  /// be used to reorder the inputs in AnalysisPredictor.ZeroCopyRun().
  ///
  /// \return bool Whether to specify the inputs' names.
  ///
268
  bool specify_input_name() const { return specify_input_name_; }
269

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
  ///
  /// \brief Turn on the TensorRT engine.
  /// The TensorRT engine will accelerate some subgraphes in the original Fluid
  /// computation graph. In some models such as resnet50, GoogleNet and so on,
  /// it gains significant performance acceleration.
  ///
  /// \param workspace_size The memory size(in byte) used for TensorRT
  /// workspace.
  /// \param max_batch_size The maximum batch size of this prediction task,
  /// better set as small as possible for less performance loss.
  /// \param min_subgrpah_size The minimum TensorRT subgraph size needed, if a
  /// subgraph is smaller than this, it will not be transferred to TensorRT
  /// engine.
  /// \param precision The precision used in TensorRT.
  /// \param use_static Serialize optimization information to disk for reusing.
  /// \param use_calib_mode Use TRT int8 calibration(post training
  /// quantization).
  ///
  ///
289 290 291 292 293
  void EnableTensorRtEngine(int workspace_size = 1 << 20,
                            int max_batch_size = 1, int min_subgraph_size = 3,
                            Precision precision = Precision::kFloat32,
                            bool use_static = false,
                            bool use_calib_mode = true);
294 295 296 297 298
  ///
  /// \brief A boolean state telling whether the TensorRT engine is used.
  ///
  /// \return bool Whether the TensorRT engine is used.
  ///
299
  bool tensorrt_engine_enabled() const { return use_tensorrt_; }
300 301 302 303 304 305 306 307
  ///
  /// \brief Set min, max, opt shape for TensorRT Dynamic shape mode.
  /// \param min_input_shape The min input shape of the subgraph input.
  /// \param max_input_shape The max input shape of the subgraph input.
  /// \param opt_input_shape The opt input shape of the subgraph input.
  /// \param disable_trt_plugin_fp16 Setting this parameter to true means that
  /// TRT plugin will not run fp16.
  ///
308 309 310 311 312
  void SetTRTDynamicShapeInfo(
      std::map<std::string, std::vector<int>> min_input_shape,
      std::map<std::string, std::vector<int>> max_input_shape,
      std::map<std::string, std::vector<int>> optim_input_shape,
      bool disable_trt_plugin_fp16 = false);
313 314 315 316 317 318 319
  ///
  /// \brief Turn on the usage of Lite sub-graph engine.
  ///
  /// \param precision_mode Precion used in Lite sub-graph engine.
  /// \param passes_filter Set the passes used in Lite sub-graph engine.
  /// \param ops_filter Operators not supported by Lite.
  ///
石晓伟 已提交
320 321 322 323 324
  void EnableLiteEngine(
      AnalysisConfig::Precision precision_mode = Precision::kFloat32,
      const std::vector<std::string>& passes_filter = {},
      const std::vector<std::string>& ops_filter = {});

325 326 327 328 329 330
  ///
  /// \brief A boolean state indicating whether the Lite sub-graph engine is
  /// used.
  ///
  /// \return bool whether the Lite sub-graph engine is used.
  ///
石晓伟 已提交
331 332
  bool lite_engine_enabled() const { return use_lite_; }

333 334 335 336 337 338 339
  ///
  /// \brief Control whether to debug IR graph analysis phase.
  /// This will generate DOT files for visualizing the computation graph after
  /// each analysis pass applied.
  ///
  /// \param x whether to debug IR graph analysis phase.
  ///
Y
Yan Chunwei 已提交
340
  void SwitchIrDebug(int x = true);
341

342 343 344 345
  ///
  /// \brief Turn on MKLDNN.
  ///
  ///
L
luotao1 已提交
346
  void EnableMKLDNN();
347 348 349 350 351 352
  ///
  /// \brief Set the cache capacity of different input shapes for MKLDNN.
  /// Default value 0 means not caching any shape.
  ///
  /// \param capacity The cache capacity.
  ///
353
  void SetMkldnnCacheCapacity(int capacity);
354 355 356 357 358
  ///
  /// \brief A boolean state telling whether to use the MKLDNN.
  ///
  /// \return bool Whether to use the MKLDNN.
  ///
359 360
  bool mkldnn_enabled() const { return use_mkldnn_; }

361 362 363 364 365 366
  ///
  /// \brief Set the number of cpu math library threads.
  ///
  /// \param cpu_math_library_num_threads The number of cpu math library
  /// threads.
  ///
367
  void SetCpuMathLibraryNumThreads(int cpu_math_library_num_threads);
368 369 370 371 372 373
  ///
  /// \brief An int state telling how many threads are used in the CPU math
  /// library.
  ///
  /// \return int The number of threads used in the CPU math library.
  ///
374 375 376 377
  int cpu_math_library_num_threads() const {
    return cpu_math_library_num_threads_;
  }

378 379 380 381 382
  ///
  /// \brief Transform the AnalysisConfig to NativeConfig.
  ///
  /// \return NativeConfig The NativeConfig transformed.
  ///
Y
Yan Chunwei 已提交
383
  NativeConfig ToNativeConfig() const;
384 385 386 387 388
  ///
  /// \brief Specify the operator type list to use MKLDNN acceleration.
  ///
  /// \param op_list The operator type list.
  ///
389 390 391
  void SetMKLDNNOp(std::unordered_set<std::string> op_list) {
    mkldnn_enabled_op_types_ = op_list;
  }
392

393 394 395 396
  ///
  /// \brief Turn on MKLDNN quantization.
  ///
  ///
397 398
  void EnableMkldnnQuantizer();

399 400 401 402 403 404 405 406
  ///
  /// \brief A boolean state telling whether the thread local CUDA stream is
  /// enabled.
  ///
  /// \return bool Whether the thread local CUDA stream is enabled.
  ///
  bool thread_local_stream_enabled() const { return thread_local_stream_; }

407 408 409 410 411
  ///
  /// \brief A boolean state telling whether the MKLDNN quantization is enabled.
  ///
  /// \return bool Whether the MKLDNN quantization is enabled.
  ///
412 413
  bool mkldnn_quantizer_enabled() const { return use_mkldnn_quantizer_; }

414 415 416 417 418
  ///
  /// \brief Get MKLDNN quantizer config.
  ///
  /// \return MkldnnQuantizerConfig* MKLDNN quantizer config.
  ///
419
  MkldnnQuantizerConfig* mkldnn_quantizer_config() const;
420

421 422 423 424 425 426 427 428 429
  ///
  /// \brief Specify the memory buffer of program and parameter.
  /// Used when model and params are loaded directly from memory.
  ///
  /// \param prog_buffer The memory buffer of program.
  /// \param prog_buffer_size The size of the model data.
  /// \param params_buffer The memory buffer of the combined parameters file.
  /// \param params_buffer_size The size of the combined parameters data.
  ///
T
Tao Luo 已提交
430
  void SetModelBuffer(const char* prog_buffer, size_t prog_buffer_size,
431
                      const char* params_buffer, size_t params_buffer_size);
432 433 434 435 436 437
  ///
  /// \brief A boolean state telling whether the model is set from the CPU
  /// memory.
  ///
  /// \return bool Whether model and params are loaded directly from memory.
  ///
T
Tao Luo 已提交
438
  bool model_from_memory() const { return model_from_memory_; }
T
Tao Luo 已提交
439

440 441 442 443
  ///
  /// \brief Turn on memory optimize
  /// NOTE still in development.
  ///
444
  void EnableMemoryOptim();
445 446 447 448 449 450
  ///
  /// \brief A boolean state telling whether the memory optimization is
  /// activated.
  ///
  /// \return bool Whether the memory optimization is activated.
  ///
Y
Yan Chunwei 已提交
451
  bool enable_memory_optim() const;
452

453 454 455 456
  ///
  /// \brief Turn on profiling report.
  /// If not turned on, no profiling report will be generated.
  ///
457
  void EnableProfile();
458 459 460 461 462
  ///
  /// \brief A boolean state telling whether the profiler is activated.
  ///
  /// \return bool Whether the profiler is activated.
  ///
463 464
  bool profile_enabled() const { return with_profile_; }

465 466 467
  ///
  /// \brief Mute all logs in Paddle inference.
  ///
468
  void DisableGlogInfo();
469 470 471 472 473
  ///
  /// \brief A boolean state telling whether logs in Paddle inference are muted.
  ///
  /// \return bool Whether logs in Paddle inference are muted.
  ///
474 475
  bool glog_info_disabled() const { return !with_glog_info_; }

476 477 478 479 480
  ///
  /// \brief Set the AnalysisConfig to be invalid.
  /// This is to ensure that an AnalysisConfig can only be used in one
  /// AnalysisPredictor.
  ///
481
  void SetInValid() const { is_valid_ = false; }
482 483 484 485 486
  ///
  /// \brief A boolean state telling whether the AnalysisConfig is valid.
  ///
  /// \return bool Whether the AnalysisConfig is valid.
  ///
487
  bool is_valid() const { return is_valid_; }
Y
Yan Chunwei 已提交
488

489 490
  friend class ::paddle::AnalysisPredictor;

491 492 493 494 495
  ///
  /// \brief Get a pass builder for customize the passes in IR analysis phase.
  /// NOTE: Just for developer, not an official API, easy to be broken.
  ///
  ///
496
  PassStrategy* pass_builder() const;
497 498 499 500 501 502 503

  ///
  /// \brief Enable the GPU multi-computing stream feature.
  /// NOTE: The current behavior of this interface is to bind the computation
  /// stream to the thread, and this behavior may be changed in the future.
  ///
  void EnableGpuMultiStream();
504
  void PartiallyRelease();
505 506 507 508 509 510 511

 protected:
  // Update the config.
  void Update();

  std::string SerializeInfoCache();

512
 protected:
513 514
  // Model pathes.
  std::string model_dir_;
515 516
  mutable std::string prog_file_;
  mutable std::string params_file_;
517

S
Sylwester Fraczek 已提交
518
  // GPU related.
519 520 521 522
  bool use_gpu_{false};
  int device_id_{0};
  uint64_t memory_pool_init_size_mb_{100};  // initial size is 100MB.

523 524
  bool use_cudnn_{false};

525 526 527
  // Padding related
  bool use_fc_padding_{true};

S
Sylwester Fraczek 已提交
528
  // TensorRT related.
529
  bool use_tensorrt_{false};
530 531
  // For workspace_size, refer it from here:
  // https://docs.nvidia.com/deeplearning/sdk/tensorrt-developer-guide/index.html#troubleshooting
532
  int tensorrt_workspace_size_{1 << 30};
533 534 535 536
  // While TensorRT allows an engine optimized for a given max batch size
  // to run at any smaller size, the performance for those smaller
  // sizes may not be as well-optimized. Therefore, Max batch is best
  // equivalent to the runtime batch size.
537
  int tensorrt_max_batchsize_{1};
538 539 540 541 542
  //  We transform the Ops that can be converted into TRT layer in the model,
  //  and aggregate these Ops into subgraphs for TRT execution.
  //  We set this variable to control the minimum number of nodes in the
  //  subgraph, 3 as default value.
  int tensorrt_min_subgraph_size_{3};
543 544 545
  Precision tensorrt_precision_mode_{Precision::kFloat32};
  bool trt_use_static_engine_{false};
  bool trt_use_calib_mode_{true};
546 547 548 549
  std::map<std::string, std::vector<int>> min_input_shape_{};
  std::map<std::string, std::vector<int>> max_input_shape_{};
  std::map<std::string, std::vector<int>> optim_input_shape_{};
  bool disable_trt_plugin_fp16_{false};
550

Y
Yan Chunwei 已提交
551 552 553
  // memory reuse related.
  bool enable_memory_optim_{false};

554 555 556
  bool use_mkldnn_{false};
  std::unordered_set<std::string> mkldnn_enabled_op_types_;

T
Tao Luo 已提交
557
  bool model_from_memory_{false};
558

559 560 561 562 563 564 565 566
  bool enable_ir_optim_{true};
  bool use_feed_fetch_ops_{true};
  bool ir_debug_{false};

  bool specify_input_name_{false};

  int cpu_math_library_num_threads_{1};

567 568
  bool with_profile_{false};

569 570
  bool with_glog_info_{true};

571 572 573 574
  // A runtime cache, shouldn't be transferred to others.
  std::string serialized_info_cache_;

  mutable std::unique_ptr<PassStrategy> pass_builder_;
575

石晓伟 已提交
576 577 578 579 580
  bool use_lite_{false};
  std::vector<std::string> lite_passes_filter_;
  std::vector<std::string> lite_ops_filter_;
  Precision lite_precision_mode_;

581 582
  bool thread_local_stream_{false};

583 584
  // mkldnn related.
  int mkldnn_cache_capacity_{0};
585 586
  bool use_mkldnn_quantizer_{false};
  std::shared_ptr<MkldnnQuantizerConfig> mkldnn_quantizer_config_;
587

588 589 590 591
  // If the config is already used on a predictor, it becomes invalid.
  // Any config can only be used with one predictor.
  // Variables held by config can take up a lot of memory in some cases.
  // So we release the memory when the predictor is set up.
592 593
  mutable bool is_valid_{true};
  std::string opt_cache_dir_;
594 595 596
};

}  // namespace paddle