analysis_config.cc 14.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/scope.h"
17 18
#include "paddle/fluid/inference/api/paddle_analysis_config.h"
#include "paddle/fluid/inference/api/paddle_pass_builder.h"
19
#include "paddle/fluid/platform/enforce.h"
20
#include "paddle/fluid/platform/gpu_info.h"
21 22

namespace paddle {
23
extern const std::vector<std::string> kTRTSubgraphPasses;
石晓伟 已提交
24
extern const std::vector<std::string> kLiteSubgraphPasses;
25

26
PassStrategy *AnalysisConfig::pass_builder() const {
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
  if (!pass_builder_.get()) {
    if (use_gpu_) {
      LOG(INFO) << "Create GPU IR passes";
      pass_builder_.reset(new GpuPassStrategy);
    } else {
      LOG(INFO) << "Create CPU IR passes";
      pass_builder_.reset(new CpuPassStrategy);
    }
  } else if (pass_builder_->use_gpu() ^ use_gpu()) {
    LOG(WARNING) << "The use_gpu flag is not compatible between Config and "
                    "PassBuilder, the flags are "
                 << use_gpu() << " " << pass_builder_->use_gpu();
    LOG(WARNING) << "Please make them compatible, still use the existing "
                    "PassBuilder.";
  }

43 44 45
  return pass_builder_.get();
}

46
AnalysisConfig::AnalysisConfig(const std::string &model_dir) {
47
  model_dir_ = model_dir;
Y
Yan Chunwei 已提交
48 49

  Update();
50
}
51 52
AnalysisConfig::AnalysisConfig(const std::string &prog_file,
                               const std::string &params_file) {
53 54
  prog_file_ = prog_file;
  params_file_ = params_file;
Y
Yan Chunwei 已提交
55 56

  Update();
57
}
58 59
void AnalysisConfig::SetModel(const std::string &prog_file_path,
                              const std::string &params_file_path) {
60 61
  prog_file_ = prog_file_path;
  params_file_ = params_file_path;
Y
Yan Chunwei 已提交
62 63

  Update();
64
}
65 66
void AnalysisConfig::EnableUseGpu(uint64_t memory_pool_init_size_mb,
                                  int device_id) {
67 68 69 70 71
#ifdef PADDLE_WITH_CUDA
  use_gpu_ = true;
  memory_pool_init_size_mb_ = memory_pool_init_size_mb;
  device_id_ = device_id;
#else
Y
Yan Chunwei 已提交
72
  LOG(ERROR) << "Please compile with gpu to EnableGpu()";
73 74
  use_gpu_ = false;
#endif
Y
Yan Chunwei 已提交
75 76 77

  Update();
}
78
void AnalysisConfig::DisableGpu() {
Y
Yan Chunwei 已提交
79 80 81
  use_gpu_ = false;

  Update();
82 83
}

84 85 86 87 88 89
void AnalysisConfig::DisableFCPadding() {
  use_fc_padding_ = false;

  Update();
}

90 91 92 93 94 95
void AnalysisConfig::EnableXpu(int l3_workspace_size) {
  use_xpu_ = true;
  xpu_l3_workspace_size_ = l3_workspace_size;
  Update();
}

96
AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
97 98 99 100 101 102
#define CP_MEMBER(member__) member__ = other.member__;

  // Model related.
  CP_MEMBER(model_dir_);
  CP_MEMBER(model_from_memory_);  // the memory model reuses prog_file_ and
                                  // params_file_ fields.
103

104
  CP_MEMBER(opt_cache_dir_);
W
Wilber 已提交
105 106
  CP_MEMBER(prog_file_);
  CP_MEMBER(params_file_);
107

108
  CP_MEMBER(use_fc_padding_);
109
  // GPU related.
110
  CP_MEMBER(use_gpu_);
111
  CP_MEMBER(use_cudnn_);
112 113
  CP_MEMBER(device_id_);
  CP_MEMBER(memory_pool_init_size_mb_);
Y
Yan Chunwei 已提交
114 115

  CP_MEMBER(enable_memory_optim_);
S
Sylwester Fraczek 已提交
116
  // TensorRT related.
117 118 119 120
  CP_MEMBER(use_tensorrt_);
  CP_MEMBER(tensorrt_workspace_size_);
  CP_MEMBER(tensorrt_max_batchsize_);
  CP_MEMBER(tensorrt_min_subgraph_size_);
N
nhzlx 已提交
121
  CP_MEMBER(tensorrt_precision_mode_);
N
nhzlx 已提交
122
  CP_MEMBER(trt_use_static_engine_);
123
  CP_MEMBER(trt_use_calib_mode_);
S
Sylwester Fraczek 已提交
124
  // MKLDNN related.
125 126
  CP_MEMBER(use_mkldnn_);
  CP_MEMBER(mkldnn_enabled_op_types_);
127
  CP_MEMBER(mkldnn_cache_capacity_);
128 129 130
  // Quantization related.
  CP_MEMBER(use_mkldnn_quantizer_);
  CP_MEMBER(mkldnn_quantizer_config_);
131 132 133
  CP_MEMBER(min_input_shape_);
  CP_MEMBER(max_input_shape_);
  CP_MEMBER(optim_input_shape_);
134
  CP_MEMBER(disable_trt_plugin_fp16_);
135

石晓伟 已提交
136 137 138 139
  CP_MEMBER(use_lite_);
  CP_MEMBER(lite_precision_mode_);
  CP_MEMBER(lite_passes_filter_);
  CP_MEMBER(lite_ops_filter_);
140 141 142 143
  CP_MEMBER(lite_zero_copy_);

  CP_MEMBER(use_xpu_);
  CP_MEMBER(xpu_l3_workspace_size_);
石晓伟 已提交
144

145 146 147
  // profile related.
  CP_MEMBER(with_profile_);

148 149 150
  // glog related.
  CP_MEMBER(with_glog_info_);

151 152 153 154 155 156 157 158 159 160
  // Ir related.
  CP_MEMBER(enable_ir_optim_);
  CP_MEMBER(use_feed_fetch_ops_);
  CP_MEMBER(ir_debug_);
  CP_MEMBER(specify_input_name_);

  CP_MEMBER(cpu_math_library_num_threads_);

  CP_MEMBER(serialized_info_cache_);

161 162
  CP_MEMBER(thread_local_stream_);

163
  if (use_gpu_) {
164 165 166 167 168 169 170
    pass_builder_.reset(new GpuPassStrategy(
        *static_cast<GpuPassStrategy *>(other.pass_builder())));
  } else {
    pass_builder_.reset(new CpuPassStrategy(
        *static_cast<CpuPassStrategy *>(other.pass_builder())));
  }

171
#undef CP_MEMBER
Y
Yan Chunwei 已提交
172 173

  Update();
174 175
}

176 177 178 179 180 181 182 183 184 185 186
void AnalysisConfig::EnableCUDNN() {
#ifdef PADDLE_WITH_CUDA
  use_cudnn_ = use_gpu_;
#else
  LOG(ERROR) << "Please compile with CUDA first to use cuDNN";
  use_cudnn_ = false;
#endif

  Update();
}

187
void AnalysisConfig::EnableMKLDNN() {
188 189 190 191 192 193
#ifdef PADDLE_WITH_MKLDNN
  use_mkldnn_ = true;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MKLDNN";
  use_mkldnn_ = false;
#endif
Y
Yan Chunwei 已提交
194 195

  Update();
196 197
}

198 199 200 201 202 203 204 205 206
void AnalysisConfig::SetMkldnnCacheCapacity(int capacity) {
#ifdef PADDLE_WITH_MKLDNN
  mkldnn_cache_capacity_ = capacity;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to set MKLDNN Thread Id";
  mkldnn_cache_capacity_ = 0;
#endif
}

207 208 209 210 211 212 213 214 215 216 217 218 219
void AnalysisConfig::EnableMkldnnQuantizer() {
#ifdef PADDLE_WITH_MKLDNN
  if (!mkldnn_quantizer_config_)
    mkldnn_quantizer_config_.reset(new MkldnnQuantizerConfig());
  use_mkldnn_quantizer_ = true;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MkldnnQuantizer";
  use_mkldnn_quantizer_ = false;
#endif

  Update();
}

220 221 222 223 224 225 226 227 228 229 230
void AnalysisConfig::EnableMkldnnBfloat16() {
#ifdef PADDLE_WITH_MKLDNN
  use_mkldnn_bfloat16_ = true;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MkldnnBfloat16";
  use_mkldnn_bfloat16_ = false;
#endif

  Update();
}

231
MkldnnQuantizerConfig *AnalysisConfig::mkldnn_quantizer_config() const {
232 233
  PADDLE_ENFORCE_NOT_NULL(mkldnn_quantizer_config_,
                          "MkldnnQuantizer was not enabled yet.");
234
  return mkldnn_quantizer_config_.get();
235 236
}

237
void AnalysisConfig::EnableTensorRtEngine(
N
nhzlx 已提交
238
    int workspace_size, int max_batch_size, int min_subgraph_size,
239
    AnalysisConfig::Precision precision_mode, bool use_static,
240
    bool use_calib_mode) {
Y
Yan Chunwei 已提交
241 242 243 244 245 246
#ifdef PADDLE_WITH_CUDA
  if (!use_gpu()) {
    LOG(ERROR) << "To use TensorRT engine, please call EnableGpu() first";
    return;
  }

247 248 249
  use_tensorrt_ = true;
  tensorrt_workspace_size_ = workspace_size;
  tensorrt_max_batchsize_ = max_batch_size;
N
nhzlx 已提交
250
  tensorrt_min_subgraph_size_ = min_subgraph_size;
N
nhzlx 已提交
251
  tensorrt_precision_mode_ = precision_mode;
N
nhzlx 已提交
252
  trt_use_static_engine_ = use_static;
253
  trt_use_calib_mode_ = use_calib_mode;
Y
Yan Chunwei 已提交
254

255
  Update();
Y
Yan Chunwei 已提交
256 257 258 259
#else
  LOG(ERROR)
      << "To use TensorRT engine, please compile inference lib with GPU first.";
#endif
260 261
}

262 263 264 265 266 267 268 269 270 271 272
void AnalysisConfig::SetTRTDynamicShapeInfo(
    std::map<std::string, std::vector<int>> min_input_shape,
    std::map<std::string, std::vector<int>> max_input_shape,
    std::map<std::string, std::vector<int>> optim_input_shape,
    bool disable_trt_plugin_fp16) {
  min_input_shape_ = min_input_shape;
  max_input_shape_ = max_input_shape;
  optim_input_shape_ = optim_input_shape;
  disable_trt_plugin_fp16_ = disable_trt_plugin_fp16;
}

Y
Yan Chunwei 已提交
273
// TODO(Superjomn) refactor this, buggy.
274
void AnalysisConfig::Update() {
275 276 277
  auto info = SerializeInfoCache();
  if (info == serialized_info_cache_) return;

Y
Yan Chunwei 已提交
278 279 280 281 282 283 284 285 286 287 288 289
  // Transfer pass_builder and copy the existing compatible passes.
  if (!pass_builder_ || ((use_gpu() ^ pass_builder_->use_gpu()))) {
    if (use_gpu()) {
      pass_builder_.reset(new GpuPassStrategy);

      if (use_tensorrt_) {
        // Append after the Affine_channel_conv_fuse pass.
        pass_builder()->InsertPass(3, "tensorrt_subgraph_pass");
      }
    } else {
      pass_builder_.reset(new CpuPassStrategy);
    }
290

291
  } else {
Y
Yan Chunwei 已提交
292 293 294 295 296 297 298 299
    if (use_gpu()) {
      pass_builder_.reset(new GpuPassStrategy(
          *static_cast<GpuPassStrategy *>(pass_builder_.get())));

    } else {
      pass_builder_.reset(new CpuPassStrategy(
          *static_cast<CpuPassStrategy *>(pass_builder_.get())));
    }
300 301 302
  }

  if (use_tensorrt_) {
303 304
    pass_builder()->ClearPasses();
    for (const auto &pass : kTRTSubgraphPasses) {
305
      if (tensorrt_precision_mode_ == AnalysisConfig::Precision::kInt8 &&
306 307 308
          (pass == "conv_bn_fuse_pass" || pass == "fc_fuse_pass")) {
        continue;
      }
309
      pass_builder()->AppendPass(pass);
310 311
    }
  }
312 313 314 315 316 317 318 319 320 321
  if (use_gpu() && use_cudnn_) {
#ifdef PADDLE_WITH_CUDA
    if (!enable_ir_optim_) {
      LOG(ERROR) << "EnableCUDNN() only works when IR optimization is enabled.";
    } else {
      pass_builder()->EnableCUDNN();
    }
#endif
  }

322
  if (use_mkldnn_) {
W
Wojciech Uss 已提交
323
#ifdef PADDLE_WITH_MKLDNN
324 325 326
    if (!enable_ir_optim_) {
      LOG(ERROR)
          << "EnableMKLDNN() only works when IR optimization is enabled.";
W
Wojciech Uss 已提交
327 328
    } else {
      pass_builder()->EnableMKLDNN();
329 330 331 332
    }
#endif
  }

333 334 335 336 337
  // Quantization passes must come after all other optimization passes
  if (use_mkldnn_quantizer_) {
    if (!enable_ir_optim_) {
      LOG(ERROR) << "EnableMkldnnQuantizer() only works when IR optimization "
                    "is enabled.";
338 339
    }
#ifdef PADDLE_WITH_MKLDNN
340
    pass_builder()->EnableMkldnnQuantizer();
341 342 343
#endif
  }

344 345 346 347 348 349
  if (use_mkldnn_bfloat16_) {
#ifdef PADDLE_WITH_MKLDNN
    pass_builder()->EnableMkldnnBfloat16();
#endif
  }

350
#ifdef PADDLE_WITH_MKLDNN
351 352
  // Do not optimize when mkldnn is on
  if (enable_memory_optim_ && !use_mkldnn_) {
353
#else
Y
Yan Chunwei 已提交
354
  if (enable_memory_optim_) {
355 356
#endif
    pass_builder()->AppendAnalysisPass("memory_optimize_pass");
Y
Yan Chunwei 已提交
357 358
  }

石晓伟 已提交
359 360 361 362 363 364 365 366 367 368 369 370 371 372
  if (use_lite_) {
#ifndef PADDLE_WITH_LITE
    LOG(WARNING) << "You tried to enable the lite subgraph "
                    "but did not have the option -DWITH_LITE compiled.";
#endif
    pass_builder()->ClearPasses();
    for (const auto &pass : kLiteSubgraphPasses) {
      if (std::find(lite_passes_filter_.begin(), lite_passes_filter_.end(),
                    pass) == lite_passes_filter_.end()) {
        pass_builder()->AppendPass(pass);
      }
    }
  }

373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
  if (use_xpu_) {
#ifndef PADDLE_WITH_XPU
    PADDLE_THROW(platform::errors::Unavailable(
        "You tried to use an XPU device, but Paddle was not compiled "
        "with XPU-runtime."));
#endif
    if (!use_lite_) {
      LOG(WARNING) << "Because XPU currently only works in Paddle-Lite "
                      "subgraph mode, please make sure you have enabled it.";
    }
    PADDLE_ENFORCE_EQ(use_gpu_, false,
                      platform::errors::Unavailable(
                          "Currently, XPU and GPU cannot be enabled in the "
                          "same analysis configuration."));
  }

389 390 391 392 393
  if (ir_debug_) {
    pass_builder()->TurnOnDebug();
  }
}

394
std::string AnalysisConfig::SerializeInfoCache() {
395
  std::stringstream ss;
Y
Yan Chunwei 已提交
396 397 398 399
  ss << model_dir_;
  ss << prog_file_;
  ss << params_file_;

400
  ss << use_gpu_;
401
  ss << use_fc_padding_;
Y
Yan Chunwei 已提交
402
  ss << device_id_;
403 404 405 406 407
  ss << memory_pool_init_size_mb_;

  ss << use_tensorrt_;
  ss << tensorrt_workspace_size_;
  ss << tensorrt_max_batchsize_;
Y
Yan Chunwei 已提交
408 409 410
  ss << tensorrt_min_subgraph_size_;

  ss << enable_memory_optim_;
411 412

  ss << use_mkldnn_;
413
  ss << mkldnn_cache_capacity_;
Y
Yan Chunwei 已提交
414 415 416
  for (auto &item : mkldnn_enabled_op_types_) ss << item;
  ss << ";";

417
  ss << use_mkldnn_quantizer_;
418
  ss << use_mkldnn_bfloat16_;
Y
Yan Chunwei 已提交
419 420
  ss << model_from_memory_;

421 422
  ss << with_profile_;

423 424
  ss << with_glog_info_;

425 426 427 428
  ss << enable_ir_optim_;
  ss << use_feed_fetch_ops_;
  ss << ir_debug_;

Y
Yan Chunwei 已提交
429 430
  ss << specify_input_name_;
  ss << cpu_math_library_num_threads_;
石晓伟 已提交
431 432

  ss << use_lite_;
433 434
  ss << use_xpu_;
  ss << xpu_l3_workspace_size_;
435

436 437
  ss << thread_local_stream_;

438 439 440
  return ss.str();
}

441
void AnalysisConfig::SetCpuMathLibraryNumThreads(
442 443
    int cpu_math_library_num_threads) {
  cpu_math_library_num_threads_ = cpu_math_library_num_threads;
Y
Yan Chunwei 已提交
444 445

  Update();
446 447
}

448
float AnalysisConfig::fraction_of_gpu_memory_for_pool() const {
449 450 451 452
#ifdef PADDLE_WITH_CUDA
  // Get the GPU memory details and calculate the fraction of memory for the
  // GPU memory pool.
  size_t gpu_used, gpu_available;
453
  platform::SetDeviceId(device_id_);
454 455 456 457 458 459 460 461
  platform::GpuMemoryUsage(&gpu_used, &gpu_available);
  double total_gpu_memory = (gpu_used + gpu_available) / 1024. / 1024.;
  float fraction_of_gpu_memory =
      static_cast<double>(memory_pool_init_size_mb()) / total_gpu_memory;
  return fraction_of_gpu_memory;
#else
  return 0.;
#endif
462 463
}

464
void AnalysisConfig::EnableMemoryOptim() {
Y
Yan Chunwei 已提交
465 466 467 468
  enable_memory_optim_ = true;
  Update();
}

469
bool AnalysisConfig::enable_memory_optim() const {
Y
Yan Chunwei 已提交
470 471 472
  return enable_memory_optim_;
}

473 474 475 476
void AnalysisConfig::SetModelBuffer(const char *prog_buffer,
                                    size_t prog_buffer_size,
                                    const char *param_buffer,
                                    size_t param_buffer_size) {
477 478
  prog_file_ = std::string(prog_buffer, prog_buffer + prog_buffer_size);
  params_file_ = std::string(param_buffer, param_buffer + param_buffer_size);
T
Tao Luo 已提交
479
  model_from_memory_ = true;
Y
Yan Chunwei 已提交
480 481

  Update();
T
Tao Luo 已提交
482 483
}

484
NativeConfig AnalysisConfig::ToNativeConfig() const {
Y
Yan Chunwei 已提交
485 486 487 488 489 490 491 492 493 494 495
  NativeConfig config;
  config.model_dir = model_dir_;
  config.prog_file = prog_file_;
  config.param_file = params_file_;
  config.use_gpu = use_gpu_;
  config.device = device_id_;
  config.fraction_of_gpu_memory = fraction_of_gpu_memory_for_pool();
  config.specify_input_name = specify_input_name_;
  return config;
}

Y
Yan Chunwei 已提交
496 497 498 499
void AnalysisConfig::SwitchIrDebug(int x) {
  ir_debug_ = x;
  Update();
}
500 501 502 503 504 505

void AnalysisConfig::EnableProfile() {
  with_profile_ = true;
  Update();
}

506 507 508 509 510
void AnalysisConfig::DisableGlogInfo() {
  with_glog_info_ = false;
  Update();
}

石晓伟 已提交
511
void AnalysisConfig::EnableLiteEngine(
512
    AnalysisConfig::Precision precision_mode, bool zero_copy,
石晓伟 已提交
513 514 515 516 517 518
    const std::vector<std::string> &passes_filter,
    const std::vector<std::string> &ops_filter) {
  use_lite_ = true;
  lite_precision_mode_ = precision_mode;
  lite_passes_filter_ = passes_filter;
  lite_ops_filter_ = ops_filter;
519
  lite_zero_copy_ = zero_copy;
石晓伟 已提交
520 521 522
  Update();
}

523 524 525 526 527 528 529
void AnalysisConfig::PartiallyRelease() {
  prog_file_.clear();
  prog_file_.shrink_to_fit();
  params_file_.clear();
  params_file_.shrink_to_fit();
}

530 531
void AnalysisConfig::EnableGpuMultiStream() { thread_local_stream_ = true; }

532
}  // namespace paddle