analysis_config.cc 24.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
#include <string>
16 17
#include "paddle/fluid/inference/api/paddle_analysis_config.h"
#include "paddle/fluid/inference/api/paddle_pass_builder.h"
18
#include "paddle/fluid/inference/utils/table_printer.h"
19
#include "paddle/fluid/platform/cpu_info.h"
20
#include "paddle/fluid/platform/enforce.h"
21
#include "paddle/fluid/platform/gpu_info.h"
22

23
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
24 25 26
DECLARE_uint64(initial_gpu_memory_in_mb);
#endif

27
namespace paddle {
W
wanghuancoder 已提交
28 29
struct MkldnnQuantizerConfig;

30
extern const std::vector<std::string> kTRTSubgraphPasses;
D
denglin-github 已提交
31
extern const std::vector<std::string> kDlnneSubgraphPasses;
石晓伟 已提交
32
extern const std::vector<std::string> kLiteSubgraphPasses;
33

34
PassStrategy *AnalysisConfig::pass_builder() const {
35 36 37 38
  if (!pass_builder_.get()) {
    if (use_gpu_) {
      LOG(INFO) << "Create GPU IR passes";
      pass_builder_.reset(new GpuPassStrategy);
39 40
    } else if (use_xpu_) {
      pass_builder_.reset(new XpuPassStrategy);
W
Wilber 已提交
41 42
    } else if (use_npu_) {
      pass_builder_.reset(new NpuPassStrategy);
43 44 45 46 47 48 49 50 51 52 53 54
    } else {
      LOG(INFO) << "Create CPU IR passes";
      pass_builder_.reset(new CpuPassStrategy);
    }
  } else if (pass_builder_->use_gpu() ^ use_gpu()) {
    LOG(WARNING) << "The use_gpu flag is not compatible between Config and "
                    "PassBuilder, the flags are "
                 << use_gpu() << " " << pass_builder_->use_gpu();
    LOG(WARNING) << "Please make them compatible, still use the existing "
                    "PassBuilder.";
  }

55 56 57
  return pass_builder_.get();
}

58
AnalysisConfig::AnalysisConfig(const std::string &model_dir) {
59
  model_dir_ = model_dir;
Y
Yan Chunwei 已提交
60 61

  Update();
62
}
63 64
AnalysisConfig::AnalysisConfig(const std::string &prog_file,
                               const std::string &params_file) {
65 66
  prog_file_ = prog_file;
  params_file_ = params_file;
Y
Yan Chunwei 已提交
67 68

  Update();
69
}
70 71
void AnalysisConfig::SetModel(const std::string &prog_file_path,
                              const std::string &params_file_path) {
72 73
  prog_file_ = prog_file_path;
  params_file_ = params_file_path;
Y
Yan Chunwei 已提交
74 75

  Update();
76
}
77 78
void AnalysisConfig::EnableUseGpu(uint64_t memory_pool_init_size_mb,
                                  int device_id) {
79
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
80 81
  use_gpu_ = true;
  memory_pool_init_size_mb_ = memory_pool_init_size_mb;
82
  FLAGS_initial_gpu_memory_in_mb = memory_pool_init_size_mb_;
83
  gpu_device_id_ = device_id;
84
#else
Y
Yan Chunwei 已提交
85
  LOG(ERROR) << "Please compile with gpu to EnableGpu()";
86 87
  use_gpu_ = false;
#endif
Y
Yan Chunwei 已提交
88 89 90

  Update();
}
91
void AnalysisConfig::DisableGpu() {
Y
Yan Chunwei 已提交
92 93 94
  use_gpu_ = false;

  Update();
95 96
}

97 98 99 100 101 102
void AnalysisConfig::DisableFCPadding() {
  use_fc_padding_ = false;

  Update();
}

W
Wilber 已提交
103 104 105 106
void AnalysisConfig::EnableXpu(int l3_workspace_size, bool locked,
                               bool autotune, const std::string &autotune_file,
                               const std::string &precision,
                               bool adaptive_seqlen) {
107 108
  use_xpu_ = true;
  xpu_l3_workspace_size_ = l3_workspace_size;
W
Wilber 已提交
109 110 111 112 113
  xpu_locked_ = locked;
  xpu_autotune_ = autotune;
  xpu_autotune_file_ = autotune_file;
  xpu_precision_ = precision;
  xpu_adaptive_seqlen_ = adaptive_seqlen;
114 115 116
  Update();
}

W
Wilber 已提交
117 118 119 120 121 122 123 124 125 126 127 128
void AnalysisConfig::EnableNpu(int device_id) {
#ifdef PADDLE_WITH_ASCEND_CL
  use_npu_ = true;
  npu_device_id_ = device_id;
#else
  LOG(ERROR) << "Please compile with npu to EnableNpu()";
  use_npu_ = false;
#endif

  Update();
}

129
AnalysisConfig::AnalysisConfig(const AnalysisConfig &other) {
130 131 132 133 134 135
#define CP_MEMBER(member__) member__ = other.member__;

  // Model related.
  CP_MEMBER(model_dir_);
  CP_MEMBER(model_from_memory_);  // the memory model reuses prog_file_ and
                                  // params_file_ fields.
136

137
  CP_MEMBER(opt_cache_dir_);
W
Wilber 已提交
138 139
  CP_MEMBER(prog_file_);
  CP_MEMBER(params_file_);
140

141
  CP_MEMBER(use_fc_padding_);
142
  // GPU related.
143
  CP_MEMBER(use_gpu_);
144
  CP_MEMBER(use_cudnn_);
145
  CP_MEMBER(gpu_device_id_);
146
  CP_MEMBER(memory_pool_init_size_mb_);
Y
Yan Chunwei 已提交
147 148

  CP_MEMBER(enable_memory_optim_);
S
Sylwester Fraczek 已提交
149
  // TensorRT related.
150 151 152 153
  CP_MEMBER(use_tensorrt_);
  CP_MEMBER(tensorrt_workspace_size_);
  CP_MEMBER(tensorrt_max_batchsize_);
  CP_MEMBER(tensorrt_min_subgraph_size_);
N
nhzlx 已提交
154
  CP_MEMBER(tensorrt_precision_mode_);
155
  CP_MEMBER(trt_disabled_ops_);
156 157
  CP_MEMBER(trt_use_dla_);
  CP_MEMBER(trt_dla_core_);
N
nhzlx 已提交
158
  CP_MEMBER(trt_use_static_engine_);
159
  CP_MEMBER(trt_use_calib_mode_);
160
  CP_MEMBER(trt_use_oss_);
D
denglin-github 已提交
161 162 163
  // Dlnne related
  CP_MEMBER(use_dlnne_);
  CP_MEMBER(dlnne_min_subgraph_size_);
S
Sylwester Fraczek 已提交
164
  // MKLDNN related.
165 166
  CP_MEMBER(use_mkldnn_);
  CP_MEMBER(mkldnn_enabled_op_types_);
167
  CP_MEMBER(mkldnn_cache_capacity_);
168 169 170
  // Bfloat16 related.
  CP_MEMBER(use_mkldnn_bfloat16_);
  CP_MEMBER(bfloat16_enabled_op_types_);
171 172 173
  // Quantization related.
  CP_MEMBER(use_mkldnn_quantizer_);
  CP_MEMBER(mkldnn_quantizer_config_);
174 175 176
  CP_MEMBER(min_input_shape_);
  CP_MEMBER(max_input_shape_);
  CP_MEMBER(optim_input_shape_);
177
  CP_MEMBER(disable_trt_plugin_fp16_);
178

石晓伟 已提交
179 180 181 182
  CP_MEMBER(use_lite_);
  CP_MEMBER(lite_precision_mode_);
  CP_MEMBER(lite_passes_filter_);
  CP_MEMBER(lite_ops_filter_);
183 184
  CP_MEMBER(lite_zero_copy_);

W
Wilber 已提交
185
  // XPU related.
186
  CP_MEMBER(use_xpu_);
W
Wilber 已提交
187
  CP_MEMBER(xpu_device_id_);
188
  CP_MEMBER(xpu_l3_workspace_size_);
W
Wilber 已提交
189 190 191 192 193
  CP_MEMBER(xpu_locked_);
  CP_MEMBER(xpu_autotune_);
  CP_MEMBER(xpu_autotune_file_);
  CP_MEMBER(xpu_precision_);
  CP_MEMBER(xpu_adaptive_seqlen_);
石晓伟 已提交
194

W
Wilber 已提交
195 196 197 198
  // NPU related.
  CP_MEMBER(use_npu_);
  CP_MEMBER(npu_device_id_);

199 200 201
  // profile related.
  CP_MEMBER(with_profile_);

202 203 204
  // glog related.
  CP_MEMBER(with_glog_info_);

205 206 207 208 209 210 211 212 213 214
  // Ir related.
  CP_MEMBER(enable_ir_optim_);
  CP_MEMBER(use_feed_fetch_ops_);
  CP_MEMBER(ir_debug_);
  CP_MEMBER(specify_input_name_);

  CP_MEMBER(cpu_math_library_num_threads_);

  CP_MEMBER(serialized_info_cache_);

215 216
  CP_MEMBER(thread_local_stream_);

217
  if (use_gpu_) {
218 219 220
    PADDLE_ENFORCE_EQ(use_xpu_, false,
                      platform::errors::InvalidArgument(
                          "Only one choice can be made between CPU and XPU."));
221 222
    pass_builder_.reset(new GpuPassStrategy(
        *static_cast<GpuPassStrategy *>(other.pass_builder())));
223 224 225
  } else if (use_xpu_) {
    pass_builder_.reset(new XpuPassStrategy(
        *static_cast<XpuPassStrategy *>(other.pass_builder())));
W
Wilber 已提交
226 227 228
  } else if (use_npu_) {
    pass_builder_.reset(new NpuPassStrategy(
        *static_cast<NpuPassStrategy *>(other.pass_builder())));
229 230 231 232 233
  } else {
    pass_builder_.reset(new CpuPassStrategy(
        *static_cast<CpuPassStrategy *>(other.pass_builder())));
  }

234
#undef CP_MEMBER
Y
Yan Chunwei 已提交
235

W
Wilber 已提交
236 237 238 239 240 241 242
  Update();
  if (use_tensorrt_) {
    // Update() will reset all the passes, when some tensorRT pass is deleted in
    // other.pass_builder(), it will set again, so we just remove the
    // deleted_pass.
    auto all_passes = kTRTSubgraphPasses;
    auto other_passes = other.pass_builder()->AllPasses();
W
Wilber 已提交
243 244 245 246
    // We should sort them, because the user may call the SwitchIrDebug
    // interface, which will change the pass.
    std::sort(all_passes.begin(), all_passes.end());
    std::sort(other_passes.begin(), other_passes.end());
W
Wilber 已提交
247 248 249 250 251 252 253
    std::vector<std::string> deleted_passes;
    std::set_difference(all_passes.begin(), all_passes.end(),
                        other_passes.begin(), other_passes.end(),
                        std::inserter(deleted_passes, deleted_passes.begin()));
    for (auto ps : deleted_passes) {
      pass_builder_->DeletePass(ps);
    }
254
  }
D
denglin-github 已提交
255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
  if (use_dlnne_) {
    auto all_passes = kDlnneSubgraphPasses;
    auto other_passes = other.pass_builder()->AllPasses();
    // We should sort them, because the user may call the SwitchIrDebug
    // interface, which will change the pass.
    std::sort(all_passes.begin(), all_passes.end());
    std::sort(other_passes.begin(), other_passes.end());
    std::vector<std::string> deleted_passes;
    std::set_difference(all_passes.begin(), all_passes.end(),
                        other_passes.begin(), other_passes.end(),
                        std::inserter(deleted_passes, deleted_passes.begin()));
    for (auto ps : deleted_passes) {
      pass_builder_->DeletePass(ps);
    }
  }
270 271
}

272
void AnalysisConfig::EnableCUDNN() {
273
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
274 275 276 277 278 279 280 281 282
  use_cudnn_ = use_gpu_;
#else
  LOG(ERROR) << "Please compile with CUDA first to use cuDNN";
  use_cudnn_ = false;
#endif

  Update();
}

283
void AnalysisConfig::EnableMKLDNN() {
284 285 286 287 288 289
#ifdef PADDLE_WITH_MKLDNN
  use_mkldnn_ = true;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MKLDNN";
  use_mkldnn_ = false;
#endif
Y
Yan Chunwei 已提交
290 291

  Update();
292 293
}

294 295 296 297 298 299 300 301 302
void AnalysisConfig::SetMkldnnCacheCapacity(int capacity) {
#ifdef PADDLE_WITH_MKLDNN
  mkldnn_cache_capacity_ = capacity;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to set MKLDNN Thread Id";
  mkldnn_cache_capacity_ = 0;
#endif
}

303 304 305 306 307 308 309 310 311 312 313 314 315
void AnalysisConfig::EnableMkldnnQuantizer() {
#ifdef PADDLE_WITH_MKLDNN
  if (!mkldnn_quantizer_config_)
    mkldnn_quantizer_config_.reset(new MkldnnQuantizerConfig());
  use_mkldnn_quantizer_ = true;
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MkldnnQuantizer";
  use_mkldnn_quantizer_ = false;
#endif

  Update();
}

316 317
void AnalysisConfig::EnableMkldnnBfloat16() {
#ifdef PADDLE_WITH_MKLDNN
318 319
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core)) {
    use_mkldnn_bfloat16_ = true;
320 321 322 323
    LOG(INFO) << "Hardware support for BFLOAT16"
              << (platform::MayIUse(platform::cpu_isa_t::avx512_bf16)
                      ? " is enabled"
                      : " is disabled. Simulation will be used");
324 325 326 327
  } else {
    LOG(INFO) << "CPU does not support BFLOAT16 calculations";
    use_mkldnn_bfloat16_ = false;
  }
328 329 330 331 332 333 334 335
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MkldnnBfloat16";
  use_mkldnn_bfloat16_ = false;
#endif

  Update();
}

336
MkldnnQuantizerConfig *AnalysisConfig::mkldnn_quantizer_config() const {
337
  PADDLE_ENFORCE_NOT_NULL(mkldnn_quantizer_config_,
338 339
                          platform::errors::PreconditionNotMet(
                              "MkldnnQuantizer was not enabled yet."));
340
  return mkldnn_quantizer_config_.get();
341 342
}

343
void AnalysisConfig::EnableTensorRtEngine(
N
nhzlx 已提交
344
    int workspace_size, int max_batch_size, int min_subgraph_size,
345
    AnalysisConfig::Precision precision_mode, bool use_static,
346
    bool use_calib_mode) {
347
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
Y
Yan Chunwei 已提交
348 349 350 351 352
  if (!use_gpu()) {
    LOG(ERROR) << "To use TensorRT engine, please call EnableGpu() first";
    return;
  }

353 354 355
  use_tensorrt_ = true;
  tensorrt_workspace_size_ = workspace_size;
  tensorrt_max_batchsize_ = max_batch_size;
N
nhzlx 已提交
356
  tensorrt_min_subgraph_size_ = min_subgraph_size;
N
nhzlx 已提交
357
  tensorrt_precision_mode_ = precision_mode;
N
nhzlx 已提交
358
  trt_use_static_engine_ = use_static;
359
  trt_use_calib_mode_ = use_calib_mode;
Y
Yan Chunwei 已提交
360

361
  Update();
Y
Yan Chunwei 已提交
362 363 364 365
#else
  LOG(ERROR)
      << "To use TensorRT engine, please compile inference lib with GPU first.";
#endif
366 367
}

D
denglin-github 已提交
368 369 370 371 372 373
void AnalysisConfig::EnableDlnne(int min_subgraph_size) {
  use_dlnne_ = true;
  dlnne_min_subgraph_size_ = min_subgraph_size;
  Update();
}

374 375 376 377 378 379 380 381 382 383 384
void AnalysisConfig::SetTRTDynamicShapeInfo(
    std::map<std::string, std::vector<int>> min_input_shape,
    std::map<std::string, std::vector<int>> max_input_shape,
    std::map<std::string, std::vector<int>> optim_input_shape,
    bool disable_trt_plugin_fp16) {
  min_input_shape_ = min_input_shape;
  max_input_shape_ = max_input_shape;
  optim_input_shape_ = optim_input_shape;
  disable_trt_plugin_fp16_ = disable_trt_plugin_fp16;
}

385 386 387 388 389
void AnalysisConfig::EnableTensorRtDLA(int dla_core) {
  trt_use_dla_ = true;
  trt_dla_core_ = dla_core;
}

390 391 392 393 394
void AnalysisConfig::Exp_DisableTensorRtOPs(
    const std::vector<std::string> &ops) {
  trt_disabled_ops_.insert(trt_disabled_ops_.end(), ops.begin(), ops.end());
}

395
void AnalysisConfig::EnableTensorRtOSS() { trt_use_oss_ = true; }
396

Y
Yan Chunwei 已提交
397
// TODO(Superjomn) refactor this, buggy.
398
void AnalysisConfig::Update() {
399 400 401
  auto info = SerializeInfoCache();
  if (info == serialized_info_cache_) return;

Y
Yan Chunwei 已提交
402
  // Transfer pass_builder and copy the existing compatible passes.
W
Wilber 已提交
403 404 405
  if (!pass_builder_ || ((use_gpu() ^ pass_builder_->use_gpu())) ||
      ((use_xpu() ^ pass_builder_->use_xpu())) ||
      ((use_npu() ^ pass_builder_->use_npu()))) {
Y
Yan Chunwei 已提交
406 407 408 409 410 411 412
    if (use_gpu()) {
      pass_builder_.reset(new GpuPassStrategy);

      if (use_tensorrt_) {
        // Append after the Affine_channel_conv_fuse pass.
        pass_builder()->InsertPass(3, "tensorrt_subgraph_pass");
      }
413 414 415 416 417 418
    } else if (use_xpu()) {
      PADDLE_ENFORCE_EQ(
          use_gpu(), false,
          platform::errors::InvalidArgument(
              "Only one choice can be made between CPU and XPU."));
      pass_builder_.reset(new XpuPassStrategy);
W
Wilber 已提交
419 420 421 422 423 424
    } else if (use_npu()) {
      PADDLE_ENFORCE_EQ(
          use_gpu(), false,
          platform::errors::InvalidArgument(
              "Only one choice can be made between GPU and NPU."));
      pass_builder_.reset(new NpuPassStrategy);
Y
Yan Chunwei 已提交
425 426 427
    } else {
      pass_builder_.reset(new CpuPassStrategy);
    }
428

429
  } else {
Y
Yan Chunwei 已提交
430 431 432
    if (use_gpu()) {
      pass_builder_.reset(new GpuPassStrategy(
          *static_cast<GpuPassStrategy *>(pass_builder_.get())));
433 434 435 436 437 438 439
    } else if (use_xpu()) {
      PADDLE_ENFORCE_EQ(
          use_gpu(), false,
          platform::errors::InvalidArgument(
              "Only one choice can be made between CPU and XPU."));
      pass_builder_.reset(new XpuPassStrategy(
          *static_cast<XpuPassStrategy *>(pass_builder_.get())));
W
Wilber 已提交
440 441 442 443 444 445 446
    } else if (use_npu()) {
      PADDLE_ENFORCE_EQ(
          use_gpu(), false,
          platform::errors::InvalidArgument(
              "Only one choice can be made between GPU and NPU."));
      pass_builder_.reset(new NpuPassStrategy(
          *static_cast<NpuPassStrategy *>(pass_builder_.get())));
Y
Yan Chunwei 已提交
447 448 449 450
    } else {
      pass_builder_.reset(new CpuPassStrategy(
          *static_cast<CpuPassStrategy *>(pass_builder_.get())));
    }
451 452 453
  }

  if (use_tensorrt_) {
454 455
    pass_builder()->ClearPasses();
    for (const auto &pass : kTRTSubgraphPasses) {
456
      if (tensorrt_precision_mode_ == AnalysisConfig::Precision::kInt8 &&
457
          (pass == "conv_bn_fuse_pass")) {
458 459
        continue;
      }
460
      pass_builder()->AppendPass(pass);
461 462
    }
  }
D
denglin-github 已提交
463 464 465 466 467 468 469
  if (use_dlnne_) {
    pass_builder()->ClearPasses();
    for (const auto &pass : kDlnneSubgraphPasses) {
      pass_builder()->AppendPass(pass);
    }
  }

470
  if (use_gpu() && use_cudnn_) {
471
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
472 473 474 475 476 477 478 479
    if (!enable_ir_optim_) {
      LOG(ERROR) << "EnableCUDNN() only works when IR optimization is enabled.";
    } else {
      pass_builder()->EnableCUDNN();
    }
#endif
  }

480
  if (use_mkldnn_) {
W
Wojciech Uss 已提交
481
#ifdef PADDLE_WITH_MKLDNN
482 483 484
    if (!enable_ir_optim_) {
      LOG(ERROR)
          << "EnableMKLDNN() only works when IR optimization is enabled.";
W
Wojciech Uss 已提交
485 486
    } else {
      pass_builder()->EnableMKLDNN();
487 488 489 490
    }
#endif
  }

491 492 493 494 495
  // Quantization passes must come after all other optimization passes
  if (use_mkldnn_quantizer_) {
    if (!enable_ir_optim_) {
      LOG(ERROR) << "EnableMkldnnQuantizer() only works when IR optimization "
                    "is enabled.";
496 497
    }
#ifdef PADDLE_WITH_MKLDNN
498
    pass_builder()->EnableMkldnnQuantizer();
499 500 501
#endif
  }

502 503 504 505 506 507
  if (use_mkldnn_bfloat16_) {
#ifdef PADDLE_WITH_MKLDNN
    pass_builder()->EnableMkldnnBfloat16();
#endif
  }

508
#ifdef PADDLE_WITH_MKLDNN
509 510
  // Do not optimize when mkldnn is on
  if (enable_memory_optim_ && !use_mkldnn_) {
511
#else
Y
Yan Chunwei 已提交
512
  if (enable_memory_optim_) {
513 514
#endif
    pass_builder()->AppendAnalysisPass("memory_optimize_pass");
Y
Yan Chunwei 已提交
515 516
  }

石晓伟 已提交
517 518 519 520 521 522 523 524 525 526 527 528 529 530
  if (use_lite_) {
#ifndef PADDLE_WITH_LITE
    LOG(WARNING) << "You tried to enable the lite subgraph "
                    "but did not have the option -DWITH_LITE compiled.";
#endif
    pass_builder()->ClearPasses();
    for (const auto &pass : kLiteSubgraphPasses) {
      if (std::find(lite_passes_filter_.begin(), lite_passes_filter_.end(),
                    pass) == lite_passes_filter_.end()) {
        pass_builder()->AppendPass(pass);
      }
    }
  }

531
  if (use_xpu_) {
532
#if (defined LITE_SUBGRAPH_WITH_XPU) || (defined PADDLE_WITH_XPU)
533 534 535 536
    PADDLE_ENFORCE_EQ(use_gpu_, false,
                      platform::errors::Unavailable(
                          "Currently, XPU and GPU cannot be enabled in the "
                          "same analysis configuration."));
537 538 539 540 541
#else
    PADDLE_THROW(platform::errors::Unavailable(
        "You tried to use an XPU device, but Paddle was not compiled "
        "with XPU-runtime."));
#endif
542 543
  }

W
Wilber 已提交
544 545 546 547 548 549 550 551 552 553 554 555 556
  if (use_npu_) {
#ifdef PADDLE_WITH_ASCEND_CL
    PADDLE_ENFORCE_EQ(use_gpu_, false,
                      platform::errors::Unavailable(
                          "Currently, NPU and GPU cannot be enabled in the "
                          "same analysis configuration."));
#else
    PADDLE_THROW(platform::errors::Unavailable(
        "You tried to use an NPU device, but Paddle was not compiled "
        "with NPU-runtime."));
#endif
  }

557 558 559 560 561
  if (ir_debug_) {
    pass_builder()->TurnOnDebug();
  }
}

562
std::string AnalysisConfig::SerializeInfoCache() {
563
  std::stringstream ss;
Y
Yan Chunwei 已提交
564 565 566 567
  ss << model_dir_;
  ss << prog_file_;
  ss << params_file_;

568
  ss << use_gpu_;
569
  ss << use_fc_padding_;
570 571
  ss << gpu_device_id_;
  ss << xpu_device_id_;
572 573 574 575 576
  ss << memory_pool_init_size_mb_;

  ss << use_tensorrt_;
  ss << tensorrt_workspace_size_;
  ss << tensorrt_max_batchsize_;
Y
Yan Chunwei 已提交
577 578
  ss << tensorrt_min_subgraph_size_;

D
denglin-github 已提交
579 580 581
  ss << use_dlnne_;
  ss << dlnne_min_subgraph_size_;

582 583 584
  for (auto &op : trt_disabled_ops_) ss << op.c_str();
  ss << ";";

585 586 587
  ss << trt_use_dla_;
  ss << trt_dla_core_;

Y
Yan Chunwei 已提交
588
  ss << enable_memory_optim_;
589 590

  ss << use_mkldnn_;
591
  ss << mkldnn_cache_capacity_;
Y
Yan Chunwei 已提交
592 593 594
  for (auto &item : mkldnn_enabled_op_types_) ss << item;
  ss << ";";

595
  ss << use_mkldnn_quantizer_;
596
  ss << use_mkldnn_bfloat16_;
597 598
  for (auto &item : bfloat16_enabled_op_types_) ss << item;
  ss << ";";
Y
Yan Chunwei 已提交
599 600
  ss << model_from_memory_;

601 602
  ss << with_profile_;

603 604
  ss << with_glog_info_;

605 606 607 608
  ss << enable_ir_optim_;
  ss << use_feed_fetch_ops_;
  ss << ir_debug_;

Y
Yan Chunwei 已提交
609 610
  ss << specify_input_name_;
  ss << cpu_math_library_num_threads_;
石晓伟 已提交
611 612

  ss << use_lite_;
613 614
  ss << use_xpu_;
  ss << xpu_l3_workspace_size_;
W
Wilber 已提交
615 616 617 618 619
  ss << xpu_locked_;
  ss << xpu_autotune_;
  ss << xpu_autotune_file_;
  ss << xpu_precision_;
  ss << xpu_adaptive_seqlen_;
620

W
Wilber 已提交
621 622 623
  ss << use_npu_;
  ss << npu_device_id_;

624 625
  ss << thread_local_stream_;

626 627 628
  return ss.str();
}

629
void AnalysisConfig::SetCpuMathLibraryNumThreads(
630 631
    int cpu_math_library_num_threads) {
  cpu_math_library_num_threads_ = cpu_math_library_num_threads;
Y
Yan Chunwei 已提交
632 633

  Update();
634 635
}

636
float AnalysisConfig::fraction_of_gpu_memory_for_pool() const {
637
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
638 639
  // Get the GPU memory details and calculate the fraction of memory for the
  // GPU memory pool.
640
  size_t gpu_total, gpu_available;
641
  platform::SetDeviceId(gpu_device_id_);
642 643
  platform::GpuMemoryUsage(&gpu_available, &gpu_total);
  double total_gpu_memory = gpu_total / 1024. / 1024.;
644 645
  float fraction_of_gpu_memory =
      static_cast<double>(memory_pool_init_size_mb()) / total_gpu_memory;
646 647 648 649
  VLOG(3) << "total_gpu_memory is " << total_gpu_memory
          << "M, gpu_available is " << gpu_available / 1024. / 1024.
          << "M, memory_pool_init_size is " << memory_pool_init_size_mb()
          << "M.";
650 651 652 653
  return fraction_of_gpu_memory;
#else
  return 0.;
#endif
654 655
}

656
void AnalysisConfig::EnableMemoryOptim() {
Y
Yan Chunwei 已提交
657 658 659 660
  enable_memory_optim_ = true;
  Update();
}

661
bool AnalysisConfig::enable_memory_optim() const {
Y
Yan Chunwei 已提交
662 663 664
  return enable_memory_optim_;
}

665 666 667 668
void AnalysisConfig::SetModelBuffer(const char *prog_buffer,
                                    size_t prog_buffer_size,
                                    const char *param_buffer,
                                    size_t param_buffer_size) {
669 670
  prog_file_ = std::string(prog_buffer, prog_buffer + prog_buffer_size);
  params_file_ = std::string(param_buffer, param_buffer + param_buffer_size);
T
Tao Luo 已提交
671
  model_from_memory_ = true;
Y
Yan Chunwei 已提交
672 673

  Update();
T
Tao Luo 已提交
674 675
}

676
NativeConfig AnalysisConfig::ToNativeConfig() const {
Y
Yan Chunwei 已提交
677 678 679 680 681
  NativeConfig config;
  config.model_dir = model_dir_;
  config.prog_file = prog_file_;
  config.param_file = params_file_;
  config.use_gpu = use_gpu_;
682
  config.device = gpu_device_id_;
Y
Yan Chunwei 已提交
683 684 685 686 687
  config.fraction_of_gpu_memory = fraction_of_gpu_memory_for_pool();
  config.specify_input_name = specify_input_name_;
  return config;
}

Y
Yan Chunwei 已提交
688 689 690 691
void AnalysisConfig::SwitchIrDebug(int x) {
  ir_debug_ = x;
  Update();
}
692 693 694 695 696 697

void AnalysisConfig::EnableProfile() {
  with_profile_ = true;
  Update();
}

698 699 700 701 702
void AnalysisConfig::DisableGlogInfo() {
  with_glog_info_ = false;
  Update();
}

石晓伟 已提交
703
void AnalysisConfig::EnableLiteEngine(
704
    AnalysisConfig::Precision precision_mode, bool zero_copy,
石晓伟 已提交
705 706 707 708 709 710
    const std::vector<std::string> &passes_filter,
    const std::vector<std::string> &ops_filter) {
  use_lite_ = true;
  lite_precision_mode_ = precision_mode;
  lite_passes_filter_ = passes_filter;
  lite_ops_filter_ = ops_filter;
711
  lite_zero_copy_ = zero_copy;
石晓伟 已提交
712 713 714
  Update();
}

715 716 717 718 719 720 721
void AnalysisConfig::PartiallyRelease() {
  prog_file_.clear();
  prog_file_.shrink_to_fit();
  params_file_.clear();
  params_file_.shrink_to_fit();
}

722 723
void AnalysisConfig::EnableGpuMultiStream() { thread_local_stream_ = true; }

724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
std::string AnalysisConfig::Summary() {
  const std::vector<std::string> header{"Option", "Value"};
  paddle::inference::TablePrinter os(header);

  if (!model_dir_.empty()) {
    os.InsertRow({"model_dir", model_dir_});
  }
  if (!(prog_file_.empty() && params_file_.empty())) {
    os.InsertRow({"model_file", prog_file_});
    os.InsertRow({"params_file", params_file_});
  }
  if (model_from_memory_) {
    os.InsertRow({"model_from_memory", params_file_});
  }
  os.InsetDivider();

  // cpu info
  os.InsertRow(
      {"cpu_math_thread", std::to_string(cpu_math_library_num_threads_)});
  os.InsertRow({"enable_mkdlnn", use_mkldnn_ ? "true" : "false"});
  os.InsertRow(
      {"mkldnn_cache_capacity", std::to_string(mkldnn_cache_capacity_)});
  os.InsetDivider();

  auto Precision2String =
      [](paddle::AnalysisConfig::Precision prec) -> std::string {
    if (prec == Precision::kFloat32)
      return "fp32";
    else if (prec == Precision::kHalf)
      return "fp16";
    else if (prec == Precision::kInt8)
      return "int8";
    else
      return "None";
  };
  // gpu info
  os.InsertRow({"use_gpu", use_gpu_ ? "true" : "false"});
  if (use_gpu_) {
    os.InsertRow({"gpu_device_id", std::to_string(gpu_device_id_)});
    os.InsertRow({"memory_pool_init_size",
                  std::to_string(memory_pool_init_size_mb_) + "MB"});
    os.InsertRow(
        {"thread_local_stream", thread_local_stream_ ? "true" : "false"});

    os.InsertRow({"use_tensorrt", use_tensorrt_ ? "true" : "false"});
    if (use_tensorrt_) {
      os.InsertRow({"tensorrt_precision_mode",
                    Precision2String(tensorrt_precision_mode_)});
      os.InsertRow({"tensorrt_workspace_size",
                    std::to_string(tensorrt_workspace_size_)});
      os.InsertRow(
          {"tensorrt_max_batch_size", std::to_string(tensorrt_max_batchsize_)});
      os.InsertRow({"tensorrt_min_subgraph_size",
                    std::to_string(tensorrt_min_subgraph_size_)});
      os.InsertRow({"tensorrt_use_static_engine",
                    trt_use_static_engine_ ? "true" : "false"});
      os.InsertRow(
          {"tensorrt_use_calib_mode", trt_use_calib_mode_ ? "true" : "false"});

      // dynamic_shape
      os.InsertRow({"tensorrt_enable_dynamic_shape",
                    min_input_shape_.empty() ? "false" : "true"});

      os.InsertRow({"tensorrt_use_oss", trt_use_oss_ ? "true" : "false"});
      os.InsertRow({"tensorrt_use_dla", trt_use_dla_ ? "true" : "false"});
      if (trt_use_dla_) {
        os.InsertRow({"tensorrt_dla_core", std::to_string(trt_dla_core_)});
      }
    }
  }
  os.InsetDivider();

  // xpu info
  os.InsertRow({"use_xpu", use_xpu_ ? "true" : "false"});
  if (use_xpu_) {
    os.InsertRow({"xpu_device_id", std::to_string(xpu_device_id_)});
    os.InsertRow(
        {"xpu_l3_workspace_size", std::to_string(xpu_l3_workspace_size_)});
  }
  os.InsetDivider();

  if (use_lite_) {
    os.InsertRow({"use_lite", use_lite_ ? "true" : "false"});
  }

  // ir info
  os.InsertRow({"ir_optim", enable_ir_optim_ ? "true" : "false"});
  os.InsertRow({"ir_debug", ir_debug_ ? "true" : "false"});
  os.InsertRow({"memory_optim", enable_memory_optim_ ? "true" : "false"});
  os.InsertRow({"enable_profile", with_profile_ ? "true" : "false"});
  os.InsertRow({"enable_log", with_glog_info_ ? "true" : "false"});

  return os.PrintTable();
}

819
}  // namespace paddle