analysis_predictor.cc 42.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

Y
Yan Chunwei 已提交
15
#include "paddle/fluid/inference/api/analysis_predictor.h"
16 17
#include <glog/logging.h>
#include <algorithm>
N
nhzlx 已提交
18
#include <fstream>
19
#include <memory>
20
#include <set>
21
#include <string>
22
#include <utility>
23
#include <vector>
24
#include "paddle/fluid/framework/feed_fetch_method.h"
25
#include "paddle/fluid/framework/feed_fetch_type.h"
Y
Yan Chunwei 已提交
26
#include "paddle/fluid/framework/ir/fuse_pass_base.h"
27
#include "paddle/fluid/framework/ir/pass.h"
28
#include "paddle/fluid/framework/naive_executor.h"
29
#include "paddle/fluid/framework/scope.h"
Y
Yan Chunwei 已提交
30
#include "paddle/fluid/framework/var_type_traits.h"
31
#include "paddle/fluid/framework/version.h"
32
#include "paddle/fluid/inference/analysis/helper.h"
Y
Yan Chunwei 已提交
33
#include "paddle/fluid/inference/analysis/passes/memory_optimize_pass.h"
34
#include "paddle/fluid/inference/api/helper.h"
L
luotao1 已提交
35
#include "paddle/fluid/inference/api/paddle_inference_pass.h"
36
#include "paddle/fluid/inference/utils/singleton.h"
37
#include "paddle/fluid/memory/memcpy.h"
38
#include "paddle/fluid/platform/cpu_helper.h"
39
#include "paddle/fluid/platform/device_context.h"
40
#include "paddle/fluid/platform/gpu_info.h"
41
#include "paddle/fluid/platform/place.h"
T
tensor-tang 已提交
42 43
#include "paddle/fluid/platform/profiler.h"

44 45 46 47
#ifdef PADDLE_WITH_MKLML
#include "paddle/fluid/platform/dynload/mklml.h"
#endif

48 49 50 51
#ifdef PADDLE_WITH_MKLDNN
#include "paddle/fluid/inference/api/mkldnn_quantizer.h"
#endif

Y
Yan Chunwei 已提交
52 53
#if PADDLE_WITH_TENSORRT
#include "paddle/fluid/inference/tensorrt/convert/op_converter.h"
54
#include "paddle/fluid/inference/tensorrt/trt_int8_calibrator.h"
Y
Yan Chunwei 已提交
55 56
#endif

57 58
namespace paddle {

N
nhzlx 已提交
59
using inference::Singleton;
N
nhzlx 已提交
60
#if PADDLE_WITH_TENSORRT
N
nhzlx 已提交
61
using inference::tensorrt::TRTInt8Calibrator;
N
nhzlx 已提交
62 63
using inference::tensorrt::TRTCalibratorEngine;
using inference::tensorrt::TRTCalibratorEngineManager;
N
nhzlx 已提交
64
#endif
65

66 67 68 69
namespace {
bool IsPersistable(const framework::VarDesc *var) {
  if (var->Persistable() &&
      var->GetType() != framework::proto::VarType::FEED_MINIBATCH &&
70 71
      var->GetType() != framework::proto::VarType::FETCH_LIST &&
      var->GetType() != framework::proto::VarType::RAW) {
72 73 74 75 76 77
    return true;
  }
  return false;
}
}  // namespace

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110
bool PaddleTensorToLoDTensor(const PaddleTensor &pt, framework::LoDTensor *t,
                             const platform::Place &place) {
  framework::DDim ddim = framework::make_ddim(pt.shape);
  void *input_ptr;
  if (pt.dtype == PaddleDType::INT64) {
    input_ptr = t->mutable_data<int64_t>(ddim, place);
  } else if (pt.dtype == PaddleDType::FLOAT32) {
    input_ptr = t->mutable_data<float>(ddim, place);
  } else if (pt.dtype == PaddleDType::INT32) {
    input_ptr = t->mutable_data<int32_t>(ddim, place);
  } else {
    LOG(ERROR) << "unsupported feed type " << pt.dtype;
    return false;
  }

  PADDLE_ENFORCE_NOT_NULL(
      input_ptr,
      paddle::platform::errors::Fatal(
          "Cannot convert to LoDTensor because LoDTensor creation failed."));
  PADDLE_ENFORCE_NOT_NULL(
      pt.data.data(),
      paddle::platform::errors::InvalidArgument(
          "The data contained in the input PaddleTensor is illegal."));

  if (platform::is_cpu_place(place)) {
    // TODO(panyx0718): Init LoDTensor from existing memcpy to save a copy.
    std::memcpy(static_cast<void *>(input_ptr), pt.data.data(),
                pt.data.length());
  } else {
#ifdef PADDLE_WITH_CUDA
    platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
    auto *dev_ctx =
        static_cast<const platform::CUDADeviceContext *>(pool.Get(place));
111
    auto dst_gpu_place = BOOST_GET_CONST(platform::CUDAPlace, place);
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
    memory::Copy(dst_gpu_place, static_cast<void *>(input_ptr),
                 platform::CPUPlace(), pt.data.data(), pt.data.length(),
                 dev_ctx->stream());
#else
    PADDLE_THROW(paddle::platform::errors::Fatal(
        "Not compile with CUDA, should not reach here."));
#endif
  }
  // TODO(Superjomn) Low performance, need optimization for heavy LoD copy.
  framework::LoD lod;
  for (auto &level : pt.lod) {
    lod.emplace_back(level);
  }
  t->set_lod(lod);
  return true;
}

Y
Yan Chunwei 已提交
129
bool AnalysisPredictor::Init(
130 131
    const std::shared_ptr<framework::Scope> &parent_scope,
    const std::shared_ptr<framework::ProgramDesc> &program) {
M
minqiyang 已提交
132
  VLOG(3) << "Predictor::init()";
133 134
  if (config_.with_profile_) {
    LOG(WARNING) << "Profiler is activated, which might affect the performance";
135 136
    auto tracking_device = config_.use_gpu() ? platform::ProfilerState::kAll
                                             : platform::ProfilerState::kCPU;
T
tensor-tang 已提交
137
    platform::EnableProfiler(tracking_device);
138 139 140
  } else {
    LOG(INFO) << "Profiler is deactivated, and no profiling report will be "
                 "generated.";
T
tensor-tang 已提交
141 142
  }

143
  // no matter with or without MKLDNN
L
luotao1 已提交
144
  paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
145

146 147 148 149 150 151 152 153 154 155 156 157 158
  if (!PrepareScope(parent_scope)) {
    return false;
  }
  if (!CreateExecutor()) {
    return false;
  }
  if (!PrepareProgram(program)) {
    return false;
  }

  // Prepare executor, create local variables.
  if (!PrepareExecutor()) {
    return true;
Y
Yan Chunwei 已提交
159
  }
160 161 162 163 164 165 166 167 168

  // Get the feed_target_names and fetch_target_names
  PrepareFeedFetch();

  return true;
}

bool AnalysisPredictor::PrepareScope(
    const std::shared_ptr<framework::Scope> &parent_scope) {
Y
Yan Chunwei 已提交
169
  if (parent_scope) {
170 171
    PADDLE_ENFORCE_NOT_NULL(
        parent_scope,
172 173
        platform::errors::PreconditionNotMet(
            "Both program and parent_scope should be set in Clone mode."));
Y
Yan Chunwei 已提交
174
    scope_ = parent_scope;
175
    status_is_cloned_ = true;
Y
Yan Chunwei 已提交
176
  } else {
177
    paddle::framework::InitDevices();
178
    scope_.reset(new paddle::framework::Scope(), [](framework::Scope *scope) {
179
      delete scope;
180 181 182 183 184 185 186
#ifdef PADDLE_WITH_CUDA
      for (int dev_id = 0; dev_id < paddle::platform::GetCUDADeviceCount();
           ++dev_id) {
        memory::Release(platform::CUDAPlace(dev_id));
      }
#endif
      memory::Release(platform::CPUPlace());
187
    });
188
    status_is_cloned_ = false;
Y
Yan Chunwei 已提交
189
  }
190 191 192 193 194
  sub_scope_ = &scope_->NewScope();
  return true;
}
bool AnalysisPredictor::PrepareProgram(
    const std::shared_ptr<framework::ProgramDesc> &program) {
195 196
  if (!program) {
    if (!LoadProgramDesc()) return false;
197 198 199 200 201 202 203 204 205
    // If not cloned, the parameters should be loaded.
    // If config_.ir_optim() is True, parameters is loaded in
    // OptimizeInferenceProgram(), but other persistable variables
    // (like RAW type var) are not created in scope.
    // If config_.ir_optim() is False, parameters is loaded in LoadParameters(),
    // still need to create other persistable variables.
    // So in both case, create persistable variables at first.
    executor_->CreateVariables(*inference_program_, 0, true, sub_scope_);

206 207 208 209
    // if enable_ir_optim_ is false,
    // the analysis pass(op fuse, graph analysis, trt subgraph, mkldnn etc) will
    // not be executed.
    OptimizeInferenceProgram();
Y
Yan Chunwei 已提交
210
  } else {
211 212
    // If the program is passed from external, no need to optimize it, this
    // logic is used in the clone scenario.
213 214
    inference_program_ = program;
  }
M
Michal Gallus 已提交
215

216 217 218 219 220
  executor_->CreateVariables(*inference_program_, 0, false, sub_scope_);

  return true;
}
bool AnalysisPredictor::CreateExecutor() {
221
  if (config_.use_gpu()) {
222
    status_use_gpu_ = true;
223 224 225 226 227 228 229 230 231 232
    place_ = paddle::platform::CUDAPlace(config_.gpu_device_id());
#ifdef PADDLE_WITH_CUDA
    if (config_.thread_local_stream_enabled()) {
      auto *ctx = static_cast<platform::CUDADeviceContext *>(
          platform::DeviceContextPool::Instance().Get(place_));
      VLOG(3) << "The prediction process will be completed using a separate "
                 "normal-priority stream on each thread.";
      ctx->ResetThreadContext(platform::stream::Priority::kNormal);
    }
#endif
233 234 235 236 237 238 239 240
  } else {
    place_ = paddle::platform::CPUPlace();
  }
  executor_.reset(new paddle::framework::NaiveExecutor(place_));
  return true;
}
bool AnalysisPredictor::PrepareExecutor() {
  executor_->Prepare(sub_scope_, *inference_program_, 0,
241
                     config_.use_feed_fetch_ops_);
242

243 244 245
  PADDLE_ENFORCE_NOT_NULL(sub_scope_,
                          platform::errors::PreconditionNotMet(
                              "The sub_scope should not be nullptr."));
Y
Yan Chunwei 已提交
246

247 248 249
  return true;
}

250 251
void AnalysisPredictor::MkldnnPreSet(const std::vector<PaddleTensor> &inputs) {
#ifdef PADDLE_WITH_MKLDNN
W
Wilber 已提交
252 253 254 255 256 257 258 259 260 261 262 263
  std::vector<std::vector<int>> inputs_shape;
  for (size_t i = 0; i < inputs.size(); ++i) {
    inputs_shape.emplace_back(inputs[i].shape);
  }
  MkldnnPreSet(inputs_shape);
#endif
}

void AnalysisPredictor::MkldnnPreSet(
    const std::vector<std::vector<int>> &inputs_shape) {
#ifdef PADDLE_WITH_MKLDNN
  VLOG(2) << "AnalysisPredictor::ZeroCopyRun get_cur_mkldnn_session_id="
264
          << platform::MKLDNNDeviceContext::tls().get_cur_mkldnn_session_id();
265 266 267
  // In cache clearing mode.
  if (config_.mkldnn_cache_capacity_ > 0) {
    VLOG(2) << "In mkldnn cache clear mode.";
268 269 270 271
    platform::MKLDNNDeviceContext::tls().set_cur_mkldnn_session_id(
        platform::MKLDNNDeviceContextThreadLocals::
            kMKLDNNSessionID_CacheClearing);
    platform::MKLDNNDeviceContext::tls().set_cur_input_shape_cache_capacity(
272 273 274
        config_.mkldnn_cache_capacity_);
    // Set current_input_shape for caching dynamic shape.
    std::stringstream ss;
W
Wilber 已提交
275 276 277
    for (size_t i = 0; i < inputs_shape.size(); ++i) {
      for (size_t j = 0; j < inputs_shape[i].size(); ++j) {
        ss << inputs_shape[i][j] << "-";
278 279 280
      }
    }
    VLOG(2) << "Set input shape=" << ss.str();
281
    platform::MKLDNNDeviceContext::tls().set_cur_input_shape_str(ss.str());
282 283 284 285 286 287 288 289
  }
#endif
}

void AnalysisPredictor::MkldnnPostReset() {
#ifdef PADDLE_WITH_MKLDNN
  // In cache clearing mode.
  if (config_.mkldnn_cache_capacity_ > 0) {
290 291 292 293 294 295 296 297
    if (VLOG_IS_ON(2)) {
      auto shape_blob_size = static_cast<platform::MKLDNNDeviceContext *>(
                                 (&platform::DeviceContextPool::Instance())
                                     ->Get(platform::CPUPlace()))
                                 ->GetShapeBlobSize();
      CHECK_LE(shape_blob_size,
               static_cast<size_t>(config_.mkldnn_cache_capacity_));
    }
298 299 300 301
    paddle::platform::MKLDNNDeviceContext::tls().set_cur_mkldnn_session_id(
        platform::MKLDNNDeviceContextThreadLocals::kMKLDNNSessionID_Default);
    platform::MKLDNNDeviceContext::tls().set_cur_input_shape_cache_capacity(0);
    platform::MKLDNNDeviceContext::tls().set_cur_input_shape_str("");
302 303 304 305
  }
#endif
}

306 307 308
bool AnalysisPredictor::Run(const std::vector<PaddleTensor> &inputs,
                            std::vector<PaddleTensor> *output_data,
                            int batch_size) {
309
  paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
310 311 312
#ifdef PADDLE_WITH_MKLDNN
  if (config_.use_mkldnn_) MkldnnPreSet(inputs);
#endif
M
minqiyang 已提交
313
  VLOG(3) << "Predictor::predict";
314 315 316 317
  inference::Timer timer;
  timer.tic();
  // set feed variable
  framework::Scope *scope = sub_scope_ ? sub_scope_ : scope_.get();
318 319
  PADDLE_ENFORCE_NOT_NULL(scope, platform::errors::PreconditionNotMet(
                                     "The scope should not be nullptr."));
320 321
  if (!SetFeed(inputs, scope)) {
    LOG(ERROR) << "fail to set feed";
Y
Yan Chunwei 已提交
322
    return false;
323
  }
M
Michal Gallus 已提交
324

325 326 327
  // Run the inference program
  // if share variables, we need not create variables
  executor_->Run();
328

329 330 331 332
  // get fetch variable
  if (!GetFetch(output_data, scope)) {
    LOG(ERROR) << "fail to get fetches";
    return false;
T
tensor-tang 已提交
333
  }
Y
Yan Chunwei 已提交
334

M
minqiyang 已提交
335
  VLOG(3) << "predict cost: " << timer.toc() << "ms";
Y
Yan Chunwei 已提交
336

Y
Yan Chunwei 已提交
337 338 339 340 341
  // All the containers in the scope will be hold in inference, but the
  // operators assume that the container will be reset after each batch.
  // Here is a bugfix, collect all the container variables, and reset then to a
  // bool; the next time, the operator will call MutableData and construct a new
  // container again, so that the container will be empty for each batch.
342 343 344
  if (sub_scope_) {
    tensor_array_batch_cleaner_.CollectNoTensorVars(sub_scope_);
  }
Y
Yan Chunwei 已提交
345
  tensor_array_batch_cleaner_.ResetNoTensorVars();
346 347 348 349

  // recover the cpu_math_library_num_threads to 1, in order to avoid thread
  // conflict when integrating it into deployment service.
  paddle::platform::SetNumThreads(1);
350 351
#ifdef PADDLE_WITH_MKLDNN
  if (config_.use_mkldnn_) MkldnnPostReset();
T
Tao Luo 已提交
352
#endif
353
#if defined(PADDLE_WITH_MKLML)
T
Tao Luo 已提交
354 355 356 357
  // Frees unused memory allocated by the Intel® MKL Memory Allocator to
  // avoid memory leak. See:
  // https://software.intel.com/en-us/mkl-developer-reference-c-mkl-free-buffers
  platform::dynload::MKL_Free_Buffers();
358
#endif
359 360
  return true;
}
361

362 363
bool AnalysisPredictor::SetFeed(const std::vector<PaddleTensor> &inputs,
                                framework::Scope *scope) {
M
minqiyang 已提交
364
  VLOG(3) << "Predictor::set_feed";
365 366 367 368 369 370 371 372 373 374
  if (inputs.size() != feeds_.size()) {
    LOG(ERROR) << "wrong feed input size, need " << feeds_.size() << " but get "
               << inputs.size();
    return false;
  }

  // Cache the inputs memory for better concurrency performance.
  feed_tensors_.resize(inputs.size());

  for (size_t i = 0; i < inputs.size(); ++i) {
375 376
    framework::LoDTensor *input = &feed_tensors_[i];
    if (!PaddleTensorToLoDTensor(inputs[i], input, place_)) {
377 378 379
      return false;
    }
    int idx = -1;
380
    if (config_.specify_input_name_) {
T
tensor-tang 已提交
381 382
      auto name = inputs[i].name;
      if (feed_names_.find(name) == feed_names_.end()) {
T
tensor-tang 已提交
383 384
        LOG(ERROR) << "feed names from program do not have name: [" << name
                   << "] from specified input";
T
tensor-tang 已提交
385 386
      }
      idx = feed_names_[name];
387
    } else {
388
      idx = BOOST_GET_CONST(int, feeds_[i]->GetAttr("col"));
389
    }
390
    framework::SetFeedVariable(scope, *input, "feed", idx);
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
  }
  return true;
}

template <typename T>
void AnalysisPredictor::GetFetchOne(const framework::LoDTensor &fetch,
                                    PaddleTensor *output) {
  // set shape.
  auto shape = framework::vectorize(fetch.dims());
  output->shape.assign(shape.begin(), shape.end());
  // set data.
  const T *data = fetch.data<T>();
  int num_elems = inference::VecReduceToInt(shape);
  output->data.Resize(num_elems * sizeof(T));
  // The fetched tensor output by fetch op, should always in CPU memory, so just
  // copy.
  memcpy(output->data.data(), data, num_elems * sizeof(T));
  // set lod
  output->lod.clear();
  for (auto &level : fetch.lod()) {
    output->lod.emplace_back(level.begin(), level.end());
  }
}

bool AnalysisPredictor::GetFetch(std::vector<PaddleTensor> *outputs,
                                 framework::Scope *scope) {
M
minqiyang 已提交
417
  VLOG(3) << "Predictor::get_fetch";
Y
Yan Chunwei 已提交
418 419
  outputs->resize(fetches_.size());
  for (size_t i = 0; i < fetches_.size(); ++i) {
420
    int idx = BOOST_GET_CONST(int, fetches_[i]->GetAttr("col"));
421 422 423 424 425
    PADDLE_ENFORCE_EQ(
        static_cast<size_t>(idx), i,
        platform::errors::InvalidArgument(
            "Fetch op's col attr(%d) should be equal to the index(%d)", idx,
            i));
426
    framework::FetchType &fetch_var =
427
        framework::GetFetchVariable(*scope, "fetch", idx);
428
    auto &fetch = BOOST_GET(framework::LoDTensor, fetch_var);
429 430
    auto type = fetch.type();
    auto output = &(outputs->at(i));
Y
Yan Chunwei 已提交
431
    output->name = fetches_[idx]->Input("X")[0];
Y
Yu Yang 已提交
432
    if (type == framework::proto::VarType::FP32) {
433 434
      GetFetchOne<float>(fetch, output);
      output->dtype = PaddleDType::FLOAT32;
Y
Yu Yang 已提交
435
    } else if (type == framework::proto::VarType::INT64) {
436 437
      GetFetchOne<int64_t>(fetch, output);
      output->dtype = PaddleDType::INT64;
438 439 440
    } else if (type == framework::proto::VarType::INT32) {
      GetFetchOne<int32_t>(fetch, output);
      output->dtype = PaddleDType::INT32;
441
    } else {
442
      LOG(ERROR) << "unknown type, only support float32, int64 and int32 now.";
443 444
    }
  }
Y
Yan Chunwei 已提交
445 446
  return true;
}
447

448
void AnalysisPredictor::PrepareArgument() {
449
  argument_.SetUseGPU(config_.use_gpu());
450
  argument_.SetUseFcPadding(config_.use_fc_padding());
451
  argument_.SetGPUDeviceId(config_.gpu_device_id());
452
  argument_.SetEnableAnalysisOptim(config_.enable_ir_optim_);
Y
Yan Chunwei 已提交
453
  argument_.SetEnableMemoryOptim(config_.enable_memory_optim());
T
Tao Luo 已提交
454
  argument_.SetModelFromMemory(config_.model_from_memory_);
Y
Yan Chunwei 已提交
455
  // Analyze inference_program
456
  argument_.SetPredictorID(predictor_id_);
457
  argument_.SetOptimCacheDir(config_.opt_cache_dir_);
458 459
  if (!config_.model_dir().empty()) {
    argument_.SetModelDir(config_.model_dir());
T
Tao Luo 已提交
460
  } else {
461 462 463 464 465 466
    PADDLE_ENFORCE_EQ(config_.params_file().empty(), false,
                      platform::errors::PreconditionNotMet(
                          "Either model_dir or param_file should be set."));
    PADDLE_ENFORCE_EQ(config_.prog_file().empty(), false,
                      platform::errors::PreconditionNotMet(
                          "Either model_dir or prog_file should be set."));
N
nhzlx 已提交
467
    std::string dir = inference::analysis::GetDirRoot(config_.prog_file());
N
nhzlx 已提交
468

469 470
    argument_.SetModelProgramPath(config_.prog_file());
    argument_.SetModelParamsPath(config_.params_file());
Y
Yan Chunwei 已提交
471
  }
472

473
  if (config_.use_gpu() && config_.tensorrt_engine_enabled()) {
Y
Yan Chunwei 已提交
474
    LOG(INFO) << "TensorRT subgraph engine is enabled";
475 476 477
    argument_.SetUseTensorRT(true);
    argument_.SetTensorRtWorkspaceSize(config_.tensorrt_workspace_size_);
    argument_.SetTensorRtMaxBatchSize(config_.tensorrt_max_batchsize_);
478
    argument_.SetTensorRtMinSubgraphSize(config_.tensorrt_min_subgraph_size_);
N
nhzlx 已提交
479
    argument_.SetTensorRtPrecisionMode(config_.tensorrt_precision_mode_);
N
nhzlx 已提交
480
    argument_.SetTensorRtUseStaticEngine(config_.trt_use_static_engine_);
481
    argument_.SetTensorRtUseCalibMode(config_.trt_use_calib_mode_);
482
    argument_.SetTensorRtUseOSS(config_.trt_use_oss_);
483 484 485
    argument_.SetMinInputShape(config_.min_input_shape_);
    argument_.SetMaxInputShape(config_.max_input_shape_);
    argument_.SetOptimInputShape(config_.optim_input_shape_);
486
    argument_.SetCloseTrtPluginFp16(config_.disable_trt_plugin_fp16_);
W
Wojciech Uss 已提交
487
  }
488

石晓伟 已提交
489
  if (config_.lite_engine_enabled()) {
W
Wilber 已提交
490 491
    argument_.SetCpuMathLibraryNumThreads(
        config_.cpu_math_library_num_threads());
石晓伟 已提交
492 493 494
    argument_.SetLitePrecisionMode(config_.lite_precision_mode_);
    argument_.SetLitePassesFilter(config_.lite_passes_filter_);
    argument_.SetLiteOpsFilter(config_.lite_ops_filter_);
495 496 497
    argument_.SetLiteZeroCopy(config_.lite_zero_copy_);
    argument_.SetUseXpu(config_.use_xpu_);
    argument_.SetXpuL3WorkspaceSize(config_.xpu_l3_workspace_size_);
石晓伟 已提交
498 499 500
    LOG(INFO) << "Lite subgraph engine is enabled";
  }

501
  if (config_.use_mkldnn_) {
Y
Yan Chunwei 已提交
502
    LOG(INFO) << "MKLDNN is enabled";
503 504 505
    argument_.SetMKLDNNEnabledOpTypes(config_.mkldnn_enabled_op_types_);
  }

506 507 508 509 510 511 512 513
#ifdef PADDLE_WITH_MKLDNN
  if (config_.mkldnn_quantizer_enabled()) {
    LOG(INFO) << "Quantization is enabled";
    argument_.SetQuantizeEnabledOpTypes(
        config_.mkldnn_quantizer_config()->enabled_op_types());
    argument_.SetQuantizeExcludedOpIds(
        config_.mkldnn_quantizer_config()->excluded_op_ids());
  }
514 515 516 517
  if (config_.use_mkldnn_bfloat16_) {
    LOG(INFO) << "Bfloat16 is enabled";
    argument_.SetBfloat16EnabledOpTypes(config_.bfloat16_enabled_op_types_);
  }
518 519
#endif

520
  auto passes = config_.pass_builder()->AllPasses();
Y
Yan Chunwei 已提交
521 522 523 524
  if (!config_.ir_optim()) {
    passes.clear();
    LOG(INFO) << "ir_optim is turned off, no IR pass will be executed";
  }
525
  argument_.SetDisableLogs(config_.glog_info_disabled());
526
  argument_.SetIrAnalysisPasses(passes);
Y
Yan Chunwei 已提交
527
  argument_.SetAnalysisPasses(config_.pass_builder()->AnalysisPasses());
528
  argument_.SetScopeNotOwned(scope_.get());
529 530 531 532 533
}

// NOTE All the members in AnalysisConfig should be copied to Argument.
void AnalysisPredictor::OptimizeInferenceProgram() {
  PrepareArgument();
534 535
  Analyzer().Run(&argument_);

536 537 538
  PADDLE_ENFORCE_EQ(
      argument_.scope_valid(), true,
      platform::errors::InvalidArgument("The argument scope should be valid."));
539 540
  VLOG(5) << "to prepare executor";
  ARGUMENT_CHECK_FIELD((&argument_), ir_analyzed_program);
Y
Yan Chunwei 已提交
541
  inference_program_.reset(
542
      new framework::ProgramDesc(argument_.ir_analyzed_program()));
543 544 545 546
  // The config and argument take a lot of storage,
  // when the predictor settings are complete, we release these stores.
  argument_.PartiallyRelease();
  config_.PartiallyRelease();
547
  LOG(INFO) << "======= optimize end =======";
Y
Yan Chunwei 已提交
548
}
549 550

template <>
551 552
std::unique_ptr<PaddlePredictor> CreatePaddlePredictor<
    AnalysisConfig, PaddleEngineKind::kAnalysis>(const AnalysisConfig &config) {
W
Wilber 已提交
553 554
  // TODO(NHZlX): Should add the link to the doc of
  // paddle_infer::CreatePredictor<paddle_infer::Config>
P
Pei Yang 已提交
555 556 557 558
  if (config.glog_info_disabled()) {
    FLAGS_logtostderr = 1;
    FLAGS_minloglevel = 2;  // GLOG_ERROR
  }
M
minqiyang 已提交
559
  VLOG(3) << "create AnalysisConfig";
560 561 562 563
  PADDLE_ENFORCE_EQ(
      config.is_valid(), true,
      platform::errors::InvalidArgument(
          "Note: Each config can only be used for one predictor."));
564

565
  if (config.use_gpu()) {
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
    static std::once_flag gflags_initialized;
    static bool process_level_allocator_enabled;

    std::call_once(gflags_initialized, [&]() {
      std::vector<std::string> gflags;
      PADDLE_ENFORCE_GE(
          config.memory_pool_init_size_mb(), 0.f,
          platform::errors::InvalidArgument(
              "The size of memory pool should be greater than 0."));
      PADDLE_ENFORCE_GE(
          config.gpu_device_id(), 0,
          platform::errors::InvalidArgument(
              "Invalid device id (%d). The device id should be greater than 0.",
              config.gpu_device_id()));
      gflags.push_back("dummy");

      float fraction_of_gpu_memory = config.fraction_of_gpu_memory_for_pool();
      if (fraction_of_gpu_memory > 0.95f) {
        LOG(ERROR)
            << "Allocate too much memory for the GPU memory pool, assigned "
            << config.memory_pool_init_size_mb() << " MB";
        LOG(ERROR) << "Try to shink the value by setting "
                      "AnalysisConfig::EnableGpu(...)";
      }
590

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
      if (fraction_of_gpu_memory >= 0.0f || fraction_of_gpu_memory <= 0.95f) {
        std::string flag = "--fraction_of_gpu_memory_to_use=" +
                           std::to_string(fraction_of_gpu_memory);
        VLOG(3) << "set flag: " << flag;
        gflags.push_back(flag);
        gflags.push_back("--cudnn_deterministic=True");
      }

      if (config.thread_local_stream_enabled()) {
        gflags.push_back("--allocator_strategy=thread_local");
        process_level_allocator_enabled = false;
      } else {
        process_level_allocator_enabled = true;
      }

      if (framework::InitGflags(gflags)) {
        VLOG(3) << "The following gpu analysis configurations only take effect "
                   "for the first predictor: ";
        for (size_t i = 1; i < gflags.size(); ++i) {
          VLOG(3) << gflags[i];
        }
      } else {
        LOG(WARNING) << "The one-time configuration of analysis predictor "
                        "failed, which may be due to native predictor called "
                        "first and its configurations taken effect.";
      }
    });

    if (config.thread_local_stream_enabled() &&
        process_level_allocator_enabled) {
621 622 623 624 625 626
      PADDLE_THROW(platform::errors::Fatal(
          "When binding threads and streams, the use of "
          "process-level allocators will result in undefined result "
          "errors due to memory asynchronous operations."
          "The thread and stream binding configuration of all "
          "predictors should be the same in a single process."));
627 628 629 630
    }
  }

  std::unique_ptr<PaddlePredictor> predictor(new AnalysisPredictor(config));
631 632
  // Each config can only be used for one predictor.
  config.SetInValid();
633 634 635 636 637 638 639
  auto predictor_p = dynamic_cast<AnalysisPredictor *>(predictor.get());

  if (!predictor_p->Init(nullptr)) {
    return nullptr;
  }

  if (config.mkldnn_quantizer_enabled() && !predictor_p->MkldnnQuantize()) {
640 641
    return nullptr;
  }
642

G
Gabor Buella 已提交
643
  return predictor;
644 645
}

646 647 648 649 650 651 652 653 654 655 656 657
bool AnalysisPredictor::MkldnnQuantize() {
#if PADDLE_WITH_MKLDNN
  if (!mkldnn_quantizer_)
    mkldnn_quantizer_ = new AnalysisPredictor::MkldnnQuantizer(
        *this, config_.mkldnn_quantizer_config());
  return mkldnn_quantizer_->Quantize();
#else
  LOG(ERROR) << "Please compile with MKLDNN first to use MkldnnQuantizer";
  return false;
#endif
}

658
void AnalysisPredictor::PrepareFeedFetch() {
659 660 661
  PADDLE_ENFORCE_NOT_NULL(sub_scope_,
                          platform::errors::InvalidArgument(
                              "The sub_scope should not be nullptr."));
662
  CreateFeedFetchVar(sub_scope_);
663 664
  for (auto *op : inference_program_->Block(0).AllOps()) {
    if (op->Type() == "feed") {
665
      int idx = BOOST_GET_CONST(int, op->GetAttr("col"));
666 667 668 669 670
      if (feeds_.size() <= static_cast<size_t>(idx)) {
        feeds_.resize(idx + 1);
      }
      feeds_[idx] = op;
      feed_names_[op->Output("Out")[0]] = idx;
N
nhzlx 已提交
671
      idx2feeds_[idx] = op->Output("Out")[0];
672
    } else if (op->Type() == "fetch") {
673
      int idx = BOOST_GET_CONST(int, op->GetAttr("col"));
Y
Yan Chunwei 已提交
674 675
      if (fetches_.size() <= static_cast<size_t>(idx)) {
        fetches_.resize(idx + 1);
676
      }
Y
Yan Chunwei 已提交
677
      fetches_[idx] = op;
N
nhzlx 已提交
678
      idx2fetches_[idx] = op->Input("X")[0];
679 680 681 682
    }
  }
}

683
void AnalysisPredictor::CreateFeedFetchVar(framework::Scope *scope) {
684 685
  PADDLE_ENFORCE_NOT_NULL(scope, platform::errors::InvalidArgument(
                                     "The scope should not be nullptr."));
686
  auto *var = scope->Var("feed");
687
  var->GetMutable<framework::FeedList>();
688
  var = scope->Var("fetch");
689
  var->GetMutable<framework::FetchList>();
690 691
}

N
nhzlx 已提交
692 693 694 695 696 697 698 699
std::vector<std::string> AnalysisPredictor::GetInputNames() {
  std::vector<std::string> input_names;
  for (auto &item : idx2feeds_) {
    input_names.push_back(item.second);
  }
  return input_names;
}

700 701 702 703 704 705
std::map<std::string, std::vector<int64_t>>
AnalysisPredictor::GetInputTensorShape() {
  std::map<std::string, std::vector<int64_t>> input_shapes;
  std::vector<std::string> names = GetInputNames();
  for (std::string name : names) {
    auto *var = inference_program_->Block(0).FindVar(name);
706 707
    PADDLE_ENFORCE_NOT_NULL(var, platform::errors::PreconditionNotMet(
                                     "Input %s does not exist.", name));
708 709 710 711 712
    input_shapes[name] = var->GetShape();
  }
  return input_shapes;
}

N
nhzlx 已提交
713 714 715 716 717 718 719 720
std::vector<std::string> AnalysisPredictor::GetOutputNames() {
  std::vector<std::string> output_names;
  for (auto &item : idx2fetches_) {
    output_names.push_back(item.second);
  }
  return output_names;
}

721 722
std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetInputTensor(
    const std::string &name) {
723 724 725 726 727
  PADDLE_ENFORCE_NOT_NULL(
      executor_->scope()->FindVar(name),
      platform::errors::PreconditionNotMet(
          "The variable named %s is not found in the scope of the exector.",
          name));
728 729 730 731
  std::unique_ptr<ZeroCopyTensor> res(
      new ZeroCopyTensor(static_cast<void *>(executor_->scope())));
  res->input_or_output_ = true;
  res->SetName(name);
N
nhzlx 已提交
732 733 734
  if (platform::is_cpu_place(place_)) {
    res->SetPlace(PaddlePlace::kCPU);
  } else {
735
    auto gpu_place = BOOST_GET_CONST(platform::CUDAPlace, place_);
N
nhzlx 已提交
736 737 738
    res->SetPlace(PaddlePlace::kGPU, gpu_place.GetDeviceId());
  }

739 740 741 742 743
  return res;
}

std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetOutputTensor(
    const std::string &name) {
744 745 746 747 748
  PADDLE_ENFORCE_NOT_NULL(
      executor_->scope()->FindVar(name),
      platform::errors::PreconditionNotMet(
          "he variable named %s is not found in the scope of the exector.",
          name));
749 750 751 752
  std::unique_ptr<ZeroCopyTensor> res(
      new ZeroCopyTensor(static_cast<void *>(executor_->scope())));
  res->input_or_output_ = false;
  res->SetName(name);
N
nhzlx 已提交
753 754 755
  if (platform::is_cpu_place(place_)) {
    res->SetPlace(PaddlePlace::kCPU);
  } else {
756
    auto gpu_place = BOOST_GET_CONST(platform::CUDAPlace, place_);
N
nhzlx 已提交
757 758
    res->SetPlace(PaddlePlace::kGPU, gpu_place.GetDeviceId());
  }
759 760 761 762
  return res;
}

bool AnalysisPredictor::ZeroCopyRun() {
763
  paddle::platform::SetNumThreads(config_.cpu_math_library_num_threads());
W
Wilber 已提交
764 765 766 767 768 769 770 771 772 773 774 775
#ifdef PADDLE_WITH_MKLDNN
  if (config_.use_mkldnn_) {
    std::vector<std::vector<int>> shape_vector;
    auto names = GetInputNames();
    for (size_t i = 0; i < names.size(); ++i) {
      auto in_tensor = GetInputTensor(names[i]);
      shape_vector.emplace_back(in_tensor->shape());
    }
    MkldnnPreSet(shape_vector);
  }
#endif

776
  executor_->Run();
Y
Yan Chunwei 已提交
777
  // Fix TensorArray reuse not cleaned bug.
Y
Yan Chunwei 已提交
778
  tensor_array_batch_cleaner_.CollectTensorArrays(sub_scope_);
Y
Yan Chunwei 已提交
779
  tensor_array_batch_cleaner_.ResetTensorArray();
780 781 782 783

  // recover the cpu_math_library_num_threads to 1, in order to avoid thread
  // conflict when integrating it into deployment service.
  paddle::platform::SetNumThreads(1);
W
Wilber 已提交
784 785 786
#ifdef PADDLE_WITH_MKLDNN
  if (config_.use_mkldnn_) MkldnnPostReset();
#endif
787
#if defined(PADDLE_WITH_MKLML)
T
Tao Luo 已提交
788 789 790 791 792
  // Frees unused memory allocated by the Intel® MKL Memory Allocator to
  // avoid memory leak. See:
  // https://software.intel.com/en-us/mkl-developer-reference-c-mkl-free-buffers
  platform::dynload::MKL_Free_Buffers();
#endif
793 794 795 796 797
  return true;
}

bool AnalysisPredictor::LoadProgramDesc() {
  // Initialize the inference program
798
  std::string filename;
799 800 801
  if (!config_.model_dir().empty()) {
    filename = config_.model_dir() + "/__model__";
  } else if (!config_.prog_file().empty() && !config_.params_file().empty()) {
802 803 804
    // All parameters are saved in a single file.
    // The file names should be consistent with that used
    // in Python API `fluid.io.save_inference_model`.
805
    filename = config_.prog_file();
806
  } else {
807
    if (config_.model_dir().empty() && config_.prog_file().empty()) {
808 809 810 811
      LOG(ERROR)
          << "Either model_dir or (prog_file, param_file) should be set.";
      return false;
    }
812
    LOG(ERROR) << string::Sprintf(
813 814
        "not valid model path '%s' or program path '%s'.", config_.model_dir(),
        config_.params_file());
815 816
    return false;
  }
817 818 819

  // Create ProgramDesc
  framework::proto::ProgramDesc proto;
T
Tao Luo 已提交
820
  if (!config_.model_from_memory()) {
T
Tao Luo 已提交
821 822 823
    std::string pb_content;
    // Read binary
    std::ifstream fin(filename, std::ios::in | std::ios::binary);
824 825 826 827 828
    PADDLE_ENFORCE_EQ(
        static_cast<bool>(fin.is_open()), true,
        platform::errors::NotFound(
            "Cannot open file %s, please confirm whether the file is normal.",
            filename));
T
Tao Luo 已提交
829 830 831 832 833 834 835 836
    fin.seekg(0, std::ios::end);
    pb_content.resize(fin.tellg());
    fin.seekg(0, std::ios::beg);
    fin.read(&(pb_content.at(0)), pb_content.size());
    fin.close();

    proto.ParseFromString(pb_content);
  } else {
837
    proto.ParseFromString(config_.prog_file());
T
Tao Luo 已提交
838
  }
839 840 841 842 843 844
  inference_program_.reset(new framework::ProgramDesc(proto));
  return true;
}

bool AnalysisPredictor::LoadParameters() {
  PADDLE_ENFORCE_NOT_NULL(inference_program_.get(),
845 846
                          platform::errors::PreconditionNotMet(
                              "The inference program should be loaded first."));
T
Tao Luo 已提交
847

848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
  const auto &global_block = inference_program_->MutableBlock(0);

  // create a temporary program to load parameters.

  std::unique_ptr<framework::ProgramDesc> load_program(
      new framework::ProgramDesc());
  framework::BlockDesc *load_block = load_program->MutableBlock(0);
  std::vector<std::string> params;

  for (auto *var : global_block->AllVars()) {
    if (IsPersistable(var)) {
      VLOG(3) << "persistable variable's name: " << var->Name();

      framework::VarDesc *new_var = load_block->Var(var->Name());
      new_var->SetShape(var->GetShape());
      new_var->SetDataType(var->GetDataType());
      new_var->SetType(var->GetType());
      new_var->SetLoDLevel(var->GetLoDLevel());
      new_var->SetPersistable(true);

868
      if (!config_.params_file().empty()) {
869 870 871 872 873 874
        params.push_back(new_var->Name());
      } else {
        // append_op
        framework::OpDesc *op = load_block->AppendOp();
        op->SetType("load");
        op->SetOutput("Out", {new_var->Name()});
875
        op->SetAttr("file_path", {config_.model_dir() + "/" + new_var->Name()});
876 877 878 879 880
        op->CheckAttrs();
      }
    }
  }

881
  if (!config_.params_file().empty()) {
882 883 884 885 886 887
    // sort paramlist to have consistent ordering
    std::sort(params.begin(), params.end());
    // append just the load_combine op
    framework::OpDesc *op = load_block->AppendOp();
    op->SetType("load_combine");
    op->SetOutput("Out", params);
888
    op->SetAttr("file_path", {config_.params_file()});
889 890 891 892
    op->CheckAttrs();
  }

  // Use NaiveExecutor to Load parameters.
S
superjomn 已提交
893
  framework::NaiveExecutor e(place_);
894 895 896 897
  e.Prepare(scope_.get(), *load_program, 0, false);
  e.Run();
  VLOG(3) << "get " << scope_->LocalVarNames().size() << " vars after load";

898 899
  return true;
}
900

901 902 903 904 905
uint64_t AnalysisPredictor::TryShrinkMemory() {
  ClearIntermediateTensor();
  return paddle::memory::Release(place_);
}

906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
void AnalysisPredictor::ClearIntermediateTensor() {
  PADDLE_ENFORCE_NOT_NULL(inference_program_.get(),
                          platform::errors::PreconditionNotMet(
                              "The inference program should be loaded first."));
  const auto &global_block = inference_program_->MutableBlock(0);
  for (auto *var : global_block->AllVars()) {
    if (!IsPersistable(var)) {
      const std::string name = var->Name();
      auto *variable = executor_->scope()->FindVar(name);
      if (variable != nullptr && variable->IsType<framework::LoDTensor>() &&
          name != "feed" && name != "fetch") {
        VLOG(3) << "Clear Intermediate Tensor: " << name;
        auto *t = variable->GetMutable<framework::LoDTensor>();
        t->clear();
      }
    }
  }
}

N
nhzlx 已提交
925
#if PADDLE_WITH_TENSORRT
N
nhzlx 已提交
926
bool AnalysisPredictor::SaveTrtCalibToDisk() {
927 928 929
  PADDLE_ENFORCE_EQ(config_.tensorrt_engine_enabled(), true,
                    platform::errors::PreconditionNotMet(
                        "This func can be invoked only in trt mode"));
N
nhzlx 已提交
930 931 932 933
  auto &block = inference_program_->Block(0);
  for (auto &op_desc : block.AllOps()) {
    if (op_desc->Type() == "tensorrt_engine") {
      std::string engine_name =
934
          BOOST_GET_CONST(std::string, op_desc->GetAttr("engine_key"));
N
nhzlx 已提交
935
      if (!Singleton<TRTCalibratorEngineManager>::Global().Has(engine_name)) {
N
nhzlx 已提交
936 937 938 939
        LOG(ERROR) << "You should run the predictor(with trt) on the real data "
                      "to generate calibration info";
        return false;
      }
N
nhzlx 已提交
940 941
      TRTCalibratorEngine *calib_engine =
          Singleton<TRTCalibratorEngineManager>::Global().Get(engine_name);
N
nhzlx 已提交
942
      LOG(INFO) << "Wait for calib threads done.";
N
nhzlx 已提交
943
      calib_engine->calib_->waitAndSetDone();
N
nhzlx 已提交
944 945
      LOG(INFO) << "Generating TRT Calibration table data, this may cost a lot "
                   "of time...";
N
nhzlx 已提交
946 947 948
      calib_engine->thr_->join();
      std::string calibration_table_data =
          calib_engine->calib_->getCalibrationTableAsString();
N
nhzlx 已提交
949

N
nhzlx 已提交
950
      if (calibration_table_data.empty()) {
N
nhzlx 已提交
951 952 953
        LOG(ERROR) << "the calibration table is empty.";
        return false;
      }
N
nhzlx 已提交
954

N
nhzlx 已提交
955 956 957 958 959
      std::string model_opt_cache_dir =
          argument_.Has("model_dir")
              ? argument_.model_dir()
              : inference::analysis::GetDirRoot(argument_.model_program_path());

N
nhzlx 已提交
960
      std::string calibration_table_data_path =
N
nhzlx 已提交
961 962 963 964
          inference::analysis::GetTrtCalibPath(
              inference::analysis::GetOrCreateModelOptCacheDir(
                  model_opt_cache_dir),
              engine_name);
N
nhzlx 已提交
965 966 967 968 969

      std::ofstream ofile(calibration_table_data_path, std::ios::out);
      LOG(INFO) << "Write Paddle-TRT INT8 calibration table data to file "
                << calibration_table_data_path;
      ofile << calibration_table_data;
N
nhzlx 已提交
970 971 972 973
      ofile.close();
    }
  }
  // Free all calibrator resources.
N
nhzlx 已提交
974
  Singleton<TRTCalibratorEngineManager>::Global().DeleteALL();
N
nhzlx 已提交
975 976
  return true;
}
N
nhzlx 已提交
977
#endif
N
nhzlx 已提交
978

979
AnalysisPredictor::~AnalysisPredictor() {
N
nhzlx 已提交
980
#if PADDLE_WITH_TENSORRT
N
nhzlx 已提交
981
  if (config_.tensorrt_engine_enabled() &&
N
nhzlx 已提交
982 983
      config_.tensorrt_precision_mode_ == AnalysisConfig::Precision::kInt8 &&
      Singleton<TRTCalibratorEngineManager>::Global().Has()) {
N
nhzlx 已提交
984 985
    SaveTrtCalibToDisk();
  }
N
nhzlx 已提交
986
#endif
987
  if (config_.with_profile_) {
988 989 990 991 992 993
    platform::DisableProfiler(platform::EventSortingKey::kTotal,
                              "./profile.log");
  }
  if (sub_scope_) {
    scope_->DeleteScope(sub_scope_);
  }
Y
Yan Chunwei 已提交
994

995 996 997 998 999 1000
#if PADDLE_WITH_MKLDNN
  if (mkldnn_quantizer_) {
    delete mkldnn_quantizer_;
    mkldnn_quantizer_ = nullptr;
  }
#endif
1001 1002

  memory::Release(place_);
1003 1004
}

1005
std::unique_ptr<PaddlePredictor> AnalysisPredictor::Clone() {
Y
Yan Chunwei 已提交
1006
  std::lock_guard<std::mutex> lk(clone_mutex_);
1007 1008 1009 1010 1011
  auto *x = new AnalysisPredictor(config_);
  x->Init(scope_, inference_program_);
  return std::unique_ptr<PaddlePredictor>(x);
}

1012
std::string AnalysisPredictor::GetSerializedProgram() const {
Y
Yan Chunwei 已提交
1013 1014 1015
  return inference_program_->Proto()->SerializeAsString();
}

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
// Add SaveOptimModel
void AnalysisPredictor::SaveOptimModel(const std::string &dir) {
  // save model
  std::string model_name = dir + "/model";
  std::ofstream outfile;
  outfile.open(model_name, std::ios::out | std::ios::binary);
  std::string inference_prog_desc = GetSerializedProgram();
  outfile << inference_prog_desc;
  // save params
  framework::ProgramDesc save_program;
  auto *save_block = save_program.MutableBlock(0);

  const framework::ProgramDesc &main_program = program();
  const framework::BlockDesc &global_block = main_program.Block(0);
  std::vector<std::string> save_var_list;
  for (framework::VarDesc *var : global_block.AllVars()) {
    if (IsPersistable(var)) {
      framework::VarDesc *new_var = save_block->Var(var->Name());
      new_var->SetShape(var->GetShape());
      new_var->SetDataType(var->GetDataType());
      new_var->SetType(var->GetType());
      new_var->SetLoDLevel(var->GetLoDLevel());
      new_var->SetPersistable(true);

      save_var_list.push_back(new_var->Name());
    }
  }
  std::sort(save_var_list.begin(), save_var_list.end());
  auto *op = save_block->AppendOp();
  op->SetType("save_combine");
  op->SetInput("X", save_var_list);
  op->SetAttr("file_path", dir + "/params");
  op->CheckAttrs();

  platform::CPUPlace place;
  framework::Executor exe(place);
  exe.Run(save_program, scope(), 0, true, true);
}

Y
Yan Chunwei 已提交
1055
template <>
1056 1057
std::unique_ptr<PaddlePredictor> CreatePaddlePredictor<AnalysisConfig>(
    const AnalysisConfig &config) {
W
Wilber 已提交
1058
  LOG(WARNING) << "Deprecated. Please use CreatePredictor instead.";
1059 1060
  return CreatePaddlePredictor<AnalysisConfig, PaddleEngineKind::kAnalysis>(
      config);
Y
Yan Chunwei 已提交
1061 1062
}

1063
}  // namespace paddle
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073

#if PADDLE_WITH_TENSORRT
USE_TRT_CONVERTER(elementwise_add_weight);
USE_TRT_CONVERTER(elementwise_add_tensor);
USE_TRT_CONVERTER(elementwise_sub_tensor);
USE_TRT_CONVERTER(elementwise_div_tensor);
USE_TRT_CONVERTER(elementwise_mul_tensor);
USE_TRT_CONVERTER(elementwise_max_tensor);
USE_TRT_CONVERTER(elementwise_min_tensor);
USE_TRT_CONVERTER(elementwise_pow_tensor);
1074
USE_TRT_CONVERTER(matmul);
1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
USE_TRT_CONVERTER(conv2d);
USE_TRT_CONVERTER(relu);
USE_TRT_CONVERTER(sigmoid);
USE_TRT_CONVERTER(tanh);
USE_TRT_CONVERTER(fc);
USE_TRT_CONVERTER(pool2d);
USE_TRT_CONVERTER(softmax);
USE_TRT_CONVERTER(batch_norm);
USE_TRT_CONVERTER(concat);
USE_TRT_CONVERTER(dropout);
USE_TRT_CONVERTER(pad);
1086 1087
USE_TRT_CONVERTER(hard_sigmoid);
USE_TRT_CONVERTER(hard_swish);
1088
USE_TRT_CONVERTER(split);
1089 1090
USE_TRT_CONVERTER(prelu);
USE_TRT_CONVERTER(conv2d_transpose);
H
hjchen2 已提交
1091
USE_TRT_CONVERTER(leaky_relu);
1092 1093
USE_TRT_CONVERTER(shuffle_channel);
USE_TRT_CONVERTER(swish);
1094
USE_TRT_CONVERTER(instance_norm);
P
Pei Yang 已提交
1095 1096 1097
USE_TRT_CONVERTER(layer_norm);
USE_TRT_CONVERTER(gelu);
USE_TRT_CONVERTER(multihead_matmul);
1098 1099
USE_TRT_CONVERTER(fused_embedding_eltwise_layernorm);
USE_TRT_CONVERTER(skip_layernorm);
1100
USE_TRT_CONVERTER(slice);
1101
USE_TRT_CONVERTER(scale);
1102
USE_TRT_CONVERTER(stack);
P
Pei Yang 已提交
1103
USE_TRT_CONVERTER(clip);
1104
#endif
W
Wilber 已提交
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160

namespace paddle_infer {

void Tensor::Reshape(const std::vector<int> &shape) { tensor_->Reshape(shape); }

std::vector<int> Tensor::shape() const { return tensor_->shape(); }

void Tensor::SetLoD(const std::vector<std::vector<size_t>> &x) {
  return tensor_->SetLoD(x);
}

std::vector<std::vector<size_t>> Tensor::lod() const { return tensor_->lod(); }

const std::string &Tensor::name() const { return tensor_->name(); }

DataType Tensor::type() const { return tensor_->type(); }

Predictor::Predictor(const Config &config) {
  const_cast<Config *>(&config)->SwitchUseFeedFetchOps(false);
  // The second parameter indicates that the discard log is not printed
  predictor_ = paddle::CreatePaddlePredictor<
      Config, paddle::PaddleEngineKind::kAnalysis>(config);
}

std::vector<std::string> Predictor::GetInputNames() {
  return predictor_->GetInputNames();
}

std::unique_ptr<Tensor> Predictor::GetInputHandle(const std::string &name) {
  auto zero_copy_tensor = predictor_->GetInputTensor(name);
  std::unique_ptr<Tensor> tensor(new Tensor(std::move(zero_copy_tensor)));
  return tensor;
}

std::vector<std::string> Predictor::GetOutputNames() {
  return predictor_->GetOutputNames();
}

std::unique_ptr<Tensor> Predictor::GetOutputHandle(const std::string &name) {
  auto zero_copy_tensor = predictor_->GetOutputTensor(name);
  std::unique_ptr<Tensor> tensor(new Tensor(std::move(zero_copy_tensor)));
  return tensor;
}

bool Predictor::Run() { return predictor_->ZeroCopyRun(); }

std::unique_ptr<Predictor> Predictor::Clone() {
  auto analysis_pred = predictor_->Clone();
  std::unique_ptr<Predictor> pred(new Predictor(std::move(analysis_pred)));
  return pred;
}

void Predictor::ClearIntermediateTensor() {
  predictor_->ClearIntermediateTensor();
}

1161 1162
uint64_t Predictor::TryShrinkMemory() { return predictor_->TryShrinkMemory(); }

W
Wilber 已提交
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
int GetNumBytesOfDataType(DataType dtype) {
  switch (dtype) {
    case DataType::FLOAT32:
      return sizeof(float);
    case DataType::INT64:
      return sizeof(int64_t);
    case DataType::INT32:
      return sizeof(int32_t);
    case DataType::UINT8:
      return sizeof(uint8_t);
    default:
      assert(false);
      return -1;
  }
}

std::string GetVersion() { return paddle::get_version(); }

std::string UpdateDllFlag(const char *name, const char *value) {
  return paddle::UpdateDllFlag(name, value);
}

}  // namespace paddle_infer

namespace paddle_infer {
std::shared_ptr<Predictor> CreatePredictor(const Config &config) {  // NOLINT
  std::shared_ptr<Predictor> predictor(new Predictor(config));
  return predictor;
}

namespace services {
PredictorPool::PredictorPool(const Config &config, size_t size) {
  PADDLE_ENFORCE_GE(
      size, 1UL,
      paddle::platform::errors::InvalidArgument(
          "The predictor pool size should be greater than 1, but it's (%d)",
          size));
  Config copy_config(config);
  main_pred_.reset(new Predictor(config));
  for (size_t i = 0; i < size - 1; i++) {
    if (config.tensorrt_engine_enabled()) {
      Config config_tmp(copy_config);
      preds_.push_back(
          std::move(std::unique_ptr<Predictor>(new Predictor(config_tmp))));
    } else {
      preds_.push_back(std::move(main_pred_->Clone()));
    }
  }
}

Predictor *PredictorPool::Retrive(size_t idx) {
  PADDLE_ENFORCE_LT(
      idx, preds_.size() + 1,
      paddle::platform::errors::InvalidArgument(
          "There are (%d) predictors in the pool, but the idx is (%d)", idx,
          preds_.size() + 1));
  if (idx == 0) {
    return main_pred_.get();
  }
  return preds_[idx - 1].get();
}
}  // namespace services
}  // namespace paddle_infer