analysis_predictor.h 15.7 KB
Newer Older
Y
Yan Chunwei 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
#pragma once
16 17
#include <algorithm>
#include <map>
N
nhzlx 已提交
18
#include <memory>
19 20
#include <string>
#include <vector>
21
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
22 23
#include "paddle/fluid/distributed/fleet_executor/fleet_executor.h"
#endif
24
#include "paddle/fluid/framework/naive_executor.h"
25
#include "paddle/fluid/framework/op_compatible_info.h"
Y
Yan Chunwei 已提交
26 27
#include "paddle/fluid/inference/analysis/analyzer.h"
#include "paddle/fluid/inference/api/api_impl.h"
Y
Yan Chunwei 已提交
28
#include "paddle/fluid/inference/api/details/reset_tensor_array.h"
N
nhzlx 已提交
29
#include "paddle/fluid/inference/api/helper.h"
Y
Yan Chunwei 已提交
30
#include "paddle/fluid/inference/api/paddle_inference_api.h"
31
#include "paddle/fluid/inference/api/resource_manager.h"
W
Wilber 已提交
32
#include "paddle/fluid/platform/device/gpu/gpu_types.h"
33
#include "paddle/fluid/platform/float16.h"
34
#include "paddle/fluid/string/printf.h"
35 36 37 38
#ifdef PADDLE_WITH_TESTING
#include <gtest/gtest.h>
#include <gtest/gtest_prod.h>
#endif
39

40 41
namespace paddle_infer {
using float16 = paddle::platform::float16;
W
Wilber 已提交
42 43 44
namespace experimental {
class InternalUtils;
};
45
}  // namespace paddle_infer
46 47 48 49 50 51 52 53 54 55 56
///
/// \file analysis_predictor.h
///
/// \brief Compared to NativePredictor, AnalysisPredictor is a high-performance
/// predictor that includes many optimizations
///
/// \author paddle-infer@baidu.com
/// \date 2020-01-01
/// \since 1.7.0
///

Y
Yan Chunwei 已提交
57 58
namespace paddle {

59
using framework::NaiveExecutor;
60 61 62
using framework::proto::ProgramDesc;
using inference::analysis::Analyzer;
using inference::analysis::Argument;
Y
Yan Chunwei 已提交
63

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
///
/// \class AnalysisPredictor
///
/// \brief The analysis predictor is based on the original native predictor with
/// IR and Analysis support. It will optimize IR and Parameters in the runtime.
///
/// The predictor has the following typical uses:
///
/// Get predictor
/// \code{cpp}
///   auto predictor = CreatePaddlePredictor(config);
/// \endcode
///
/// Get input or output names
/// \code{cpp}
///   auto input_names = predictor->GetInputNames();
///   auto output_names = predictor->GetOutputNames();
/// \endcode
///
/// Get input or output tensors
/// \code{cpp}
///   auto input_t = predictor->GetInputTensor(input_names[0]);
///   auto output_t = predictor->GetOutputTensor(output_names[0]);
/// \endcode
///
/// Run predictor
/// \code{cpp}
///   predictor->ZeroCopyRun();
/// \endcode
///
94
class AnalysisPredictor : public PaddlePredictor {
Y
Yan Chunwei 已提交
95
 public:
96 97 98 99 100
  ///
  /// \brief Construct a new Analysis Predictor object
  ///
  /// \param[in] AnalysisConfig config
  ///
101
  explicit AnalysisPredictor(const AnalysisConfig &config) : config_(config) {
102 103 104 105
    if (config_.shape_range_info_collected()) {
      config_.SwitchIrOptim(false);
      config_.EnableMemoryOptim(false);
    }
106 107
    predictor_id_ = inference::GetUniqueId();
  }
108 109 110
  ///
  /// \brief Destroy the Analysis Predictor object
  ///
F
flame 已提交
111
  ~AnalysisPredictor();
Y
Yan Chunwei 已提交
112

113 114 115 116 117 118 119 120 121 122 123 124
  ///
  /// \brief Initialize predictor
  ///
  /// Initializing predictor mainly includes the following tasks:
  /// preparing scope, creating executor, preparing program, initializing the
  /// variables required by the executor, getting the feed_target_names and
  /// fetch_target_names, etc.
  ///
  /// \param[in] parent_scope parent scope
  /// \param[in] program program
  /// \return Whether the init function executed successfully
  ///
125 126
  bool Init(const std::shared_ptr<framework::Scope> &parent_scope,
            const std::shared_ptr<framework::ProgramDesc> &program = nullptr);
Y
Yan Chunwei 已提交
127

128 129 130 131 132 133 134 135
  ///
  /// \brief Run the prediction engine. Deprecated. Please refer to ZeroCopyRun
  ///
  /// \param[in] inputs input tensors
  /// \param[out] output_data output tensors
  /// \param[in] batch_size data's batch size
  /// \return Whether the function executed successfully
  ///
136 137 138 139
  bool Run(const std::vector<PaddleTensor> &inputs,
           std::vector<PaddleTensor> *output_data,
           int batch_size = -1) override;

140 141 142 143 144
  ///
  /// \brief Get the input names
  ///
  /// \return input names
  ///
N
nhzlx 已提交
145
  std::vector<std::string> GetInputNames();
146 147 148 149 150
  ///
  /// \brief Get the output names
  ///
  /// \return output names
  ///
N
nhzlx 已提交
151 152
  std::vector<std::string> GetOutputNames();

153 154 155 156 157 158
  ///
  /// \brief Get the Input Tensor object
  ///
  /// \param[in] name input name
  /// \return input tensor
  ///
159 160
  std::unique_ptr<ZeroCopyTensor> GetInputTensor(
      const std::string &name) override;
161 162 163 164 165 166
  ///
  /// \brief Get the Output Tensor object
  ///
  /// \param[in] name otuput name
  /// \return output tensor
  ///
167 168
  std::unique_ptr<ZeroCopyTensor> GetOutputTensor(
      const std::string &name) override;
169 170 171 172 173
  ///
  /// \brief Get all input names and their corresponding shapes
  ///
  /// \return the map of input names and shapes
  ///
174 175
  std::map<std::string, std::vector<int64_t>> GetInputTensorShape() override;

176 177 178 179 180
  ///
  /// \brief Run the prediction engine
  ///
  /// \return Whether the function executed successfully
  ///
181 182
  bool ZeroCopyRun() override;

W
Wilber 已提交
183 184 185 186 187
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
  // Note: Can only be used under thread_local semantics.
  bool ExpRunWithExternalStream(const gpuStream_t stream);
#endif

188 189 190 191 192 193 194 195
  ///
  /// \brief Get the execution stream on devices with a concept of stream,
  /// otherwise returns nullptr.
  ///
  /// \return The execution stream or nullptr (CPU).
  ///
  void *GetExecStream() const override;

196 197 198 199 200
  ///
  /// \brief Create feed fetch variables
  ///
  /// \param[in] scope Scope needed to create variables
  ///
201
  void CreateFeedFetchVar(framework::Scope *scope);
202 203 204 205
  ///
  /// \brief Determine the model's inputs and outputs based on the program's
  /// feed fetch op
  ///
206
  void PrepareFeedFetch();
Y
Yan Chunwei 已提交
207

208 209 210 211
  ///
  /// \brief Set predictor's argument according to config, which mainly includes
  /// execution information and graph optimization related pass information
  ///
212
  void PrepareArgument();
213 214 215 216
  ///
  /// \brief According to argument information, execute the relevant pass
  /// to get the optimized model program
  ///
Y
Yan Chunwei 已提交
217 218
  void OptimizeInferenceProgram();

219 220 221 222 223 224
  ///
  /// \brief Clear the intermediate tensors of the predictor
  ///
  ///
  void ClearIntermediateTensor();

225 226 227 228 229 230 231 232 233 234 235
  ///
  /// \brief Release all tmp tensor to compress the size of the memory pool.
  /// The memory pool is considered to be composed of a list of chunks, if
  /// the chunk is not occupied, it can be released.
  ///
  /// \return Number of bytes released. It may be smaller than the actual
  /// released memory, because part of the memory is not managed by the
  /// MemoryPool.
  ///
  uint64_t TryShrinkMemory() override;

236 237 238 239 240
  ///
  /// \brief Get the argument used by predictor
  ///
  /// \return the argument obtained by config
  ///
241
  Argument &analysis_argument() { return argument_; }
242 243 244 245 246
  ///
  /// \brief Clone to get the new predictor. thread safe.
  ///
  /// \return get a new predictor
  ///
247
  std::unique_ptr<PaddlePredictor> Clone(void *stream = nullptr) override;
248 249 250 251 252
  ///
  /// \brief Get the scope used by predictor
  ///
  /// \return scope
  ///
253
  framework::Scope *scope() { return scope_.get(); }
254 255 256 257 258
  ///
  /// \brief Get the inference program
  ///
  /// \return the inference program
  ///
259 260
  framework::ProgramDesc &program() { return *inference_program_; }

261 262 263 264 265
  ///
  /// \brief Get the serialized program
  ///
  /// \return the serialized program
  ///
266
  std::string GetSerializedProgram() const override;
Y
Yan Chunwei 已提交
267

268 269 270 271 272
  ///
  /// \brief Initialize mkldnn quantizer and execute mkldnn quantization pass
  ///
  /// \return Whether the function executed successfully
  ///
273 274
  bool MkldnnQuantize();

275 276 277 278 279
  ///
  /// \brief save program to model and save parameters to params
  ///
  /// \param[in] dir path to save the model
  ///
280 281
  void SaveOptimModel(const std::string &dir);

282
 protected:
283 284 285 286 287 288 289
  ///
  /// \brief Prepare predictor's required programs, including loading model
  /// information, graph optimization, and executor creation variables, etc.
  ///
  /// \param[in] program paddle program
  /// \return Whether the function executed successfully
  ///
290
  bool PrepareProgram(const std::shared_ptr<framework::ProgramDesc> &program);
291 292 293 294 295 296
  ///
  /// \brief Prepare scope environment, each predictor has its own scope
  ///
  /// \param[in] parent_scope The scope of the predictor to be cloned, or null
  /// \return Whether the function executed successfully
  ///
297
  bool PrepareScope(const std::shared_ptr<framework::Scope> &parent_scope);
298 299 300 301 302
  ///
  /// \brief Create an Executor object
  ///
  /// \return Whether the function executed successfully
  ///
303
  bool CreateExecutor();
304 305 306 307 308
  ///
  /// \brief According to the model's program, the executor creates ops
  ///
  /// \return Whether the function executed successfully
  ///
309 310
  bool PrepareExecutor();

311 312 313 314 315
  ///
  /// \brief Load model program.
  ///
  /// \return Whether the function executed successfully
  ///
316
  bool LoadProgramDesc();
317 318 319 320 321
  ///
  /// \brief Load model parameters.
  ///
  /// \return Whether the function executed successfully
  ///
322
  bool LoadParameters();
323

324 325 326 327 328 329 330
  ///
  /// \brief Prepare input data, only used in Run()
  ///
  /// \param[in] input_datas inpute tensors
  /// \param[in] scope the scope used by predictor
  /// \return Whether the function executed successfully
  ///
331 332
  bool SetFeed(const std::vector<PaddleTensor> &input_datas,
               framework::Scope *scope);
333 334 335 336 337 338 339
  ///
  /// \brief Get the output data, only used in Run()
  ///
  /// \param[out] output_data output tensors
  /// \param[in] scope the scope used by predictor
  /// \return Whether the function executed successfully
  ///
340 341
  bool GetFetch(std::vector<PaddleTensor> *output_data,
                framework::Scope *scope);
342 343 344 345 346 347
  ///
  /// \brief Get the output data, only used in GetFetch()
  ///
  /// \param[in] tensor for fetch op
  /// \param[out] output_data output tensor
  ///
348 349 350
  template <typename T>
  void GetFetchOne(const framework::LoDTensor &fetchs,
                   PaddleTensor *output_data);
351 352 353 354 355 356 357 358
  ///
  /// \brief PreSet for Mkldnn multi-thread and dynamic shape input.
  ///
  /// Used in AnalysisPredictor::Run(), do not support
  /// AnalysisPredictor::ZeroCopyRun() now.
  ///
  /// \param[in] inputs tensors
  ///
359
  void MkldnnPreSet(const std::vector<PaddleTensor> &inputs);
W
Wilber 已提交
360 361 362 363 364 365 366 367 368 369 370

  ///
  /// \brief PreSet for Mkldnn multi-thread and dynamic shape input.
  ///
  /// Used in AnalysisPredictor::Run(), do not support
  /// AnalysisPredictor::ZeroCopyRun() now.
  ///
  /// \param[in] inputs tensor shape
  ///
  void MkldnnPreSet(const std::vector<std::vector<int>> &inputs_shape);

371 372 373 374 375 376
  ///
  /// \brief PostReset for Mkldnn multi-thread and dynamic shape input.
  ///
  /// Used in AnalysisPredictor::Run(), do not support
  /// AnalysisPredictor::ZeroCopyRun() now.
  ///
377
  void MkldnnPostReset();
Y
Yan Chunwei 已提交
378

N
nhzlx 已提交
379
#if PADDLE_WITH_TENSORRT
380 381 382 383 384 385 386 387 388 389 390 391 392 393
  ///
  /// \brief save calibration table
  ///
  /// When we use Paddle-TRT INT8 engine, we need to generate calibration table
  /// data first,
  /// the calibration table contains the range for each op's input and output,
  /// this whole process can be divided into several steps:
  /// 1. Builds a 32-bit engine, runs it on the calibration set, and records a
  ///  histogram for each tensor of the distribution of activation values.
  /// 2. Builds a calibration table from the histograms.
  /// After step 2, we need to store the calibration table on disk.
  ///
  /// \return Whether the function executed successfully
  ///
N
nhzlx 已提交
394
  bool SaveTrtCalibToDisk();
N
nhzlx 已提交
395
#endif
N
nhzlx 已提交
396

397 398 399 400 401 402 403 404
// Some more detailed tests, they are made the friends of the predictor, so that
// the all the details can be tested.
#if PADDLE_WITH_TESTING
  FRIEND_TEST(AnalysisPredictor, analysis_off);
  FRIEND_TEST(AnalysisPredictor, analysis_on);
  FRIEND_TEST(AnalysisPredictor, with_gpu);
#endif

405 406 407
 protected:
  const void *GetDeviceContexts() const override;

408 409 410 411
 private:
  void StatisticShapeRangeInfo();
  void CollectShapeRangeInfo();

412 413 414 415
  void InitPlace();
  void InitDeviceContexts();
  void InitResourceManager(void *stream);

416
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
  // fleet exe related

  ///
  /// \brief prepare for fleet executor to run
  ///
  /// Used in AnalysisPredictor::Init(),
  ///
  bool PrepareFleetExecutor();

  ///
  /// \brief init NCCL env for multi gpus inference
  ///
  /// Used in AnalysisPredictor::PrepareFleetExecutor()
  ///
  bool CommInit();

  ///
  /// \brief read the config to init NCCL env
  ///
  /// Used in AnalysisPredictor::CommInit()
  ///
  /// \param[in] ring_id_to_ranks: a ptr to ring_id_to_ranks
  /// \param[in] rank_to_ring_ids: a ptr to rank_to_ring_ids
  ///
  bool LoadConverterConfig(
      std::map<int64_t, std::vector<int64_t>> *ring_id_to_ranks,
      std::map<int64_t, std::vector<int64_t>> *rank_to_ring_ids);

  ///
  /// \brief add ops and run them with NaiveExecutor to init NCCL env
  ///
  /// Used in AnalysisPredictor::CommInit()
  ///
  /// \param[in] tmp_var_name: var name to hold NCCL unique id
  /// \param[in] nranks: number of ranks in one comm group
  /// \param[in] rank: relative rank of current rank in the comm group
  /// \param[in] peer_endpoints: group's peers' endpoints
  /// \param[in] block: the block to insert comm ops
  /// \param[in] ring_id: the ring id to be used to init NCCL env
  ///
457 458 459
  void InsertCommOp(std::string tmp_var_name,
                    int nranks,
                    int rank,
460
                    const std::vector<std::string> &peer_endpoints,
461 462
                    framework::BlockDesc *block,
                    int ring_id);
463 464
#endif

Y
Yan Chunwei 已提交
465
 private:
466
  AnalysisConfig config_;
Y
Yan Chunwei 已提交
467
  Argument argument_;
468 469 470 471 472
  std::unique_ptr<NaiveExecutor> executor_;
  platform::Place place_;
  std::shared_ptr<framework::Scope> scope_;
  framework::Scope *sub_scope_{nullptr};
  std::shared_ptr<framework::ProgramDesc> inference_program_;
473
  framework::OpCompatibleMap op_compatible_map_;
474 475
  std::vector<framework::OpDesc *> feeds_;
  std::map<std::string, size_t> feed_names_;
N
nhzlx 已提交
476 477
  // Sorted according to the idx.
  std::map<size_t, std::string> idx2feeds_;
Y
Yan Chunwei 已提交
478
  std::vector<framework::OpDesc *> fetches_;
N
nhzlx 已提交
479 480
  std::map<size_t, std::string> idx2fetches_;

481 482 483 484 485 486 487 488 489 490
#if PADDLE_WITH_MKLDNN
  // Helper class to perform quantization
  class MkldnnQuantizer;
  MkldnnQuantizer *mkldnn_quantizer_{nullptr};

#if PADDLE_WITH_TESTING
  friend class MkldnnQuantizerTest;
#endif
#endif

491
  // Memory buffer for feed inputs. The temporary LoDTensor will cause serious
492
  // concurrency problems, wrong results and memory leak, so cache them.
493
  std::vector<framework::LoDTensor> feed_tensors_;
Y
Yan Chunwei 已提交
494
  details::TensorArrayBatchCleaner tensor_array_batch_cleaner_;
Y
Yan Chunwei 已提交
495 496
  // A mutex help to make Clone thread safe.
  std::mutex clone_mutex_;
497

Y
Yan Chunwei 已提交
498 499 500 501
  // For memory optimization.
  const size_t max_shape_collect_count_{1000};
  int need_collect_var_shapes_{-1};  // -1 for default, 0 for false, 1 for true.
  std::vector<std::map<std::string, std::vector<int>>> batch_var_shapes_;
502
  int predictor_id_;
Y
Yan Chunwei 已提交
503

504 505 506
 private:
  // Some status here that help to determine the status inside the predictor.
  bool status_is_cloned_{false};
507 508

  std::map<std::string, std::vector<std::vector<int32_t>>> shape_info_;
509
  static int clone_num_;
510

511 512 513 514 515
  bool private_context_{false};
  void *predictor_stream_{nullptr};
  std::map<phi::Place, std::shared_future<std::unique_ptr<phi::DeviceContext>>>
      device_contexts_;

516
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
517 518 519 520 521
  // fleet executor related
  distributed::FleetExecutorDesc executor_desc_;
  std::shared_ptr<distributed::FleetExecutor> fleet_exe_;
  std::shared_ptr<distributed::TaskNode> task_node_;
#endif
W
Wilber 已提交
522
  friend class paddle_infer::experimental::InternalUtils;
Y
Yan Chunwei 已提交
523 524 525
};

}  // namespace paddle