inference_api.cc 42.4 KB
Newer Older
F
flame 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/pybind/inference_api.h"
16

17
#include <pybind11/functional.h>
18
#include <pybind11/numpy.h>
F
flame 已提交
19
#include <pybind11/stl.h>
20

F
flame 已提交
21
#include <cstring>
22
#include <functional>
F
flame 已提交
23
#include <iostream>
24
#include <iterator>
25
#include <map>
26
#include <memory>
F
flame 已提交
27
#include <string>
28
#include <type_traits>
29
#include <unordered_set>
30
#include <utility>
F
flame 已提交
31
#include <vector>
32

F
flame 已提交
33
#include "paddle/fluid/inference/api/analysis_predictor.h"
34
#include "paddle/fluid/inference/api/helper.h"
35
#include "paddle/fluid/inference/api/paddle_infer_contrib.h"
F
flame 已提交
36
#include "paddle/fluid/inference/api/paddle_inference_api.h"
37
#include "paddle/fluid/inference/api/paddle_pass_builder.h"
38
#include "paddle/fluid/inference/utils/io_utils.h"
39
#include "paddle/phi/core/compat/convert_utils.h"
F
flame 已提交
40

41 42 43 44
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#include "paddle/phi/core/cuda_stream.h"
#endif

45 46 47 48
#ifdef PADDLE_WITH_ONNXRUNTIME
#include "paddle/fluid/inference/api/onnxruntime_predictor.h"
#endif

F
flame 已提交
49 50
namespace py = pybind11;

51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
namespace pybind11 {
namespace detail {

// Note: use same enum number of float16 in numpy.
// import numpy as np
// print np.dtype(np.float16).num  # 23
constexpr int NPY_FLOAT16_ = 23;
constexpr int NPY_UINT16_ = 4;

// Note: Since float16 is not a builtin type in C++, we register
// paddle::platform::float16 as numpy.float16.
// Ref: https://github.com/pybind/pybind11/issues/1776
template <>
struct npy_format_descriptor<paddle_infer::float16> {
  static py::dtype dtype() {
    handle ptr = npy_api::get().PyArray_DescrFromType_(NPY_FLOAT16_);
    return reinterpret_borrow<py::dtype>(ptr);
  }
  static std::string format() {
    // Note: "e" represents float16.
    // Details at:
    // https://docs.python.org/3/library/struct.html#format-characters.
    return "e";
  }
  static constexpr auto name = _("float16");
};

}  // namespace detail
}  // namespace pybind11

F
flame 已提交
81 82
namespace paddle {
namespace pybind {
83 84 85
using paddle::AnalysisPredictor;
using paddle::NativeConfig;
using paddle::NativePaddlePredictor;
F
flame 已提交
86
using paddle::PaddleBuf;
87
using paddle::PaddleDataLayout;
88
using paddle::PaddleDType;
89
using paddle::PaddlePassBuilder;
F
flame 已提交
90 91
using paddle::PaddlePlace;
using paddle::PaddlePredictor;
92 93 94
using paddle::PaddleTensor;
using paddle::PassStrategy;
using paddle::ZeroCopyTensor;
F
flame 已提交
95

96 97
namespace {
void BindPaddleDType(py::module *m);
98
void BindPaddleDataLayout(py::module *m);
99 100 101 102 103 104
void BindPaddleBuf(py::module *m);
void BindPaddleTensor(py::module *m);
void BindPaddlePlace(py::module *m);
void BindPaddlePredictor(py::module *m);
void BindNativeConfig(py::module *m);
void BindNativePredictor(py::module *m);
105
void BindLiteNNAdapterConfig(py::module *m);
106 107
void BindAnalysisConfig(py::module *m);
void BindAnalysisPredictor(py::module *m);
108 109
void BindZeroCopyTensor(py::module *m);
void BindPaddlePassBuilder(py::module *m);
W
Wilber 已提交
110 111 112
void BindPaddleInferPredictor(py::module *m);
void BindPaddleInferTensor(py::module *m);
void BindPredictorPool(py::module *m);
F
flame 已提交
113

114
#ifdef PADDLE_WITH_MKLDNN
115
void BindMkldnnQuantizerConfig(py::module *m);
116
#endif
117 118

template <typename T>
119 120
PaddleBuf PaddleBufCreate(
    py::array_t<T, py::array::c_style | py::array::forcecast> data) {
121
  PaddleBuf buf(data.size() * sizeof(T));
W
Wilber 已提交
122 123
  std::copy_n(static_cast<const T *>(data.data()),
              data.size(),
124 125 126 127 128
              static_cast<T *>(buf.data()));
  return buf;
}

template <typename T>
129 130 131
void PaddleBufReset(
    PaddleBuf &buf,                                                    // NOLINT
    py::array_t<T, py::array::c_style | py::array::forcecast> data) {  // NOLINT
132
  buf.Resize(data.size() * sizeof(T));
W
Wilber 已提交
133 134
  std::copy_n(static_cast<const T *>(data.data()),
              data.size(),
135 136 137 138 139
              static_cast<T *>(buf.data()));
}

template <typename T>
PaddleTensor PaddleTensorCreate(
140 141
    py::array_t<T, py::array::c_style | py::array::forcecast> data,
    const std::string name = "",
W
Wilber 已提交
142 143
    const std::vector<std::vector<size_t>> &lod = {},
    bool copy = true) {
144 145 146 147
  PaddleTensor tensor;

  if (copy) {
    PaddleBuf buf(data.size() * sizeof(T));
W
Wilber 已提交
148 149
    std::copy_n(static_cast<const T *>(data.data()),
                data.size(),
150 151 152 153 154 155
                static_cast<T *>(buf.data()));
    tensor.data = std::move(buf);
  } else {
    tensor.data = PaddleBuf(data.mutable_data(), data.size() * sizeof(T));
  }

156
  tensor.dtype = inference::PaddleTensorGetDType<T>();
157 158 159 160 161 162 163 164
  tensor.name = name;
  tensor.lod = lod;
  tensor.shape.resize(data.ndim());
  std::copy_n(data.shape(), data.ndim(), tensor.shape.begin());

  return tensor;
}

165
py::dtype PaddleDTypeToNumpyDType(PaddleDType dtype) {
166
  py::dtype dt;
167
  switch (dtype) {
168 169 170 171 172 173 174 175 176
    case PaddleDType::INT32:
      dt = py::dtype::of<int32_t>();
      break;
    case PaddleDType::INT64:
      dt = py::dtype::of<int64_t>();
      break;
    case PaddleDType::FLOAT32:
      dt = py::dtype::of<float>();
      break;
W
Wilber 已提交
177 178 179
    case PaddleDType::UINT8:
      dt = py::dtype::of<uint8_t>();
      break;
180 181 182
    case PaddleDType::FLOAT16:
      dt = py::dtype::of<paddle_infer::float16>();
      break;
183
    default:
184
      PADDLE_THROW(platform::errors::Unimplemented(
W
Wilber 已提交
185
          "Unsupported data type. Now only supports INT32, INT64, UINT8 and "
186
          "FLOAT32."));
187
  }
188 189 190 191 192 193 194 195 196 197

  return dt;
}

py::array PaddleTensorGetData(PaddleTensor &tensor) {  // NOLINT
  py::dtype dt = PaddleDTypeToNumpyDType(tensor.dtype);
  return py::array(std::move(dt), {tensor.shape}, tensor.data.data());
}

template <typename T>
198 199 200
void ZeroCopyTensorCreate(
    ZeroCopyTensor &tensor,  // NOLINT
    py::array_t<T, py::array::c_style | py::array::forcecast> data) {
201 202 203 204 205 206
  std::vector<int> shape;
  std::copy_n(data.shape(), data.ndim(), std::back_inserter(shape));
  tensor.Reshape(std::move(shape));
  tensor.copy_from_cpu(static_cast<const T *>(data.data()));
}

S
Steffy-zxf 已提交
207 208 209 210 211 212 213 214 215 216 217 218
/// \brief Experimental interface.
/// Create the Strings tensor from data.
/// \param tensor The tensor will be created and
/// the tensor value is same as data.
/// \param data The input text.
void ZeroCopyStringTensorCreate(ZeroCopyTensor &tensor,  // NOLINT
                                const paddle_infer::Strings *data) {
  size_t shape = data->size();
  tensor.ReshapeStrings(shape);
  tensor.copy_strings_from_cpu(data);
}

W
Wilber 已提交
219
template <typename T>
220 221 222
void PaddleInferTensorCreate(
    paddle_infer::Tensor &tensor,  // NOLINT
    py::array_t<T, py::array::c_style | py::array::forcecast> data) {
W
Wilber 已提交
223 224 225 226 227 228
  std::vector<int> shape;
  std::copy_n(data.shape(), data.ndim(), std::back_inserter(shape));
  tensor.Reshape(std::move(shape));
  tensor.CopyFromCpu(static_cast<const T *>(data.data()));
}

229 230 231 232 233 234 235 236 237 238 239 240
paddle_infer::PlaceType ToPaddleInferPlace(
    phi::AllocationType allocation_type) {
  if (allocation_type == phi::AllocationType::CPU) {
    return paddle_infer::PlaceType::kCPU;
  } else if (allocation_type == phi::AllocationType::GPU) {
    return paddle_infer::PlaceType::kGPU;
  } else {
    return paddle_infer::PlaceType::kCPU;
  }
}

void PaddleInferShareExternalData(paddle_infer::Tensor &tensor,  // NOLINT
241
                                  phi::DenseTensor input_tensor) {
242 243 244 245 246 247
  std::vector<int> shape;
  for (int i = 0; i < input_tensor.dims().size(); ++i) {
    shape.push_back(input_tensor.dims()[i]);
  }
  if (input_tensor.dtype() == phi::DataType::FLOAT32) {
    tensor.ShareExternalData(
W
Wilber 已提交
248 249
        static_cast<float *>(input_tensor.data()),
        shape,
250 251 252
        ToPaddleInferPlace(input_tensor.place().GetType()));
  } else if (input_tensor.dtype() == phi::DataType::FLOAT16) {
    tensor.ShareExternalData(
W
Wilber 已提交
253 254
        static_cast<paddle::platform::float16 *>(input_tensor.data()),
        shape,
255 256 257 258
        ToPaddleInferPlace(input_tensor.place().GetType()));
  }
}

S
Steffy-zxf 已提交
259 260 261 262 263 264 265 266 267 268 269 270 271
/// \brief Experimental interface.
/// Create the Strings tensor from data.
/// \param tensor The tensor will be created and
/// the tensor value is same as data.
/// \param data The input text.
void PaddleInferStringTensorCreate(paddle_infer::Tensor &tensor,  // NOLINT
                                   const paddle_infer::Strings *data) {
  VLOG(3) << "Create PaddleInferTensor, dtype = Strings ";
  size_t shape = data->size();
  tensor.ReshapeStrings(shape);
  tensor.CopyStringsFromCpu(data);
}

272 273 274 275 276 277 278 279 280 281 282 283 284
size_t PaddleGetDTypeSize(PaddleDType dt) {
  size_t size{0};
  switch (dt) {
    case PaddleDType::INT32:
      size = sizeof(int32_t);
      break;
    case PaddleDType::INT64:
      size = sizeof(int64_t);
      break;
    case PaddleDType::FLOAT32:
      size = sizeof(float);
      break;
    default:
285 286 287
      PADDLE_THROW(platform::errors::Unimplemented(
          "Unsupported data type. Now only supports INT32, INT64 and "
          "FLOAT32."));
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
  }
  return size;
}

py::array ZeroCopyTensorToNumpy(ZeroCopyTensor &tensor) {  // NOLINT
  py::dtype dt = PaddleDTypeToNumpyDType(tensor.type());
  auto tensor_shape = tensor.shape();
  py::array::ShapeContainer shape(tensor_shape.begin(), tensor_shape.end());
  py::array array(dt, std::move(shape));

  switch (tensor.type()) {
    case PaddleDType::INT32:
      tensor.copy_to_cpu(static_cast<int32_t *>(array.mutable_data()));
      break;
    case PaddleDType::INT64:
      tensor.copy_to_cpu(static_cast<int64_t *>(array.mutable_data()));
      break;
    case PaddleDType::FLOAT32:
      tensor.copy_to_cpu<float>(static_cast<float *>(array.mutable_data()));
      break;
308 309 310 311
    case PaddleDType::FLOAT16:
      tensor.copy_to_cpu<paddle::platform::float16>(
          static_cast<paddle::platform::float16 *>(array.mutable_data()));
      break;
W
Wilber 已提交
312 313 314
    case PaddleDType::UINT8:
      tensor.copy_to_cpu<uint8_t>(static_cast<uint8_t *>(array.mutable_data()));
      break;
315 316 317
    case PaddleDType::INT8:
      tensor.copy_to_cpu<int8_t>(static_cast<int8_t *>(array.mutable_data()));
      break;
318
    default:
319
      PADDLE_THROW(platform::errors::Unimplemented(
W
Wilber 已提交
320
          "Unsupported data type. Now only supports INT32, INT64, UINT8 and "
321
          "FLOAT32."));
322 323
  }
  return array;
324
}
325

W
Wilber 已提交
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
py::array PaddleInferTensorToNumpy(paddle_infer::Tensor &tensor) {  // NOLINT
  py::dtype dt = PaddleDTypeToNumpyDType(tensor.type());
  auto tensor_shape = tensor.shape();
  py::array::ShapeContainer shape(tensor_shape.begin(), tensor_shape.end());
  py::array array(dt, std::move(shape));

  switch (tensor.type()) {
    case PaddleDType::INT32:
      tensor.CopyToCpu(static_cast<int32_t *>(array.mutable_data()));
      break;
    case PaddleDType::INT64:
      tensor.CopyToCpu(static_cast<int64_t *>(array.mutable_data()));
      break;
    case PaddleDType::FLOAT32:
      tensor.CopyToCpu<float>(static_cast<float *>(array.mutable_data()));
      break;
342 343 344 345
    case PaddleDType::FLOAT16:
      tensor.CopyToCpu<paddle::platform::float16>(
          static_cast<paddle::platform::float16 *>(array.mutable_data()));
      break;
346 347 348 349 350 351
    case PaddleDType::UINT8:
      tensor.CopyToCpu(static_cast<uint8_t *>(array.mutable_data()));
      break;
    case PaddleDType::INT8:
      tensor.CopyToCpu(static_cast<int8_t *>(array.mutable_data()));
      break;
W
Wilber 已提交
352 353 354 355 356 357 358 359
    default:
      PADDLE_THROW(platform::errors::Unimplemented(
          "Unsupported data type. Now only supports INT32, INT64 and "
          "FLOAT32."));
  }
  return array;
}

360 361 362 363 364
py::bytes SerializePDTensorToBytes(PaddleTensor &tensor) {  // NOLINT
  std::stringstream ss;
  paddle::inference::SerializePDTensorToStream(&ss, tensor);
  return static_cast<py::bytes>(ss.str());
}
365

366
void CopyPaddleInferTensor(paddle_infer::Tensor &dst,  // NOLINT
367 368 369 370
                           const paddle_infer::Tensor &src) {
  return paddle_infer::contrib::TensorUtils::CopyTensor(&dst, src);
}

371
}  // namespace
372

F
flame 已提交
373 374
void BindInferenceApi(py::module *m) {
  BindPaddleDType(m);
375
  BindPaddleDataLayout(m);
F
flame 已提交
376 377 378 379 380 381
  BindPaddleBuf(m);
  BindPaddleTensor(m);
  BindPaddlePlace(m);
  BindPaddlePredictor(m);
  BindNativeConfig(m);
  BindNativePredictor(m);
382
  BindLiteNNAdapterConfig(m);
F
flame 已提交
383 384
  BindAnalysisConfig(m);
  BindAnalysisPredictor(m);
W
Wilber 已提交
385
  BindPaddleInferPredictor(m);
386
  BindZeroCopyTensor(m);
W
Wilber 已提交
387
  BindPaddleInferTensor(m);
388
  BindPaddlePassBuilder(m);
W
Wilber 已提交
389
  BindPredictorPool(m);
390 391 392
#ifdef PADDLE_WITH_MKLDNN
  BindMkldnnQuantizerConfig(m);
#endif
F
flame 已提交
393
  m->def("create_paddle_predictor",
W
Wilber 已提交
394 395
         &paddle::CreatePaddlePredictor<AnalysisConfig>,
         py::arg("config"));
F
flame 已提交
396
  m->def("create_paddle_predictor",
W
Wilber 已提交
397 398
         &paddle::CreatePaddlePredictor<NativeConfig>,
         py::arg("config"));
399 400 401 402 403 404 405
  m->def("create_predictor",
         [](const paddle_infer::Config &config)
             -> std::unique_ptr<paddle_infer::Predictor> {
           auto pred = std::unique_ptr<paddle_infer::Predictor>(
               new paddle_infer::Predictor(config));
           return pred;
         });
406 407 408 409 410 411
  m->def(
      "_get_phi_kernel_name",
      [](const std::string &fluid_op_name) {
        return phi::TransToPhiKernelName(fluid_op_name);
      },
      py::return_value_policy::reference);
412
  m->def("copy_tensor", &CopyPaddleInferTensor);
F
flame 已提交
413
  m->def("paddle_dtype_size", &paddle::PaddleDtypeSize);
414
  m->def("paddle_tensor_to_bytes", &SerializePDTensorToBytes);
W
Wilber 已提交
415
  m->def("get_version", &paddle_infer::GetVersion);
416 417
  m->def("get_trt_compile_version", &paddle_infer::GetTrtCompileVersion);
  m->def("get_trt_runtime_version", &paddle_infer::GetTrtRuntimeVersion);
W
Wilber 已提交
418
  m->def("get_num_bytes_of_data_type", &paddle_infer::GetNumBytesOfDataType);
419 420 421 422 423 424 425 426 427 428
  m->def("convert_to_mixed_precision_bind",
         &paddle_infer::ConvertToMixedPrecision,
         py::arg("model_file"),
         py::arg("params_file"),
         py::arg("mixed_model_file"),
         py::arg("mixed_params_file"),
         py::arg("mixed_precision"),
         py::arg("backend"),
         py::arg("keep_io_types") = true,
         py::arg("black_list") = std::unordered_set<std::string>());
F
flame 已提交
429 430
}

431
namespace {
F
flame 已提交
432 433 434
void BindPaddleDType(py::module *m) {
  py::enum_<PaddleDType>(*m, "PaddleDType")
      .value("FLOAT32", PaddleDType::FLOAT32)
435 436
      .value("INT64", PaddleDType::INT64)
      .value("INT32", PaddleDType::INT32);
F
flame 已提交
437 438
}

439 440 441 442 443 444 445 446
void BindPaddleDataLayout(py::module *m) {
  py::enum_<PaddleDataLayout>(*m, "PaddleDataLayout")
      .value("UNK", PaddleDataLayout::kUNK)
      .value("Any", PaddleDataLayout::kAny)
      .value("NHWC", PaddleDataLayout::kNHWC)
      .value("NCHW", PaddleDataLayout::kNCHW);
}

F
flame 已提交
447 448 449 450 451 452
void BindPaddleBuf(py::module *m) {
  py::class_<PaddleBuf>(*m, "PaddleBuf")
      .def(py::init<size_t>())
      .def(py::init([](std::vector<float> &data) {
        auto buf = PaddleBuf(data.size() * sizeof(float));
        std::memcpy(buf.data(), static_cast<void *>(data.data()), buf.length());
G
Gabor Buella 已提交
453
        return buf;
F
flame 已提交
454
      }))
455 456 457
      .def(py::init(&PaddleBufCreate<int32_t>))
      .def(py::init(&PaddleBufCreate<int64_t>))
      .def(py::init(&PaddleBufCreate<float>))
F
flame 已提交
458 459 460 461 462 463
      .def("resize", &PaddleBuf::Resize)
      .def("reset",
           [](PaddleBuf &self, std::vector<float> &data) {
             self.Resize(data.size() * sizeof(float));
             std::memcpy(self.data(), data.data(), self.length());
           })
464 465 466
      .def("reset", &PaddleBufReset<int32_t>)
      .def("reset", &PaddleBufReset<int64_t>)
      .def("reset", &PaddleBufReset<float>)
467
      .def("empty", &PaddleBuf::empty)
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
      .def("tolist",
           [](PaddleBuf &self, const std::string &dtype) -> py::list {
             py::list l;
             if (dtype == "int32") {
               auto *data = static_cast<int32_t *>(self.data());
               auto size = self.length() / sizeof(int32_t);
               l = py::cast(std::vector<int32_t>(data, data + size));
             } else if (dtype == "int64") {
               auto *data = static_cast<int64_t *>(self.data());
               auto size = self.length() / sizeof(int64_t);
               l = py::cast(std::vector<int64_t>(data, data + size));
             } else if (dtype == "float32") {
               auto *data = static_cast<float *>(self.data());
               auto size = self.length() / sizeof(float);
               l = py::cast(std::vector<float>(data, data + size));
             } else {
484 485 486
               PADDLE_THROW(platform::errors::Unimplemented(
                   "Unsupported data type. Now only supports INT32, INT64 and "
                   "FLOAT32."));
487 488 489
             }
             return l;
           })
F
flame 已提交
490 491 492 493 494 495 496 497 498 499
      .def("float_data",
           [](PaddleBuf &self) -> std::vector<float> {
             auto *data = static_cast<float *>(self.data());
             return {data, data + self.length() / sizeof(*data)};
           })
      .def("int64_data",
           [](PaddleBuf &self) -> std::vector<int64_t> {
             int64_t *data = static_cast<int64_t *>(self.data());
             return {data, data + self.length() / sizeof(*data)};
           })
500 501 502 503
      .def("int32_data",
           [](PaddleBuf &self) -> std::vector<int32_t> {
             int32_t *data = static_cast<int32_t *>(self.data());
             return {data, data + self.length() / sizeof(*data)};
F
flame 已提交
504 505 506 507 508 509 510
           })
      .def("length", &PaddleBuf::length);
}

void BindPaddleTensor(py::module *m) {
  py::class_<PaddleTensor>(*m, "PaddleTensor")
      .def(py::init<>())
W
Wilber 已提交
511 512
      .def(py::init(&PaddleTensorCreate<int32_t>),
           py::arg("data"),
513 514 515
           py::arg("name") = "",
           py::arg("lod") = std::vector<std::vector<size_t>>(),
           py::arg("copy") = true)
W
Wilber 已提交
516 517
      .def(py::init(&PaddleTensorCreate<int64_t>),
           py::arg("data"),
518 519 520
           py::arg("name") = "",
           py::arg("lod") = std::vector<std::vector<size_t>>(),
           py::arg("copy") = true)
W
Wilber 已提交
521 522
      .def(py::init(&PaddleTensorCreate<float>),
           py::arg("data"),
523 524 525 526
           py::arg("name") = "",
           py::arg("lod") = std::vector<std::vector<size_t>>(),
           py::arg("copy") = true)
      .def("as_ndarray", &PaddleTensorGetData)
F
flame 已提交
527 528 529 530 531 532 533 534 535 536 537
      .def_readwrite("name", &PaddleTensor::name)
      .def_readwrite("shape", &PaddleTensor::shape)
      .def_readwrite("data", &PaddleTensor::data)
      .def_readwrite("dtype", &PaddleTensor::dtype)
      .def_readwrite("lod", &PaddleTensor::lod);
}

void BindPaddlePlace(py::module *m) {
  py::enum_<PaddlePlace>(*m, "PaddlePlace")
      .value("UNK", PaddlePlace::kUNK)
      .value("CPU", PaddlePlace::kCPU)
538
      .value("GPU", PaddlePlace::kGPU)
W
Wilber 已提交
539 540
      .value("XPU", PaddlePlace::kXPU)
      .value("NPU", PaddlePlace::kNPU);
F
flame 已提交
541 542 543 544 545 546 547 548 549 550 551 552 553
}

void BindPaddlePredictor(py::module *m) {
  auto paddle_predictor = py::class_<PaddlePredictor>(*m, "PaddlePredictor");
  paddle_predictor
      .def("run",
           [](PaddlePredictor &self, const std::vector<PaddleTensor> &inputs) {
             std::vector<PaddleTensor> outputs;
             self.Run(inputs, &outputs);
             return outputs;
           })
      .def("get_input_tensor", &PaddlePredictor::GetInputTensor)
      .def("get_output_tensor", &PaddlePredictor::GetOutputTensor)
554 555
      .def("get_input_names", &PaddlePredictor::GetInputNames)
      .def("get_output_names", &PaddlePredictor::GetOutputNames)
F
flame 已提交
556
      .def("zero_copy_run", &PaddlePredictor::ZeroCopyRun)
557
      .def("clone", [](PaddlePredictor &self) { return self.Clone(nullptr); })
558 559 560
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
      .def("clone",
           [](PaddlePredictor &self, phi::CUDAStream &stream) {
561
             return self.Clone(stream.raw_stream());
562 563
           })
#endif
564
      .def("get_serialized_program", &PaddlePredictor::GetSerializedProgram);
F
flame 已提交
565 566 567 568 569 570 571 572 573 574

  auto config = py::class_<PaddlePredictor::Config>(paddle_predictor, "Config");
  config.def(py::init<>())
      .def_readwrite("model_dir", &PaddlePredictor::Config::model_dir);
}

void BindNativeConfig(py::module *m) {
  py::class_<NativeConfig, PaddlePredictor::Config>(*m, "NativeConfig")
      .def(py::init<>())
      .def_readwrite("use_gpu", &NativeConfig::use_gpu)
575
      .def_readwrite("use_xpu", &NativeConfig::use_xpu)
W
Wilber 已提交
576
      .def_readwrite("use_npu", &NativeConfig::use_npu)
F
flame 已提交
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
      .def_readwrite("device", &NativeConfig::device)
      .def_readwrite("fraction_of_gpu_memory",
                     &NativeConfig::fraction_of_gpu_memory)
      .def_readwrite("prog_file", &NativeConfig::prog_file)
      .def_readwrite("param_file", &NativeConfig::param_file)
      .def_readwrite("specify_input_name", &NativeConfig::specify_input_name)
      .def("set_cpu_math_library_num_threads",
           &NativeConfig::SetCpuMathLibraryNumThreads)
      .def("cpu_math_library_num_threads",
           &NativeConfig::cpu_math_library_num_threads);
}

void BindNativePredictor(py::module *m) {
  py::class_<NativePaddlePredictor, PaddlePredictor>(*m,
                                                     "NativePaddlePredictor")
      .def(py::init<const NativeConfig &>())
      .def("init", &NativePaddlePredictor::Init)
      .def("run",
           [](NativePaddlePredictor &self,
              const std::vector<PaddleTensor> &inputs) {
             std::vector<PaddleTensor> outputs;
             self.Run(inputs, &outputs);
             return outputs;
           })
      .def("get_input_tensor", &NativePaddlePredictor::GetInputTensor)
      .def("get_output_tensor", &NativePaddlePredictor::GetOutputTensor)
      .def("zero_copy_run", &NativePaddlePredictor::ZeroCopyRun)
604 605
      .def("clone",
           [](NativePaddlePredictor &self) { return self.Clone(nullptr); })
606 607 608
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
      .def("clone",
           [](NativePaddlePredictor &self, phi::CUDAStream &stream) {
609
             return self.Clone(stream.raw_stream());
610 611
           })
#endif
W
Wilber 已提交
612 613
      .def("scope",
           &NativePaddlePredictor::scope,
F
flame 已提交
614 615 616 617
           py::return_value_policy::reference);
}

void BindAnalysisConfig(py::module *m) {
618 619 620 621 622
  py::class_<AnalysisConfig> analysis_config(*m, "AnalysisConfig");

  py::enum_<AnalysisConfig::Precision>(analysis_config, "Precision")
      .value("Float32", AnalysisConfig::Precision::kFloat32)
      .value("Int8", AnalysisConfig::Precision::kInt8)
Z
Zhaolong Xing 已提交
623
      .value("Half", AnalysisConfig::Precision::kHalf)
624 625 626
      .value("Bfloat16", AnalysisConfig::Precision::kBf16)
      .export_values();

627 628
  analysis_config.def(py::init<>())
      .def(py::init<const AnalysisConfig &>())
F
flame 已提交
629 630
      .def(py::init<const std::string &>())
      .def(py::init<const std::string &, const std::string &>())
631
      .def("summary", &AnalysisConfig::Summary)
W
Wilber 已提交
632 633 634
      .def("set_model",
           (void(AnalysisConfig::*)(const std::string &)) &
               AnalysisConfig::SetModel)
635 636 637
      .def("set_model",
           (void(AnalysisConfig::*)(const std::string &, const std::string &)) &
               AnalysisConfig::SetModel)
F
flame 已提交
638 639 640 641 642
      .def("set_prog_file", &AnalysisConfig::SetProgFile)
      .def("set_params_file", &AnalysisConfig::SetParamsFile)
      .def("model_dir", &AnalysisConfig::model_dir)
      .def("prog_file", &AnalysisConfig::prog_file)
      .def("params_file", &AnalysisConfig::params_file)
W
Wilber 已提交
643 644 645 646
      .def("enable_use_gpu",
           &AnalysisConfig::EnableUseGpu,
           py::arg("memory_pool_init_size_mb"),
           py::arg("device_id") = 0)
647 648 649 650 651 652
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
      .def("set_exec_stream",
           [](AnalysisConfig &self, phi::CUDAStream &stream) {
             self.SetExecStream(stream.raw_stream());
           })
#endif
W
Wilber 已提交
653 654
      .def("enable_xpu",
           &AnalysisConfig::EnableXpu,
W
Wilber 已提交
655
           py::arg("l3_workspace_size") = 16 * 1024 * 1024,
W
Wilber 已提交
656 657 658 659
           py::arg("locked") = false,
           py::arg("autotune") = true,
           py::arg("autotune_file") = "",
           py::arg("precision") = "int16",
660 661
           py::arg("adaptive_seqlen") = false,
           py::arg("enable_multi_stream") = false)
W
Wilber 已提交
662 663
      .def("set_xpu_device_id",
           &AnalysisConfig::SetXpuDeviceId,
664
           py::arg("device_id") = 0)
665 666 667 668
      .def("enable_custom_device",
           &AnalysisConfig::EnableCustomDevice,
           py::arg("device_type"),
           py::arg("device_id") = 0)
W
Wilber 已提交
669
      .def("enable_npu", &AnalysisConfig::EnableNpu, py::arg("device_id") = 0)
W
Wilber 已提交
670 671 672 673
      .def("enable_ipu",
           &AnalysisConfig::EnableIpu,
           py::arg("ipu_device_num") = 1,
           py::arg("ipu_micro_batch_size") = 1,
674 675
           py::arg("ipu_enable_pipelining") = false,
           py::arg("ipu_batches_per_step") = 1)
W
Wilber 已提交
676 677 678 679
      .def("set_ipu_config",
           &AnalysisConfig::SetIpuConfig,
           py::arg("ipu_enable_fp16") = false,
           py::arg("ipu_replica_num") = 1,
680
           py::arg("ipu_available_memory_proportion") = 1.0,
681 682
           py::arg("ipu_enable_half_partial") = false,
           py::arg("ipu_enable_model_runtime_executor") = false)
683 684 685 686 687 688 689 690
      .def("set_ipu_custom_info",
           &AnalysisConfig::SetIpuCustomInfo,
           py::arg("ipu_custom_ops_info") =
               std::vector<std::vector<std::string>>({}),
           py::arg("ipu_custom_patterns") = std::map<std::string, bool>({}))
      .def("load_ipu_config",
           &AnalysisConfig::LoadIpuConfig,
           py::arg("config_path"))
F
flame 已提交
691
      .def("disable_gpu", &AnalysisConfig::DisableGpu)
692 693 694
      .def("enable_onnxruntime", &AnalysisConfig::EnableONNXRuntime)
      .def("disable_onnxruntime", &AnalysisConfig::DisableONNXRuntime)
      .def("onnxruntime_enabled", &AnalysisConfig::use_onnxruntime)
695
      .def("use_opencl", &AnalysisConfig::use_opencl)
696
      .def("enable_ort_optimization", &AnalysisConfig::EnableORTOptimization)
F
flame 已提交
697
      .def("use_gpu", &AnalysisConfig::use_gpu)
698
      .def("use_xpu", &AnalysisConfig::use_xpu)
W
Wilber 已提交
699
      .def("use_npu", &AnalysisConfig::use_npu)
F
flame 已提交
700
      .def("gpu_device_id", &AnalysisConfig::gpu_device_id)
701
      .def("xpu_device_id", &AnalysisConfig::xpu_device_id)
W
Wilber 已提交
702
      .def("npu_device_id", &AnalysisConfig::npu_device_id)
F
flame 已提交
703 704 705 706
      .def("memory_pool_init_size_mb",
           &AnalysisConfig::memory_pool_init_size_mb)
      .def("fraction_of_gpu_memory_for_pool",
           &AnalysisConfig::fraction_of_gpu_memory_for_pool)
W
Wilber 已提交
707 708
      .def("switch_ir_optim",
           &AnalysisConfig::SwitchIrOptim,
F
flame 已提交
709 710
           py::arg("x") = true)
      .def("ir_optim", &AnalysisConfig::ir_optim)
W
Wilber 已提交
711 712
      .def("enable_memory_optim",
           &AnalysisConfig::EnableMemoryOptim,
713
           py::arg("x") = true)
714
      .def("enable_profile", &AnalysisConfig::EnableProfile)
715
      .def("disable_glog_info", &AnalysisConfig::DisableGlogInfo)
716
      .def("glog_info_disabled", &AnalysisConfig::glog_info_disabled)
717
      .def("set_optim_cache_dir", &AnalysisConfig::SetOptimCacheDir)
W
Wilber 已提交
718 719
      .def("switch_use_feed_fetch_ops",
           &AnalysisConfig::SwitchUseFeedFetchOps,
F
flame 已提交
720 721 722 723
           py::arg("x") = true)
      .def("use_feed_fetch_ops_enabled",
           &AnalysisConfig::use_feed_fetch_ops_enabled)
      .def("switch_specify_input_names",
W
Wilber 已提交
724 725
           &AnalysisConfig::SwitchSpecifyInputNames,
           py::arg("x") = true)
F
flame 已提交
726
      .def("specify_input_name", &AnalysisConfig::specify_input_name)
W
Wilber 已提交
727 728
      .def("enable_tensorrt_engine",
           &AnalysisConfig::EnableTensorRtEngine,
729
           py::arg("workspace_size") = 1 << 30,
W
Wilber 已提交
730
           py::arg("max_batch_size") = 1,
731
           py::arg("min_subgraph_size") = 3,
N
nhzlx 已提交
732
           py::arg("precision_mode") = AnalysisConfig::Precision::kFloat32,
W
Wilber 已提交
733 734
           py::arg("use_static") = false,
           py::arg("use_calib_mode") = true)
735
      .def("tensorrt_precision_mode", &AnalysisConfig::tensorrt_precision_mode)
736 737
      .def("set_trt_dynamic_shape_info",
           &AnalysisConfig::SetTRTDynamicShapeInfo,
738 739 740 741 742
           py::arg("min_input_shape") =
               std::map<std::string, std::vector<int>>({}),
           py::arg("max_input_shape") =
               std::map<std::string, std::vector<int>>({}),
           py::arg("optim_input_shape") =
743 744
               std::map<std::string, std::vector<int>>({}),
           py::arg("disable_trt_plugin_fp16") = false)
745 746
      .def("tensorrt_dynamic_shape_enabled",
           &AnalysisConfig::tensorrt_dynamic_shape_enabled)
747 748 749
      .def("enable_tensorrt_varseqlen", &AnalysisConfig::EnableVarseqlen)
      .def("tensorrt_varseqlen_enabled",
           &AnalysisConfig::tensorrt_varseqlen_enabled)
750 751 752 753 754 755 756 757 758 759
      .def("collect_shape_range_info", &AnalysisConfig::CollectShapeRangeInfo)
      .def("shape_range_info_path", &AnalysisConfig::shape_range_info_path)
      .def("shape_range_info_collected",
           &AnalysisConfig::shape_range_info_collected)
      .def("enable_tuned_tensorrt_dynamic_shape",
           &AnalysisConfig::EnableTunedTensorRtDynamicShape)
      .def("tuned_tensorrt_dynamic_shape",
           &AnalysisConfig::tuned_tensorrt_dynamic_shape)
      .def("trt_allow_build_at_runtime",
           &AnalysisConfig::trt_allow_build_at_runtime)
760
      .def("exp_disable_tensorrt_ops", &AnalysisConfig::Exp_DisableTensorRtOPs)
W
Wilber 已提交
761 762
      .def("enable_tensorrt_dla",
           &AnalysisConfig::EnableTensorRtDLA,
763 764
           py::arg("dla_core") = 0)
      .def("tensorrt_dla_enabled", &AnalysisConfig::tensorrt_dla_enabled)
765 766 767 768
      .def("enable_tensorrt_inspector",
           &AnalysisConfig::EnableTensorRtInspector)
      .def("tensorrt_inspector_enabled",
           &AnalysisConfig::tensorrt_inspector_enabled)
F
flame 已提交
769
      .def("tensorrt_engine_enabled", &AnalysisConfig::tensorrt_engine_enabled)
W
Wilber 已提交
770 771
      .def("enable_dlnne",
           &AnalysisConfig::EnableDlnne,
D
denglin-github 已提交
772 773 774 775 776 777 778 779 780 781
           py::arg("min_subgraph_size") = 3,
           py::arg("max_batch_size") = 1,
           py::arg("use_static_batch") = false,
           py::arg("weight_share_mode") = "0",
           py::arg("disable_nodes_by_outputs") =
               std::unordered_set<std::string>(),
           py::arg("input_shape_dict") =
               std::map<std::string, std::vector<int64_t>>(),
           py::arg("use_calib_mode") = false,
           py::arg("precision_mode") = AnalysisConfig::Precision::kFloat32)
W
Wilber 已提交
782 783
      .def("enable_lite_engine",
           &AnalysisConfig::EnableLiteEngine,
784
           py::arg("precision_mode") = AnalysisConfig::Precision::kFloat32,
W
Wilber 已提交
785
           py::arg("zero_copy") = false,
786 787
           py::arg("passes_filter") = std::vector<std::string>(),
           py::arg("ops_filter") = std::vector<std::string>())
788
      .def("enable_opencl", &AnalysisConfig::EnableOpenCL)
789
      .def("lite_engine_enabled", &AnalysisConfig::lite_engine_enabled)
W
Wilber 已提交
790 791
      .def("switch_ir_debug",
           &AnalysisConfig::SwitchIrDebug,
F
flame 已提交
792 793 794 795 796 797 798 799
           py::arg("x") = true)
      .def("enable_mkldnn", &AnalysisConfig::EnableMKLDNN)
      .def("mkldnn_enabled", &AnalysisConfig::mkldnn_enabled)
      .def("set_cpu_math_library_num_threads",
           &AnalysisConfig::SetCpuMathLibraryNumThreads)
      .def("cpu_math_library_num_threads",
           &AnalysisConfig::cpu_math_library_num_threads)
      .def("to_native_config", &AnalysisConfig::ToNativeConfig)
800
      .def("enable_quantizer", &AnalysisConfig::EnableMkldnnQuantizer)
801
      .def("enable_mkldnn_bfloat16", &AnalysisConfig::EnableMkldnnBfloat16)
802
#ifdef PADDLE_WITH_MKLDNN
W
Wilber 已提交
803 804
      .def("quantizer_config",
           &AnalysisConfig::mkldnn_quantizer_config,
805
           py::return_value_policy::reference)
W
Wilber 已提交
806 807
      .def("set_mkldnn_cache_capacity",
           &AnalysisConfig::SetMkldnnCacheCapacity,
808
           py::arg("capacity") = 0)
809
      .def("set_bfloat16_op", &AnalysisConfig::SetBfloat16Op)
W
Wilber 已提交
810 811
      .def("enable_mkldnn_int8",
           &AnalysisConfig::EnableMkldnnInt8,
B
baoachun 已提交
812 813 814
           py::arg("mkldnn_int8_enabled_op_types") =
               std::unordered_set<std::string>({}))
      .def("mkldnn_int8_enabled", &AnalysisConfig::mkldnn_int8_enabled)
P
Paulina Gacek 已提交
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
      .def("disable_mkldnn_fc_passes",
           &AnalysisConfig::DisableMkldnnFcPasses,
           R"DOC(
           Disable Mkldnn FC
           Args:
                None.
           Returns:
                None.
           Examples:
               .. code-block:: python
                from paddle.inference import Config

                config = Config("")
                config.enable_mkldnn()
                config.disable_mkldnn_fc_passes()
           )DOC")
831
#endif
F
flame 已提交
832 833 834
      .def("set_mkldnn_op", &AnalysisConfig::SetMKLDNNOp)
      .def("set_model_buffer", &AnalysisConfig::SetModelBuffer)
      .def("model_from_memory", &AnalysisConfig::model_from_memory)
835 836 837 838
      .def("delete_pass",
           [](AnalysisConfig &self, const std::string &pass) {
             self.pass_builder()->DeletePass(pass);
           })
839 840 841 842 843 844
      .def(
          "pass_builder",
          [](AnalysisConfig &self) {
            return dynamic_cast<PaddlePassBuilder *>(self.pass_builder());
          },
          py::return_value_policy::reference)
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
      .def("nnadapter", &AnalysisConfig::NNAdapter)
      .def("set_dist_config", &AnalysisConfig::SetDistConfig)
      .def("dist_config", &AnalysisConfig::dist_config);

  py::class_<DistConfig>(*m, "DistConfig")
      .def(py::init<>())
      .def("set_carrier_id", &DistConfig::SetCarrierId)
      .def("set_comm_init_config", &DistConfig::SetCommInitConfig)
      .def("set_endpoints", &DistConfig::SetEndpoints)
      .def("set_ranks", &DistConfig::SetRanks)
      .def("enable_dist_model", &DistConfig::EnableDistModel)
      .def("carrier_id", &DistConfig::carrier_id)
      .def("current_endpoint", &DistConfig::current_endpoint)
      .def("trainer_endpoints", &DistConfig::trainer_endpoints)
      .def("nranks", &DistConfig::nranks)
      .def("rank", &DistConfig::rank)
      .def("comm_init_config", &DistConfig::comm_init_config)
      .def("use_dist_model", &DistConfig::use_dist_model);
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880
}

void BindLiteNNAdapterConfig(py::module *m) {
  py::class_<LiteNNAdapterConfig> lite_nnadapter_config(*m,
                                                        "LiteNNAdapterConfig");

  lite_nnadapter_config
      .def("set_device_names", &LiteNNAdapterConfig::SetDeviceNames)
      .def("set_context_properties", &LiteNNAdapterConfig::SetContextProperties)
      .def("set_model_cache_dir", &LiteNNAdapterConfig::SetModelCacheDir)
      .def("set_model_cache_buffers",
           &LiteNNAdapterConfig::SetModelCacheBuffers)
      .def("set_subgraph_partition_config_path",
           &LiteNNAdapterConfig::SetSubgraphPartitionConfigPath)
      .def("set_subgraph_partition_config_buffer",
           &LiteNNAdapterConfig::SetSubgraphPartitionConfigBuffer)
      .def("enable", &LiteNNAdapterConfig::Enable)
      .def("disable", &LiteNNAdapterConfig::Disable);
F
flame 已提交
881 882
}

883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
#ifdef PADDLE_WITH_MKLDNN
void BindMkldnnQuantizerConfig(py::module *m) {
  py::class_<MkldnnQuantizerConfig> quantizer_config(*m,
                                                     "MkldnnQuantizerConfig");
  quantizer_config.def(py::init<const MkldnnQuantizerConfig &>())
      .def(py::init<>())
      .def("set_quant_data",
           [](MkldnnQuantizerConfig &self,
              const std::vector<PaddleTensor> &data) {
             auto warmup_data =
                 std::make_shared<std::vector<PaddleTensor>>(data);
             self.SetWarmupData(warmup_data);
             return;
           })
      .def("set_quant_batch_size", &MkldnnQuantizerConfig::SetWarmupBatchSize)
898
      .def("set_enabled_op_types", &MkldnnQuantizerConfig::SetEnabledOpTypes);
899 900 901
}
#endif

F
flame 已提交
902 903 904 905 906 907 908 909 910 911 912 913 914
void BindAnalysisPredictor(py::module *m) {
  py::class_<AnalysisPredictor, PaddlePredictor>(*m, "AnalysisPredictor")
      .def(py::init<const AnalysisConfig &>())
      .def("init", &AnalysisPredictor::Init)
      .def(
          "run",
          [](AnalysisPredictor &self, const std::vector<PaddleTensor> &inputs) {
            std::vector<PaddleTensor> outputs;
            self.Run(inputs, &outputs);
            return outputs;
          })
      .def("get_input_tensor", &AnalysisPredictor::GetInputTensor)
      .def("get_output_tensor", &AnalysisPredictor::GetOutputTensor)
915 916 917
      .def("get_input_names", &AnalysisPredictor::GetInputNames)
      .def("get_output_names", &AnalysisPredictor::GetOutputNames)
      .def("get_input_tensor_shape", &AnalysisPredictor::GetInputTensorShape)
F
flame 已提交
918
      .def("zero_copy_run", &AnalysisPredictor::ZeroCopyRun)
919 920
      .def("clear_intermediate_tensor",
           &AnalysisPredictor::ClearIntermediateTensor)
921
      .def("try_shrink_memory", &AnalysisPredictor::TryShrinkMemory)
922 923 924 925 926
      .def("create_feed_fetch_var", &AnalysisPredictor::CreateFeedFetchVar)
      .def("prepare_feed_fetch", &AnalysisPredictor::PrepareFeedFetch)
      .def("prepare_argument", &AnalysisPredictor::PrepareArgument)
      .def("optimize_inference_program",
           &AnalysisPredictor::OptimizeInferenceProgram)
W
Wilber 已提交
927 928
      .def("analysis_argument",
           &AnalysisPredictor::analysis_argument,
929
           py::return_value_policy::reference)
930
      .def("clone", [](AnalysisPredictor &self) { return self.Clone(nullptr); })
931 932 933
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
      .def("clone",
           [](AnalysisPredictor &self, phi::CUDAStream &stream) {
934
             return self.Clone(stream.raw_stream());
935 936
           })
#endif
W
Wilber 已提交
937 938
      .def("scope",
           &AnalysisPredictor::scope,
939
           py::return_value_policy::reference)
W
Wilber 已提交
940 941
      .def("program",
           &AnalysisPredictor::program,
942 943 944
           py::return_value_policy::reference)
      .def("get_serialized_program", &AnalysisPredictor::GetSerializedProgram)
      .def("mkldnn_quantize", &AnalysisPredictor::MkldnnQuantize)
W
Wilber 已提交
945 946
      .def(
          "SaveOptimModel", &AnalysisPredictor::SaveOptimModel, py::arg("dir"));
F
flame 已提交
947
}
948

W
Wilber 已提交
949 950 951 952 953 954 955
void BindPaddleInferPredictor(py::module *m) {
  py::class_<paddle_infer::Predictor>(*m, "PaddleInferPredictor")
      .def(py::init<const paddle_infer::Config &>())
      .def("get_input_names", &paddle_infer::Predictor::GetInputNames)
      .def("get_output_names", &paddle_infer::Predictor::GetOutputNames)
      .def("get_input_handle", &paddle_infer::Predictor::GetInputHandle)
      .def("get_output_handle", &paddle_infer::Predictor::GetOutputHandle)
W
Wilber 已提交
956 957 958 959 960 961 962
      .def("run",
           [](paddle_infer::Predictor &self) {
#ifdef PADDLE_WITH_ASCEND_CL
             pybind11::gil_scoped_release release;
#endif
             self.Run();
           })
963 964
      .def("clone",
           [](paddle_infer::Predictor &self) { return self.Clone(nullptr); })
965 966 967
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
      .def("clone",
           [](paddle_infer::Predictor &self, phi::CUDAStream &stream) {
968
             return self.Clone(stream.raw_stream());
969 970
           })
#endif
971
      .def("try_shrink_memory", &paddle_infer::Predictor::TryShrinkMemory)
W
Wilber 已提交
972
      .def("clear_intermediate_tensor",
973 974 975
           &paddle_infer::Predictor::ClearIntermediateTensor)
      .def("register_output_hook",
           &paddle_infer::Predictor::RegisterOutputHook);
W
Wilber 已提交
976 977
}

978 979
void BindZeroCopyTensor(py::module *m) {
  py::class_<ZeroCopyTensor>(*m, "ZeroCopyTensor")
W
Wilber 已提交
980 981 982 983 984 985
      .def(
          "reshape",
          py::overload_cast<const std::vector<int> &>(&ZeroCopyTensor::Reshape))
      .def("reshape",
           py::overload_cast<const std::size_t &>(
               &paddle_infer::Tensor::ReshapeStrings))
986 987 988
      .def("copy_from_cpu", &ZeroCopyTensorCreate<int32_t>)
      .def("copy_from_cpu", &ZeroCopyTensorCreate<int64_t>)
      .def("copy_from_cpu", &ZeroCopyTensorCreate<float>)
989
      .def("copy_from_cpu", &ZeroCopyTensorCreate<paddle_infer::float16>)
S
Steffy-zxf 已提交
990
      .def("copy_from_cpu", &ZeroCopyStringTensorCreate)
991 992 993 994 995 996 997
      .def("copy_to_cpu", &ZeroCopyTensorToNumpy)
      .def("shape", &ZeroCopyTensor::shape)
      .def("set_lod", &ZeroCopyTensor::SetLoD)
      .def("lod", &ZeroCopyTensor::lod)
      .def("type", &ZeroCopyTensor::type);
}

W
Wilber 已提交
998 999
void BindPaddleInferTensor(py::module *m) {
  py::class_<paddle_infer::Tensor>(*m, "PaddleInferTensor")
W
Wilber 已提交
1000 1001 1002 1003 1004 1005
      .def("reshape",
           py::overload_cast<const std::vector<int> &>(
               &paddle_infer::Tensor::Reshape))
      .def("reshape",
           py::overload_cast<const std::size_t &>(
               &paddle_infer::Tensor::ReshapeStrings))
1006 1007 1008 1009 1010
      .def("copy_from_cpu_bind", &PaddleInferTensorCreate<int32_t>)
      .def("copy_from_cpu_bind", &PaddleInferTensorCreate<int64_t>)
      .def("copy_from_cpu_bind", &PaddleInferTensorCreate<float>)
      .def("copy_from_cpu_bind",
           &PaddleInferTensorCreate<paddle_infer::float16>)
S
Steffy-zxf 已提交
1011
      .def("copy_from_cpu_bind", &PaddleInferStringTensorCreate)
1012
      .def("share_external_data_bind", &PaddleInferShareExternalData)
W
Wilber 已提交
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
      .def("copy_to_cpu", &PaddleInferTensorToNumpy)
      .def("shape", &paddle_infer::Tensor::shape)
      .def("set_lod", &paddle_infer::Tensor::SetLoD)
      .def("lod", &paddle_infer::Tensor::lod)
      .def("type", &paddle_infer::Tensor::type);
}

void BindPredictorPool(py::module *m) {
  py::class_<paddle_infer::services::PredictorPool>(*m, "PredictorPool")
      .def(py::init<const paddle_infer::Config &, size_t>())
W
Wilber 已提交
1023 1024
      .def("retrive",
           &paddle_infer::services::PredictorPool::Retrive,
W
Wilber 已提交
1025 1026 1027
           py::return_value_policy::reference);
}

1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
void BindPaddlePassBuilder(py::module *m) {
  py::class_<PaddlePassBuilder>(*m, "PaddlePassBuilder")
      .def(py::init<const std::vector<std::string> &>())
      .def("set_passes",
           [](PaddlePassBuilder &self, const std::vector<std::string> &passes) {
             self.ClearPasses();
             for (auto pass : passes) {
               self.AppendPass(std::move(pass));
             }
           })
      .def("append_pass", &PaddlePassBuilder::AppendPass)
      .def("insert_pass", &PaddlePassBuilder::InsertPass)
      .def("delete_pass",
           [](PaddlePassBuilder &self, const std::string &pass_type) {
             self.DeletePass(pass_type);
           })
      .def("append_analysis_pass", &PaddlePassBuilder::AppendAnalysisPass)
      .def("turn_on_debug", &PaddlePassBuilder::TurnOnDebug)
      .def("debug_string", &PaddlePassBuilder::DebugString)
W
Wilber 已提交
1047 1048
      .def("all_passes",
           &PaddlePassBuilder::AllPasses,
1049 1050 1051 1052 1053 1054 1055 1056
           py::return_value_policy::reference)
      .def("analysis_passes", &PaddlePassBuilder::AnalysisPasses);

  py::class_<PassStrategy, PaddlePassBuilder>(*m, "PassStrategy")
      .def(py::init<const std::vector<std::string> &>())
      .def("enable_cudnn", &PassStrategy::EnableCUDNN)
      .def("enable_mkldnn", &PassStrategy::EnableMKLDNN)
      .def("enable_mkldnn_quantizer", &PassStrategy::EnableMkldnnQuantizer)
1057
      .def("enable_mkldnn_bfloat16", &PassStrategy::EnableMkldnnBfloat16)
1058 1059 1060 1061 1062 1063 1064
      .def("use_gpu", &PassStrategy::use_gpu);

  py::class_<CpuPassStrategy, PassStrategy>(*m, "CpuPassStrategy")
      .def(py::init<>())
      .def(py::init<const CpuPassStrategy &>())
      .def("enable_cudnn", &CpuPassStrategy::EnableCUDNN)
      .def("enable_mkldnn", &CpuPassStrategy::EnableMKLDNN)
1065 1066
      .def("enable_mkldnn_quantizer", &CpuPassStrategy::EnableMkldnnQuantizer)
      .def("enable_mkldnn_bfloat16", &CpuPassStrategy::EnableMkldnnBfloat16);
1067 1068 1069 1070 1071 1072

  py::class_<GpuPassStrategy, PassStrategy>(*m, "GpuPassStrategy")
      .def(py::init<>())
      .def(py::init<const GpuPassStrategy &>())
      .def("enable_cudnn", &GpuPassStrategy::EnableCUDNN)
      .def("enable_mkldnn", &GpuPassStrategy::EnableMKLDNN)
1073 1074
      .def("enable_mkldnn_quantizer", &GpuPassStrategy::EnableMkldnnQuantizer)
      .def("enable_mkldnn_bfloat16", &GpuPassStrategy::EnableMkldnnBfloat16);
1075
}
1076
}  // namespace
F
flame 已提交
1077 1078
}  // namespace pybind
}  // namespace paddle