pybind.cc 156.0 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
15

C
chengduoZH 已提交
16
#include <algorithm>
17
#include <cctype>
18
#include <cstdlib>
19
#include <iterator>
C
chengduoZH 已提交
20
#include <map>
S
sneaxiy 已提交
21
#include <memory>
C
chengduoZH 已提交
22 23
#include <mutex>  // NOLINT // for call_once
#include <string>
24 25
#include <tuple>
#include <type_traits>
C
chengduoZH 已提交
26
#include <unordered_map>
27
#include <unordered_set>
C
chengduoZH 已提交
28 29
#include <utility>
#include <vector>
30

31
#include "paddle/fluid/framework/convert_utils.h"
32
#include "paddle/fluid/framework/custom_operator.h"
33
#include "paddle/fluid/framework/data_layout.h"
L
Leo Chen 已提交
34
#include "paddle/fluid/framework/data_type_transform.h"
Y
Yi Wang 已提交
35
#include "paddle/fluid/framework/executor.h"
36
#include "paddle/fluid/framework/executor_cache.h"
37
#include "paddle/fluid/framework/executor_gc_helper.h"
Y
Yi Wang 已提交
38
#include "paddle/fluid/framework/feed_fetch_method.h"
Z
Zhen Wang 已提交
39
#include "paddle/fluid/framework/feed_fetch_type.h"
S
sneaxiy 已提交
40
#include "paddle/fluid/framework/garbage_collector.h"
H
hutuxian 已提交
41
#include "paddle/fluid/framework/io/fs.h"
42
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
H
Huihuang Zheng 已提交
43
#include "paddle/fluid/framework/ir/cost_model.h"
44
#include "paddle/fluid/framework/ir/generate_pass.h"
45
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
46 47
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
48
#include "paddle/fluid/framework/new_executor/standalone_executor.h"
S
sneaxiy 已提交
49
#include "paddle/fluid/framework/op_info.h"
50
#include "paddle/fluid/framework/op_registry.h"
51
#include "paddle/fluid/framework/op_version_registry.h"
Y
Yu Yang 已提交
52
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
53
#include "paddle/fluid/framework/prune.h"
54
#include "paddle/fluid/framework/pten_utils.h"
Y
Refine  
Yu Yang 已提交
55
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
56
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
57
#include "paddle/fluid/framework/scope_pool.h"
58
#include "paddle/fluid/framework/selected_rows_utils.h"
59
#include "paddle/fluid/framework/tensor_util.h"
60
#include "paddle/fluid/framework/trainer.h"
61
#include "paddle/fluid/framework/type_defs.h"
X
Xin Pan 已提交
62
#include "paddle/fluid/framework/version.h"
L
Leo Chen 已提交
63
#include "paddle/fluid/imperative/amp_auto_cast.h"
H
hong 已提交
64
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
65
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
66
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
D
dzhwinter 已提交
67
#include "paddle/fluid/operators/activation_op.h"
L
Leo Chen 已提交
68
#include "paddle/fluid/operators/common_infer_shape_functions.h"
S
sneaxiy 已提交
69
#include "paddle/fluid/operators/py_func_op.h"
70
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
71
#include "paddle/fluid/platform/cpu_info.h"
72
#include "paddle/fluid/platform/device/device_wrapper.h"
73
#include "paddle/fluid/platform/device_context.h"
74
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
75
#include "paddle/fluid/platform/enforce.h"
76
#include "paddle/fluid/platform/init.h"
H
hutuxian 已提交
77
#include "paddle/fluid/platform/monitor.h"
Y
Yi Wang 已提交
78 79
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
80
#include "paddle/fluid/pybind/cuda_streams_py.h"
81
#include "paddle/fluid/pybind/distributed_py.h"
82 83
#include "paddle/phi/core/compat/convert_utils.h"
#include "paddle/phi/core/lod_utils.h"
W
wanghuancoder 已提交
84
#ifndef PADDLE_ON_INFERENCE
85
#include "paddle/fluid/pybind/eager.h"
W
wanghuancoder 已提交
86
#endif
87
#include "paddle/fluid/pybind/io.h"
88
#include "paddle/utils/none.h"
89 90 91
#ifdef PADDLE_WITH_ASCEND
#include "paddle/fluid/pybind/ascend_wrapper_py.h"
#endif
H
Huihuang Zheng 已提交
92
#include "paddle/fluid/pybind/bind_cost_model.h"
L
LiYuRio 已提交
93
#include "paddle/fluid/pybind/bind_fleet_executor.h"
H
hutuxian 已提交
94
#include "paddle/fluid/pybind/box_helper_py.h"
95
#include "paddle/fluid/pybind/communication.h"
96
#include "paddle/fluid/pybind/compatible.h"
Y
Yi Wang 已提交
97
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
98
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
99
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
100
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
Y
yaoxuefeng 已提交
101
#include "paddle/fluid/pybind/generator_py.h"
102
#include "paddle/fluid/pybind/global_value_getter_setter.h"
103
#include "paddle/fluid/pybind/gloo_context_py.h"
104
#include "paddle/fluid/pybind/gloo_wrapper_py.h"
T
Thunderbrook 已提交
105
#include "paddle/fluid/pybind/heter_wrapper_py.h"
106
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
107
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
108
#include "paddle/fluid/pybind/ir.h"
109
#include "paddle/fluid/pybind/metrics_py.h"
T
Thunderbrook 已提交
110
#include "paddle/fluid/pybind/ps_gpu_wrapper_py.h"
111
#include "paddle/fluid/pybind/pybind_boost_headers.h"
112

113
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
114
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
115
#endif
116
#include "paddle/fluid/framework/data_type.h"
117 118
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
119
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
120
#include "paddle/fluid/pybind/tensor_py.h"
121
#include "paddle/fluid/string/to_string.h"
122 123
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
Y
Yi Wang 已提交
124
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
125
#endif
126
#ifndef PADDLE_WITH_HIP
127
#include "paddle/fluid/platform/device/gpu/cuda/cuda_profiler.h"
128
#endif
129
#include "paddle/fluid/platform/device/gpu/gpu_info.h"
D
Dong Zhihong 已提交
130 131
#endif

132
#ifdef PADDLE_WITH_ASCEND_CL
133
#include "paddle/fluid/platform/collective_helper.h"
134 135
#include "paddle/fluid/platform/device/npu/npu_info.h"
#include "paddle/fluid/platform/device/npu/npu_profiler.h"
136 137
#endif

138
#ifdef PADDLE_WITH_XPU
139
#include "paddle/fluid/platform/device/xpu/xpu_info.h"
T
TTerror 已提交
140
#include "paddle/fluid/platform/device/xpu/xpu_op_list.h"
141 142
#endif

143
#include "paddle/fluid/platform/cuda_graph_with_memory_pool.h"
A
Allen Guo 已提交
144

J
jianghaicheng 已提交
145
#ifdef PADDLE_WITH_IPU
A
Allen Guo 已提交
146 147
#include "paddle/fluid/platform/device/ipu/ipu_backend.h"
#include "paddle/fluid/platform/device/ipu/ipu_info.h"
J
jianghaicheng 已提交
148
#endif
149

150 151 152 153
#ifdef PADDLE_WITH_MLU
#include "paddle/fluid/platform/device/mlu/mlu_info.h"
#endif

Y
Yanghello 已提交
154 155 156 157
#ifdef PADDLE_WITH_CRYPTO
#include "paddle/fluid/pybind/crypto.h"
#endif

T
tangwei12 已提交
158
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
159 160 161
#include "paddle/fluid/pybind/fleet_py.h"
#endif

M
minqiyang 已提交
162 163
#include "pybind11/stl.h"

164
DECLARE_bool(use_mkldnn);
165

Q
Qiao Longfei 已提交
166 167
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
168 169 170
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchUnmergedList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType);
Q
Qiao Longfei 已提交
171

172
namespace paddle {
173
namespace pybind {
174 175 176 177 178 179 180

PyTypeObject *g_place_pytype = nullptr;
PyTypeObject *g_cudaplace_pytype = nullptr;
PyTypeObject *g_cpuplace_pytype = nullptr;
PyTypeObject *g_xpuplace_pytype = nullptr;
PyTypeObject *g_npuplace_pytype = nullptr;
PyTypeObject *g_cudapinnedplace_pytype = nullptr;
181
PyTypeObject *g_mluplace_pytype = nullptr;
182
PyTypeObject *g_framework_tensor_pytype = nullptr;
183
PyTypeObject *g_framework_lodtensorarray_pytype = nullptr;
184

185
bool IsCompiledWithCUDA() {
186 187 188 189 190 191 192
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
  return false;
#else
  return true;
#endif
}

193 194 195 196 197 198 199 200
bool IsCompiledWithNCCL() {
#ifdef PADDLE_WITH_NCCL
  return true;
#else
  return false;
#endif
}

201 202
bool IsCompiledWithROCM() {
#ifndef PADDLE_WITH_HIP
Q
qijun 已提交
203 204 205 206 207 208
  return false;
#else
  return true;
#endif
}

209 210 211 212 213 214 215 216
bool IsCompiledWithAscend() {
#ifndef PADDLE_WITH_ASCEND
  return false;
#else
  return true;
#endif
}

217 218 219 220 221 222 223 224
bool IsCompiledWithXPU() {
#ifndef PADDLE_WITH_XPU
  return false;
#else
  return true;
#endif
}

225 226 227 228 229 230 231 232
bool IsCompiledWithNPU() {
#ifndef PADDLE_WITH_ASCEND_CL
  return false;
#else
  return true;
#endif
}

J
jianghaicheng 已提交
233 234 235 236 237 238 239 240
bool IsCompiledWithIPU() {
#ifndef PADDLE_WITH_IPU
  return false;
#else
  return true;
#endif
}

241 242 243 244 245 246 247 248
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

249 250 251 252 253 254 255 256
bool IsCompiledWithCINN() {
#ifndef PADDLE_WITH_CINN
  return false;
#else
  return true;
#endif
}

257 258 259 260 261 262 263 264
bool IsCompiledWithMLU() {
#ifndef PADDLE_WITH_MLU
  return false;
#else
  return true;
#endif
}

265 266 267 268 269 270 271 272
bool IsCompiledWithHETERPS() {
#ifndef PADDLE_WITH_HETERPS
  return false;
#else
  return true;
#endif
}

273 274 275 276 277 278 279 280 281 282 283
bool SupportsBfloat16() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core))
    return true;
  else
    return false;
#endif
}

284 285 286 287 288 289 290 291 292 293 294
bool SupportsBfloat16FastPerformance() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_bf16))
    return true;
  else
    return false;
#endif
}

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
bool SupportsInt8() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return (platform::MayIUse(platform::cpu_isa_t::avx2) ||
          platform::MayIUse(platform::cpu_isa_t::avx512f));
#endif
}

bool SupportsVNNI() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return platform::MayIUse(platform::cpu_isa_t::avx512_core_vnni);
#endif
}

312
bool IsCompiledWithBrpc() {
313
#ifndef PADDLE_WITH_DISTRIBUTE
314 315
  return false;
#endif
316
  return true;
317 318
}

Y
update  
Yancey1989 已提交
319
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
320
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
321 322 323 324 325 326
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
327 328 329 330 331 332 333
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
334
  return static_cast<int>(paddle::platform::Place(p).GetType());
S
sneaxiy 已提交
335 336
}

H
hong 已提交
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
359 360 361
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Python object is not type of %s, the real type is %s",
        typeid(T).name(), obj->ob_type->tp_name));
H
hong 已提交
362 363 364 365 366 367 368 369 370 371 372 373 374
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
375 376
      PADDLE_THROW(platform::errors::InvalidArgument(
          "The parameter [%s] to save is None", para.first));
H
hong 已提交
377 378
    }
    vec_res.emplace_back(
379
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
H
hong 已提交
380 381 382 383 384 385 386 387 388 389 390 391
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
392 393
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameter list to save is None"));
H
hong 已提交
394 395 396 397 398 399 400 401 402 403 404 405
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
406 407 408
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to save is None"));
H
hong 已提交
409 410 411 412
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
413 414
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to save is not a list"));
H
hong 已提交
415 416 417 418
  }
  return vec_res;
}

419 420 421 422 423 424 425 426
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
427 428
    PADDLE_THROW(
        platform::errors::InvalidArgument("The parameter list to set is None"));
429 430 431 432 433 434 435 436 437 438 439 440 441
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
442 443 444
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to set is None"));
445 446 447 448 449
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
450 451 452 453 454
        PADDLE_ENFORCE_NOT_NULL(exe,
                                platform::errors::InvalidArgument(
                                    "Parameter not Initialized, "
                                    "Please set argument [executor] not None "
                                    "or run startup program first"));
455 456
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
457 458 459
        PADDLE_ENFORCE_NOT_NULL(
            py_var_desc, platform::errors::InvalidArgument(
                             "The var_desc of parameter to set is None"));
460 461 462 463
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
464
        tensor_temp->Resize(phi::make_ddim(var_desc.GetShape()));
465 466 467
        tensor_temp->mutable_data(
            exe->GetPlace(),
            framework::TransToPtenDataType(var_desc.GetDataType()));
468 469 470
      }
    }
  } else {
471 472
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to set is not a list"));
473 474 475 476 477
  }

  return;
}

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
static void AssertStaticGraphAndDygraphGradMakerNoDiff() {
  std::set<std::string> ops;
  for (auto &pair : framework::OpInfoMap::Instance().map()) {
    bool has_static_grad_maker = (pair.second.grad_op_maker_ != nullptr);
    bool has_dygraph_grad_maker =
        (pair.second.dygraph_grad_op_maker_ != nullptr);
    if (has_static_grad_maker ^ has_dygraph_grad_maker) {
      bool has_kernel =
          (framework::OperatorWithKernel::AllOpKernels().count(pair.first) > 0);
      if (has_kernel) {
        ops.insert(pair.first);
      } else {
        VLOG(5) << pair.first << " has no kernels, skip";
      }
    }
  }
  PADDLE_ENFORCE_EQ(ops.empty(), true,
                    platform::errors::Unimplemented(
                        "OperatorWithKernel [%s] have only static graph grad "
                        "maker or have only dygraph grad maker, which is not "
                        "allowed",
                        string::join_strings(ops, ',')));
}

Z
Zeng Jinle 已提交
502 503 504 505
#ifdef PADDLE_WITH_NCCL
static int GetNCCLVersion() {
#if NCCL_VERSION_CODE >= 2304
  int ver;
506
  PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclGetVersion(&ver));
Z
Zeng Jinle 已提交
507 508 509 510 511 512 513 514
  return ver;
#else
  PADDLE_THROW(platform::errors::External(
      "Cannot get NCCL version successfully when nccl version < 2.3.4"));
#endif
}
#endif

Z
Zeng Jinle 已提交
515 516 517 518 519 520 521 522 523 524 525
template <typename PlaceType>
static void TensorCopyFrom(framework::Tensor *dst, const framework::Tensor &src,
                           const PlaceType &place, int64_t batch_size) {
  if (batch_size < 0) {
    framework::TensorCopy(src, place, dst);
  } else {
    auto sliced = src.Slice(0, batch_size);
    framework::TensorCopy(sliced, place, dst);
  }
}

526 527 528 529 530 531
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

W
wanghuancoder 已提交
532
#ifndef PADDLE_ON_INFERENCE
533
  BindEager(&m);
W
wanghuancoder 已提交
534
#endif
535 536
  BindCudaStream(&m);

Y
Yu Yang 已提交
537 538 539
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
540
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
541

542 543
  AssertStaticGraphAndDygraphGradMakerNoDiff();

544
  m.doc() = "C++ core of PaddlePaddle";
545

546 547 548 549
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

550
  BindException(&m);
Y
Yu Yang 已提交
551

552 553
  m.def("set_num_threads", &platform::SetNumThreads);

554 555
  m.def("disable_signal_handler", &DisableSignalHandler);

556 557 558 559 560 561 562 563
  m.def("clear_gradients",
        [](std::vector<std::shared_ptr<imperative::VarBase>> param_list,
           bool set_to_zero) {
          for (auto param : param_list) {
            param->ClearGradient(set_to_zero);
          }
        });

564
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
565
  m.def("cudnn_version", &platform::DnnVersion);
566 567 568 569 570 571
  m.def("gpu_memory_available", []() {
    size_t available = 0;
    size_t total = 0;
    paddle::platform::GpuMemoryUsage(&available, &total);
    return available;
  });
572
#endif
573

Z
Zeng Jinle 已提交
574 575 576 577
#ifdef PADDLE_WITH_NCCL
  m.def("nccl_version", &GetNCCLVersion);
#endif

578 579 580 581 582 583 584 585 586 587
  m.def("is_cuda_graph_capturing", &platform::IsCUDAGraphCapturing);
#ifdef PADDLE_WITH_CUDA
  py::class_<platform::CUDAGraph>(m, "CUDAGraph")
      .def_static("begin_capture",
                  [](platform::CUDAPlace place, int mode) {
                    platform::BeginCUDAGraphCapture(
                        place, static_cast<cudaStreamCaptureMode>(mode));
                  })
      .def_static("end_capture", &platform::EndCUDAGraphCapture)
      .def("replay", &platform::CUDAGraph::Replay)
588 589
      .def("reset", &platform::CUDAGraph::Reset)
      .def("print_to_dot_files", &platform::CUDAGraph::PrintToDotFiles);
590 591
#endif

Z
Zeng Jinle 已提交
592 593 594 595
  m.def("wait_device", [](const platform::Place &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });

6
633WHU 已提交
596 597 598
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
599 600 601 602 603 604

    PADDLE_ENFORCE_NOT_NULL(
        dmt, platform::errors::InvalidArgument(
                 "from_dlpack received an invalid capsule. "
                 "Note that a DLPack tensor can be consumed only once."));

6
633WHU 已提交
605 606
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
607
    framework::Tensor tensor;
6
633WHU 已提交
608

S
Siming Dai 已提交
609
    if (dl.device.device_type == kDLCPU) {
6
633WHU 已提交
610 611
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
612
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
S
Siming Dai 已提交
613
    if (dl.device.device_type == kDLGPU) {
6
633WHU 已提交
614 615 616 617 618
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });
H
hong 已提交
619

620 621 622 623 624 625
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

626 627 628 629 630 631
  m.def("save_op_version_info", [](framework::ProgramDesc &desc) {
    framework::compatible::pb::OpVersionMap pb_vmap{desc.OpVersionMap()};
    framework::compatible::SaveOpVersions(
        framework::compatible::OpVersionRegistrar::GetInstance()
            .GetVersionMap(),
        &pb_vmap);
632 633
  });

634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
  m.def("set_printoptions", [](const py::kwargs &kwargs) {
    auto &print_opt = framework::PrintOptions::Instance();
    if (kwargs.contains("precision")) {
      print_opt.precision = kwargs["precision"].cast<int>();
    }
    if (kwargs.contains("threshold")) {
      print_opt.threshold = kwargs["threshold"].cast<int>();
    }
    if (kwargs.contains("edgeitems")) {
      print_opt.edgeitems = kwargs["edgeitems"].cast<int>();
    }
    if (kwargs.contains("linewidth")) {
      print_opt.linewidth = kwargs["linewidth"].cast<int>();
    }
    if (kwargs.contains("sci_mode")) {
      print_opt.sci_mode = kwargs["sci_mode"].cast<bool>();
    }

    VLOG(4) << "Set printoptions: precision=" << print_opt.precision
            << ", threshold=" << print_opt.threshold
            << ", edgeitems=" << print_opt.edgeitems
            << ", linewidth=" << print_opt.linewidth
            << ", sci_mode=" << print_opt.sci_mode;
  });

L
Leo Chen 已提交
659 660
  m.def("broadcast_shape", [](const std::vector<int64_t> &x_dim,
                              const std::vector<int64_t> &y_dim) {
661 662
    return phi::vectorize(operators::details::BroadcastTwoDims(
        phi::make_ddim(x_dim), phi::make_ddim(y_dim), -1));
L
Leo Chen 已提交
663 664
  });

S
sneaxiy 已提交
665
  m.def(
S
sneaxiy 已提交
666
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
667 668 669 670
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
671 672 673
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
  m.def(
      "_get_all_register_op_kernels",
      [](const std::string &lib) {
        std::unordered_map<std::string, std::vector<std::string>>
            all_kernels_info;
        if (lib == "fluid" || lib == "all") {
          auto &all_kernels =
              paddle::framework::OperatorWithKernel::AllOpKernels();

          for (auto &kernel_pair : all_kernels) {
            auto op_type = kernel_pair.first;
            std::vector<std::string> kernel_types;
            for (auto &info_pair : kernel_pair.second) {
              paddle::framework::OpKernelType kernel_type = info_pair.first;
              kernel_types.emplace_back(
                  paddle::framework::KernelTypeToString(kernel_type));
            }
            all_kernels_info.emplace(op_type, kernel_types);
          }
        }
        if (lib == "pten" || lib == "all") {
695
          auto pten_kernels = phi::KernelFactory::Instance().kernels();
696
          for (auto &kernel_pair : pten_kernels) {
697
            auto op_type = phi::TransToFluidOpName(kernel_pair.first);
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
            std::vector<std::string> kernel_types;
            for (auto &info_pair : kernel_pair.second) {
              framework::OpKernelType kernel_type =
                  framework::TransPtenKernelKeyToOpKernelType(info_pair.first);
              auto kernel_type_str = framework::KernelTypeToString(kernel_type);
              if (all_kernels_info.count(op_type)) {
                if (std::find(all_kernels_info[op_type].begin(),
                              all_kernels_info[op_type].end(),
                              kernel_type_str) ==
                    all_kernels_info[op_type].end()) {
                  all_kernels_info[op_type].emplace_back(kernel_type_str);
                }
              } else {
                kernel_types.emplace_back(kernel_type_str);
              }
            }
            if (!kernel_types.empty()) {
              all_kernels_info.emplace(op_type, kernel_types);
            }
          }
        }

        return all_kernels_info;
      },
      py::arg("lib") = "all",
      R"DOC(
           Return the registered kernels in paddle.

           Args:
               lib[string]: the libarary, could be 'pten', 'fluid' and 'all'.
           )DOC");
729

S
sneaxiy 已提交
730 731 732
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
733
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
734

735
  m.def("_set_fuse_parameter_group_size",
736
        &paddle::framework::ir::SetFuseParameterGroupsSize);
737
  m.def("_set_fuse_parameter_memory_size",
738
        &paddle::framework::ir::SetFuseParameterMemorySize);
739

S
sneaxiy 已提交
740 741 742
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

743 744
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

745 746 747
  m.def("_promote_types_if_complex_exists",
        &paddle::framework::PromoteTypesIfComplexExists);

748
  BindImperative(&m);
749

750 751 752 753 754
  py::class_<framework::Tensor> framework_tensor(m, "Tensor",
                                                 py::buffer_protocol());
  g_framework_tensor_pytype =
      reinterpret_cast<PyTypeObject *>(framework_tensor.ptr());
  framework_tensor
755 756
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
757 758 759 760
      .def("_ptr",
           [](const framework::Tensor &self) {
             return reinterpret_cast<uintptr_t>(self.data());
           })
S
sneaxiy 已提交
761
      .def("_is_initialized",
762
           [](const framework::Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
763
      .def("_get_dims",
764
           [](const framework::Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
765
      .def("_set_dims",
766
           [](framework::Tensor &self, const std::vector<int64_t> &dim) {
767
             self.Resize(phi::make_ddim(dim));
Y
Yu Yang 已提交
768
           })
Y
yuyang18 已提交
769
      .def("_set_layout",
770
           [](framework::Tensor &self, const std::string &layout) {
D
dzhwinter 已提交
771 772
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
773
      .def("_alloc_float",
774
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
775
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
776
           })
777
      .def("_alloc_float",
778
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
779 780
             self.mutable_data<float>(place);
           })
Y
yuyang18 已提交
781
      .def("_alloc_float",
782
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
783
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
784
           })
785 786 787 788
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place) {
             self.mutable_data<float>(place);
           })
789 790 791 792
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place) {
             self.mutable_data<float>(place);
           })
793
      .def("_alloc_double",
794
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
795 796
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
797
      .def("_alloc_int",
798
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
799
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
800
           })
801
      .def("_alloc_int",
802
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
803 804
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
805
      .def("_alloc_int",
806
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
807
             self.mutable_data<int>(place);
Q
qijun 已提交
808
           })
809 810 811 812
      .def("_alloc_int",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
813
      .def("_alloc_int",
814 815
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
816 817
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
818
      .def("_alloc_float",
819 820
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
821 822
             self.mutable_data<float>(place);
           })
823
      .def("_mutable_data",
824
           [](framework::Tensor &self, paddle::platform::CPUPlace &place,
825
              paddle::framework::proto::VarType::Type type) {
826 827
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
828
           })
829
      .def("_mutable_data",
830
           [](framework::Tensor &self, paddle::platform::XPUPlace &place,
831
              paddle::framework::proto::VarType::Type type) {
832 833
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
834
           })
835
      .def("_mutable_data",
836
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place,
837
              paddle::framework::proto::VarType::Type type) {
838 839
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
840 841
           })
      .def("_mutable_data",
842
           [](framework::Tensor &self, paddle::platform::CUDAPinnedPlace &place,
843
              paddle::framework::proto::VarType::Type type) {
844 845
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
846
           })
847 848 849
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place,
              paddle::framework::proto::VarType::Type type) {
850 851
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
852
           })
853
      .def("_clear", &framework::Tensor::clear)
854 855 856
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
857 858
             return reinterpret_cast<uintptr_t>(self.mutable_data(
                 place, framework::TransToPtenDataType(type)));
859
           })
Z
Zeng Jinle 已提交
860 861 862 863 864 865 866 867 868 869
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::XPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CUDAPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::NPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CUDAPinnedPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
870 871
      .def("_copy_from", &TensorCopyFrom<paddle::platform::MLUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
Z
Zeng Jinle 已提交
872
      .def("_copy_from", &TensorCopyFrom<paddle::platform::Place>,
873
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
874
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
875
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
876 877
      .def("set", SetTensorFromPyArray<paddle::platform::XPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
878
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
879
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
880 881
      .def("set", SetTensorFromPyArray<paddle::platform::NPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
J
jianghaicheng 已提交
882 883
      .def("set", SetTensorFromPyArray<paddle::platform::IPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
884 885
      .def("set", SetTensorFromPyArray<paddle::platform::MLUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
886
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
887 888
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
889
        Set the data of Tensor on place with given numpy array.
L
Leo Chen 已提交
890 891 892
        
        Args:
          lod (numpy.ndarray): The data to set.
893
          place (CPUPlace|CUDAPlace|XPUPlace|IPUPlace|CUDAPinnedPlace|NPUPlace|MLUPlace): The place where the
894
          Tensor is to be set.
895 896
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
897 898 899 900 901 902 903 904 905 906

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

907
                t = fluid.Tensor()
L
Leo Chen 已提交
908 909
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
910

911 912 913
      .def("shape",
           [](framework::Tensor &self) { return vectorize(self.dims()); },
           R"DOC(
914
           Return the shape of Tensor.
L
Leo Chen 已提交
915 916

           Returns:
917
               list[int]: The shape of Tensor.
L
Leo Chen 已提交
918 919 920 921 922 923 924 925


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

926
                  t = fluid.Tensor()
L
Leo Chen 已提交
927 928 929
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
930
      .def("_to_dlpack",
931
           [](framework::Tensor &self) {
6
633WHU 已提交
932
             DLPackTensor dlpack_tensor(self, 1);
S
Siming Dai 已提交
933
             DLManagedTensor *dmt = dlpack_tensor.ToDLManagedTensor();
6
633WHU 已提交
934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
951 952 953 954
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
955
      .def("_place", [](framework::Tensor &self) { return self.place(); })
956 957 958 959
      .def("_dtype",
           [](framework::Tensor &self) {
             return framework::TransToProtoVarType(self.type());
           })
960
      .def("_layout",
961 962 963 964
           [](framework::Tensor &self) {
             return DataLayoutToString(self.layout());
           })
      .def("_share_data_with", &framework::Tensor::ShareDataWith)
965
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
      .def("__str__",
           [](const framework::Tensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           }) /* ------ End of original Tensor ------ */
      .def(
          "__init__",
          [](framework::Tensor &instance, const std::vector<std::vector<size_t>>
                                              &recursive_sequence_lengths) {
            LoD new_lod;
            new_lod.reserve(recursive_sequence_lengths.size());
            std::copy(recursive_sequence_lengths.begin(),
                      recursive_sequence_lengths.end(),
                      std::back_inserter(new_lod));
            LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
            PADDLE_ENFORCE_EQ(
                CheckLoD(new_offset_lod, -1), true,
                platform::errors::InvalidArgument(
985 986
                    "The provided recursive_sequence_lengths info is "
                    "invalid, "
987 988 989 990
                    "the LoD converted by recursive_sequence_lengths is %s",
                    new_lod));
            new (&instance) framework::Tensor(new_offset_lod);
          })
991
      .def("__init__",
992 993
           [](framework::Tensor &instance) {
             new (&instance) framework::Tensor();
994
           })
G
gongweibao 已提交
995
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
996 997
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
998 999 1000
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
1001
      .def("set_lod",
1002 1003
           [](framework::Tensor &self,
              const std::vector<std::vector<size_t>> &lod) {
1004
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
1005
             LoD new_lod;
1006 1007
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
1008 1009
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
1010 1011
                 platform::errors::InvalidArgument(
                     "The provided LoD is invalid, the LoD is %s", new_lod));
1012
             self.set_lod(new_lod);
S
sneaxiy 已提交
1013 1014
           },
           py::arg("lod"), R"DOC(
1015
           Set LoD of the Tensor.
S
sneaxiy 已提交
1016 1017

           Args:
L
Leo Chen 已提交
1018 1019 1020 1021
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
1022 1023 1024 1025 1026 1027 1028

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1029
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1030 1031
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
1032
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1033
           )DOC")
1034
      .def("set_recursive_sequence_lengths",
1035 1036
           [](framework::Tensor &self, const std::vector<std::vector<size_t>>
                                           &recursive_sequence_lengths) {
1037 1038 1039 1040 1041 1042 1043 1044
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
1045 1046
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
1047
                 platform::errors::InvalidArgument(
1048 1049
                     "The provided recursive_sequence_lengths info is "
                     "invalid, "
1050 1051 1052
                     "the LoD converted by recursive_sequence_lengths is "
                     "%s",
                     new_lod));
1053
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
1054 1055
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
1056
           Set LoD of the Tensor according to recursive sequence lengths.
S
sneaxiy 已提交
1057

L
Leo Chen 已提交
1058
           For example, if recursive_sequence_lengths=[[2, 3]], which means
1059
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
1060
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
1061 1062

           Args:
L
Leo Chen 已提交
1063 1064 1065 1066
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
1067 1068 1069 1070 1071 1072 1073

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1074
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1075 1076
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
1077
                 print(t.recursive_sequence_lengths())  # [[2, 3]]
L
Leo Chen 已提交
1078
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
1079
           )DOC")
1080
      .def("lod",
1081
           [](framework::Tensor &self) -> std::vector<std::vector<size_t>> {
1082 1083 1084 1085 1086 1087
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1088 1089
           },
           R"DOC(
1090
           Return the LoD of the Tensor.
S
sneaxiy 已提交
1091 1092

           Returns:
1093
               list[list[int]]: The lod of the Tensor.
L
Leo Chen 已提交
1094
           
Z
Zeng Jinle 已提交
1095 1096 1097 1098 1099 1100
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1101
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1102 1103 1104
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1105
           )DOC")
G
gongweibao 已提交
1106
      // Set above comments of set_lod.
1107
      .def("recursive_sequence_lengths",
1108
           [](framework::Tensor &self) -> std::vector<std::vector<size_t>> {
1109
             // output the length-based lod info
1110
             LoD lod = phi::ConvertToLengthBasedLoD(self.lod());
1111 1112 1113 1114
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1115 1116
           },
           R"DOC(
L
Leo Chen 已提交
1117
           Return the recursive sequence lengths corresponding to of the LodD 
1118
           of the Tensor.
S
sneaxiy 已提交
1119 1120

           Returns:
L
Leo Chen 已提交
1121
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
1122 1123 1124 1125 1126 1127 1128

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1129
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1130 1131 1132
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
1133 1134
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
1135
           [](framework::Tensor &self) -> bool {
S
sneaxiy 已提交
1136
             // Check that the lod info is valid and match the outermost
1137
             // dimension of the Tensor data
S
sneaxiy 已提交
1138 1139 1140
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
1141
           Check whether the LoD of the Tensor is valid.
S
sneaxiy 已提交
1142 1143

           Returns:
L
Leo Chen 已提交
1144
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
1145 1146 1147 1148 1149 1150 1151

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1152
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1153 1154 1155
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
1156
           )DOC")
L
Leo Chen 已提交
1157
      .def("_as_type",
1158
           [](const framework::Tensor &self,
L
Leo Chen 已提交
1159
              paddle::framework::proto::VarType::Type type) {
1160
             framework::Tensor dst;
L
Leo Chen 已提交
1161 1162 1163 1164 1165
             if (self.IsInitialized() && self.numel() > 0) {
               TransDataType(self, type, &dst);
             }
             return dst;
           })
1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178
      .def("_copy",
           [](const framework::Tensor &self, const platform::Place &place) {
             // follow fetch_op's inplementation
             framework::Tensor dst;
             if (self.IsInitialized() && self.numel() > 0) {
               TensorCopySync(self, place, &dst);
             } else {
               // Not copy, if the src tensor is empty.
               dst.clear();
               dst.Resize({0});
             }
             dst.set_lod(self.lod());
             return dst;
1179
#ifdef _WIN32
1180
           });
1181 1182 1183
#else
           })
      .def(py::pickle(
1184
          [](const framework::Tensor &t) {  // __getstate__
1185
            auto holder = t.Holder();
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
            PADDLE_ENFORCE_EQ(platform::is_cpu_place(holder->place()), true,
                              platform::errors::PreconditionNotMet(
                                  "Tensor is not on CPU."
                                  "Now only Tensor on CPU can be serialized."));
            auto *mmap_writer_allocation =
                dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
                    holder.get());
            PADDLE_ENFORCE_NOT_NULL(
                mmap_writer_allocation,
                platform::errors::PreconditionNotMet(
                    "Tensor is not in shared memory."
                    "Now only Tensor on shared memory can be serialized."));
1198 1199 1200
            int type_idx = static_cast<int>(t.type());

            return py::make_tuple(mmap_writer_allocation->ipc_name(),
1201 1202
                                  mmap_writer_allocation->size(), type_idx,
                                  vectorize(t.dims()), t.lod());
1203 1204 1205
          },
          [](py::tuple t) {  // __setstate__
            if (t.size() != 5)
1206
              throw std::runtime_error("Invalid Tensor state!");
1207 1208

            // 1. Create a new C++ instance
1209
            framework::Tensor tensor;
1210 1211 1212 1213 1214

            // 2. Rebuild Allocation
            const std::string &ipc_name = t[0].cast<std::string>();
            size_t size = t[1].cast<size_t>();
            auto shared_reader_holder =
1215 1216
                memory::allocation::RebuildMemoryMapReaderAllocation(ipc_name,
                                                                     size);
1217 1218

            // 3. Maintain global fd set
1219
            VLOG(3) << "Tensor ipc name: " << ipc_name;
1220 1221
            memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);

1222 1223 1224
            // 4. Rebuild Tensor
            tensor.ResetHolderWithType(
                shared_reader_holder,
1225
                static_cast<paddle::experimental::DataType>(t[2].cast<int>()));
1226
            tensor.Resize(phi::make_ddim(t[3].cast<std::vector<int>>()));
1227 1228 1229 1230 1231
            tensor.set_lod(t[4].cast<framework::LoD>());

            return tensor;
          }));
#endif
D
dangqingqing 已提交
1232

1233
  py::class_<phi::SelectedRows>(m, "SelectedRows")
Q
qijun 已提交
1234
      .def("__init__",
1235 1236
           [](phi::SelectedRows &instance) {
             new (&instance) phi::SelectedRows();
1237
           })
Q
qijun 已提交
1238
      .def("__init__",
1239
           [](phi::SelectedRows &instance, const std::vector<int64_t> rows,
Q
qijun 已提交
1240
              const int64_t &height) {
1241
             new (&instance) phi::SelectedRows(rows, height);
Q
qijun 已提交
1242 1243
           })
      .def("get_tensor",
1244
           [](phi::SelectedRows &self) { return self.mutable_value(); },
Q
qijun 已提交
1245
           py::return_value_policy::reference)
1246
      .def("numel",
1247
           [](phi::SelectedRows &self) -> int64_t {
1248 1249
             return self.value().numel();
           })
1250 1251
      .def("set_height", &phi::SelectedRows::set_height)
      .def("height", &phi::SelectedRows::height)
Q
qijun 已提交
1252
      .def("set_rows",
1253
           [](phi::SelectedRows &self, std::vector<int64_t> rows) {
1254
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
Q
qijun 已提交
1255 1256 1257 1258 1259 1260
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
1261
      .def("sync_index",
1262 1263
           [](phi::SelectedRows &instance) { instance.SyncIndex(); })
      .def("rows", [](phi::SelectedRows &self) {
1264 1265 1266 1267 1268
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
1269
      });
Q
qijun 已提交
1270

1271
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
1272 1273 1274

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
1275
      .def(py::init<>())
1276
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
1277
      .def("set_int",
1278 1279
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
1280 1281 1282 1283 1284 1285 1286
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
1287
      .def("get_tensor",
1288 1289
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
1290 1291
           },
           py::return_value_policy::reference)
1292 1293 1294 1295
      .def("get_bytes",
           [](Variable &self) {
             return py::bytes(*self.GetMutable<std::string>());
           })
S
Steffy-zxf 已提交
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
      .def("set_string_list",
           [](Variable &self, Strings str_list) {
             *self.GetMutable<Strings>() = str_list;
           })
      .def("set_vocab", [](Variable &self,
                           Vocab vocab) { *self.GetMutable<Vocab>() = vocab; })
      .def("get_string_tensor",
           [](Variable &self) { return self.GetMutable<Strings>(); },
           py::return_value_policy::reference)
      .def("get_map_tensor",
           [](Variable &self) { return self.GetMutable<Vocab>(); },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1308 1309 1310
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
1311
      .def("get_selected_rows",
1312 1313
           [](Variable &self) -> phi::SelectedRows * {
             return self.GetMutable<phi::SelectedRows>();
Q
qijun 已提交
1314 1315
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1316 1317 1318
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
1319 1320 1321
      .def("get_fetch_list",
           [](Variable &self) { return self.GetMutable<FetchList>(); },
           py::return_value_policy::reference)
1322
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1323 1324 1325 1326 1327
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1328
#endif
Y
Refine  
Yu Yang 已提交
1329 1330
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
1331 1332 1333 1334
             PADDLE_ENFORCE_EQ(
                 self.IsType<framework::ReaderHolder>(), true,
                 platform::errors::InvalidArgument(
                     "The variable is not type of ReaderHolder."));
Y
Refine  
Yu Yang 已提交
1335 1336
             return self.GetMutable<framework::ReaderHolder>();
           },
1337
           py::return_value_policy::reference)
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
      .def("get_scope",
           [](Variable &self) -> Scope * {
             auto scope_vec =
                 self.GetMutable<std::vector<framework::Scope *>>();
             PADDLE_ENFORCE_GT(
                 scope_vec->size(), 0,
                 platform::errors::InvalidArgument(
                     "The size of scope_vec should be greater than 0"));
             return scope_vec->front();
           },
           py::return_value_policy::reference)
1349 1350 1351 1352
      .def("set_scope", [](Variable &self, Scope &scope) {
        auto scope_vec = self.GetMutable<std::vector<framework::Scope *>>();
        scope_vec->emplace_back(&scope);
      });
1353

S
sneaxiy 已提交
1354
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
1355

S
sneaxiy 已提交
1356
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

1370
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
1371 1372 1373 1374 1375 1376
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
1377 1378
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
1379
      .def("var",
1380
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
1381
             return self.Var(name);
Y
Yu Yang 已提交
1382
           },
S
sneaxiy 已提交
1383 1384
           py::arg("name"),
           R"DOC(
1385
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
1386

1387
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
1388
           current scope, the variable would be created. Otherwise,
1389
           return the existing variable.
S
sneaxiy 已提交
1390 1391

           Args:
1392 1393
               name (str): the variable name.

S
sneaxiy 已提交
1394
           Returns:
1395
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
1396 1397 1398 1399
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
1400
           Find variable named :code:`name` in the current scope or
1401
           its parent scope. Return None if not found. 
1402

S
sneaxiy 已提交
1403 1404
           Args:
               name (str): the variable name.
1405

S
sneaxiy 已提交
1406
           Returns:
1407
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1408
           )DOC",
1409
           py::return_value_policy::reference)
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
      .def("erase", &Scope::EraseVars, py::arg("names"),
           R"DOC(
           Find variable named :code:`name` in the current scope or
           its parent scope. Return None if not found. 

           Args:
               name (str): the variable names to be erase.

           Returns:
               None
           )DOC",
           py::return_value_policy::reference)
1422
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1423 1424 1425 1426 1427 1428
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1429
           py::return_value_policy::reference)
S
sneaxiy 已提交
1430 1431 1432
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1433 1434
           )DOC")
      .def("_kids", &Scope::kids);
1435

S
sneaxiy 已提交
1436 1437 1438 1439 1440 1441
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1442 1443
        R"DOC(
        Create a new scope.
1444

S
sneaxiy 已提交
1445 1446 1447
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1448 1449
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1450 1451
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1452 1453
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1454 1455 1456 1457
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1458 1459
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1460 1461
            platform::errors::Fatal(
                "Serialize OpProto Error. This could be a bug of Paddle."));
1462 1463 1464
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1465 1466
    return ret_values;
  });
1467 1468 1469 1470 1471 1472 1473 1474
  m.def("get_op_attrs_default_value",
        [](py::bytes byte_name) -> paddle::framework::AttributeMap {
          std::string op_type = byte_name;
          paddle::framework::AttributeMap res;
          auto info = OpInfoMap::Instance().GetNullable(op_type);
          if (info != nullptr) {
            if (info->HasOpProtoAndChecker()) {
              auto op_checker = info->Checker();
1475
              res = op_checker->GetDefaultAttrsMap();
1476 1477 1478 1479
            }
          }
          return res;
        });
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1496 1497 1498
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
1499 1500 1501 1502 1503
  m.def("has_non_empty_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance()
        .Get(op_type)
        .HasNonEmptyGradOpMaker();
  });
1504 1505 1506
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520
  m.def("infer_no_need_buffer_slots",
        [](const std::string op_type, const framework::VariableNameMap &inputs,
           const framework::VariableNameMap &outputs,
           const framework::AttributeMap &attrs) {
          auto infer_func = framework::OpInfoMap::Instance()
                                .Get(op_type)
                                .NoNeedBufferVarsInferer();
          if (infer_func) {
            return infer_func(inputs, outputs, attrs);
          } else {
            std::unordered_set<std::string> empty = {};
            return empty;
          }
        });
Y
Yu Yang 已提交
1521
  m.def("prune", [](const ProgramDesc &origin,
1522
                    const std::set<std::string> &feeded_var_names,
1523
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1524
    ProgramDesc prog_with_targets(origin);
1525

1526
    for (const auto &t : targets) {
1527
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1528
    }
1529
    proto::ProgramDesc pruned_desc;
1530 1531 1532 1533
    auto pruned_origin_block_id_map =
        Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
    return std::make_tuple(ProgramDesc(pruned_desc),
                           pruned_origin_block_id_map);
1534
  });
1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551
  m.def("prune_backward",
        [](const framework::ProgramDesc &program) {
          return PruneBackward(program);
        },
        R"DOC(
             Prune the backward part of a program, mostly called in
             program.clone(for_test=True).
              
             Args:
                   program (ProgramDesc): The original program.

             Returns:
                   tuple(ProgramDesc, map<int, int>): The first part is 
                   the pruned program desc, and the second part is a map
                   which contains the id pair of pruned block and corresponding
                   origin block.
           )DOC");
1552 1553 1554 1555
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1556 1557 1558
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1559 1560
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
1561

Q
qijun 已提交
1562
  // clang-format off
Y
Yu Yang 已提交
1563
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1564 1565
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1566
                      -> paddle::platform::DeviceContext* {
W
Wilber 已提交
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
    auto* context = new paddle::platform::CPUDeviceContext();
    context->SetAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetAllocator(place)
        .get());
    context->SetHostAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetAllocator(paddle::platform::CPUPlace())
        .get());
    context->SetZeroAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetZeroAllocator(place)
        .get());
    return context;
Q
qijun 已提交
1581
                  })
1582 1583 1584 1585 1586 1587 1588 1589 1590
      .def_static("create",
                  [](paddle::platform::XPUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_XPU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use XPUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with XPU support."));
#else
W
Wilber 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
      auto* context = new paddle::platform::XPUDeviceContext(place);
      context->SetAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(place)
          .get());
      context->SetHostAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(paddle::platform::CPUPlace())
          .get());
      context->SetZeroAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetZeroAllocator(place)
          .get());
      return context;
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
#endif
                  })
        .def_static("create",
                  [](paddle::platform::MLUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_MLU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use MLUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with MLU support."));
#else
                    return new paddle::platform::MLUDeviceContext(place);
1617 1618
#endif
                  })
1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
        .def_static("create",
                    [](paddle::platform::NPUPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_ASCEND_CL
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use NPUPlace in CPU/GPU/XPU version, "
                 "Please recompile or reinstall Paddle with NPU support."));
#else
                return new paddle::platform::NPUDeviceContext(place);
#endif
        })
Q
qijun 已提交
1631
      .def_static("create",
D
dzhwinter 已提交
1632
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1633
                      -> paddle::platform::DeviceContext* {
1634
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1635 1636 1637 1638
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
Q
qijun 已提交
1639
#else
W
Wilber 已提交
1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
      auto* context = new paddle::platform::CUDADeviceContext(place);
      context->SetAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(place, context->stream())
          .get());
      context->SetHostAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(paddle::platform::CPUPlace())
          .get());
      context->SetZeroAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
        .GetZeroAllocator(place)
        .get());
      context->PartialInitWithAllocator();
      return context;
Q
qijun 已提交
1655
#endif
C
chengduoZH 已提交
1656 1657 1658 1659
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
1660
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1661 1662 1663 1664
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
C
chengduoZH 已提交
1665 1666 1667 1668
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
1669
// clang-format on
1670
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1671 1672
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
  m.def("get_all_device_type", []() {
    std::vector<std::string> device_types;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
    device_types = platform::DeviceManager::GetAllDeviceTypes();
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_all_device_type because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_all_device_type, please try to install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return device_types;
  });
  m.def("get_all_custom_device_type", []() {
    std::vector<std::string> device_types;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
    device_types = platform::DeviceManager::GetAllCustomDeviceTypes();
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_all_custom_device_type because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_all_custom_device_type, please try to "
              "install CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return device_types;
  });
  m.def("get_available_device", [] {
    std::vector<std::string> devices;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
    devices = platform::DeviceManager::GetAllDeviceList();
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_available_device because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_available_device, please try to install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return devices;
  });
  m.def("get_available_custom_device", [] {
    std::vector<std::string> devices;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
    devices = platform::DeviceManager::GetAllCustomDeviceList();
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_available_custom_device because you have "
              "installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_available_custom_device, please try to "
              "install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return devices;
  });
  py::class_<platform::CustomPlace>(m, "CustomPlace",
                                    R"DOC(
    CustomPlace is a descriptor of a device.
    It represents a custom device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python

          import paddle
          fake_cpu_place = paddle.CustomPlace("FakeCPU", 0)
                                             )DOC")
      .def("__init__",
           [](platform::CustomPlace &self, const std::string &device_type,
              int dev_id) {
#ifdef PADDLE_WITH_CUSTOM_DEVICE
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CustomPlace(%s, %d), device id must be 0 "
                   "or "
                   "positive integer",
                   device_type, dev_id);
               std::exit(-1);
             }

             if (LIKELY(platform::DeviceManager::HasDeviceType(device_type) &&
                        platform::DeviceManager::IsCustom(device_type))) {
               int dev_count = static_cast<int>(
                   platform::DeviceManager::GetDeviceCount(device_type));
               if (UNLIKELY(dev_id >= dev_count)) {
                 if (dev_count == 0) {
                   LOG(ERROR) << "Cannot use " << device_type
                              << " because there is no " << device_type
                              << " detected on your "
                                 "machine.";
                   std::exit(-1);
                 } else {
                   LOG(ERROR) << string::Sprintf(
                       "Invalid CustomPlace(%s, %d), dev_id must "
                       "inside "
                       "[0, %d), because %s "
                       "number on your machine is %d",
                       device_type, dev_id, dev_count, device_type, dev_count);
                   std::exit(-1);
                 }
               }
               new (&self) platform::CustomPlace(device_type, dev_id);
             } else {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CustomPlace(%s, %d), the device type is "
                   "not registered "
                   "as a custom device.",
                   device_type, dev_id);
               std::exit(-1);
             }
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use CustomDevice because you have installed CPU/GPU"
                 "version PaddlePaddle.\n"
                 "If you want to use CustomDevice, please try to install"
                 "CustomDevice version "
                 "PaddlePaddle by: pip install paddlepaddle-core\n"
                 "If you only have CPU, please change "
                 "CustomPlace(%s, %d) to be CPUPlace().\n",
                 device_type, dev_id);
             std::exit(-1);
#endif
           })
      .def("get_device_id",
           [](const platform::CustomPlace &self) { return self.GetDeviceId(); })
      .def("get_device_type",
           [](const platform::CustomPlace &self) {
             return self.GetDeviceType();
           })
      .def("__repr__", string::to_string<const platform::CustomPlace &>)
      .def("__str__", string::to_string<const platform::CustomPlace &>);
1806
  py::class_<platform::CUDAPlace> cudaplace(m, "CUDAPlace", R"DOC(
1807 1808 1809 1810 1811

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
1812
    The memory of CUDAPlace with different dev_id is not accessible.
1813 1814 1815 1816 1817 1818 1819 1820
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
1821 1822 1823 1824

    Examples:
        .. code-block:: python

1825 1826 1827
          import paddle

          place = paddle.CUDAPlace(0)
L
lujun 已提交
1828

1829 1830 1831
        )DOC");
  g_cudaplace_pytype = reinterpret_cast<PyTypeObject *>(cudaplace.ptr());
  cudaplace
S
sneaxiy 已提交
1832 1833
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
1834
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1835 1836 1837 1838 1839 1840 1841 1842
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

1843 1844
             if (UNLIKELY(dev_id >= platform::GetGPUDeviceCount())) {
               if (platform::GetGPUDeviceCount() == 0) {
1845 1846 1847 1848 1849 1850 1851 1852
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
1853 1854
                     dev_id, platform::GetGPUDeviceCount(),
                     platform::GetGPUDeviceCount());
1855 1856 1857 1858
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
1859 1860
             new (&self) platform::CUDAPlace(dev_id);
#else
1861 1862 1863 1864 1865 1866 1867 1868 1869
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
1870 1871
#endif
           })
1872
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1873 1874
      .def("get_device_id",
           [](const platform::CUDAPlace &self) { return self.GetDeviceId(); })
S
sneaxiy 已提交
1875 1876 1877 1878
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
1879
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
1880
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
1881
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::MLUPlace>)
S
sneaxiy 已提交
1882 1883
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
1884 1885 1886
      .def("_get_device_id",
           [](platform::CUDAPlace &self) -> int { return self.GetDeviceId(); })
#endif
1887
      .def("__repr__", string::to_string<const platform::CUDAPlace &>)
D
dzhwinter 已提交
1888
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
1889

1890
  py::class_<platform::XPUPlace> xpuplace(m, "XPUPlace", R"DOC(
1891 1892 1893 1894 1895
    **Note**:
    Examples:
        .. code-block:: python
          import paddle.fluid as fluid
          xpu_place = fluid.XPUPlace(0)
1896 1897 1898
        )DOC");
  g_xpuplace_pytype = reinterpret_cast<PyTypeObject *>(xpuplace.ptr());
  xpuplace
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936
      .def("__init__",
           [](platform::XPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_XPU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid XPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetXPUDeviceCount())) {
               if (platform::GetXPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use XPU because there is no XPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid XPUPlace(%d), must inside [0, %d), because XPU "
                     "number on your machine is %d",
                     dev_id, platform::GetXPUDeviceCount(),
                     platform::GetXPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::XPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use XPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use XPU, please try to install XPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change XPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
1937
#ifdef PADDLE_WITH_XPU
1938 1939 1940 1941 1942 1943 1944
      .def("_type", &PlaceIndex<platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::XPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::XPUPlace, platform::CUDAPinnedPlace>)
1945 1946 1947
      .def("get_device_id",
           [](const platform::XPUPlace &self) { return self.GetDeviceId(); })
#endif
1948
      .def("__repr__", string::to_string<const platform::XPUPlace &>)
1949
      .def("__str__", string::to_string<const platform::XPUPlace &>);
1950
#ifdef PADDLE_WITH_XPU
1951 1952 1953
  py::enum_<phi::backends::xpu::XPUVersion>(m, "XPUVersion", py::arithmetic())
      .value("XPU1", phi::backends::xpu::XPUVersion::XPU1)
      .value("XPU2", phi::backends::xpu::XPUVersion::XPU2)
T
TTerror 已提交
1954
      .export_values();
1955
  m.def("get_xpu_device_count", platform::GetXPUDeviceCount);
T
TTerror 已提交
1956 1957
  m.def("get_xpu_device_version",
        [](int device_id) { return platform::get_xpu_version(device_id); });
1958 1959 1960 1961 1962
  m.def("get_xpu_device_op_support_types",
        [](const std::string &op_name, phi::backends::xpu::XPUVersion version) {
          return platform::get_xpu_op_support_type(op_name, version);
        });
  m.def("get_xpu_device_op_list", [](phi::backends::xpu::XPUVersion version) {
T
TTerror 已提交
1963 1964
    return platform::get_xpu_op_list(version);
  });
T
taixiurong 已提交
1965 1966
  m.def("is_float16_supported", [](const platform::XPUPlace &place) -> bool {
    // XPUs with Compute Capability > xpu2 support float16 and bfloat16
W
Wilber 已提交
1967
    return platform::get_xpu_version(place.device) >
1968
           phi::backends::xpu::XPUVersion::XPU1;
T
taixiurong 已提交
1969 1970 1971
  });
  m.def("is_bfloat16_supported", [](const platform::XPUPlace &place) -> bool {
    // XPUs with Compute Capability > xpu2 support float16 and bfloat16
W
Wilber 已提交
1972
    return platform::get_xpu_version(place.device) >
1973
           phi::backends::xpu::XPUVersion::XPU1;
T
taixiurong 已提交
1974
  });
1975
#endif
1976

1977
  py::class_<paddle::platform::CPUPlace> cpuplace(m, "CPUPlace", R"DOC(
1978
    CPUPlace is a descriptor of a device.
1979
    It represents a CPU device on which a tensor will be allocated and a model will run.
L
lujun 已提交
1980 1981 1982 1983

    Examples:
        .. code-block:: python

1984 1985
          import paddle
          cpu_place = paddle.CPUPlace()
L
lujun 已提交
1986

1987 1988 1989
        )DOC");
  g_cpuplace_pytype = reinterpret_cast<PyTypeObject *>(cpuplace.ptr());
  cpuplace.def(py::init<>())
S
sneaxiy 已提交
1990 1991
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
1992
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
1993
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1994 1995 1996 1997
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
1998
      .def("__repr__", string::to_string<const platform::CPUPlace &>)
1999
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
2000

2001 2002
  py::class_<paddle::platform::CUDAPinnedPlace> cudapinnedplace(
      m, "CUDAPinnedPlace", R"DOC(
2003 2004 2005 2006 2007 2008
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
2009 2010 2011 2012

    Examples:
        .. code-block:: python

2013 2014
          import paddle
          place = paddle.CUDAPinnedPlace()
L
lujun 已提交
2015

2016 2017 2018 2019
        )DOC");
  g_cudapinnedplace_pytype =
      reinterpret_cast<PyTypeObject *>(cudapinnedplace.ptr());
  cudapinnedplace
S
sneaxiy 已提交
2020
      .def("__init__",
S
sneaxiy 已提交
2021
           [](platform::CUDAPinnedPlace &self) {
2022
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
2023 2024 2025
             PADDLE_THROW(platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
S
sneaxiy 已提交
2026
#endif
S
sneaxiy 已提交
2027
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
2028
           })
S
sneaxiy 已提交
2029 2030 2031 2032
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
2033 2034
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
2035 2036
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
S
sneaxiy 已提交
2037 2038 2039 2040
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
2041
      .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
C
chengduoZH 已提交
2042 2043
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

2044
  // NPUPlace
2045
  py::class_<platform::NPUPlace> npuplace(m, "NPUPlace", R"DOC(
2046 2047 2048 2049 2050 2051 2052 2053
    NPUPlace is a descriptor of a device.
    It represents a NPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          npu_place = paddle.NPUPlace(0)

2054 2055 2056
        )DOC");
  g_npuplace_pytype = reinterpret_cast<PyTypeObject *>(npuplace.ptr());
  npuplace
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
      .def("__init__",
           [](platform::NPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_ASCEND_CL
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid NPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetNPUDeviceCount())) {
               if (platform::GetNPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use NPU because there is no NPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid NPUPlace(%d), must inside [0, %d), because NPU "
                     "number on your machine is %d",
                     dev_id, platform::GetNPUDeviceCount(),
                     platform::GetNPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::NPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use NPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use NPU, please try to install NPU version "
2088
                 "PaddlePaddle by: pip install paddlepaddle-npu\n"
2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102
                 "If you only have CPU, please change NPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
H
houj04 已提交
2103 2104
      .def("get_device_id",
           [](const platform::NPUPlace &self) { return self.GetDeviceId(); })
2105 2106
      .def("__str__", string::to_string<const platform::NPUPlace &>);

J
jianghaicheng 已提交
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158
  // IPUPlace
  py::class_<platform::IPUPlace>(m, "IPUPlace", R"DOC(
    IPUPlace is a descriptor of a device.
    It represents a IPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle

          # required: ipu

          ipu_place = paddle.IPUPlace()

        )DOC")
      .def("__init__",
           [](platform::IPUPlace &self) {
#ifdef PADDLE_WITH_IPU
             if (platform::GetIPUDeviceCount() == 0) {
               LOG(ERROR) << "Cannot use IPU because there is no IPU "
                             "detected on your "
                             "machine.";
               std::exit(-1);
             }
             // use ipu(0) to comile, while run with the number user configure
             // in sharding and pipline.
             new (&self) platform::IPUPlace(0);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use IPU because you didn't install IPU version "
                 "PaddlePaddle.\n"
                 "If you want to use IPU, please try to install IPU version "
                 "PaddlePaddle by: pip install paddlepaddle*\n"
                 "If you only have CPU, please change IPUPlace to be "
                 "CPUPlace().\n");
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::IPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::IPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::IPUPlace, platform::CUDAPinnedPlace>)
#ifdef PADDLE_WITH_IPU
      .def("get_device_id",
           [](const platform::IPUPlace &self) { return self.GetDeviceId(); })
#endif
      .def("__str__", string::to_string<const platform::IPUPlace &>);

2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
  // MLUPlace
  py::class_<platform::MLUPlace> mluplace(m, "MLUPlace", R"DOC(
    MLUPlace is a descriptor of a device.
    It represents a MLU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          # required: mlu
          mlu_place = paddle.MLUPlace(0)

        )DOC");
  g_mluplace_pytype = reinterpret_cast<PyTypeObject *>(mluplace.ptr());
  mluplace
      .def("__init__",
           [](platform::MLUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_MLU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid MLUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetMLUDeviceCount())) {
               if (platform::GetMLUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use MLU because there is no MLU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid MLUPlace(%d), must inside [0, %d), because MLU "
                     "number on your machine is %d",
                     dev_id, platform::GetMLUDeviceCount(),
                     platform::GetMLUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::MLUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use MLU because you have installed CPU/GPU/... "
                 "version "
                 "PaddlePaddle.\n"
                 "If you want to use MLU, please try to install MLU version "
                 "PaddlePaddle by: pip install paddlepaddle-mlu\n"
                 "If you only have CPU, please change MLUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::MLUPlace>)
#ifdef PADDLE_WITH_MLU
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::IPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::MLUPlace>)
      .def("_equals",
           &IsSamePlace<platform::MLUPlace, platform::CUDAPinnedPlace>)
      .def("get_device_id",
           [](const platform::MLUPlace &self) { return self.GetDeviceId(); })
#endif
      .def("__str__", string::to_string<const platform::MLUPlace &>);

2228 2229 2230
  py::class_<platform::Place> platformplace(m, "Place");
  g_place_pytype = reinterpret_cast<PyTypeObject *>(platformplace.ptr());
  platformplace.def(py::init<>())
S
sneaxiy 已提交
2231 2232 2233 2234
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
2235
      .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
2236
      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
J
jianghaicheng 已提交
2237
      .def("_equals", &IsSamePlace<platform::Place, platform::IPUPlace>)
S
sneaxiy 已提交
2238
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
2239
      .def("_equals", &IsSamePlace<platform::Place, platform::MLUPlace>)
X
xuezhong 已提交
2240 2241
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
2242 2243
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
2244 2245
      .def("is_xpu_place",
           [](platform::Place &self) { return platform::is_xpu_place(self); })
2246 2247
      .def("is_npu_place",
           [](platform::Place &self) { return platform::is_npu_place(self); })
J
jianghaicheng 已提交
2248 2249
      .def("is_ipu_place",
           [](platform::Place &self) { return platform::is_ipu_place(self); })
S
sneaxiy 已提交
2250 2251 2252 2253
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
2254 2255
      .def("is_mlu_place",
           [](platform::Place &self) { return platform::is_mlu_place(self); })
2256 2257 2258
      .def(
          "is_custom_place",
          [](platform::Place &self) { return platform::is_custom_place(self); })
2259 2260 2261 2262 2263
      .def("gpu_device_id", [](platform::Place &self) { return self.device; })
      .def("xpu_device_id", [](platform::Place &self) { return self.device; })
      .def("npu_device_id", [](platform::Place &self) { return self.device; })
      .def("ipu_device_id", [](platform::Place &self) { return self.device; })
      .def("mlu_device_id", [](platform::Place &self) { return self.device; })
2264 2265
      .def("custom_device_id",
           [](platform::Place &self) { return self.device; })
S
sneaxiy 已提交
2266 2267
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
2268 2269 2270 2271
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
2272 2273 2274 2275
      .def("set_place",
           [](platform::Place &self, const platform::XPUPlace &xpu_place) {
             self = xpu_place;
           })
Y
Yu Yang 已提交
2276
      .def("set_place",
D
dzhwinter 已提交
2277
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
2278
             self = gpu_place;
C
chengduoZH 已提交
2279
           })
2280 2281 2282 2283 2284
      .def("set_place",
           [](platform::Place &self,
              const platform::CUDAPinnedPlace &cuda_pinned_place) {
             self = cuda_pinned_place;
           })
2285 2286 2287 2288
      .def("set_place",
           [](platform::Place &self, const platform::NPUPlace &npu_place) {
             self = npu_place;
           })
J
jianghaicheng 已提交
2289 2290 2291 2292
      .def("set_place",
           [](platform::Place &self, const platform::IPUPlace &ipu_place) {
             self = ipu_place;
           })
2293 2294 2295 2296
      .def("set_place",
           [](platform::Place &self, const platform::MLUPlace &mlu_place) {
             self = mlu_place;
           })
2297 2298 2299 2300
      .def("set_place",
           [](platform::Place &self, const platform::CustomPlace &plug_place) {
             self = plug_place;
           })
2301 2302
      .def("__repr__", string::to_string<const platform::Place &>)
      .def("__str__", string::to_string<const platform::Place &>);
Y
Yu Yang 已提交
2303

Y
Yu Yang 已提交
2304
  py::class_<OperatorBase>(m, "Operator")
2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
      .def_static("create",
                  [](py::bytes protobin) {
                    proto::OpDesc desc;
                    PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin),
                                      true,
                                      platform::errors::InvalidArgument(
                                          "Cannot parse user input to OpDesc"));
                    PADDLE_ENFORCE_EQ(desc.IsInitialized(), true,
                                      platform::errors::InvalidArgument(
                                          "The provided OpDesc is not "
                                          "initialized, the reason is: %s",
                                          desc.InitializationErrorString()));
                    return OpRegistry::CreateOp(desc);
                  })
2319
      .def("run",
2320
           [](OperatorBase &self, const Scope &scope,
2321 2322 2323 2324
              const platform::CPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
2325 2326
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2327 2328 2329 2330
              const platform::XPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
2331 2332
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2333 2334 2335 2336
              const platform::NPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
D
dzhwinter 已提交
2337 2338
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2339 2340 2341 2342
              const platform::CUDAPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
C
chengduoZH 已提交
2343 2344 2345
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
2346
             pybind11::gil_scoped_release release;
C
chengduoZH 已提交
2347 2348
             self.Run(scope, place);
           })
2349 2350 2351 2352 2353 2354
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::MLUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
2355 2356 2357 2358 2359 2360 2361
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
2362 2363
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
2364
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
2365
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
2366 2367 2368 2369
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
2370

2371 2372 2373
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

2374 2375 2376 2377 2378 2379 2380
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
2381 2382
      .def("finalize", &TrainerBase::Finalize)
      .def("ResetDataset", &TrainerBase::ResetDataset);
2383

2384 2385
  m.def("_get_eager_deletion_vars", &framework::GetEagerDeletionCleanVars);

F
fengjiayi 已提交
2386
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
2387
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
2388
      .def("close", &Executor::Close)
2389 2390
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
2391 2392
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
2393 2394 2395 2396
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
2397
             pybind11::gil_scoped_release release;
2398 2399 2400 2401 2402 2403 2404
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
2405 2406 2407
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
2408
              std::map<std::string, FetchType *> *fetch_targets,
2409 2410 2411 2412 2413 2414 2415 2416
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
2417
      .def("run_prepared_ctx",
G
guru4elephant 已提交
2418 2419 2420 2421 2422 2423 2424
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
2435
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
2436 2437
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
2438
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
2439 2440
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
2441
      });
S
sneaxiy 已提交
2442

2443
  py::class_<framework::interpreter::CostInfo>(m, "CostInfo")
2444
      .def(py::init<>())
2445 2446 2447 2448 2449
      .def("total_time",
           [](interpreter::CostInfo &self) { return self.total_time; })
      .def("device_memory_bytes", [](interpreter::CostInfo &self) {
        return self.device_memory_bytes;
      });
2450

2451
  py::class_<framework::StandaloneExecutor>(m, "StandaloneExecutor")
H
hong 已提交
2452 2453 2454
      .def(py::init<const platform::Place &, const ProgramDesc &,
                    const ProgramDesc &, Scope *>())
      .def("run",
2455
           [](StandaloneExecutor &self,
H
hong 已提交
2456
              const std::unordered_map<std::string, py::array> &input_dict,
2457
              std::vector<std::string> fetch_names) {
2458
             std::vector<framework::LoDTensor> feed_tensors;
2459
             std::vector<std::string> feed_names;
H
hong 已提交
2460 2461 2462 2463 2464

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
2465 2466
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
H
hong 已提交
2467 2468
             }

2469 2470 2471 2472 2473 2474 2475 2476 2477
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
             }
             return py::cast(std::move(ret));
           })
      .def("run",
           [](StandaloneExecutor &self,
2478
              const std::unordered_map<std::string, framework::LoDTensor>
2479 2480
                  &input_dict,
              std::vector<std::string> fetch_names) {
2481
             std::vector<framework::LoDTensor> feed_tensors;
2482 2483 2484 2485 2486 2487 2488
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               feed_names.push_back(item.first);
               feed_tensors.push_back(item.second);
             }

W
wanghuancoder 已提交
2489 2490 2491 2492
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
H
hong 已提交
2493
             }
W
wanghuancoder 已提交
2494
             return py::cast(std::move(ret));
2495
           })
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505
      .def("run",
           [](StandaloneExecutor &self, std::vector<std::string> feed_names,
              std::vector<std::string> fetch_names) {
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, fetch_names);
             }
             return py::cast(std::move(ret));
           })
2506 2507 2508
      .def("dry_run",
           [](StandaloneExecutor &self,
              const std::unordered_map<std::string, py::array> &input_dict) {
2509
             std::vector<framework::LoDTensor> feed_tensors;
2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
             }

2520
             framework::interpreter::CostInfo cost_info;
2521 2522 2523 2524 2525
             {
               pybind11::gil_scoped_release release;
               cost_info = self.DryRun(feed_names, feed_tensors);
             }
             return cost_info;
H
hong 已提交
2526 2527
           });

D
dzhwinter 已提交
2528
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
2529
  m.def("init_glog", framework::InitGLOG);
2530 2531
  m.def("load_op_meta_info_and_register_op",
        framework::LoadOpMetaInfoAndRegisterOp);
2532
  m.def("init_devices", []() { framework::InitDevices(); });
2533

2534
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
2535
  m.def("is_compiled_with_ascend", IsCompiledWithAscend);
2536
  m.def("is_compiled_with_rocm", IsCompiledWithROCM);
2537
  m.def("is_compiled_with_npu", IsCompiledWithNPU);
J
jianghaicheng 已提交
2538
  m.def("is_compiled_with_ipu", IsCompiledWithIPU);
2539
  m.def("is_compiled_with_xpu", IsCompiledWithXPU);
2540
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
2541
  m.def("is_compiled_with_nccl", IsCompiledWithNCCL);
2542
  m.def("is_compiled_with_cinn", IsCompiledWithCINN);
2543
  m.def("is_compiled_with_mlu", IsCompiledWithMLU);
2544
  m.def("_is_compiled_with_heterps", IsCompiledWithHETERPS);
2545
  m.def("supports_bfloat16", SupportsBfloat16);
2546
  m.def("supports_bfloat16_fast_performance", SupportsBfloat16FastPerformance);
2547 2548
  m.def("supports_int8", SupportsInt8);
  m.def("supports_vnni", SupportsVNNI);
L
Leo Chen 已提交
2549
  m.def("op_supported_infos", imperative::OpSupportedInfos);
2550
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
2551
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
2552 2553 2554
  m.def("_cuda_synchronize", [](const platform::CUDAPlace &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });
H
hutuxian 已提交
2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573

  m.def("get_float_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<float>> float_stats;
    paddle::platform::StatRegistry<float>::Instance().publish(float_stats);
    std::unordered_map<std::string, float> stats_map;
    for (const auto &stat : float_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
  m.def("get_int_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<int64_t>> int_stats;
    paddle::platform::StatRegistry<int64_t>::Instance().publish(int_stats);
    std::unordered_map<std::string, int64_t> stats_map;
    for (const auto &stat : int_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
H
hutuxian 已提交
2574 2575 2576 2577 2578 2579 2580
  m.def("run_cmd",
        [](const std::string &cmd, int time_out = -1,
           int sleep_inter = -1) -> const std::string {
          return paddle::framework::shell_get_command_output(cmd, time_out,
                                                             sleep_inter);
        },
        py::arg("cmd"), py::arg("time_out") = -1, py::arg("sleep_inter") = -1);
G
gongweibao 已提交
2581 2582 2583 2584 2585 2586 2587 2588 2589
  m.def("shell_execute_cmd",
        [](const std::string &cmd, int time_out = 0, int sleep_inter = 0,
           bool redirect_stderr = false) -> std::vector<std::string> {
          return paddle::framework::shell_execute_cmd(
              cmd, time_out, sleep_inter, redirect_stderr);
        },
        py::arg("cmd"), py::arg("time_out") = 0, py::arg("sleep_inter") = 0,
        py::arg("redirect_stderr") = false);

2590
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2591 2592
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
2593
    return platform::GetGPUComputeCapability(place.device) >= 53;
2594 2595
  });
#endif
2596

S
Steffy-zxf 已提交
2597 2598 2599 2600 2601 2602
  m.def("set_feed_variable",
        static_cast<void (*)(Scope *, const LoDTensor &, const std::string &,
                             size_t)>(&framework::SetFeedVariable));
  m.def("set_feed_variable",
        static_cast<void (*)(Scope *, const Strings &, const std::string &,
                             size_t)>(&framework::SetFeedVariable));
2603 2604 2605 2606 2607
  m.def("get_fetch_variable",
        [](const Scope &scope, const std::string &var_name,
           size_t index) -> py::object {
          auto &var = framework::GetFetchVariable(scope, var_name, index);
          if (data_is_lod_tensor(var)) {
2608
            return py::cast(BOOST_GET(LoDTensor, var));
2609
          } else {
2610
            return py::cast(BOOST_GET(LoDTensorArray, var));
2611 2612
          }
        });
2613
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
2614

X
Xin Pan 已提交
2615 2616
  m.def("_is_program_version_supported", IsProgramVersionSupported);

2617 2618 2619 2620
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
H
Huihuang Zheng 已提交
2621
  BindCostModel(&m);
2622
  BindConstValue(&m);
2623
  BindGlobalValueGetterSetter(&m);
2624
  BindProcessMeshDesc(&m);
L
LiYuRio 已提交
2625
  BindFleetExecutor(&m);
2626
  BindTCPStore(&m);
Y
Yu Yang 已提交
2627

Y
Yu Yang 已提交
2628 2629 2630 2631 2632 2633 2634 2635 2636
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

2637
  py::class_<LoDTensorArray> pylodtensorarray(m, "LoDTensorArray", R"DOC(
2638
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
2639 2640 2641

    Examples:
        .. code-block:: python
2642

Z
Zeng Jinle 已提交
2643 2644 2645
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
2646 2647 2648 2649
)DOC");
  g_framework_lodtensorarray_pytype =
      reinterpret_cast<PyTypeObject *>(pylodtensorarray.ptr());
  pylodtensorarray
S
sneaxiy 已提交
2650 2651
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
2652 2653 2654 2655 2656 2657
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
2658 2659 2660 2661
             PADDLE_ENFORCE_LT(i, self.size(),
                               platform::errors::InvalidArgument(
                                   "The index to set is larger than the size "
                                   "of LoDTensorArray."));
Y
Yu Yang 已提交
2662 2663 2664
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
2665 2666 2667 2668 2669 2670
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
2671 2672
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
2673 2674 2675 2676 2677 2678
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
2701

2702 2703 2704 2705 2706 2707 2708 2709
  py::class_<FetchList>(m, "FetchList", R"DOC( FetchList is a
        vector of boost::variant<LoDTensor, LoDTensorArray>.
        )DOC")
      .def("_move_to_list",
           [](FetchList &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               if (data_is_lod_tensor(self[i])) {
2710
                 auto &data = BOOST_GET(LoDTensor, self[i]);
2711 2712
                 res[i] = py::cast(std::move(data));
               } else {
2713
                 auto &data = BOOST_GET(LoDTensorArray, self[i]);
2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728
                 py::list tmp(data.size());
                 for (size_t j = 0; j < data.size(); ++j) {
                   tmp[j] = py::cast(std::move(data[j]));
                 }
                 res[i] = std::move(tmp);
               }
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership)

      .def("append",
           [](FetchList &self, const LoDTensor &t) {
             self.emplace_back();
2729
             auto &lod_tensor = BOOST_GET(LoDTensor, self.back());
2730 2731 2732 2733 2734 2735 2736 2737
             lod_tensor.ShareDataWith(t);
             lod_tensor.set_lod(t.lod());
           },
           py::arg("var"))

      .def("append",
           [](FetchList &self, const LoDTensorArray &t) {
             self.emplace_back();
2738
             auto &lod_tensor_array = BOOST_GET(LoDTensorArray, self.back());
2739 2740 2741 2742 2743 2744 2745 2746 2747
             for (size_t i = 0; i < t.size(); ++i) {
               lod_tensor_array[i].ShareDataWith(t[i]);
               lod_tensor_array[i].set_lod(t[i].lod());
             }
           },
           py::arg("var"));

  py::class_<FetchUnmergedList>(m, "FetchUnmergedList", R"DOC(
        FetchUnmergedList is 2-D array of FetchType(boost::variant(LoDTensor, LoDTensorArray)).
Z
Zhen Wang 已提交
2748 2749
        )DOC")
      .def("_move_to_list",
2750
           [](FetchUnmergedList &self) -> py::list {
Z
Zhen Wang 已提交
2751 2752 2753 2754
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               py::list tmp(self[i].size());
               for (size_t j = 0; j < self[i].size(); ++j) {
2755
                 if (data_is_lod_tensor(self[i][j])) {
2756
                   auto &var = BOOST_GET(LoDTensor, self[i][j]);
2757 2758
                   tmp[j] = py::cast(std::move(var));
                 } else {
2759
                   auto &var = BOOST_GET(LoDTensorArray, self[i][j]);
2760 2761 2762 2763 2764 2765
                   py::list tmp_array(var.size());
                   for (size_t k = 0; k < var.size(); ++k) {
                     tmp_array[k] = std::move(var[k]);
                   }
                   tmp[j] = std::move(tmp_array);
                 }
Z
Zhen Wang 已提交
2766 2767 2768 2769 2770 2771 2772 2773 2774
               }
               res[i] = std::move(tmp);
               self[i].clear();
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);

Y
Yu Yang 已提交
2775
  m.def("op_support_gpu", OpSupportGPU);
2776
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2777
  m.def("get_cuda_device_count", platform::GetGPUDeviceCount);
2778 2779 2780 2781 2782 2783 2784 2785
  m.def("cuda_empty_cache", [] {
    for (int dev_id : platform::GetSelectedDevices()) {
      auto *dev_ctx = platform::DeviceContextPool::Instance().GetByPlace(
          platform::CUDAPlace(dev_id));
      dev_ctx->cudnn_workspace_handle().ResetWorkspace();
    }
    platform::EmptyCache();
  });
2786 2787 2788 2789 2790 2791 2792
  m.def("get_device_properties",
        [](int id) -> const gpuDeviceProp & {
          return platform::GetDeviceProperties(id);
        },
        py::return_value_policy::copy);

  py::class_<gpuDeviceProp>(m, "_gpuDeviceProperties")
Y
Yanxing Shi 已提交
2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817
      .def_property_readonly(
          "name", [](const gpuDeviceProp &prop) { return prop.name; })
      .def_property_readonly(
          "major", [](const gpuDeviceProp &prop) { return prop.major; })
      .def_property_readonly(
          "minor", [](const gpuDeviceProp &prop) { return prop.minor; })
      .def_property_readonly(
          "total_memory",
          [](const gpuDeviceProp &prop) { return prop.totalGlobalMem; })
      .def_property_readonly(
          "multi_processor_count",
          [](const gpuDeviceProp &prop) { return prop.multiProcessorCount; })
      .def_property_readonly(
          "is_multi_gpu_board",
          [](const gpuDeviceProp &prop) { return prop.isMultiGpuBoard; })
      .def_property_readonly(
          "is_integrated",
          [](const gpuDeviceProp &prop) { return prop.integrated; })
      .def("__repr__", [](const gpuDeviceProp &prop) {
        std::stringstream ostr;
        ostr << "_gpuDeviceProperties(name='" << prop.name
             << "', major=" << prop.major << ", minor=" << prop.minor
             << ", total_memory=" << prop.totalGlobalMem / (1024 * 1024)
             << "MB, multi_processor_count=" << prop.multiProcessorCount << ")";
        return ostr.str();
2818
      });
D
dangqingqing 已提交
2819

2820
#if !defined(PADDLE_WITH_HIP) && !defined(_WIN32)
D
dangqingqing 已提交
2821 2822 2823
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
2824 2825 2826 2827
  m.def("nvprof_nvtx_push", platform::CudaNvtxRangePush);
  m.def("nvprof_nvtx_pop", platform::CudaNvtxRangePop);
  m.def("nvprof_enable_record_event", platform::NvprofEnableRecordEvent);
  m.def("nvprof_disable_record_event", platform::NvprofDisableRecordEvent);
D
Dong Zhihong 已提交
2828
#endif
P
peizhilin 已提交
2829
#endif
Y
Yu Yang 已提交
2830

2831 2832
#ifdef PADDLE_WITH_ASCEND_CL
  m.def("get_npu_device_count", platform::GetNPUDeviceCount);
2833
  m.def("npu_finalize", []() {
2834 2835
    platform::HCCLCommContext::Instance().ReleaseHCCLComms();

2836 2837 2838
    auto &pool = platform::DeviceContextPool::Instance();
    auto devices = platform::GetSelectedNPUDevices();
    for (size_t i = 0; i < devices.size(); ++i) {
R
ronnywang 已提交
2839
      platform::NPUDeviceGuard guard(devices[i]);
2840 2841 2842 2843
      pool.Get(platform::NPUPlace(devices[i]))->Wait();
    }
    platform::AclInstance::Instance().Finalize();
  });
2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863

  py::class_<platform::NPUProfConfigWrapper>(m, "NPUProfConfigWrapper");

  m.def("npu_prof_init", platform::NPUProfilerInit);
  m.def("npu_prof_start", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStart(c.ptr());
  });
  m.def("npu_prof_stop", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStop(c.ptr());
  });
  m.def("npu_prof_finalize", platform::NPUProfilerFinalize);
  m.def("npu_prof_create_config", []() {
    return platform::NPUProfConfigWrapper(platform::NPUProfilerCreateConfig());
  });

  m.def("npu_prof_destropy_config", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerDestroyConfig(c.ptr());
  });
#endif

J
jianghaicheng 已提交
2864 2865 2866 2867
#ifdef PADDLE_WITH_IPU
  m.def("get_ipu_device_count", platform::GetIPUDeviceCount);
#endif

2868 2869 2870 2871
#ifdef PADDLE_WITH_MLU
  m.def("get_mlu_device_count", platform::GetMLUDeviceCount);
#endif

2872 2873 2874 2875 2876 2877
  py::enum_<platform::TracerOption>(m, "TracerOption", py::arithmetic())
      .value("kDefault", platform::TracerOption::kDefault)
      .value("kOpDetail", platform::TracerOption::kOpDetail)
      .value("kAllOpDetail", platform::TracerOption::kAllOpDetail)
      .export_values();

2878 2879 2880 2881
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
2882
      .value("kAll", platform::ProfilerState::kAll)
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

2894
  m.def("set_tracer_option", platform::SetTracerOption);
2895 2896
  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
2897
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
2898
  m.def("reset_profiler", platform::ResetProfiler);
W
wuhuanzhou 已提交
2899
  m.def("register_pass", [](const std::string &pass_type, py::object callable) {
2900 2901
    PADDLE_ENFORCE_EQ(
        framework::ir::PassRegistry::Instance().Has(pass_type), false,
2902 2903 2904
        platform::errors::AlreadyExists("Pass '%s' is registered more than "
                                        "once. Please use another name.",
                                        pass_type));
W
wuhuanzhou 已提交
2905
    callable.inc_ref();
2906 2907 2908 2909 2910 2911 2912 2913
    framework::ir::PassRegistry::Instance().Insert(pass_type, [pass_type,
                                                               callable]() {
      py::gil_scoped_acquire guard;
      std::unique_ptr<framework::ir::Pass> pass(
          new framework::ir::GeneratePass(py::cast<std::string>(callable())));
      return pass;
    });
  });
2914
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
2915 2916 2917
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
2918

2919 2920
  m.def("size_of_dtype", framework::SizeOfType);

2921
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2922 2923
  m.def("set_cublas_switch", platform::SetAllowTF32Cublas);
  m.def("get_cublas_switch", platform::AllowTF32Cublas);
A
AshburnLee 已提交
2924 2925
  m.def("set_cudnn_switch", platform::SetAllowTF32Cudnn);
  m.def("get_cudnn_switch", platform::AllowTF32Cudnn);
2926
#endif  // PADDLE_WITH_CUDA
2927 2928
  m.def("clear_executor_cache",
        []() { framework::ExecutorInfoCache::Instance().Finalize(); });
2929

2930 2931 2932
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

2933 2934
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
2935
      .def("has", &ir::Pass::Has)
2936 2937 2938
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
2939
           })
2940
      .def(
2941
          "set",
2942 2943 2944
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
2945 2946
      .def("set", [](ir::Pass &self, const std::string &name,
                     bool val) { self.Set<bool>(name, new bool(val)); })
2947 2948
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
J
jianghaicheng 已提交
2949 2950 2951 2952 2953
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::vector<std::string> set) {
             self.Set(name, new std::vector<std::string>(set));
           })
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
2968 2969
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
2970
        self.Apply(graph.get());
F
flame 已提交
2971
      });
2972

X
fix  
Xin Pan 已提交
2973 2974
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
2989
  // -- python binds for parallel executor.
Y
yuyang18 已提交
2990
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
2991 2992 2993 2994
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

2995 2996 2997
    Returns:
        ExecutionStrategy: An ExecutionStrategy object.

C
chengduo 已提交
2998 2999 3000
    Examples:
        .. code-block:: python

3001 3002 3003 3004 3005 3006 3007 3008 3009
          import paddle
          import paddle.static as static
          import paddle.nn.functional as F

          paddle.enable_static()

          x = static.data(name='x', shape=[None, 13], dtype='float32')
          y = static.data(name='y', shape=[None, 1], dtype='float32')
          y_predict = static.nn.fc(input=x, size=1, act=None)
3010

3011 3012
          cost = F.square_error_cost(input=y_predict, label=y)
          avg_loss = paddle.mean(cost)
3013

3014
          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
3015 3016
          sgd_optimizer.minimize(avg_loss)

3017
          exec_strategy = static.ExecutionStrategy()
C
chengduo 已提交
3018 3019
          exec_strategy.num_threads = 4

3020 3021 3022
          train_exe = static.ParallelExecutor(use_cuda=False,
                                              loss_name=avg_loss.name,
                                              exec_strategy=exec_strategy)
C
chengduo 已提交
3023 3024
        )DOC");

3025 3026 3027 3028
  py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
      .value("CPU", paddle::platform::DeviceType::CPU)
      .value("CUDA", paddle::platform::DeviceType::CUDA)
      .value("XPU", paddle::platform::DeviceType::XPU);
3029

Y
yuyang18 已提交
3030
  exec_strategy.def(py::init())
Y
yuyang18 已提交
3031 3032 3033 3034 3035
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
3036
          },
3037 3038
          R"DOC(
            The type is INT, num_threads represents the size of thread pool that
C
chengduo 已提交
3039 3040 3041 3042 3043 3044 3045
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058
            `multiprocessing.cpu_count()`. Default 0.

            Examples:
                .. code-block:: python

                    import paddle
                    import paddle.static as static

                    paddle.enable_static()

                    exec_strategy = static.ExecutionStrategy()
                    exec_strategy.num_threads = 4
            )DOC")
Y
yuyang18 已提交
3059
      .def_property(
3060 3061
          "_use_device",
          [](const ExecutionStrategy &self) { return self.use_device_; },
3062
          [](ExecutionStrategy &self, paddle::platform::DeviceType use_device) {
3063 3064 3065
            self.use_device_ = use_device;
          })  // NOTE(liuyuhui): Doesn't add doc for 'use_device', because
              // use_device isn‘t exposed to users.
Y
yuyang18 已提交
3066 3067 3068 3069 3070
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
3071 3072 3073
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
3074 3075
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
3076 3077 3078 3079 3080 3081 3082
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
3083 3084 3085 3086
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
3087
                because the temp variable's shape maybe the same between two iterations.
3088 3089 3090 3091 3092 3093 3094 3095 3096 3097
                Default 100.

                .. note::
                    1. If you fetch data when calling the 'run', the ParallelExecutor 
                    will clean up the temp variables at the end of the current iteration. 
                    2. In some NLP model, it may cause the GPU memory is insufficient, 
                    in this case, you should reduce `num_iteration_per_drop_scope`.

                Examples:
                    .. code-block:: python
C
chengduo 已提交
3098

3099 3100 3101 3102 3103 3104 3105
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_drop_scope = 10
3106
              )DOC")
Q
Qiao Longfei 已提交
3107 3108 3109 3110 3111 3112 3113 3114 3115
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127
                user call exe.run() in python。Default: 1.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_run = 10
Q
Qiao Longfei 已提交
3128
              )DOC")
3129 3130 3131 3132 3133 3134 3135 3136
      .def_property(
          "use_thread_barrier",
          [](const ExecutionStrategy &self) { return self.thread_barrier_; },
          [](ExecutionStrategy &self, bool use_thread_barrier) {
            self.thread_barrier_ = use_thread_barrier;
          },
          R"DOC(This config that the this is distributed training with parameter server
              )DOC")
3137 3138 3139 3140 3141
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
3142

Y
yuyang18 已提交
3143
  exec_strategy.def_property(
Y
yuyang18 已提交
3144 3145 3146 3147 3148 3149 3150
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
3151 3152
      });

C
chengduo 已提交
3153 3154 3155 3156
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

3157 3158 3159
    Returns:
        BuildStrategy: An BuildStrategy object.

C
chengduo 已提交
3160 3161 3162
    Examples:
        .. code-block:: python

3163
            import os
3164 3165 3166 3167
            import paddle
            import paddle.static as static

            paddle.enable_static()
3168

3169 3170
            os.environ['CPU_NUM'] = str(2)
            places = static.cpu_places()
3171

3172 3173 3174 3175
            data = static.data(name="x", shape=[None, 1], dtype="float32")
            hidden = static.nn.fc(input=data, size=10)
            loss = paddle.mean(hidden)
            paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
3176

3177
            build_strategy = static.BuildStrategy()
3178 3179
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
3180 3181
            build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
            program = static.CompiledProgram(static.default_main_program())
3182
            program = program.with_data_parallel(loss_name=loss.name,
3183 3184
                                                  build_strategy=build_strategy,
                                                  places=places)
C
chengduo 已提交
3185
)DOC");
Y
yuyang18 已提交
3186 3187 3188

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
3189 3190
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce)
      .value("_NoReduce", BuildStrategy::ReduceStrategy::kNoReduce);
Y
yuyang18 已提交
3191 3192 3193 3194 3195 3196 3197 3198
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
3199
      .def("_clear_finalized", &BuildStrategy::ClearFinalized)
Y
yuyang18 已提交
3200 3201 3202 3203
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
3204 3205 3206 3207
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3208
            self.reduce_ = strategy;
C
chengduo 已提交
3209
          },
3210
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
3211 3212
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
3213
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
3214 3215
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
3216
                Default is 'AllReduce'.
F
flame 已提交
3217 3218 3219 3220

                Examples:
                    .. code-block:: python

3221 3222 3223 3224 3225 3226 3227
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
F
flame 已提交
3228
                  )DOC")
Y
yuyang18 已提交
3229 3230 3231 3232 3233
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
3234 3235 3236 3237
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3238
            self.gradient_scale_ = strategy;
C
chengduo 已提交
3239
          },
3240
          R"DOC((paddle.static.BuildStrategy.GradientScaleStrategy, optional): there are three
3241
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
3242 3243
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
3244
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
3245 3246 3247 3248

                Examples:
                    .. code-block:: python

C
chengduo 已提交
3249 3250
                        import numpy
                        import os
3251 3252 3253 3254
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
3255 3256

                        use_cuda = True
3257 3258
                        place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace()
                        exe = static.Executor(place)
C
chengduo 已提交
3259 3260

                        # NOTE: If you use CPU to run the program, you need
3261
                        # to specify the CPU_NUM, otherwise, paddle will use
C
chengduo 已提交
3262 3263 3264 3265 3266 3267
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
3268
                            places = static.cpu_places()
C
chengduo 已提交
3269
                        else:
3270
                            places = static.cuda_places()
C
chengduo 已提交
3271

3272 3273 3274 3275
                        data = static.data(name='X', shape=[None, 1], dtype='float32')
                        hidden = static.nn.fc(input=data, size=10)
                        loss = paddle.mean(hidden)
                        paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
C
chengduo 已提交
3276

3277
                        exe.run(static.default_startup_program())
C
chengduo 已提交
3278

3279
                        build_strategy = static.BuildStrategy()
C
chengduo 已提交
3280
                        build_strategy.gradient_scale_strategy = \
3281 3282 3283
                                  static.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = static.CompiledProgram(
                                  static.default_main_program()).with_data_parallel(
C
chengduo 已提交
3284
                                          loss_name=loss.name, build_strategy=build_strategy,
3285
                                          places=places)
C
chengduo 已提交
3286 3287 3288 3289 3290 3291

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
3292 3293
                                              feed={"X": x, loss_grad_name : loss_grad},
                                              fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
3294
                   )DOC")
Y
yuyang18 已提交
3295 3296 3297 3298
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
3299 3300 3301 3302
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3303
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
3304
          },
3305
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
3306
                writing the SSA Graph to file in the form of graphviz.
3307
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
3308 3309 3310 3311

                Examples:
                    .. code-block:: python

3312 3313 3314 3315
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
3316

3317 3318
                        build_strategy = static.BuildStrategy()
                        build_strategy.debug_graphviz_path = "./graph"
F
flame 已提交
3319
                    )DOC")
S
sneaxiy 已提交
3320 3321 3322 3323 3324 3325
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
3326 3327 3328 3329
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
3330 3331
            self.enable_sequential_execution_ = b;
          },
3332 3333
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
3334 3335 3336 3337

                Examples:
                    .. code-block:: python

3338 3339 3340 3341 3342 3343
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3344 3345
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
3346 3347 3348 3349 3350 3351
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
3352 3353 3354 3355
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
3356 3357
            self.remove_unnecessary_lock_ = b;
          },
3358 3359
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
3360 3361 3362 3363

                Examples:
                    .. code-block:: python

3364 3365 3366 3367 3368 3369
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3370 3371
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
3372 3373 3374 3375
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
3376
#ifdef WIN32
3377
            PADDLE_THROW(platform::errors::Unavailable(
3378
                "Distribution mode is not supported on Windows platform."));
3379
#endif
3380 3381
            self.num_trainers_ = num_trainers;
          })
3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
3394 3395 3396 3397 3398 3399
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
3400 3401 3402 3403 3404 3405
      .def_property(
          "bkcl_comm_num",
          [](const BuildStrategy &self) { return self.bkcl_comm_num_; },
          [](BuildStrategy &self, int bkcl_comm_num) {
            self.bkcl_comm_num_ = bkcl_comm_num;
          })
3406
      .def_property("use_hierarchical_allreduce",
3407 3408 3409 3410 3411 3412
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
3413
      .def_property("hierarchical_allreduce_inter_nranks",
3414 3415 3416 3417 3418 3419 3420
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
3421 3422 3423 3424 3425 3426
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
3427 3428 3429 3430
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
C
chengduo 已提交
3431 3432
            self.fuse_elewise_add_act_ops_ = b;
          },
3433
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
3434
                to fuse elementwise_add_op and activation_op,
3435
                it may make the execution faster. Default is False.
F
flame 已提交
3436 3437 3438 3439

                Examples:
                    .. code-block:: python

3440 3441 3442 3443 3444 3445
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3446 3447
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
Z
Zhen Wang 已提交
3448 3449 3450 3451
      .def_property(
          "fuse_bn_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_act_ops_; },
          [](BuildStrategy &self, bool b) {
3452
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
Z
Zhen Wang 已提交
3453
                              platform::errors::PreconditionNotMet(
3454 3455
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Z
Zhen Wang 已提交
3456 3457 3458 3459 3460 3461 3462 3463 3464
            self.fuse_bn_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_act_ops indicate whether
                to fuse batch_norm and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

3465 3466 3467 3468 3469 3470
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
Z
Zhen Wang 已提交
3471 3472
                        build_strategy.fuse_bn_act_ops = True
                     )DOC")
Z
Zhang Ting 已提交
3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497
      .def_property(
          "fuse_bn_add_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_add_act_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_bn_add_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_add_act_ops indicate whether
                to fuse batch_norm, elementwise_add and activation_op,
                it may make the execution faster. Default is True

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_bn_add_act_ops = True
                     )DOC")
3498 3499 3500 3501
      .def_property(
          "enable_auto_fusion",
          [](const BuildStrategy &self) { return self.enable_auto_fusion_; },
          [](BuildStrategy &self, bool b) {
3502
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
3503
                              platform::errors::PreconditionNotMet(
3504 3505
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
3506 3507 3508 3509 3510 3511 3512 3513 3514 3515
            self.enable_auto_fusion_ = b;
          },
          R"DOC((bool, optional): Whether to enable fusing subgraph to a
                fusion_group. Now we only support fusing subgraph that composed
                of elementwise-like operators, such as elementwise_add/mul
                without broadcast and activations.

                Examples:
                    .. code-block:: python

3516 3517 3518 3519 3520 3521
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
3522 3523
                        build_strategy.enable_auto_fusion = True
                    )DOC")
3524 3525 3526 3527 3528 3529
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
3530 3531 3532 3533
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
3534 3535
            self.fuse_relu_depthwise_conv_ = b;
          },
3536
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
3537 3538 3539
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
3540
                Default is False.
F
flame 已提交
3541 3542 3543 3544

                Examples:
                    .. code-block:: python

3545 3546 3547 3548 3549 3550
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3551 3552
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
3553 3554 3555
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
3556
                             self.fuse_broadcast_ops_ == paddle::none;
C
chengduo 已提交
3557 3558
                    },
                    [](BuildStrategy &self, bool b) {
3559 3560 3561 3562
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
3563 3564
                      self.fuse_broadcast_ops_ = b;
                    },
3565
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
3566 3567 3568 3569
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
3570 3571 3572 3573 3574
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

3575 3576 3577 3578 3579 3580
                              import paddle
                              import paddle.static as static

                              paddle.enable_static()

                              build_strategy = static.BuildStrategy()
3581 3582
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
3583 3584
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
3585
                      return self.fuse_all_optimizer_ops_ == true ||
3586
                             self.fuse_all_optimizer_ops_ == paddle::none;
C
chengduo 已提交
3587 3588
                    },
                    [](BuildStrategy &self, bool b) {
3589 3590 3591 3592
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
3593 3594
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
3595 3596 3597 3598
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
3599 3600 3601 3602
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Q
qingqing01 已提交
3603 3604
            self.sync_batch_norm_ = b;
          },
3605
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
3606 3607 3608
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
3609 3610
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
3611 3612 3613 3614

                Examples:
                    .. code-block:: python

3615 3616 3617 3618 3619 3620
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3621 3622
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
3623 3624
      .def_property(
          "memory_optimize",
3625 3626 3627 3628 3629 3630 3631 3632 3633 3634
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
3635
              self.memory_optimize_ = paddle::none;
3636 3637 3638
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
3639
              PADDLE_THROW(platform::errors::InvalidArgument(
Z
Zeng Jinle 已提交
3640 3641
                  "BuildStrategy.memory_optimize must be set to None, False "
                  "or True"));
3642 3643
            }
          },
3644
          R"DOC((bool, optional): memory opitimize aims to save total memory
3645
                consumption, set to True to enable it.
3646

3647 3648 3649
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663
                True means enabling and False means disabling. Default is None.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.memory_optimize = True
                
                )DOC")
3664 3665 3666
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
3667 3668 3669
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
3670
              PADDLE_THROW(platform::errors::Unavailable(
3671
                  "Distribution mode is not supported on Windows platform."));
3672 3673 3674 3675 3676
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
3677 3678 3679
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
3680
      .def_property(
D
dzhwinter 已提交
3681 3682 3683
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
3684 3685 3686 3687
      .def_property(
          "enable_addto",
          [](const BuildStrategy &self) { return self.enable_addto_; },
          [](BuildStrategy &self, bool b) { self.enable_addto_ = b; })
C
chengduo 已提交
3688 3689
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
3690 3691
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
3692
                   self.fuse_all_reduce_ops_ == paddle::none;
C
chengduo 已提交
3693
          },
C
chengduo 已提交
3694
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
3695 3696 3697 3698 3699 3700 3701
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
3702 3703 3704 3705
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
3706 3707 3708 3709 3710 3711 3712 3713 3714
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
Z
Zeng Jinle 已提交
3715 3716 3717 3718 3719 3720
      .def_property(
          "fix_op_run_order",
          [](const BuildStrategy &self) { return self.fix_op_run_order_; },
          [](BuildStrategy &self, bool fix_op_run_order) {
            self.fix_op_run_order_ = fix_op_run_order;
          })
3721 3722 3723 3724 3725 3726 3727
      .def_property("allow_cuda_graph_capture",
                    [](const BuildStrategy &self) {
                      return self.allow_cuda_graph_capture_;
                    },
                    [](BuildStrategy &self, bool allow_cuda_graph_capture) {
                      self.allow_cuda_graph_capture_ = allow_cuda_graph_capture;
                    })
3728 3729 3730 3731 3732 3733
      .def("_copy",
           [](const BuildStrategy &self) {
             auto new_bs = self;
             new_bs.ClearFinalized();
             return new_bs;
           })
3734
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
3735
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
3736 3737 3738 3739 3740
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
3741

3742 3743 3744 3745 3746 3747
  m.def("_set_cached_executor_build_strategy",
        [](int64_t program_id, const BuildStrategy &build_strategy) {
          auto &cached_exe_info = framework::ExecutorInfoCache::Instance();
          cached_exe_info.SetBuildStrategy(program_id, build_strategy);
        });

Y
yuyang18 已提交
3748
  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
3749
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
3750
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
3751
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
3752 3753 3754 3755
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
3756 3757 3758 3759 3760
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
3761 3762 3763
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
3764 3765 3766 3767
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
3768 3769
      .def("run",
           [](ParallelExecutor &self,
Z
Zhen Wang 已提交
3770 3771 3772 3773 3774 3775 3776 3777
              const std::vector<std::string> &fetch_tensors,
              bool return_merged) -> py::object {
             paddle::framework::FetchResultType ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(fetch_tensors, return_merged);
             }
             if (return_merged) {
3778
               return py::cast(
3779
                   std::move(BOOST_GET(paddle::framework::FetchList, ret)));
Z
Zhen Wang 已提交
3780 3781
             } else {
               return py::cast(std::move(
3782
                   BOOST_GET(paddle::framework::FetchUnmergedList, ret)));
Z
Zhen Wang 已提交
3783
             }
3784 3785
           })
      .def("device_count", &ParallelExecutor::DeviceCount);
Y
Yu Yang 已提交
3786

J
jianghaicheng 已提交
3787 3788 3789 3790 3791 3792 3793 3794
#ifdef PADDLE_WITH_IPU
  py::class_<platform::ipu::IpuBackend,
             std::shared_ptr<platform::ipu::IpuBackend>>(m, "IpuBackend")
      .def(py::init(&platform::ipu::IpuBackend::GetNewInstance))
      .def("clear", &platform::ipu::IpuBackend::Clear)
      .def("set_scope", &platform::ipu::IpuBackend::SetScope)
      .def("set_ipu_strategy", &platform::ipu::IpuBackend::SetIpuStrategy);

J
jianghaicheng 已提交
3795 3796
  py::class_<platform::ipu::IpuStrategy> ipu_strategy(m, "IpuStrategy");
  ipu_strategy.def(py::init())
J
jianghaicheng 已提交
3797 3798 3799 3800 3801
      .def_property(
          "num_ipus",
          [](const platform::ipu::IpuStrategy &self) { return self.num_ipus; },
          [](platform::ipu::IpuStrategy &self, int num_ipus) {
            self.num_ipus = num_ipus;
J
jianghaicheng 已提交
3802
          })
J
jianghaicheng 已提交
3803 3804 3805 3806 3807 3808 3809
      .def_property(
          "accumulationFactor",
          [](const platform::ipu::IpuStrategy &self) {
            return self.popart_options_.accumulationFactor;
          },
          [](platform::ipu::IpuStrategy &self, int accumulationFactor) {
            self.popart_options_.accumulationFactor = accumulationFactor;
J
jianghaicheng 已提交
3810
          })
J
jianghaicheng 已提交
3811 3812 3813 3814 3815 3816
      .def_property("batches_per_step",
                    [](const platform::ipu::IpuStrategy &self) {
                      return self.batches_per_step;
                    },
                    [](platform::ipu::IpuStrategy &self, int batches_per_step) {
                      self.batches_per_step = batches_per_step;
J
jianghaicheng 已提交
3817
                    })
J
jianghaicheng 已提交
3818 3819 3820 3821 3822 3823
      .def_property("is_training",
                    [](const platform::ipu::IpuStrategy &self) {
                      return self.is_training;
                    },
                    [](platform::ipu::IpuStrategy &self, bool is_training) {
                      self.is_training = is_training;
J
jianghaicheng 已提交
3824
                    })
J
jianghaicheng 已提交
3825 3826 3827 3828 3829 3830 3831
      .def_property(
          "enable_pipelining",
          [](const platform::ipu::IpuStrategy &self) {
            return self.popart_options_.enablePipelining;
          },
          [](platform::ipu::IpuStrategy &self, bool enable_pipelining) {
            self.popart_options_.enablePipelining = enable_pipelining;
J
jianghaicheng 已提交
3832
          })
J
jianghaicheng 已提交
3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
      .def_property(
          "enable_manual_shard",
          [](const platform::ipu::IpuStrategy &self) {
            return self.popart_options_.virtualGraphMode ==
                   platform::ipu::VirtualGraphMode::Manual;
          },
          [](platform::ipu::IpuStrategy &self, bool enable_ipu_shard) {
            if (enable_ipu_shard) {
              self.popart_options_.virtualGraphMode =
                  platform::ipu::VirtualGraphMode::Manual;
            } else {
              self.popart_options_.virtualGraphMode =
                  platform::ipu::VirtualGraphMode::Off;
            }
J
jianghaicheng 已提交
3847
          })
J
jianghaicheng 已提交
3848 3849 3850 3851 3852 3853
      .def_property("need_avg_shard",
                    [](const platform::ipu::IpuStrategy &self) {
                      return self.need_avg_shard;
                    },
                    [](platform::ipu::IpuStrategy &self, bool need_avg_shard) {
                      self.need_avg_shard = need_avg_shard;
J
jianghaicheng 已提交
3854
                    })
J
jianghaicheng 已提交
3855 3856 3857 3858 3859 3860
      .def_property("batch_size",
                    [](const platform::ipu::IpuStrategy &self) {
                      return self.batch_size;
                    },
                    [](platform::ipu::IpuStrategy &self, int batch_size) {
                      self.batch_size = batch_size;
J
jianghaicheng 已提交
3861
                    })
J
jianghaicheng 已提交
3862 3863 3864 3865 3866 3867
      .def_property("enable_fp16",
                    [](const platform::ipu::IpuStrategy &self) {
                      return self.enable_fp16;
                    },
                    [](platform::ipu::IpuStrategy &self, bool enable_fp16) {
                      self.enable_fp16 = enable_fp16;
J
jianghaicheng 已提交
3868
                    });
J
jianghaicheng 已提交
3869 3870
#endif

D
dongdaxiang 已提交
3871
  BindFleetWrapper(&m);
3872
  BindIO(&m);
T
Thunderbrook 已提交
3873

T
Thunderbrook 已提交
3874
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
T
Thunderbrook 已提交
3875
  BindHeterWrapper(&m);
3876
  BindMetrics(&m);
T
Thunderbrook 已提交
3877
#endif
T
Thunderbrook 已提交
3878
#ifdef PADDLE_WITH_HETERPS
T
Thunderbrook 已提交
3879
  BindPSGPUWrapper(&m);
T
Thunderbrook 已提交
3880
#endif
3881
  BindGlooWrapper(&m);
H
hutuxian 已提交
3882
  BindBoxHelper(&m);
H
hutuxian 已提交
3883 3884 3885
#ifdef PADDLE_WITH_BOX_PS
  BindBoxWrapper(&m);
#endif
3886
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
3887
  BindNCCLWrapper(&m);
3888 3889 3890
#endif
#ifdef PADDLE_WITH_GLOO
  BindGlooContext(&m);
W
wopeizl 已提交
3891
#endif
F
flame 已提交
3892 3893
  BindGraph(&m);
  BindNode(&m);
3894
  BindPass(&m);
F
flame 已提交
3895
  BindInferenceApi(&m);
3896
  BindCompatible(&m);
3897
  BindDataset(&m);
Y
yaoxuefeng 已提交
3898
  BindGenerator(&m);
3899 3900 3901
#ifndef PADDLE_ON_INFERENCE
  BindDistributed(&m);
#endif
3902 3903 3904
#ifdef PADDLE_WITH_ASCEND
  BindAscendWrapper(&m);
  BindAscendGraph(&m);
3905
  BindAscendDevice(&m);
3906
#endif
Y
Yanghello 已提交
3907 3908 3909
#ifdef PADDLE_WITH_CRYPTO
  BindCrypto(&m);
#endif
T
tangwei12 已提交
3910

T
tangwei12 已提交
3911
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
3912 3913
  BindDistFleetWrapper(&m);
  BindPSHost(&m);
3914
  BindCommunicatorContext(&m);
T
tangwei12 已提交
3915 3916
  BindDistCommunicator(&m);
  BindHeterClient(&m);
S
seemingwang 已提交
3917 3918 3919 3920 3921
  BindGraphPyFeatureNode(&m);
  BindGraphNode(&m);
  BindGraphPyService(&m);
  BindGraphPyServer(&m);
  BindGraphPyClient(&m);
1
123malin 已提交
3922 3923 3924 3925
  BindIndexNode(&m);
  BindTreeIndex(&m);
  BindIndexWrapper(&m);
  BindIndexSampler(&m);
3926
  BindSparseShardingTools(&m);
3927
#endif
L
Luo Tao 已提交
3928
}
3929
}  // namespace pybind
3930
}  // namespace paddle