pybind.cc 182.1 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2
Copyright (c) 2022 NVIDIA Authors. All Rights Reserved.
3 4 5 6 7

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

8
http://www.apache.org/licenses/LICENSE-2.0
9 10 11 12 13 14

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
15
#include <Python.h>
16

C
chengduoZH 已提交
17
#include <algorithm>
18
#include <cctype>
19
#include <cstdlib>
20
#include <iterator>
C
chengduoZH 已提交
21
#include <map>
S
sneaxiy 已提交
22
#include <memory>
C
chengduoZH 已提交
23 24
#include <mutex>  // NOLINT // for call_once
#include <string>
25 26
#include <tuple>
#include <type_traits>
C
chengduoZH 已提交
27
#include <unordered_map>
28
#include <unordered_set>
C
chengduoZH 已提交
29 30
#include <utility>
#include <vector>
31

32
#include "paddle/fluid/framework/convert_utils.h"
33
#include "paddle/fluid/framework/custom_operator.h"
34
#include "paddle/fluid/framework/data_layout.h"
L
Leo Chen 已提交
35
#include "paddle/fluid/framework/data_type_transform.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/executor.h"
37
#include "paddle/fluid/framework/executor_cache.h"
38
#include "paddle/fluid/framework/executor_gc_helper.h"
Y
Yi Wang 已提交
39
#include "paddle/fluid/framework/feed_fetch_method.h"
Z
Zhen Wang 已提交
40
#include "paddle/fluid/framework/feed_fetch_type.h"
S
sneaxiy 已提交
41
#include "paddle/fluid/framework/garbage_collector.h"
H
hutuxian 已提交
42
#include "paddle/fluid/framework/io/fs.h"
43
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
H
Huihuang Zheng 已提交
44
#include "paddle/fluid/framework/ir/cost_model.h"
45
#include "paddle/fluid/framework/ir/generate_pass.h"
46
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
47 48
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
49
#include "paddle/fluid/framework/new_executor/standalone_executor.h"
S
sneaxiy 已提交
50
#include "paddle/fluid/framework/op_info.h"
51
#include "paddle/fluid/framework/op_registry.h"
52
#include "paddle/fluid/framework/op_version_registry.h"
Y
Yu Yang 已提交
53
#include "paddle/fluid/framework/parallel_executor.h"
54
#include "paddle/fluid/framework/phi_utils.h"
Y
Yi Wang 已提交
55
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
56
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
57
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
58
#include "paddle/fluid/framework/scope_pool.h"
59
#include "paddle/fluid/framework/selected_rows_utils.h"
60
#include "paddle/fluid/framework/tensor_util.h"
61
#include "paddle/fluid/framework/trainer.h"
62
#include "paddle/fluid/framework/type_defs.h"
X
Xin Pan 已提交
63
#include "paddle/fluid/framework/version.h"
L
Leo Chen 已提交
64
#include "paddle/fluid/imperative/amp_auto_cast.h"
H
hong 已提交
65
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
66
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
67 68 69
#ifdef PADDLE_WITH_CUDA
#include "paddle/fluid/memory/allocation/cuda_ipc_allocator.h"
#endif
70
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
D
dzhwinter 已提交
71
#include "paddle/fluid/operators/activation_op.h"
L
Leo Chen 已提交
72
#include "paddle/fluid/operators/common_infer_shape_functions.h"
S
sneaxiy 已提交
73
#include "paddle/fluid/operators/py_func_op.h"
74
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
75
#include "paddle/fluid/platform/cpu_info.h"
76
#include "paddle/fluid/platform/device/device_wrapper.h"
77
#include "paddle/fluid/platform/device_context.h"
78
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
79
#include "paddle/fluid/platform/enforce.h"
80
#include "paddle/fluid/platform/init.h"
H
hutuxian 已提交
81
#include "paddle/fluid/platform/monitor.h"
Y
Yi Wang 已提交
82 83
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
C
chenjian 已提交
84 85 86
#include "paddle/fluid/platform/profiler/event_python.h"
#include "paddle/fluid/platform/profiler/event_tracing.h"
#include "paddle/fluid/platform/profiler/profiler.h"
87
#include "paddle/fluid/pybind/cuda_streams_py.h"
88
#include "paddle/fluid/pybind/distributed_py.h"
89
#include "paddle/fluid/pybind/eager.h"
J
Jiabin Yang 已提交
90
#include "paddle/fluid/pybind/imperative.h"
91
#include "paddle/fluid/pybind/io.h"
92 93
#include "paddle/phi/core/compat/convert_utils.h"
#include "paddle/phi/core/lod_utils.h"
94
#include "paddle/utils/none.h"
95 96 97
#ifdef PADDLE_WITH_ASCEND
#include "paddle/fluid/pybind/ascend_wrapper_py.h"
#endif
H
Huihuang Zheng 已提交
98
#include "paddle/fluid/pybind/bind_cost_model.h"
L
LiYuRio 已提交
99
#include "paddle/fluid/pybind/bind_fleet_executor.h"
H
hutuxian 已提交
100
#include "paddle/fluid/pybind/box_helper_py.h"
101
#include "paddle/fluid/pybind/communication.h"
102
#include "paddle/fluid/pybind/compatible.h"
Y
Yi Wang 已提交
103
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
104
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
105
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
106
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
Y
yaoxuefeng 已提交
107
#include "paddle/fluid/pybind/generator_py.h"
108
#include "paddle/fluid/pybind/global_value_getter_setter.h"
109
#include "paddle/fluid/pybind/gloo_context_py.h"
110
#include "paddle/fluid/pybind/gloo_wrapper_py.h"
T
Thunderbrook 已提交
111
#include "paddle/fluid/pybind/heter_wrapper_py.h"
F
flame 已提交
112
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
113
#include "paddle/fluid/pybind/ir.h"
114
#include "paddle/fluid/pybind/metrics_py.h"
T
Thunderbrook 已提交
115
#include "paddle/fluid/pybind/ps_gpu_wrapper_py.h"
116
#include "paddle/fluid/pybind/pybind_boost_headers.h"
117
#include "paddle/phi/backends/device_manager.h"
118

119
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
120
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
121
#endif
122
#include "paddle/fluid/framework/data_type.h"
123 124
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
125
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
126
#include "paddle/fluid/pybind/tensor_py.h"
127
#include "paddle/fluid/string/to_string.h"
128 129
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
Y
Yi Wang 已提交
130
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
131
#endif
132
#ifndef PADDLE_WITH_HIP
133
#include "paddle/fluid/platform/device/gpu/cuda/cuda_profiler.h"
134
#endif
135
#include "paddle/fluid/platform/device/gpu/gpu_info.h"
D
Dong Zhihong 已提交
136 137
#endif

138
#ifdef PADDLE_WITH_ASCEND_CL
139
#include "paddle/fluid/platform/collective_helper.h"
140 141
#include "paddle/fluid/platform/device/npu/npu_info.h"
#include "paddle/fluid/platform/device/npu/npu_profiler.h"
142 143
#endif

144
#ifdef PADDLE_WITH_XPU
145
#include "paddle/fluid/platform/device/xpu/xpu_info.h"
T
TTerror 已提交
146
#include "paddle/fluid/platform/device/xpu/xpu_op_list.h"
147 148
#endif

149
#include "paddle/fluid/platform/cuda_graph_with_memory_pool.h"
A
Allen Guo 已提交
150

J
jianghaicheng 已提交
151
#ifdef PADDLE_WITH_IPU
A
Allen Guo 已提交
152 153
#include "paddle/fluid/platform/device/ipu/ipu_backend.h"
#include "paddle/fluid/platform/device/ipu/ipu_info.h"
J
jianghaicheng 已提交
154
#endif
155

156 157 158 159
#ifdef PADDLE_WITH_MLU
#include "paddle/fluid/platform/device/mlu/mlu_info.h"
#endif

Y
Yanghello 已提交
160 161 162 163
#ifdef PADDLE_WITH_CRYPTO
#include "paddle/fluid/pybind/crypto.h"
#endif

T
tangwei12 已提交
164
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
165 166 167
#include "paddle/fluid/pybind/fleet_py.h"
#endif

168 169 170
#include "paddle/fluid/eager/api/utils/global_utils.h"
#include "paddle/fluid/pybind/eager_utils.h"
#include "paddle/phi/api/ext/op_meta_info.h"
171 172
#include "paddle/phi/kernels/autotune/cache.h"
#include "paddle/phi/kernels/autotune/switch_autotune.h"
M
minqiyang 已提交
173 174
#include "pybind11/stl.h"

175
DECLARE_bool(use_mkldnn);
176

Q
Qiao Longfei 已提交
177 178
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
179 180 181
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchUnmergedList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType);
Q
Qiao Longfei 已提交
182

183
namespace paddle {
184
namespace pybind {
185 186

PyTypeObject *g_place_pytype = nullptr;
0
0x45f 已提交
187
PyTypeObject *g_framework_scope_pytype = nullptr;
188 189 190 191 192
PyTypeObject *g_cudaplace_pytype = nullptr;
PyTypeObject *g_cpuplace_pytype = nullptr;
PyTypeObject *g_xpuplace_pytype = nullptr;
PyTypeObject *g_npuplace_pytype = nullptr;
PyTypeObject *g_cudapinnedplace_pytype = nullptr;
193
PyTypeObject *g_mluplace_pytype = nullptr;
194
PyTypeObject *g_framework_tensor_pytype = nullptr;
195
PyTypeObject *g_framework_lodtensorarray_pytype = nullptr;
196
PyTypeObject *g_custom_op_kernel_ctx_pytype = nullptr;
197

198
bool IsCompiledWithCUDA() {
199 200 201 202 203 204 205
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
  return false;
#else
  return true;
#endif
}

206 207 208 209 210 211 212 213
bool IsCompiledWithNCCL() {
#ifdef PADDLE_WITH_NCCL
  return true;
#else
  return false;
#endif
}

214 215
bool IsCompiledWithROCM() {
#ifndef PADDLE_WITH_HIP
Q
qijun 已提交
216 217 218 219 220 221
  return false;
#else
  return true;
#endif
}

222 223 224 225 226 227 228 229
bool IsCompiledWithAscend() {
#ifndef PADDLE_WITH_ASCEND
  return false;
#else
  return true;
#endif
}

230 231 232 233 234 235 236 237
bool IsCompiledWithXPU() {
#ifndef PADDLE_WITH_XPU
  return false;
#else
  return true;
#endif
}

238 239 240 241 242 243 244 245
bool IsCompiledWithNPU() {
#ifndef PADDLE_WITH_ASCEND_CL
  return false;
#else
  return true;
#endif
}

J
jianghaicheng 已提交
246 247 248 249 250 251 252 253
bool IsCompiledWithIPU() {
#ifndef PADDLE_WITH_IPU
  return false;
#else
  return true;
#endif
}

254 255 256 257 258 259 260 261
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

262 263 264 265 266 267 268 269
bool IsCompiledWithCINN() {
#ifndef PADDLE_WITH_CINN
  return false;
#else
  return true;
#endif
}

270 271 272 273 274 275 276 277
bool IsCompiledWithMLU() {
#ifndef PADDLE_WITH_MLU
  return false;
#else
  return true;
#endif
}

278 279 280 281 282 283 284 285
bool IsCompiledWithHETERPS() {
#ifndef PADDLE_WITH_HETERPS
  return false;
#else
  return true;
#endif
}

286 287 288 289 290 291 292 293 294 295 296
bool SupportsBfloat16() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core))
    return true;
  else
    return false;
#endif
}

297 298 299 300 301 302 303 304 305 306 307
bool SupportsBfloat16FastPerformance() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_bf16))
    return true;
  else
    return false;
#endif
}

308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
bool SupportsInt8() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return (platform::MayIUse(platform::cpu_isa_t::avx2) ||
          platform::MayIUse(platform::cpu_isa_t::avx512f));
#endif
}

bool SupportsVNNI() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return platform::MayIUse(platform::cpu_isa_t::avx512_core_vnni);
#endif
}

325
bool IsCompiledWithBrpc() {
326
#ifndef PADDLE_WITH_DISTRIBUTE
327 328
  return false;
#endif
329
  return true;
330 331
}

Y
update  
Yancey1989 已提交
332
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
333
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
334 335 336 337 338 339
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
340 341 342 343 344 345 346
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
347
  return static_cast<int>(paddle::platform::Place(p).GetType());
S
sneaxiy 已提交
348 349
}

H
hong 已提交
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
372 373 374
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Python object is not type of %s, the real type is %s",
        typeid(T).name(), obj->ob_type->tp_name));
H
hong 已提交
375 376 377 378 379 380 381 382 383 384 385 386 387
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
388 389
      PADDLE_THROW(platform::errors::InvalidArgument(
          "The parameter [%s] to save is None", para.first));
H
hong 已提交
390 391
    }
    vec_res.emplace_back(
392
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
H
hong 已提交
393 394 395 396 397 398 399 400 401 402 403 404
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
405 406
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameter list to save is None"));
H
hong 已提交
407 408 409 410 411 412 413 414 415 416 417 418
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
419 420 421
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to save is None"));
H
hong 已提交
422 423 424 425
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
426 427
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to save is not a list"));
H
hong 已提交
428 429 430 431
  }
  return vec_res;
}

432 433 434 435 436 437 438 439
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
440 441
    PADDLE_THROW(
        platform::errors::InvalidArgument("The parameter list to set is None"));
442 443 444 445 446 447 448 449 450 451 452 453 454
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
455 456 457
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to set is None"));
458 459 460 461 462
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
463 464 465 466 467
        PADDLE_ENFORCE_NOT_NULL(exe,
                                platform::errors::InvalidArgument(
                                    "Parameter not Initialized, "
                                    "Please set argument [executor] not None "
                                    "or run startup program first"));
468 469
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
470 471 472
        PADDLE_ENFORCE_NOT_NULL(
            py_var_desc, platform::errors::InvalidArgument(
                             "The var_desc of parameter to set is None"));
473 474 475 476
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
477
        tensor_temp->Resize(phi::make_ddim(var_desc.GetShape()));
478 479
        tensor_temp->mutable_data(
            exe->GetPlace(),
480
            framework::TransToPhiDataType(var_desc.GetDataType()));
481 482 483
      }
    }
  } else {
484 485
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to set is not a list"));
486 487 488 489 490
  }

  return;
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
static void AssertStaticGraphAndDygraphGradMakerNoDiff() {
  std::set<std::string> ops;
  for (auto &pair : framework::OpInfoMap::Instance().map()) {
    bool has_static_grad_maker = (pair.second.grad_op_maker_ != nullptr);
    bool has_dygraph_grad_maker =
        (pair.second.dygraph_grad_op_maker_ != nullptr);
    if (has_static_grad_maker ^ has_dygraph_grad_maker) {
      bool has_kernel =
          (framework::OperatorWithKernel::AllOpKernels().count(pair.first) > 0);
      if (has_kernel) {
        ops.insert(pair.first);
      } else {
        VLOG(5) << pair.first << " has no kernels, skip";
      }
    }
  }
  PADDLE_ENFORCE_EQ(ops.empty(), true,
                    platform::errors::Unimplemented(
                        "OperatorWithKernel [%s] have only static graph grad "
                        "maker or have only dygraph grad maker, which is not "
                        "allowed",
                        string::join_strings(ops, ',')));
}

Z
Zeng Jinle 已提交
515 516 517 518
#ifdef PADDLE_WITH_NCCL
static int GetNCCLVersion() {
#if NCCL_VERSION_CODE >= 2304
  int ver;
519
  PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::ncclGetVersion(&ver));
Z
Zeng Jinle 已提交
520 521 522 523 524 525 526 527
  return ver;
#else
  PADDLE_THROW(platform::errors::External(
      "Cannot get NCCL version successfully when nccl version < 2.3.4"));
#endif
}
#endif

Z
Zeng Jinle 已提交
528 529 530 531 532 533 534 535 536 537 538
template <typename PlaceType>
static void TensorCopyFrom(framework::Tensor *dst, const framework::Tensor &src,
                           const PlaceType &place, int64_t batch_size) {
  if (batch_size < 0) {
    framework::TensorCopy(src, place, dst);
  } else {
    auto sliced = src.Slice(0, batch_size);
    framework::TensorCopy(sliced, place, dst);
  }
}

539 540 541 542 543 544
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

J
Jiabin Yang 已提交
545
  BindImperative(&m);
546
  BindEager(&m);
547
  BindEagerStringTensor(&m);
548 549
  BindCudaStream(&m);

Y
Yu Yang 已提交
550 551 552
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
553
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
554

555 556
  AssertStaticGraphAndDygraphGradMakerNoDiff();

557
  m.doc() = "C++ core of PaddlePaddle";
558

559 560 561 562
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

563
  BindException(&m);
Y
Yu Yang 已提交
564

565 566
  m.def("set_num_threads", &platform::SetNumThreads);

567 568
  m.def("disable_signal_handler", &DisableSignalHandler);

569 570 571 572 573 574 575 576
  m.def("clear_gradients",
        [](std::vector<std::shared_ptr<imperative::VarBase>> param_list,
           bool set_to_zero) {
          for (auto param : param_list) {
            param->ClearGradient(set_to_zero);
          }
        });

577
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
578
  m.def("cudnn_version", &platform::DnnVersion);
579 580 581 582 583 584
  m.def("gpu_memory_available", []() {
    size_t available = 0;
    size_t total = 0;
    paddle::platform::GpuMemoryUsage(&available, &total);
    return available;
  });
585
#endif
586

Z
Zeng Jinle 已提交
587 588 589 590
#ifdef PADDLE_WITH_NCCL
  m.def("nccl_version", &GetNCCLVersion);
#endif

591 592 593 594 595 596 597 598 599 600
  m.def("is_cuda_graph_capturing", &platform::IsCUDAGraphCapturing);
#ifdef PADDLE_WITH_CUDA
  py::class_<platform::CUDAGraph>(m, "CUDAGraph")
      .def_static("begin_capture",
                  [](platform::CUDAPlace place, int mode) {
                    platform::BeginCUDAGraphCapture(
                        place, static_cast<cudaStreamCaptureMode>(mode));
                  })
      .def_static("end_capture", &platform::EndCUDAGraphCapture)
      .def("replay", &platform::CUDAGraph::Replay)
601 602
      .def("reset", &platform::CUDAGraph::Reset)
      .def("print_to_dot_files", &platform::CUDAGraph::PrintToDotFiles);
603 604
#endif

Z
Zeng Jinle 已提交
605 606 607 608
  m.def("wait_device", [](const platform::Place &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });

6
633WHU 已提交
609 610 611
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
612 613 614 615 616 617

    PADDLE_ENFORCE_NOT_NULL(
        dmt, platform::errors::InvalidArgument(
                 "from_dlpack received an invalid capsule. "
                 "Note that a DLPack tensor can be consumed only once."));

6
633WHU 已提交
618 619
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
620
    framework::Tensor tensor;
6
633WHU 已提交
621

S
Siming Dai 已提交
622
    if (dl.device.device_type == kDLCPU) {
6
633WHU 已提交
623 624
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
625
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
S
Siming Dai 已提交
626
    if (dl.device.device_type == kDLGPU) {
6
633WHU 已提交
627 628 629 630 631
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });
H
hong 已提交
632

633 634 635 636 637 638
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

639 640 641 642 643 644
  m.def("save_op_version_info", [](framework::ProgramDesc &desc) {
    framework::compatible::pb::OpVersionMap pb_vmap{desc.OpVersionMap()};
    framework::compatible::SaveOpVersions(
        framework::compatible::OpVersionRegistrar::GetInstance()
            .GetVersionMap(),
        &pb_vmap);
645 646
  });

647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
  m.def("set_printoptions", [](const py::kwargs &kwargs) {
    auto &print_opt = framework::PrintOptions::Instance();
    if (kwargs.contains("precision")) {
      print_opt.precision = kwargs["precision"].cast<int>();
    }
    if (kwargs.contains("threshold")) {
      print_opt.threshold = kwargs["threshold"].cast<int>();
    }
    if (kwargs.contains("edgeitems")) {
      print_opt.edgeitems = kwargs["edgeitems"].cast<int>();
    }
    if (kwargs.contains("linewidth")) {
      print_opt.linewidth = kwargs["linewidth"].cast<int>();
    }
    if (kwargs.contains("sci_mode")) {
      print_opt.sci_mode = kwargs["sci_mode"].cast<bool>();
    }

    VLOG(4) << "Set printoptions: precision=" << print_opt.precision
            << ", threshold=" << print_opt.threshold
            << ", edgeitems=" << print_opt.edgeitems
            << ", linewidth=" << print_opt.linewidth
            << ", sci_mode=" << print_opt.sci_mode;
  });

L
Leo Chen 已提交
672 673
  m.def("broadcast_shape", [](const std::vector<int64_t> &x_dim,
                              const std::vector<int64_t> &y_dim) {
674 675
    return phi::vectorize(operators::details::BroadcastTwoDims(
        phi::make_ddim(x_dim), phi::make_ddim(y_dim), -1));
L
Leo Chen 已提交
676 677
  });

S
sneaxiy 已提交
678
  m.def(
S
sneaxiy 已提交
679
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
680 681 682 683
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
684 685 686
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
  m.def("_get_all_register_op_kernels",
        [](const std::string &lib) {
          std::unordered_map<std::string, std::vector<std::string>>
              all_kernels_info;
          if (lib == "fluid" || lib == "all") {
            auto &all_kernels =
                paddle::framework::OperatorWithKernel::AllOpKernels();

            for (auto &kernel_pair : all_kernels) {
              auto op_type = kernel_pair.first;
              std::vector<std::string> kernel_types;
              for (auto &info_pair : kernel_pair.second) {
                paddle::framework::OpKernelType kernel_type = info_pair.first;
                kernel_types.emplace_back(
                    paddle::framework::KernelTypeToString(kernel_type));
              }
              all_kernels_info.emplace(op_type, kernel_types);
704 705
            }
          }
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724
          if (lib == "phi" || lib == "all") {
            auto phi_kernels = phi::KernelFactory::Instance().kernels();
            for (auto &kernel_pair : phi_kernels) {
              auto op_type = phi::TransToFluidOpName(kernel_pair.first);
              std::vector<std::string> kernel_types;
              for (auto &info_pair : kernel_pair.second) {
                framework::OpKernelType kernel_type =
                    framework::TransPhiKernelKeyToOpKernelType(info_pair.first);
                auto kernel_type_str =
                    framework::KernelTypeToString(kernel_type);
                if (all_kernels_info.count(op_type)) {
                  if (std::find(all_kernels_info[op_type].begin(),
                                all_kernels_info[op_type].end(),
                                kernel_type_str) ==
                      all_kernels_info[op_type].end()) {
                    all_kernels_info[op_type].emplace_back(kernel_type_str);
                  }
                } else {
                  kernel_types.emplace_back(kernel_type_str);
725 726
                }
              }
727 728 729
              if (!kernel_types.empty()) {
                all_kernels_info.emplace(op_type, kernel_types);
              }
730 731 732
            }
          }

733 734 735 736
          return all_kernels_info;
        },
        py::arg("lib") = "all",
        R"DOC(
737 738 739
           Return the registered kernels in paddle.

           Args:
740
               lib[string]: the libarary, could be 'phi', 'fluid' and 'all'.
741
           )DOC");
742

743 744 745 746 747 748
  // NOTE(Aganlengzi): KernelFactory static instance is initialized BEFORE
  // plugins are loaded for custom kernels, but de-initialized AFTER they are
  // unloaded. We need manually clear symbols(may contain plugins' symbols)
  // stored in this static instance to avoid illegal memory access.
  m.def("clear_kernel_factory",
        []() { phi::KernelFactory::Instance().kernels().clear(); });
749 750 751 752 753
  m.def("clear_device_manager", []() {
#ifdef PADDLE_WITH_CUSTOM_DEVICE
    phi::DeviceManager::Clear();
#endif
  });
754

S
sneaxiy 已提交
755 756 757
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
758
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
759

760
  m.def("_set_fuse_parameter_group_size",
761
        &paddle::framework::ir::SetFuseParameterGroupsSize);
762
  m.def("_set_fuse_parameter_memory_size",
763
        &paddle::framework::ir::SetFuseParameterMemorySize);
764

S
sneaxiy 已提交
765 766 767
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

768 769
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

770 771 772
  m.def("_promote_types_if_complex_exists",
        &paddle::framework::PromoteTypesIfComplexExists);

773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823
  py::class_<paddle::CustomOpKernelContext> custom_op_kernel_ctx(
      m, "CustomOpKernelContext", R"DOC()DOC");
  g_custom_op_kernel_ctx_pytype =
      reinterpret_cast<PyTypeObject *>(custom_op_kernel_ctx.ptr());
  custom_op_kernel_ctx.def(py::init<>())
      .def("add_inputs",
           [](paddle::CustomOpKernelContext &self, const py::handle &input) {
             PyObject *obj = input.ptr();
             if (PyList_Check(obj) || PyTuple_Check(obj)) {
               self.EmplaceBackInputs(
                   std::move(CastPyArg2VectorOfTensor(obj, 1)));
             } else {
               self.EmplaceBackInput(std::move(CastPyArg2Tensor(obj, 1)));
             }
           })
      .def("add_outputs",
           [](paddle::CustomOpKernelContext &self, py::handle &outputs) {
             PyObject *obj = outputs.ptr();
             if (PyList_Check(obj) || PyTuple_Check(obj)) {
               self.EmplaceBackOutputs(
                   std::move(CastPyArg2VectorOfTensor(obj, 1)));
             } else {
               self.EmplaceBackOutput(std::move(CastPyArg2Tensor(obj, 1)));
             }
           })
      .def("add_attr", [](paddle::CustomOpKernelContext &self,
                          bool attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr", [](paddle::CustomOpKernelContext &self,
                          int attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr", [](paddle::CustomOpKernelContext &self,
                          float attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr", [](paddle::CustomOpKernelContext &self,
                          int64_t attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr",
           [](paddle::CustomOpKernelContext &self, const std::string &attr) {
             self.EmplaceBackAttr(attr);
           })
      .def("add_attr",
           [](paddle::CustomOpKernelContext &self,
              const std::vector<int> &attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr",
           [](paddle::CustomOpKernelContext &self,
              const std::vector<float> &attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr",
           [](paddle::CustomOpKernelContext &self,
              const std::vector<int64_t> &attr) { self.EmplaceBackAttr(attr); })
      .def("add_attr", [](paddle::CustomOpKernelContext &self,
                          const std::vector<std::string> &attr) {
        self.EmplaceBackAttr(attr);
      });

824 825 826 827 828
  py::class_<framework::Tensor> framework_tensor(m, "Tensor",
                                                 py::buffer_protocol());
  g_framework_tensor_pytype =
      reinterpret_cast<PyTypeObject *>(framework_tensor.ptr());
  framework_tensor
829 830
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
831 832 833 834
      .def("_ptr",
           [](const framework::Tensor &self) {
             return reinterpret_cast<uintptr_t>(self.data());
           })
J
Jiabin Yang 已提交
835 836
      .def("_slice", &framework::Tensor::Slice)
      .def("_numel", &framework::Tensor::numel)
S
sneaxiy 已提交
837
      .def("_is_initialized",
838
           [](const framework::Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
839
      .def("_get_dims",
840
           [](const framework::Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
841
      .def("_set_dims",
842
           [](framework::Tensor &self, const std::vector<int64_t> &dim) {
843
             self.Resize(phi::make_ddim(dim));
Y
Yu Yang 已提交
844
           })
Y
yuyang18 已提交
845
      .def("_set_layout",
846
           [](framework::Tensor &self, const std::string &layout) {
D
dzhwinter 已提交
847 848
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
849
      .def("_alloc_float",
850
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
851
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
852
           })
853
      .def("_alloc_float",
854
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
855 856
             self.mutable_data<float>(place);
           })
Y
yuyang18 已提交
857
      .def("_alloc_float",
858
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
859
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
860
           })
861 862 863 864
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place) {
             self.mutable_data<float>(place);
           })
865 866 867 868
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place) {
             self.mutable_data<float>(place);
           })
869
      .def("_alloc_double",
870
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
871 872
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
873
      .def("_alloc_int",
874
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
875
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
876
           })
877
      .def("_alloc_int",
878
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
879 880
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
881
      .def("_alloc_int",
882
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
883
             self.mutable_data<int>(place);
Q
qijun 已提交
884
           })
885 886 887 888
      .def("_alloc_int",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
889
      .def("_alloc_int",
890 891
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
892 893
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
894
      .def("_alloc_float",
895 896
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
897 898
             self.mutable_data<float>(place);
           })
899
      .def("_mutable_data",
900
           [](framework::Tensor &self, paddle::platform::CPUPlace &place,
901
              paddle::framework::proto::VarType::Type type) {
902 903
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
904
           })
905
      .def("_mutable_data",
906
           [](framework::Tensor &self, paddle::platform::XPUPlace &place,
907
              paddle::framework::proto::VarType::Type type) {
908 909
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
910
           })
911
      .def("_mutable_data",
912
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place,
913
              paddle::framework::proto::VarType::Type type) {
914 915
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
916 917
           })
      .def("_mutable_data",
918
           [](framework::Tensor &self, paddle::platform::CUDAPinnedPlace &place,
919
              paddle::framework::proto::VarType::Type type) {
920 921
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
922
           })
923 924 925
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::MLUPlace &place,
              paddle::framework::proto::VarType::Type type) {
926 927
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
928
           })
929
      .def("_clear", &framework::Tensor::clear)
930 931 932
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
933 934
             return reinterpret_cast<uintptr_t>(
                 self.mutable_data(place, framework::TransToPhiDataType(type)));
935
           })
Z
Zeng Jinle 已提交
936 937 938 939 940 941 942 943 944 945
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::XPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CUDAPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::NPUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
      .def("_copy_from", &TensorCopyFrom<paddle::platform::CUDAPinnedPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
946 947
      .def("_copy_from", &TensorCopyFrom<paddle::platform::MLUPlace>,
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
Z
Zeng Jinle 已提交
948
      .def("_copy_from", &TensorCopyFrom<paddle::platform::Place>,
949
           py::arg("tensor"), py::arg("place"), py::arg("batch_size") = -1)
950
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
951
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
952 953
      .def("set", SetTensorFromPyArray<paddle::platform::XPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
954
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
955
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
956 957
      .def("set", SetTensorFromPyArray<paddle::platform::NPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
J
jianghaicheng 已提交
958 959
      .def("set", SetTensorFromPyArray<paddle::platform::IPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
960 961
      .def("set", SetTensorFromPyArray<paddle::platform::MLUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
962
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
963 964
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
965
        Set the data of Tensor on place with given numpy array.
L
Leo Chen 已提交
966 967 968
        
        Args:
          lod (numpy.ndarray): The data to set.
969
          place (CPUPlace|CUDAPlace|XPUPlace|IPUPlace|CUDAPinnedPlace|NPUPlace|MLUPlace): The place where the
970
          Tensor is to be set.
971 972
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
973 974 975 976 977 978 979 980 981 982

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

983
                t = fluid.Tensor()
L
Leo Chen 已提交
984 985
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
986

987 988 989
      .def("shape",
           [](framework::Tensor &self) { return vectorize(self.dims()); },
           R"DOC(
990
           Return the shape of Tensor.
L
Leo Chen 已提交
991 992

           Returns:
993
               list[int]: The shape of Tensor.
L
Leo Chen 已提交
994 995 996 997 998 999 1000 1001


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

1002
                  t = fluid.Tensor()
L
Leo Chen 已提交
1003 1004 1005
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
1006
      .def("_to_dlpack",
1007
           [](framework::Tensor &self) {
6
633WHU 已提交
1008
             DLPackTensor dlpack_tensor(self, 1);
S
Siming Dai 已提交
1009
             DLManagedTensor *dmt = dlpack_tensor.ToDLManagedTensor();
6
633WHU 已提交
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
1027 1028 1029 1030
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
1031
      .def("_place", [](framework::Tensor &self) { return self.place(); })
1032 1033 1034 1035
      .def("_dtype",
           [](framework::Tensor &self) {
             return framework::TransToProtoVarType(self.type());
           })
1036
      .def("_layout",
1037 1038 1039 1040
           [](framework::Tensor &self) {
             return DataLayoutToString(self.layout());
           })
      .def("_share_data_with", &framework::Tensor::ShareDataWith)
1041
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
      .def("__str__",
           [](const framework::Tensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           }) /* ------ End of original Tensor ------ */
      .def(
          "__init__",
          [](framework::Tensor &instance, const std::vector<std::vector<size_t>>
                                              &recursive_sequence_lengths) {
            LoD new_lod;
            new_lod.reserve(recursive_sequence_lengths.size());
            std::copy(recursive_sequence_lengths.begin(),
                      recursive_sequence_lengths.end(),
                      std::back_inserter(new_lod));
            LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
            PADDLE_ENFORCE_EQ(
                CheckLoD(new_offset_lod, -1), true,
                platform::errors::InvalidArgument(
1061 1062
                    "The provided recursive_sequence_lengths info is "
                    "invalid, "
1063 1064 1065 1066
                    "the LoD converted by recursive_sequence_lengths is %s",
                    new_lod));
            new (&instance) framework::Tensor(new_offset_lod);
          })
1067
      .def("__init__",
1068 1069
           [](framework::Tensor &instance) {
             new (&instance) framework::Tensor();
1070
           })
G
gongweibao 已提交
1071
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
1072 1073
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
1074 1075 1076
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
1077
      .def("set_lod",
1078 1079
           [](framework::Tensor &self,
              const std::vector<std::vector<size_t>> &lod) {
1080
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
1081
             LoD new_lod;
1082 1083
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
1084 1085
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
1086 1087
                 platform::errors::InvalidArgument(
                     "The provided LoD is invalid, the LoD is %s", new_lod));
1088
             self.set_lod(new_lod);
S
sneaxiy 已提交
1089 1090
           },
           py::arg("lod"), R"DOC(
1091
           Set LoD of the Tensor.
S
sneaxiy 已提交
1092 1093

           Args:
L
Leo Chen 已提交
1094 1095 1096 1097
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
1098 1099 1100 1101 1102 1103 1104

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1105
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1106 1107
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
1108
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1109
           )DOC")
1110
      .def("set_recursive_sequence_lengths",
1111 1112
           [](framework::Tensor &self, const std::vector<std::vector<size_t>>
                                           &recursive_sequence_lengths) {
1113 1114 1115 1116 1117 1118 1119 1120
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
1121 1122
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
1123
                 platform::errors::InvalidArgument(
1124 1125
                     "The provided recursive_sequence_lengths info is "
                     "invalid, "
1126 1127 1128
                     "the LoD converted by recursive_sequence_lengths is "
                     "%s",
                     new_lod));
1129
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
1130 1131
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
1132
           Set LoD of the Tensor according to recursive sequence lengths.
S
sneaxiy 已提交
1133

L
Leo Chen 已提交
1134
           For example, if recursive_sequence_lengths=[[2, 3]], which means
1135
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
1136
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
1137 1138

           Args:
L
Leo Chen 已提交
1139 1140 1141 1142
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
1143 1144 1145 1146 1147 1148 1149

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1150
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1151 1152
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
1153
                 print(t.recursive_sequence_lengths())  # [[2, 3]]
L
Leo Chen 已提交
1154
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
1155
           )DOC")
1156
      .def("lod",
1157
           [](framework::Tensor &self) -> std::vector<std::vector<size_t>> {
1158 1159 1160 1161 1162 1163
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1164 1165
           },
           R"DOC(
1166
           Return the LoD of the Tensor.
S
sneaxiy 已提交
1167 1168

           Returns:
1169
               list[list[int]]: The lod of the Tensor.
L
Leo Chen 已提交
1170
           
Z
Zeng Jinle 已提交
1171 1172 1173 1174 1175 1176
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1177
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1178 1179 1180
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1181
           )DOC")
G
gongweibao 已提交
1182
      // Set above comments of set_lod.
1183
      .def("recursive_sequence_lengths",
1184
           [](framework::Tensor &self) -> std::vector<std::vector<size_t>> {
1185
             // output the length-based lod info
1186
             LoD lod = phi::ConvertToLengthBasedLoD(self.lod());
1187 1188 1189 1190
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1191 1192
           },
           R"DOC(
L
Leo Chen 已提交
1193
           Return the recursive sequence lengths corresponding to of the LodD 
1194
           of the Tensor.
S
sneaxiy 已提交
1195 1196

           Returns:
L
Leo Chen 已提交
1197
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
1198 1199 1200 1201 1202 1203 1204

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1205
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1206 1207 1208
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
1209 1210
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
1211
           [](framework::Tensor &self) -> bool {
S
sneaxiy 已提交
1212
             // Check that the lod info is valid and match the outermost
1213
             // dimension of the Tensor data
S
sneaxiy 已提交
1214 1215 1216
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
1217
           Check whether the LoD of the Tensor is valid.
S
sneaxiy 已提交
1218 1219

           Returns:
L
Leo Chen 已提交
1220
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
1221 1222 1223 1224 1225 1226 1227

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

1228
                 t = fluid.Tensor()
Z
Zeng Jinle 已提交
1229 1230 1231
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
1232
           )DOC")
L
Leo Chen 已提交
1233
      .def("_as_type",
1234
           [](const framework::Tensor &self,
L
Leo Chen 已提交
1235
              paddle::framework::proto::VarType::Type type) {
1236
             framework::Tensor dst;
L
Leo Chen 已提交
1237 1238 1239 1240 1241
             if (self.IsInitialized() && self.numel() > 0) {
               TransDataType(self, type, &dst);
             }
             return dst;
           })
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
      .def("_copy",
           [](const framework::Tensor &self, const platform::Place &place) {
             // follow fetch_op's inplementation
             framework::Tensor dst;
             if (self.IsInitialized() && self.numel() > 0) {
               TensorCopySync(self, place, &dst);
             } else {
               // Not copy, if the src tensor is empty.
               dst.clear();
               dst.Resize({0});
             }
             dst.set_lod(self.lod());
             return dst;
1255
#ifdef _WIN32
1256
           });
1257 1258
#else
           })
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
#ifdef PADDLE_WITH_CUDA
      .def("_share_buffer_with",
           [](framework::Tensor &self, const framework::Tensor src,
              py::tuple t) {
             auto *cuda_ipc_allocation =
                 dynamic_cast<memory::allocation::CudaIpcAllocation *>(
                     src.Holder().get());

             PADDLE_ENFORCE_NOT_NULL(
                 cuda_ipc_allocation,
                 platform::errors::PreconditionNotMet(
                     "Tensor is not Cuda IPC shared tensor. "
                     "Now only Tensor shared by cuda ipc could use this "
                     "api."));

             size_t size = t[0].cast<size_t>();
             auto dtype =
                 static_cast<paddle::experimental::DataType>(t[1].cast<int>());
             auto dims = phi::make_ddim(t[2].cast<std::vector<int>>());
             auto lod_info = t[3].cast<framework::LoD>();
             auto device_id = t[4].cast<int>();

             auto shared_reader_holder =
                 std::make_shared<memory::allocation::Allocation>(
                     cuda_ipc_allocation->ptr(),
                     cuda_ipc_allocation->base_ptr(), size,
                     platform::CUDAPlace(device_id));

             self.ResetHolderWithType(shared_reader_holder, dtype);
             self.Resize(dims);
             self.set_lod(lod_info);

             VLOG(6) << "Reconstructed tensor with buffer shared!";
           },
           R"DOC(
           Deserialize GPU Tensor for existed shared Cuda IPC tensor.

           Params:
               tensor: Shared Cuda IPC tensor.
               tuple: contrains data size, data type,
                      tensor dims, lod information, device index.

       )DOC")
      .def("_share_cuda",
           [](framework::Tensor self) {
             if (!self.IsInitialized() || self.numel() == 0)
               throw std::runtime_error(
                   "Tensor not initialized or numel is 0.  could not pass "
                   "to shared memory. ");

             auto *holder = dynamic_cast<memory::allocation::Allocation *>(
                 self.Holder().get());
             PADDLE_ENFORCE_EQ(
                 platform::is_gpu_place(holder->place()), true,
                 platform::errors::InvalidArgument(
                     "Tensor is not on GPU. share_cuda only support GPU "
                     "Tensor, share_filename is for CPU tensor."));

             void *base_ptr = holder->base_ptr();
             ptrdiff_t offset_bytes = reinterpret_cast<char *>(holder->ptr()) -
                                      reinterpret_cast<char *>(base_ptr);

             cudaIpcMemHandle_t handle;
             PADDLE_ENFORCE_GPU_SUCCESS(cudaIpcGetMemHandle(&handle, base_ptr));

             auto _handle = py::bytes(reinterpret_cast<char *>(&handle),
                                      (py::ssize_t)CUDA_IPC_HANDLE_SIZE);

             // TODO(ZHUI): use cuda event, to avoid sync.
             const auto &device_id = paddle::platform::GetCurrentDeviceId();
             auto stream =
                 paddle::platform::stream::get_current_stream(device_id);
             stream->Synchronize();

             int type_idx = static_cast<int>(self.type());
             size_t data_size =
                 self.numel() *
                 framework::SizeOfType(
                     framework::TransToProtoVarType(self.type()));

             return py::make_tuple(_handle, (py::size_t)offset_bytes, data_size,
                                   type_idx, vectorize(self.dims()), self.lod(),
                                   device_id);
           },
           R"DOC(
           Serialize GPU Tensor by cudaIpcMemHandle.

           Returns:
               tuple: contrains handle, data size, data type,
                      tensor dims, lod information, device index.

           Examples:
               .. code-block:: python

                 import paddle
                 tensor = paddle.ones([3,3])
                 metainfo = tensor.value().get_tensor()._share_cuda()

      )DOC")
      .def("_new_shared_cuda",
           [](py::tuple t) {
             if (t.size() != 7)
               throw std::runtime_error(
                   "Invalid Tensor meta info for shared cuda tensor!");

             // 1. Create a new C++ instance
             framework::Tensor tensor;

             // 2. Rebuild Allocation from handle
             const std::string &handle = t[0].cast<std::string>();
             ptrdiff_t offset_bytes = (ptrdiff_t)t[1].cast<int64_t>();
             auto device_id = t[6].cast<int>();
             auto base_ptr = memory::allocation::GetIpcBasePtr(handle);
             size_t size = t[2].cast<size_t>();
             void *dev = base_ptr.get();
             dev = reinterpret_cast<char *>(dev) + offset_bytes;

             auto shared_reader_holder =
                 std::make_shared<memory::allocation::CudaIpcAllocation>(
                     dev, size, device_id, std::move(base_ptr));

             // 3. Rebuild Tensor
             tensor.ResetHolderWithType(
                 shared_reader_holder,
                 static_cast<paddle::experimental::DataType>(t[3].cast<int>()));
             tensor.Resize(phi::make_ddim(t[4].cast<std::vector<int>>()));
             tensor.set_lod(t[5].cast<framework::LoD>());

             return tensor;
           },
           R"DOC(
           Deserialize GPU lod tensor from cudaIpcMemHandle.

           Params:
               tuple: contrains handle, data size, data type,
                      tensor dims, lod information, device index.

           Examples:
               .. code-block:: python

                 import paddle
                 tensor = paddle.ones([3,3])
                 metainfo = tensor.value().get_tensor()._share_cuda()
                 tensor_from_shared = paddle.to_tensor(paddle.fluid.core.LoDTensor._new_shared_cuda(metainfo))

        )DOC")
#endif
      .def("_share_filename",
           [](framework::Tensor &self) {
             if (!self.IsInitialized() || self.numel() == 0)
               throw std::runtime_error(
                   "Tensor not initialized or numel is 0. could not pass to "
                   "shared memory. ");

             auto holder = self.Holder();
             PADDLE_ENFORCE_EQ(
                 platform::is_cpu_place(holder->place()) ||
                     platform::is_cuda_pinned_place(holder->place()),
                 true, platform::errors::InvalidArgument(
                           "Tensor is not on CPU. share_filename only "
                           "support CPU Tensor."));

             auto *mmap_allocation = dynamic_cast<
                 memory::allocation::RefcountedMemoryMapAllocation *>(
                 holder.get());
             // If the tensor is not shared, allocate memory map allocation.
             if (mmap_allocation == nullptr) {
               void *data_ptr = self.data();
               size_t data_size =
                   self.numel() *
                   framework::SizeOfType(
                       framework::TransToProtoVarType(self.type()));

               int flags = memory::allocation::MAPPED_SHAREDMEM |
                           memory::allocation::MAPPED_EXCLUSIVE;
               std::string handle = memory::allocation::GetIPCName();
               auto shared_holder =
                   memory::allocation::AllocateRefcountedMemoryMapAllocation(
                       handle, flags, data_size);

               // copy data & reset holder
               if (platform::is_cuda_pinned_place(holder->place())) {
#ifdef PADDLE_WITH_CUDA
                 memory::Copy(platform::CPUPlace(), shared_holder->ptr(),
                              platform::CUDAPinnedPlace(), data_ptr, data_size);
#endif
               } else {
                 memory::Copy(platform::CPUPlace(), shared_holder->ptr(),
                              platform::CPUPlace(), data_ptr, data_size);
               }
               self.ResetHolder(shared_holder);
               mmap_allocation = shared_holder.get();
             }
             int type_idx = static_cast<int>(self.type());

             return py::make_tuple(mmap_allocation->ipc_name(),
                                   mmap_allocation->size(), type_idx,
                                   vectorize(self.dims()), self.lod());
           },
           R"DOC(
           Serialize CPU lod tensor in shared memory to tuple.
           If the tensor is not in shared memory, we will copy it first.

           Returns:
               tuple: contrains ipc name, data size, data type,
                      tensor dims and lod imformation.

           Examples:
               .. code-block:: python

                 import paddle
                 tensor = paddle.ones([3,3])
                 metainfo = tensor.value().get_tensor()._share_filename()

       )DOC")
      .def("_new_shared_filename",
           [](py::tuple t) {  // __setstate__
             if (t.size() != 5)
               throw std::runtime_error("Invalid Tensor meta info state!");

             framework::Tensor tensor;

             // 2. Rebuild Allocation
             const std::string &ipc_name = t[0].cast<std::string>();
             size_t size = t[1].cast<size_t>();
             int flags = memory::allocation::MAPPED_SHAREDMEM |
                         memory::allocation::MAPPED_NOCREATE;

             auto shared_holder =
                 memory::allocation::AllocateRefcountedMemoryMapAllocation(
                     ipc_name, flags, size);

             // 3. Rebuild Tensor
             tensor.ResetHolderWithType(
                 shared_holder,
                 static_cast<paddle::experimental::DataType>(t[2].cast<int>()));
             tensor.Resize(phi::make_ddim(t[3].cast<std::vector<int>>()));
             tensor.set_lod(t[4].cast<framework::LoD>());

             return tensor;
           },
           R"DOC(
           Deserialize CPU lod tensor from shared memory.

           Params:
               tuple: contrains ipc file name, data size, data type,
                      tensor dims and lod information.

           Examples:
               .. code-block:: python

                 import paddle
                 tensor = paddle.ones([3,3])
                 metainfo = tensor.value().get_tensor()._share_filename()
                 tensor_from_shared = paddle.to_tensor(paddle.fluid.core.LoDTensor._new_shared_filename(metainfo))

        )DOC")
      .def("_shared_incref",
           [](framework::Tensor &self) {
             auto *mmap_allocation = dynamic_cast<
                 memory::allocation::RefcountedMemoryMapAllocation *>(
                 self.Holder().get());
             if (mmap_allocation) {
               mmap_allocation->incref();
             }
           },
           R"DOC(
            Increase reference count of share_filename tensor.
      )DOC")
      .def("_shared_decref",
           [](framework::Tensor &self) {
             auto *mmap_allocation = dynamic_cast<
                 memory::allocation::RefcountedMemoryMapAllocation *>(
                 self.Holder().get());
             if (mmap_allocation) {
               mmap_allocation->decref();
             }
           },
           R"DOC(
            Decrease reference count of share_filename tensor.
      )DOC")
1540
      .def(py::pickle(
1541
          [](const framework::Tensor &t) {  // __getstate__
1542
            auto holder = t.Holder();
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
            PADDLE_ENFORCE_EQ(platform::is_cpu_place(holder->place()), true,
                              platform::errors::PreconditionNotMet(
                                  "Tensor is not on CPU."
                                  "Now only Tensor on CPU can be serialized."));
            auto *mmap_writer_allocation =
                dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
                    holder.get());
            PADDLE_ENFORCE_NOT_NULL(
                mmap_writer_allocation,
                platform::errors::PreconditionNotMet(
                    "Tensor is not in shared memory."
                    "Now only Tensor on shared memory can be serialized."));
1555 1556 1557
            int type_idx = static_cast<int>(t.type());

            return py::make_tuple(mmap_writer_allocation->ipc_name(),
1558 1559
                                  mmap_writer_allocation->size(), type_idx,
                                  vectorize(t.dims()), t.lod());
1560 1561 1562
          },
          [](py::tuple t) {  // __setstate__
            if (t.size() != 5)
1563
              throw std::runtime_error("Invalid Tensor state!");
1564 1565

            // 1. Create a new C++ instance
1566
            framework::Tensor tensor;
1567 1568 1569 1570 1571

            // 2. Rebuild Allocation
            const std::string &ipc_name = t[0].cast<std::string>();
            size_t size = t[1].cast<size_t>();
            auto shared_reader_holder =
1572 1573
                memory::allocation::RebuildMemoryMapReaderAllocation(ipc_name,
                                                                     size);
1574 1575

            // 3. Maintain global fd set
1576
            VLOG(3) << "Tensor ipc name: " << ipc_name;
1577 1578
            memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);

1579 1580 1581
            // 4. Rebuild Tensor
            tensor.ResetHolderWithType(
                shared_reader_holder,
1582
                static_cast<paddle::experimental::DataType>(t[2].cast<int>()));
1583
            tensor.Resize(phi::make_ddim(t[3].cast<std::vector<int>>()));
1584 1585 1586 1587 1588
            tensor.set_lod(t[4].cast<framework::LoD>());

            return tensor;
          }));
#endif
D
dangqingqing 已提交
1589

1590
  py::class_<phi::SelectedRows>(m, "SelectedRows")
Q
qijun 已提交
1591
      .def("__init__",
1592 1593
           [](phi::SelectedRows &instance) {
             new (&instance) phi::SelectedRows();
1594
           })
Q
qijun 已提交
1595
      .def("__init__",
1596
           [](phi::SelectedRows &instance, const std::vector<int64_t> rows,
Q
qijun 已提交
1597
              const int64_t &height) {
1598
             new (&instance) phi::SelectedRows(rows, height);
Q
qijun 已提交
1599 1600
           })
      .def("get_tensor",
1601
           [](phi::SelectedRows &self) { return self.mutable_value(); },
Q
qijun 已提交
1602
           py::return_value_policy::reference)
1603
      .def("numel",
1604
           [](phi::SelectedRows &self) -> int64_t {
1605 1606
             return self.value().numel();
           })
1607 1608
      .def("set_height", &phi::SelectedRows::set_height)
      .def("height", &phi::SelectedRows::height)
Q
qijun 已提交
1609
      .def("set_rows",
1610
           [](phi::SelectedRows &self, std::vector<int64_t> rows) {
1611
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
Q
qijun 已提交
1612 1613 1614 1615 1616 1617
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
1618
      .def("sync_index",
1619 1620
           [](phi::SelectedRows &instance) { instance.SyncIndex(); })
      .def("rows", [](phi::SelectedRows &self) {
1621 1622 1623 1624 1625
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
1626
      });
Q
qijun 已提交
1627

1628
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
1629 1630 1631

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
1632
      .def(py::init<>())
1633
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
1634
      .def("set_int",
1635 1636
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
1637 1638 1639 1640 1641 1642 1643
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
1644
      .def("get_tensor",
1645 1646
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
1647 1648
           },
           py::return_value_policy::reference)
1649 1650 1651 1652
      .def("get_bytes",
           [](Variable &self) {
             return py::bytes(*self.GetMutable<std::string>());
           })
S
Steffy-zxf 已提交
1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664
      .def("set_string_list",
           [](Variable &self, Strings str_list) {
             *self.GetMutable<Strings>() = str_list;
           })
      .def("set_vocab", [](Variable &self,
                           Vocab vocab) { *self.GetMutable<Vocab>() = vocab; })
      .def("get_string_tensor",
           [](Variable &self) { return self.GetMutable<Strings>(); },
           py::return_value_policy::reference)
      .def("get_map_tensor",
           [](Variable &self) { return self.GetMutable<Vocab>(); },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1665 1666 1667
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
1668
      .def("get_selected_rows",
1669 1670
           [](Variable &self) -> phi::SelectedRows * {
             return self.GetMutable<phi::SelectedRows>();
Q
qijun 已提交
1671 1672
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1673 1674 1675
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
1676 1677 1678
      .def("get_fetch_list",
           [](Variable &self) { return self.GetMutable<FetchList>(); },
           py::return_value_policy::reference)
1679
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1680 1681 1682 1683 1684
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1685
#endif
Y
Refine  
Yu Yang 已提交
1686 1687
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
1688 1689 1690 1691
             PADDLE_ENFORCE_EQ(
                 self.IsType<framework::ReaderHolder>(), true,
                 platform::errors::InvalidArgument(
                     "The variable is not type of ReaderHolder."));
Y
Refine  
Yu Yang 已提交
1692 1693
             return self.GetMutable<framework::ReaderHolder>();
           },
1694
           py::return_value_policy::reference)
1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
      .def("get_scope",
           [](Variable &self) -> Scope * {
             auto scope_vec =
                 self.GetMutable<std::vector<framework::Scope *>>();
             PADDLE_ENFORCE_GT(
                 scope_vec->size(), 0,
                 platform::errors::InvalidArgument(
                     "The size of scope_vec should be greater than 0"));
             return scope_vec->front();
           },
           py::return_value_policy::reference)
1706 1707 1708 1709
      .def("set_scope", [](Variable &self, Scope &scope) {
        auto scope_vec = self.GetMutable<std::vector<framework::Scope *>>();
        scope_vec->emplace_back(&scope);
      });
1710

S
sneaxiy 已提交
1711
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
1712

0
0x45f 已提交
1713
  py::class_<Scope> _Scope(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

1727
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
1728 1729 1730 1731 1732
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

0
0x45f 已提交
1733 1734 1735
        )DOC");
  g_framework_scope_pytype = reinterpret_cast<PyTypeObject *>(_Scope.ptr());
  _Scope
S
sneaxiy 已提交
1736 1737
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
1738
      .def("var",
1739
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
1740
             return self.Var(name);
Y
Yu Yang 已提交
1741
           },
S
sneaxiy 已提交
1742 1743
           py::arg("name"),
           R"DOC(
1744
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
1745

1746
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
1747
           current scope, the variable would be created. Otherwise,
1748
           return the existing variable.
S
sneaxiy 已提交
1749 1750

           Args:
1751 1752
               name (str): the variable name.

S
sneaxiy 已提交
1753
           Returns:
1754
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
1755 1756 1757 1758
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
1759
           Find variable named :code:`name` in the current scope or
1760
           its parent scope. Return None if not found. 
1761

S
sneaxiy 已提交
1762 1763
           Args:
               name (str): the variable name.
1764

S
sneaxiy 已提交
1765
           Returns:
1766
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1767
           )DOC",
1768
           py::return_value_policy::reference)
1769
      .def("size", &Scope::Size)
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
      .def("erase", &Scope::EraseVars, py::arg("names"),
           R"DOC(
           Find variable named :code:`name` in the current scope or
           its parent scope. Return None if not found. 

           Args:
               name (str): the variable names to be erase.

           Returns:
               None
           )DOC",
           py::return_value_policy::reference)
1782
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1783 1784 1785 1786 1787 1788
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1789
           py::return_value_policy::reference)
S
sneaxiy 已提交
1790 1791 1792
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1793 1794
           )DOC")
      .def("_kids", &Scope::kids);
1795

S
sneaxiy 已提交
1796 1797 1798 1799 1800 1801
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1802 1803
        R"DOC(
        Create a new scope.
1804

S
sneaxiy 已提交
1805 1806 1807
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1808 1809
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1810 1811
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1812 1813
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1814 1815 1816 1817
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1818 1819
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1820 1821
            platform::errors::Fatal(
                "Serialize OpProto Error. This could be a bug of Paddle."));
1822 1823 1824
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1825 1826
    return ret_values;
  });
1827 1828 1829 1830 1831 1832 1833 1834
  m.def("get_op_attrs_default_value",
        [](py::bytes byte_name) -> paddle::framework::AttributeMap {
          std::string op_type = byte_name;
          paddle::framework::AttributeMap res;
          auto info = OpInfoMap::Instance().GetNullable(op_type);
          if (info != nullptr) {
            if (info->HasOpProtoAndChecker()) {
              auto op_checker = info->Checker();
1835
              res = op_checker->GetDefaultAttrsMap();
1836 1837 1838 1839
            }
          }
          return res;
        });
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1856 1857 1858
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
1859 1860 1861 1862 1863
  m.def("has_non_empty_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance()
        .Get(op_type)
        .HasNonEmptyGradOpMaker();
  });
1864 1865 1866
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });
1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880
  m.def("infer_no_need_buffer_slots",
        [](const std::string op_type, const framework::VariableNameMap &inputs,
           const framework::VariableNameMap &outputs,
           const framework::AttributeMap &attrs) {
          auto infer_func = framework::OpInfoMap::Instance()
                                .Get(op_type)
                                .NoNeedBufferVarsInferer();
          if (infer_func) {
            return infer_func(inputs, outputs, attrs);
          } else {
            std::unordered_set<std::string> empty = {};
            return empty;
          }
        });
Y
Yu Yang 已提交
1881
  m.def("prune", [](const ProgramDesc &origin,
1882
                    const std::set<std::string> &feeded_var_names,
1883
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1884
    ProgramDesc prog_with_targets(origin);
1885

1886
    for (const auto &t : targets) {
1887
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1888
    }
1889
    proto::ProgramDesc pruned_desc;
1890 1891 1892 1893
    auto pruned_origin_block_id_map =
        Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
    return std::make_tuple(ProgramDesc(pruned_desc),
                           pruned_origin_block_id_map);
1894
  });
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911
  m.def("prune_backward",
        [](const framework::ProgramDesc &program) {
          return PruneBackward(program);
        },
        R"DOC(
             Prune the backward part of a program, mostly called in
             program.clone(for_test=True).
              
             Args:
                   program (ProgramDesc): The original program.

             Returns:
                   tuple(ProgramDesc, map<int, int>): The first part is 
                   the pruned program desc, and the second part is a map
                   which contains the id pair of pruned block and corresponding
                   origin block.
           )DOC");
1912 1913 1914 1915
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1916 1917 1918
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1919 1920
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
1921

Q
qijun 已提交
1922
  // clang-format off
Y
Yu Yang 已提交
1923
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1924 1925
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1926
                      -> paddle::platform::DeviceContext* {
W
Wilber 已提交
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
    auto* context = new paddle::platform::CPUDeviceContext();
    context->SetAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetAllocator(place)
        .get());
    context->SetHostAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetAllocator(paddle::platform::CPUPlace())
        .get());
    context->SetZeroAllocator(
      paddle::memory::allocation::AllocatorFacade::Instance()
        .GetZeroAllocator(place)
        .get());
    return context;
Q
qijun 已提交
1941
                  })
1942 1943 1944 1945 1946 1947 1948 1949 1950
      .def_static("create",
                  [](paddle::platform::XPUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_XPU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use XPUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with XPU support."));
#else
W
Wilber 已提交
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
      auto* context = new paddle::platform::XPUDeviceContext(place);
      context->SetAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(place)
          .get());
      context->SetHostAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(paddle::platform::CPUPlace())
          .get());
      context->SetZeroAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetZeroAllocator(place)
          .get());
      return context;
1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
#endif
                  })
        .def_static("create",
                  [](paddle::platform::MLUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_MLU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use MLUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with MLU support."));
#else
                    return new paddle::platform::MLUDeviceContext(place);
1977 1978
#endif
                  })
1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
        .def_static("create",
                    [](paddle::platform::NPUPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_ASCEND_CL
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use NPUPlace in CPU/GPU/XPU version, "
                 "Please recompile or reinstall Paddle with NPU support."));
#else
                return new paddle::platform::NPUDeviceContext(place);
#endif
        })
Q
qijun 已提交
1991
      .def_static("create",
D
dzhwinter 已提交
1992
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1993
                      -> paddle::platform::DeviceContext* {
1994
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1995 1996 1997 1998
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
Q
qijun 已提交
1999
#else
W
Wilber 已提交
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
      auto* context = new paddle::platform::CUDADeviceContext(place);
      context->SetAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(place, context->stream())
          .get());
      context->SetHostAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(paddle::platform::CPUPlace())
          .get());
      context->SetZeroAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
        .GetZeroAllocator(place)
        .get());
W
wanghuancoder 已提交
2013 2014 2015 2016
      context->SetPinnedAllocator(
        paddle::memory::allocation::AllocatorFacade::Instance()
          .GetAllocator(paddle::platform::CUDAPinnedPlace())
          .get());
W
Wilber 已提交
2017 2018
      context->PartialInitWithAllocator();
      return context;
Q
qijun 已提交
2019
#endif
C
chengduoZH 已提交
2020 2021 2022 2023
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
2024
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
2025 2026 2027 2028
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
C
chengduoZH 已提交
2029 2030 2031 2032
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
2033
// clang-format on
2034
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
2035 2036
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
2037 2038 2039
  m.def("get_all_device_type", []() {
    std::vector<std::string> device_types;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
2040
    device_types = phi::DeviceManager::GetAllDeviceTypes();
2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_all_device_type because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_all_device_type, please try to install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return device_types;
  });
  m.def("get_all_custom_device_type", []() {
    std::vector<std::string> device_types;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
2054
    device_types = phi::DeviceManager::GetAllCustomDeviceTypes();
2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_all_custom_device_type because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_all_custom_device_type, please try to "
              "install CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return device_types;
  });
  m.def("get_available_device", [] {
    std::vector<std::string> devices;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
2068
    devices = phi::DeviceManager::GetAllDeviceList();
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_available_device because you have installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_available_device, please try to install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return devices;
  });
  m.def("get_available_custom_device", [] {
    std::vector<std::string> devices;
#ifdef PADDLE_WITH_CUSTOM_DEVICE
2082
    devices = phi::DeviceManager::GetAllCustomDeviceList();
2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
#else
          LOG(WARNING) << string::Sprintf(
              "Cannot use get_available_custom_device because you have "
              "installed"
              "CPU/GPU version PaddlePaddle.\n"
              "If you want to use get_available_custom_device, please try to "
              "install"
              "CustomDevice version "
              "PaddlePaddle by: pip install paddlepaddle-core\n");
#endif
    return devices;
  });
  py::class_<platform::CustomPlace>(m, "CustomPlace",
                                    R"DOC(
    CustomPlace is a descriptor of a device.
    It represents a custom device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python

          import paddle
          fake_cpu_place = paddle.CustomPlace("FakeCPU", 0)
                                             )DOC")
      .def("__init__",
           [](platform::CustomPlace &self, const std::string &device_type,
              int dev_id) {
#ifdef PADDLE_WITH_CUSTOM_DEVICE
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CustomPlace(%s, %d), device id must be 0 "
                   "or "
                   "positive integer",
                   device_type, dev_id);
               std::exit(-1);
             }

2119 2120
             if (LIKELY(phi::DeviceManager::HasDeviceType(device_type) &&
                        phi::DeviceManager::IsCustom(device_type))) {
2121
               int dev_count = static_cast<int>(
2122
                   phi::DeviceManager::GetDeviceCount(device_type));
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
               if (UNLIKELY(dev_id >= dev_count)) {
                 if (dev_count == 0) {
                   LOG(ERROR) << "Cannot use " << device_type
                              << " because there is no " << device_type
                              << " detected on your "
                                 "machine.";
                   std::exit(-1);
                 } else {
                   LOG(ERROR) << string::Sprintf(
                       "Invalid CustomPlace(%s, %d), dev_id must "
                       "inside "
                       "[0, %d), because %s "
                       "number on your machine is %d",
                       device_type, dev_id, dev_count, device_type, dev_count);
                   std::exit(-1);
                 }
               }
               new (&self) platform::CustomPlace(device_type, dev_id);
             } else {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CustomPlace(%s, %d), the device type is "
                   "not registered "
                   "as a custom device.",
                   device_type, dev_id);
               std::exit(-1);
             }
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use CustomDevice because you have installed CPU/GPU"
                 "version PaddlePaddle.\n"
                 "If you want to use CustomDevice, please try to install"
                 "CustomDevice version "
                 "PaddlePaddle by: pip install paddlepaddle-core\n"
                 "If you only have CPU, please change "
                 "CustomPlace(%s, %d) to be CPUPlace().\n",
                 device_type, dev_id);
             std::exit(-1);
#endif
           })
      .def("get_device_id",
           [](const platform::CustomPlace &self) { return self.GetDeviceId(); })
      .def("get_device_type",
           [](const platform::CustomPlace &self) {
             return self.GetDeviceType();
           })
      .def("__repr__", string::to_string<const platform::CustomPlace &>)
      .def("__str__", string::to_string<const platform::CustomPlace &>);
2170
  py::class_<platform::CUDAPlace> cudaplace(m, "CUDAPlace", R"DOC(
2171 2172 2173 2174 2175

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
2176
    The memory of CUDAPlace with different dev_id is not accessible.
2177 2178 2179 2180 2181 2182 2183 2184
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
2185 2186 2187 2188

    Examples:
        .. code-block:: python

2189 2190 2191
          import paddle

          place = paddle.CUDAPlace(0)
L
lujun 已提交
2192

2193 2194 2195
        )DOC");
  g_cudaplace_pytype = reinterpret_cast<PyTypeObject *>(cudaplace.ptr());
  cudaplace
S
sneaxiy 已提交
2196 2197
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
2198
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2199 2200 2201 2202 2203 2204 2205 2206
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

2207 2208
             if (UNLIKELY(dev_id >= platform::GetGPUDeviceCount())) {
               if (platform::GetGPUDeviceCount() == 0) {
2209 2210 2211 2212 2213 2214 2215 2216
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
2217 2218
                     dev_id, platform::GetGPUDeviceCount(),
                     platform::GetGPUDeviceCount());
2219 2220 2221 2222
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
2223 2224
             new (&self) platform::CUDAPlace(dev_id);
#else
2225 2226 2227 2228 2229 2230 2231 2232 2233
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
2234 2235
#endif
           })
2236
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2237 2238
      .def("get_device_id",
           [](const platform::CUDAPlace &self) { return self.GetDeviceId(); })
S
sneaxiy 已提交
2239 2240 2241 2242
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
2243
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
2244
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
2245
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::MLUPlace>)
S
sneaxiy 已提交
2246 2247
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
2248 2249 2250
      .def("_get_device_id",
           [](platform::CUDAPlace &self) -> int { return self.GetDeviceId(); })
#endif
2251
      .def("__repr__", string::to_string<const platform::CUDAPlace &>)
D
dzhwinter 已提交
2252
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
2253

2254
  py::class_<platform::XPUPlace> xpuplace(m, "XPUPlace", R"DOC(
2255 2256 2257 2258 2259
    **Note**:
    Examples:
        .. code-block:: python
          import paddle.fluid as fluid
          xpu_place = fluid.XPUPlace(0)
2260 2261 2262
        )DOC");
  g_xpuplace_pytype = reinterpret_cast<PyTypeObject *>(xpuplace.ptr());
  xpuplace
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300
      .def("__init__",
           [](platform::XPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_XPU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid XPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetXPUDeviceCount())) {
               if (platform::GetXPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use XPU because there is no XPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid XPUPlace(%d), must inside [0, %d), because XPU "
                     "number on your machine is %d",
                     dev_id, platform::GetXPUDeviceCount(),
                     platform::GetXPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::XPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use XPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use XPU, please try to install XPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change XPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
2301
#ifdef PADDLE_WITH_XPU
2302 2303 2304 2305 2306 2307 2308
      .def("_type", &PlaceIndex<platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::XPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::XPUPlace, platform::CUDAPinnedPlace>)
2309 2310 2311
      .def("get_device_id",
           [](const platform::XPUPlace &self) { return self.GetDeviceId(); })
#endif
2312
      .def("__repr__", string::to_string<const platform::XPUPlace &>)
2313
      .def("__str__", string::to_string<const platform::XPUPlace &>);
2314
#ifdef PADDLE_WITH_XPU
2315 2316 2317
  py::enum_<phi::backends::xpu::XPUVersion>(m, "XPUVersion", py::arithmetic())
      .value("XPU1", phi::backends::xpu::XPUVersion::XPU1)
      .value("XPU2", phi::backends::xpu::XPUVersion::XPU2)
T
TTerror 已提交
2318
      .export_values();
2319
  m.def("get_xpu_device_count", platform::GetXPUDeviceCount);
T
TTerror 已提交
2320 2321
  m.def("get_xpu_device_version",
        [](int device_id) { return platform::get_xpu_version(device_id); });
L
Lijunhui 已提交
2322 2323 2324 2325 2326 2327
#ifdef PADDLE_WITH_XPU_KP
  m.def("get_xpu_device_op_support_types",
        [](const std::string &op_name, phi::backends::xpu::XPUVersion version) {
          return platform::get_xpu_kp_op_support_type(op_name, version);
        });
#else
2328 2329 2330 2331
  m.def("get_xpu_device_op_support_types",
        [](const std::string &op_name, phi::backends::xpu::XPUVersion version) {
          return platform::get_xpu_op_support_type(op_name, version);
        });
L
Lijunhui 已提交
2332
#endif
2333
  m.def("get_xpu_device_op_list", [](phi::backends::xpu::XPUVersion version) {
T
TTerror 已提交
2334 2335
    return platform::get_xpu_op_list(version);
  });
T
taixiurong 已提交
2336 2337
  m.def("is_float16_supported", [](const platform::XPUPlace &place) -> bool {
    // XPUs with Compute Capability > xpu2 support float16 and bfloat16
W
Wilber 已提交
2338
    return platform::get_xpu_version(place.device) >
2339
           phi::backends::xpu::XPUVersion::XPU1;
T
taixiurong 已提交
2340 2341 2342
  });
  m.def("is_bfloat16_supported", [](const platform::XPUPlace &place) -> bool {
    // XPUs with Compute Capability > xpu2 support float16 and bfloat16
W
Wilber 已提交
2343
    return platform::get_xpu_version(place.device) >
2344
           phi::backends::xpu::XPUVersion::XPU1;
T
taixiurong 已提交
2345
  });
2346
#endif
2347

2348
  py::class_<paddle::platform::CPUPlace> cpuplace(m, "CPUPlace", R"DOC(
2349
    CPUPlace is a descriptor of a device.
2350
    It represents a CPU device on which a tensor will be allocated and a model will run.
L
lujun 已提交
2351 2352 2353 2354

    Examples:
        .. code-block:: python

2355 2356
          import paddle
          cpu_place = paddle.CPUPlace()
L
lujun 已提交
2357

2358 2359 2360
        )DOC");
  g_cpuplace_pytype = reinterpret_cast<PyTypeObject *>(cpuplace.ptr());
  cpuplace.def(py::init<>())
S
sneaxiy 已提交
2361 2362
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
2363
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
2364
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
S
sneaxiy 已提交
2365 2366 2367 2368
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
2369
      .def("__repr__", string::to_string<const platform::CPUPlace &>)
2370
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
2371

2372 2373
  py::class_<paddle::platform::CUDAPinnedPlace> cudapinnedplace(
      m, "CUDAPinnedPlace", R"DOC(
2374 2375 2376 2377 2378 2379
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
2380 2381 2382 2383

    Examples:
        .. code-block:: python

2384 2385
          import paddle
          place = paddle.CUDAPinnedPlace()
L
lujun 已提交
2386

2387 2388 2389 2390
        )DOC");
  g_cudapinnedplace_pytype =
      reinterpret_cast<PyTypeObject *>(cudapinnedplace.ptr());
  cudapinnedplace
S
sneaxiy 已提交
2391
      .def("__init__",
S
sneaxiy 已提交
2392
           [](platform::CUDAPinnedPlace &self) {
2393
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
2394 2395 2396
             PADDLE_THROW(platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
S
sneaxiy 已提交
2397
#endif
S
sneaxiy 已提交
2398
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
2399
           })
S
sneaxiy 已提交
2400 2401 2402 2403
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
2404 2405
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
2406 2407
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
S
sneaxiy 已提交
2408 2409 2410 2411
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
2412
      .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
C
chengduoZH 已提交
2413 2414
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

2415
  // NPUPlace
2416
  py::class_<platform::NPUPlace> npuplace(m, "NPUPlace", R"DOC(
2417 2418 2419 2420 2421 2422 2423 2424
    NPUPlace is a descriptor of a device.
    It represents a NPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          npu_place = paddle.NPUPlace(0)

2425 2426 2427
        )DOC");
  g_npuplace_pytype = reinterpret_cast<PyTypeObject *>(npuplace.ptr());
  npuplace
2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
      .def("__init__",
           [](platform::NPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_ASCEND_CL
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid NPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetNPUDeviceCount())) {
               if (platform::GetNPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use NPU because there is no NPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid NPUPlace(%d), must inside [0, %d), because NPU "
                     "number on your machine is %d",
                     dev_id, platform::GetNPUDeviceCount(),
                     platform::GetNPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::NPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use NPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use NPU, please try to install NPU version "
2459
                 "PaddlePaddle by: pip install paddlepaddle-npu\n"
2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
                 "If you only have CPU, please change NPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
H
houj04 已提交
2474 2475
      .def("get_device_id",
           [](const platform::NPUPlace &self) { return self.GetDeviceId(); })
2476 2477
      .def("__str__", string::to_string<const platform::NPUPlace &>);

J
jianghaicheng 已提交
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
  // IPUPlace
  py::class_<platform::IPUPlace>(m, "IPUPlace", R"DOC(
    IPUPlace is a descriptor of a device.
    It represents a IPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle

          # required: ipu

          ipu_place = paddle.IPUPlace()

        )DOC")
      .def("__init__",
           [](platform::IPUPlace &self) {
#ifdef PADDLE_WITH_IPU
             if (platform::GetIPUDeviceCount() == 0) {
               LOG(ERROR) << "Cannot use IPU because there is no IPU "
                             "detected on your "
                             "machine.";
               std::exit(-1);
             }
             // use ipu(0) to comile, while run with the number user configure
             // in sharding and pipline.
             new (&self) platform::IPUPlace(0);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use IPU because you didn't install IPU version "
                 "PaddlePaddle.\n"
                 "If you want to use IPU, please try to install IPU version "
                 "PaddlePaddle by: pip install paddlepaddle*\n"
                 "If you only have CPU, please change IPUPlace to be "
                 "CPUPlace().\n");
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::IPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::IPUPlace, platform::IPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::IPUPlace, platform::CUDAPinnedPlace>)
#ifdef PADDLE_WITH_IPU
      .def("get_device_id",
           [](const platform::IPUPlace &self) { return self.GetDeviceId(); })
#endif
      .def("__str__", string::to_string<const platform::IPUPlace &>);

2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
  // MLUPlace
  py::class_<platform::MLUPlace> mluplace(m, "MLUPlace", R"DOC(
    MLUPlace is a descriptor of a device.
    It represents a MLU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          # required: mlu
          mlu_place = paddle.MLUPlace(0)

        )DOC");
  g_mluplace_pytype = reinterpret_cast<PyTypeObject *>(mluplace.ptr());
  mluplace
      .def("__init__",
           [](platform::MLUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_MLU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid MLUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetMLUDeviceCount())) {
               if (platform::GetMLUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use MLU because there is no MLU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid MLUPlace(%d), must inside [0, %d), because MLU "
                     "number on your machine is %d",
                     dev_id, platform::GetMLUDeviceCount(),
                     platform::GetMLUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::MLUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use MLU because you have installed CPU/GPU/... "
                 "version "
                 "PaddlePaddle.\n"
                 "If you want to use MLU, please try to install MLU version "
                 "PaddlePaddle by: pip install paddlepaddle-mlu\n"
                 "If you only have CPU, please change MLUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::MLUPlace>)
#ifdef PADDLE_WITH_MLU
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::IPUPlace>)
      .def("_equals", &IsSamePlace<platform::MLUPlace, platform::MLUPlace>)
      .def("_equals",
           &IsSamePlace<platform::MLUPlace, platform::CUDAPinnedPlace>)
      .def("get_device_id",
           [](const platform::MLUPlace &self) { return self.GetDeviceId(); })
#endif
      .def("__str__", string::to_string<const platform::MLUPlace &>);

2599 2600 2601
  py::class_<platform::Place> platformplace(m, "Place");
  g_place_pytype = reinterpret_cast<PyTypeObject *>(platformplace.ptr());
  platformplace.def(py::init<>())
S
sneaxiy 已提交
2602 2603 2604 2605
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
2606
      .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
2607
      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
J
jianghaicheng 已提交
2608
      .def("_equals", &IsSamePlace<platform::Place, platform::IPUPlace>)
S
sneaxiy 已提交
2609
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
2610
      .def("_equals", &IsSamePlace<platform::Place, platform::MLUPlace>)
X
xuezhong 已提交
2611 2612
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
2613 2614
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
2615 2616
      .def("is_xpu_place",
           [](platform::Place &self) { return platform::is_xpu_place(self); })
2617 2618
      .def("is_npu_place",
           [](platform::Place &self) { return platform::is_npu_place(self); })
J
jianghaicheng 已提交
2619 2620
      .def("is_ipu_place",
           [](platform::Place &self) { return platform::is_ipu_place(self); })
S
sneaxiy 已提交
2621 2622 2623 2624
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
2625 2626
      .def("is_mlu_place",
           [](platform::Place &self) { return platform::is_mlu_place(self); })
2627 2628 2629
      .def(
          "is_custom_place",
          [](platform::Place &self) { return platform::is_custom_place(self); })
2630 2631 2632 2633 2634
      .def("gpu_device_id", [](platform::Place &self) { return self.device; })
      .def("xpu_device_id", [](platform::Place &self) { return self.device; })
      .def("npu_device_id", [](platform::Place &self) { return self.device; })
      .def("ipu_device_id", [](platform::Place &self) { return self.device; })
      .def("mlu_device_id", [](platform::Place &self) { return self.device; })
2635 2636
      .def("custom_device_id",
           [](platform::Place &self) { return self.device; })
S
sneaxiy 已提交
2637 2638
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
2639 2640 2641 2642
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
2643 2644 2645 2646
      .def("set_place",
           [](platform::Place &self, const platform::XPUPlace &xpu_place) {
             self = xpu_place;
           })
Y
Yu Yang 已提交
2647
      .def("set_place",
D
dzhwinter 已提交
2648
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
2649
             self = gpu_place;
C
chengduoZH 已提交
2650
           })
2651 2652 2653 2654 2655
      .def("set_place",
           [](platform::Place &self,
              const platform::CUDAPinnedPlace &cuda_pinned_place) {
             self = cuda_pinned_place;
           })
2656 2657 2658 2659
      .def("set_place",
           [](platform::Place &self, const platform::NPUPlace &npu_place) {
             self = npu_place;
           })
J
jianghaicheng 已提交
2660 2661 2662 2663
      .def("set_place",
           [](platform::Place &self, const platform::IPUPlace &ipu_place) {
             self = ipu_place;
           })
2664 2665 2666 2667
      .def("set_place",
           [](platform::Place &self, const platform::MLUPlace &mlu_place) {
             self = mlu_place;
           })
2668 2669 2670 2671
      .def("set_place",
           [](platform::Place &self, const platform::CustomPlace &plug_place) {
             self = plug_place;
           })
2672 2673
      .def("__repr__", string::to_string<const platform::Place &>)
      .def("__str__", string::to_string<const platform::Place &>);
Y
Yu Yang 已提交
2674

Y
Yu Yang 已提交
2675
  py::class_<OperatorBase>(m, "Operator")
2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689
      .def_static("create",
                  [](py::bytes protobin) {
                    proto::OpDesc desc;
                    PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin),
                                      true,
                                      platform::errors::InvalidArgument(
                                          "Cannot parse user input to OpDesc"));
                    PADDLE_ENFORCE_EQ(desc.IsInitialized(), true,
                                      platform::errors::InvalidArgument(
                                          "The provided OpDesc is not "
                                          "initialized, the reason is: %s",
                                          desc.InitializationErrorString()));
                    return OpRegistry::CreateOp(desc);
                  })
2690
      .def("run",
2691
           [](OperatorBase &self, const Scope &scope,
2692 2693 2694 2695
              const platform::CPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
2696 2697
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2698 2699 2700 2701
              const platform::XPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
2702 2703
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2704 2705 2706 2707
              const platform::NPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
D
dzhwinter 已提交
2708 2709
      .def("run",
           [](OperatorBase &self, const Scope &scope,
2710 2711 2712 2713
              const platform::CUDAPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
C
chengduoZH 已提交
2714 2715 2716
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
2717
             pybind11::gil_scoped_release release;
C
chengduoZH 已提交
2718 2719
             self.Run(scope, place);
           })
2720 2721 2722 2723 2724 2725
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::MLUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
2726 2727 2728 2729 2730 2731 2732
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
2733 2734
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
2735
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
2736
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
2737 2738 2739 2740
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
2741

2742 2743 2744
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

2745 2746 2747 2748 2749 2750 2751
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
2752 2753
      .def("finalize", &TrainerBase::Finalize)
      .def("ResetDataset", &TrainerBase::ResetDataset);
2754

2755 2756
  m.def("_get_eager_deletion_vars", &framework::GetEagerDeletionCleanVars);

F
fengjiayi 已提交
2757
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
2758
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
2759
      .def("close", &Executor::Close)
2760 2761
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
2762 2763
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
2764 2765 2766 2767
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
2768
             pybind11::gil_scoped_release release;
2769 2770 2771 2772 2773 2774 2775
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
2776 2777 2778
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
2779
              std::map<std::string, FetchType *> *fetch_targets,
2780 2781 2782 2783 2784 2785 2786 2787
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
2788
      .def("run_prepared_ctx",
G
guru4elephant 已提交
2789 2790 2791 2792 2793 2794 2795
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
2796 2797 2798 2799 2800 2801 2802 2803 2804 2805
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
2806
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
2807 2808
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
2809
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
2810 2811
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
2812
      });
S
sneaxiy 已提交
2813

2814
  py::class_<framework::interpreter::CostInfo>(m, "CostInfo")
2815
      .def(py::init<>())
2816 2817 2818 2819 2820
      .def("total_time",
           [](interpreter::CostInfo &self) { return self.total_time; })
      .def("device_memory_bytes", [](interpreter::CostInfo &self) {
        return self.device_memory_bytes;
      });
2821

2822
  py::class_<framework::StandaloneExecutor>(m, "StandaloneExecutor")
H
hong 已提交
2823 2824 2825
      .def(py::init<const platform::Place &, const ProgramDesc &,
                    const ProgramDesc &, Scope *>())
      .def("run",
2826
           [](StandaloneExecutor &self,
H
hong 已提交
2827
              const std::unordered_map<std::string, py::array> &input_dict,
2828
              std::vector<std::string> fetch_names) {
2829
             std::vector<framework::LoDTensor> feed_tensors;
2830
             std::vector<std::string> feed_names;
H
hong 已提交
2831 2832 2833 2834 2835

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
2836 2837
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
H
hong 已提交
2838 2839
             }

2840 2841 2842 2843 2844 2845 2846 2847 2848
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
             }
             return py::cast(std::move(ret));
           })
      .def("run",
           [](StandaloneExecutor &self,
2849
              const std::unordered_map<std::string, framework::LoDTensor>
2850 2851
                  &input_dict,
              std::vector<std::string> fetch_names) {
2852
             std::vector<framework::LoDTensor> feed_tensors;
2853 2854 2855 2856 2857 2858 2859
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               feed_names.push_back(item.first);
               feed_tensors.push_back(item.second);
             }

W
wanghuancoder 已提交
2860 2861 2862 2863
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
H
hong 已提交
2864
             }
W
wanghuancoder 已提交
2865
             return py::cast(std::move(ret));
2866
           })
2867 2868 2869
      .def("run",
           [](StandaloneExecutor &self, std::vector<std::string> feed_names,
              std::vector<std::string> fetch_names) {
2870 2871 2872
             platform::RecordEvent record_event(
                 "StandaloneExecutor:run",
                 platform::TracerEventType::UserDefined, 1);
2873 2874 2875 2876 2877 2878 2879
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, fetch_names);
             }
             return py::cast(std::move(ret));
           })
2880 2881 2882
      .def("dry_run",
           [](StandaloneExecutor &self,
              const std::unordered_map<std::string, py::array> &input_dict) {
2883
             std::vector<framework::LoDTensor> feed_tensors;
2884 2885 2886 2887 2888 2889 2890 2891 2892 2893
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
             }

2894
             framework::interpreter::CostInfo cost_info;
2895 2896 2897 2898 2899
             {
               pybind11::gil_scoped_release release;
               cost_info = self.DryRun(feed_names, feed_tensors);
             }
             return cost_info;
H
hong 已提交
2900 2901
           });

D
dzhwinter 已提交
2902
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
2903
  m.def("init_glog", framework::InitGLOG);
2904 2905 2906 2907
  m.def("load_op_meta_info_and_register_op", [](const std::string dso_name) {
    egr::Controller::Instance().MergeOpMetaInfoMap(
        framework::LoadOpMetaInfoAndRegisterOp(dso_name));
  });
2908
  m.def("init_devices", []() { framework::InitDevices(); });
2909
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
2910
  m.def("is_compiled_with_ascend", IsCompiledWithAscend);
2911
  m.def("is_compiled_with_rocm", IsCompiledWithROCM);
2912
  m.def("is_compiled_with_npu", IsCompiledWithNPU);
J
jianghaicheng 已提交
2913
  m.def("is_compiled_with_ipu", IsCompiledWithIPU);
2914
  m.def("is_compiled_with_xpu", IsCompiledWithXPU);
2915
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
2916
  m.def("is_compiled_with_nccl", IsCompiledWithNCCL);
2917
  m.def("is_compiled_with_cinn", IsCompiledWithCINN);
2918
  m.def("is_compiled_with_mlu", IsCompiledWithMLU);
2919
  m.def("_is_compiled_with_heterps", IsCompiledWithHETERPS);
2920
  m.def("supports_bfloat16", SupportsBfloat16);
2921
  m.def("supports_bfloat16_fast_performance", SupportsBfloat16FastPerformance);
2922 2923
  m.def("supports_int8", SupportsInt8);
  m.def("supports_vnni", SupportsVNNI);
L
Leo Chen 已提交
2924
  m.def("op_supported_infos", imperative::OpSupportedInfos);
2925
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
2926
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
2927 2928 2929
  m.def("_cuda_synchronize", [](const platform::CUDAPlace &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });
H
hutuxian 已提交
2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948

  m.def("get_float_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<float>> float_stats;
    paddle::platform::StatRegistry<float>::Instance().publish(float_stats);
    std::unordered_map<std::string, float> stats_map;
    for (const auto &stat : float_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
  m.def("get_int_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<int64_t>> int_stats;
    paddle::platform::StatRegistry<int64_t>::Instance().publish(int_stats);
    std::unordered_map<std::string, int64_t> stats_map;
    for (const auto &stat : int_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
2949 2950
  m.def("memory_stat_get_current", memory::StatGetCurrentValue);
  m.def("memory_stat_get_peak", memory::StatGetPeakValue);
H
hutuxian 已提交
2951 2952 2953 2954 2955 2956 2957
  m.def("run_cmd",
        [](const std::string &cmd, int time_out = -1,
           int sleep_inter = -1) -> const std::string {
          return paddle::framework::shell_get_command_output(cmd, time_out,
                                                             sleep_inter);
        },
        py::arg("cmd"), py::arg("time_out") = -1, py::arg("sleep_inter") = -1);
G
gongweibao 已提交
2958 2959 2960 2961 2962 2963 2964 2965 2966
  m.def("shell_execute_cmd",
        [](const std::string &cmd, int time_out = 0, int sleep_inter = 0,
           bool redirect_stderr = false) -> std::vector<std::string> {
          return paddle::framework::shell_execute_cmd(
              cmd, time_out, sleep_inter, redirect_stderr);
        },
        py::arg("cmd"), py::arg("time_out") = 0, py::arg("sleep_inter") = 0,
        py::arg("redirect_stderr") = false);

2967
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2968 2969
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
2970
    return platform::GetGPUComputeCapability(place.device) >= 53;
2971 2972
  });
#endif
2973

S
Steffy-zxf 已提交
2974 2975 2976 2977 2978 2979
  m.def("set_feed_variable",
        static_cast<void (*)(Scope *, const LoDTensor &, const std::string &,
                             size_t)>(&framework::SetFeedVariable));
  m.def("set_feed_variable",
        static_cast<void (*)(Scope *, const Strings &, const std::string &,
                             size_t)>(&framework::SetFeedVariable));
2980 2981 2982 2983 2984
  m.def("get_fetch_variable",
        [](const Scope &scope, const std::string &var_name,
           size_t index) -> py::object {
          auto &var = framework::GetFetchVariable(scope, var_name, index);
          if (data_is_lod_tensor(var)) {
2985
            return py::cast(BOOST_GET(LoDTensor, var));
2986
          } else {
2987
            return py::cast(BOOST_GET(LoDTensorArray, var));
2988 2989
          }
        });
2990
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
2991

X
Xin Pan 已提交
2992 2993
  m.def("_is_program_version_supported", IsProgramVersionSupported);

2994 2995 2996 2997
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
H
Huihuang Zheng 已提交
2998
  BindCostModel(&m);
2999
  BindConstValue(&m);
3000
  BindGlobalValueGetterSetter(&m);
3001
  BindProcessMeshDesc(&m);
L
LiYuRio 已提交
3002
  BindFleetExecutor(&m);
3003
  BindTCPStore(&m);
Y
Yu Yang 已提交
3004

Y
Yu Yang 已提交
3005 3006 3007 3008 3009 3010 3011 3012 3013
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

3014
  py::class_<LoDTensorArray> pylodtensorarray(m, "LoDTensorArray", R"DOC(
3015
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
3016 3017 3018

    Examples:
        .. code-block:: python
3019

Z
Zeng Jinle 已提交
3020 3021 3022
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
3023 3024 3025 3026
)DOC");
  g_framework_lodtensorarray_pytype =
      reinterpret_cast<PyTypeObject *>(pylodtensorarray.ptr());
  pylodtensorarray
S
sneaxiy 已提交
3027 3028
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
3029 3030 3031 3032 3033 3034
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
3035 3036 3037 3038
             PADDLE_ENFORCE_LT(i, self.size(),
                               platform::errors::InvalidArgument(
                                   "The index to set is larger than the size "
                                   "of LoDTensorArray."));
Y
Yu Yang 已提交
3039 3040 3041
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
3042 3043 3044 3045 3046 3047
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
3048 3049
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
3050 3051 3052 3053 3054 3055
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
3078

3079 3080 3081 3082 3083 3084 3085 3086
  py::class_<FetchList>(m, "FetchList", R"DOC( FetchList is a
        vector of boost::variant<LoDTensor, LoDTensorArray>.
        )DOC")
      .def("_move_to_list",
           [](FetchList &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               if (data_is_lod_tensor(self[i])) {
3087
                 auto &data = BOOST_GET(LoDTensor, self[i]);
3088 3089
                 res[i] = py::cast(std::move(data));
               } else {
3090
                 auto &data = BOOST_GET(LoDTensorArray, self[i]);
3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105
                 py::list tmp(data.size());
                 for (size_t j = 0; j < data.size(); ++j) {
                   tmp[j] = py::cast(std::move(data[j]));
                 }
                 res[i] = std::move(tmp);
               }
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership)

      .def("append",
           [](FetchList &self, const LoDTensor &t) {
             self.emplace_back();
3106
             auto &lod_tensor = BOOST_GET(LoDTensor, self.back());
3107 3108 3109 3110 3111 3112 3113 3114
             lod_tensor.ShareDataWith(t);
             lod_tensor.set_lod(t.lod());
           },
           py::arg("var"))

      .def("append",
           [](FetchList &self, const LoDTensorArray &t) {
             self.emplace_back();
3115
             auto &lod_tensor_array = BOOST_GET(LoDTensorArray, self.back());
3116 3117 3118 3119 3120 3121 3122 3123 3124
             for (size_t i = 0; i < t.size(); ++i) {
               lod_tensor_array[i].ShareDataWith(t[i]);
               lod_tensor_array[i].set_lod(t[i].lod());
             }
           },
           py::arg("var"));

  py::class_<FetchUnmergedList>(m, "FetchUnmergedList", R"DOC(
        FetchUnmergedList is 2-D array of FetchType(boost::variant(LoDTensor, LoDTensorArray)).
Z
Zhen Wang 已提交
3125 3126
        )DOC")
      .def("_move_to_list",
3127
           [](FetchUnmergedList &self) -> py::list {
Z
Zhen Wang 已提交
3128 3129 3130 3131
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               py::list tmp(self[i].size());
               for (size_t j = 0; j < self[i].size(); ++j) {
3132
                 if (data_is_lod_tensor(self[i][j])) {
3133
                   auto &var = BOOST_GET(LoDTensor, self[i][j]);
3134 3135
                   tmp[j] = py::cast(std::move(var));
                 } else {
3136
                   auto &var = BOOST_GET(LoDTensorArray, self[i][j]);
3137 3138 3139 3140 3141 3142
                   py::list tmp_array(var.size());
                   for (size_t k = 0; k < var.size(); ++k) {
                     tmp_array[k] = std::move(var[k]);
                   }
                   tmp[j] = std::move(tmp_array);
                 }
Z
Zhen Wang 已提交
3143 3144 3145 3146 3147 3148 3149 3150 3151
               }
               res[i] = std::move(tmp);
               self[i].clear();
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);

Y
Yu Yang 已提交
3152
  m.def("op_support_gpu", OpSupportGPU);
3153
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
3154
  m.def("get_cuda_device_count", platform::GetGPUDeviceCount);
3155
  m.def("get_cuda_current_device_id", &platform::GetCurrentDeviceId);
3156 3157 3158 3159 3160 3161 3162 3163
  m.def("cuda_empty_cache", [] {
    for (int dev_id : platform::GetSelectedDevices()) {
      auto *dev_ctx = platform::DeviceContextPool::Instance().GetByPlace(
          platform::CUDAPlace(dev_id));
      dev_ctx->cudnn_workspace_handle().ResetWorkspace();
    }
    platform::EmptyCache();
  });
3164 3165 3166 3167 3168 3169 3170
  m.def("get_device_properties",
        [](int id) -> const gpuDeviceProp & {
          return platform::GetDeviceProperties(id);
        },
        py::return_value_policy::copy);

  py::class_<gpuDeviceProp>(m, "_gpuDeviceProperties")
Y
Yanxing Shi 已提交
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195
      .def_property_readonly(
          "name", [](const gpuDeviceProp &prop) { return prop.name; })
      .def_property_readonly(
          "major", [](const gpuDeviceProp &prop) { return prop.major; })
      .def_property_readonly(
          "minor", [](const gpuDeviceProp &prop) { return prop.minor; })
      .def_property_readonly(
          "total_memory",
          [](const gpuDeviceProp &prop) { return prop.totalGlobalMem; })
      .def_property_readonly(
          "multi_processor_count",
          [](const gpuDeviceProp &prop) { return prop.multiProcessorCount; })
      .def_property_readonly(
          "is_multi_gpu_board",
          [](const gpuDeviceProp &prop) { return prop.isMultiGpuBoard; })
      .def_property_readonly(
          "is_integrated",
          [](const gpuDeviceProp &prop) { return prop.integrated; })
      .def("__repr__", [](const gpuDeviceProp &prop) {
        std::stringstream ostr;
        ostr << "_gpuDeviceProperties(name='" << prop.name
             << "', major=" << prop.major << ", minor=" << prop.minor
             << ", total_memory=" << prop.totalGlobalMem / (1024 * 1024)
             << "MB, multi_processor_count=" << prop.multiProcessorCount << ")";
        return ostr.str();
3196
      });
D
dangqingqing 已提交
3197

3198
#if !defined(PADDLE_WITH_HIP) && !defined(_WIN32)
D
dangqingqing 已提交
3199 3200 3201
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
3202 3203 3204 3205
  m.def("nvprof_nvtx_push", platform::CudaNvtxRangePush);
  m.def("nvprof_nvtx_pop", platform::CudaNvtxRangePop);
  m.def("nvprof_enable_record_event", platform::NvprofEnableRecordEvent);
  m.def("nvprof_disable_record_event", platform::NvprofDisableRecordEvent);
D
Dong Zhihong 已提交
3206
#endif
P
peizhilin 已提交
3207
#endif
Y
Yu Yang 已提交
3208

3209 3210
#ifdef PADDLE_WITH_ASCEND_CL
  m.def("get_npu_device_count", platform::GetNPUDeviceCount);
3211
  m.def("npu_finalize", []() {
3212 3213
    platform::HCCLCommContext::Instance().ReleaseHCCLComms();

3214 3215 3216
    auto &pool = platform::DeviceContextPool::Instance();
    auto devices = platform::GetSelectedNPUDevices();
    for (size_t i = 0; i < devices.size(); ++i) {
R
ronnywang 已提交
3217
      platform::NPUDeviceGuard guard(devices[i]);
3218 3219 3220 3221
      pool.Get(platform::NPUPlace(devices[i]))->Wait();
    }
    platform::AclInstance::Instance().Finalize();
  });
3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241

  py::class_<platform::NPUProfConfigWrapper>(m, "NPUProfConfigWrapper");

  m.def("npu_prof_init", platform::NPUProfilerInit);
  m.def("npu_prof_start", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStart(c.ptr());
  });
  m.def("npu_prof_stop", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStop(c.ptr());
  });
  m.def("npu_prof_finalize", platform::NPUProfilerFinalize);
  m.def("npu_prof_create_config", []() {
    return platform::NPUProfConfigWrapper(platform::NPUProfilerCreateConfig());
  });

  m.def("npu_prof_destropy_config", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerDestroyConfig(c.ptr());
  });
#endif

J
jianghaicheng 已提交
3242 3243 3244 3245
#ifdef PADDLE_WITH_IPU
  m.def("get_ipu_device_count", platform::GetIPUDeviceCount);
#endif

3246 3247 3248 3249
#ifdef PADDLE_WITH_MLU
  m.def("get_mlu_device_count", platform::GetMLUDeviceCount);
#endif

3250 3251 3252 3253 3254 3255
  py::enum_<platform::TracerOption>(m, "TracerOption", py::arithmetic())
      .value("kDefault", platform::TracerOption::kDefault)
      .value("kOpDetail", platform::TracerOption::kOpDetail)
      .value("kAllOpDetail", platform::TracerOption::kAllOpDetail)
      .export_values();

3256 3257 3258 3259
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
3260
      .value("kAll", platform::ProfilerState::kAll)
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

3272
  m.def("set_tracer_option", platform::SetTracerOption);
3273 3274
  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
3275
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
3276
  m.def("reset_profiler", platform::ResetProfiler);
W
wuhuanzhou 已提交
3277
  m.def("register_pass", [](const std::string &pass_type, py::object callable) {
3278 3279
    PADDLE_ENFORCE_EQ(
        framework::ir::PassRegistry::Instance().Has(pass_type), false,
3280 3281 3282
        platform::errors::AlreadyExists("Pass '%s' is registered more than "
                                        "once. Please use another name.",
                                        pass_type));
W
wuhuanzhou 已提交
3283
    callable.inc_ref();
3284 3285 3286 3287 3288 3289 3290 3291
    framework::ir::PassRegistry::Instance().Insert(pass_type, [pass_type,
                                                               callable]() {
      py::gil_scoped_acquire guard;
      std::unique_ptr<framework::ir::Pass> pass(
          new framework::ir::GeneratePass(py::cast<std::string>(callable())));
      return pass;
    });
  });
3292
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
3293 3294 3295
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
3296

3297
  m.def("size_of_dtype", framework::SizeOfType);
C
chenjian 已提交
3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336
  py::class_<paddle::platform::ProfilerResult>(m, "_ProfilerResult")
      .def(py::init<>())
      .def("get_data", &paddle::platform::ProfilerResult::GetData,
           py::return_value_policy::automatic_reference)
      .def("save", &paddle::platform::ProfilerResult::Save)
      .def("get_extra_info", &paddle::platform::ProfilerResult::GetExtraInfo);

  py::class_<paddle::platform::DevicePythonNode>(m, "DevicePythonNode")
      .def(py::init<>())
      .def_readwrite("name", &paddle::platform::DevicePythonNode::name)
      .def_readwrite("type", &paddle::platform::DevicePythonNode::type)
      .def_readwrite("start_ns", &paddle::platform::DevicePythonNode::start_ns)
      .def_readwrite("end_ns", &paddle::platform::DevicePythonNode::end_ns)
      .def_readwrite("device_id",
                     &paddle::platform::DevicePythonNode::device_id)
      .def_readwrite("context_id",
                     &paddle::platform::DevicePythonNode::context_id)
      .def_readwrite("stream_id",
                     &paddle::platform::DevicePythonNode::stream_id);

  py::class_<paddle::platform::HostPythonNode>(m, "HostPythonNode")
      .def(py::init<>())
      .def_readwrite("name", &paddle::platform::HostPythonNode::name)
      .def_readwrite("type", &paddle::platform::HostPythonNode::type)
      .def_readwrite("start_ns", &paddle::platform::HostPythonNode::start_ns)
      .def_readwrite("end_ns", &paddle::platform::HostPythonNode::end_ns)
      .def_readwrite("process_id",
                     &paddle::platform::HostPythonNode::process_id)
      .def_readwrite("thread_id", &paddle::platform::HostPythonNode::thread_id)
      .def_readwrite("children_node",
                     &paddle::platform::HostPythonNode::children_node_ptrs)
      .def_readwrite("runtime_node",
                     &paddle::platform::HostPythonNode::runtime_node_ptrs)
      .def_readwrite("device_node",
                     &paddle::platform::HostPythonNode::device_node_ptrs);

  py::class_<paddle::platform::Profiler>(m, "_Profiler")
      .def("create", &paddle::platform::Profiler::Create,
           py::return_value_policy::take_ownership)
C
chenjian 已提交
3337
      .def("is_cupti_supported", &paddle::platform::Profiler::IsCuptiSupported)
C
chenjian 已提交
3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380
      .def("prepare",
           [](paddle::platform::Profiler *profiler) {
             platform::EnableHostEventRecorder();
             profiler->Prepare();
           })
      .def("start", &paddle::platform::Profiler::Start)
      .def("stop",
           [](paddle::platform::Profiler *profiler) {
             platform::DisableHostEventRecorder();
             return profiler->Stop();
           },
           py::return_value_policy::automatic_reference);

  py::class_<paddle::platform::ProfilerOptions>(m, "ProfilerOptions")
      .def(py::init<>())
      .def_readwrite("trace_switch",
                     &paddle::platform::ProfilerOptions::trace_switch);

  py::class_<platform::RecordEvent>(m, "_RecordEvent")
      .def(py::init([](std::string name, platform::TracerEventType type) {
        return std::make_unique<platform::RecordEvent>(
            name, type, 1, paddle::platform::EventRole::kOrdinary);
      }))
      .def("end", [](platform::RecordEvent *event) { event->End(); });

  py::enum_<paddle::platform::TracerEventType>(m, "TracerEventType")
      .value("Operator", paddle::platform::TracerEventType::Operator)
      .value("Dataloader", paddle::platform::TracerEventType::Dataloader)
      .value("ProfileStep", paddle::platform::TracerEventType::ProfileStep)
      .value("CudaRuntime", paddle::platform::TracerEventType::CudaRuntime)
      .value("Kernel", paddle::platform::TracerEventType::Kernel)
      .value("Memcpy", paddle::platform::TracerEventType::Memcpy)
      .value("Memset", paddle::platform::TracerEventType::Memset)
      .value("UserDefined", paddle::platform::TracerEventType::UserDefined)
      .value("OperatorInner", paddle::platform::TracerEventType::OperatorInner)
      .value("Forward", paddle::platform::TracerEventType::Forward)
      .value("Backward", paddle::platform::TracerEventType::Backward)
      .value("Optimization", paddle::platform::TracerEventType::Optimization)
      .value("Communication", paddle::platform::TracerEventType::Communication)
      .value("PythonOp", paddle::platform::TracerEventType::PythonOp)
      .value("PythonUserDefined",
             paddle::platform::TracerEventType::PythonUserDefined);
  m.def("load_profiler_result", &paddle::platform::LoadProfilerResult);
3381

3382
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
3383 3384
  m.def("set_cublas_switch", platform::SetAllowTF32Cublas);
  m.def("get_cublas_switch", platform::AllowTF32Cublas);
A
AshburnLee 已提交
3385 3386
  m.def("set_cudnn_switch", platform::SetAllowTF32Cudnn);
  m.def("get_cudnn_switch", platform::AllowTF32Cudnn);
3387
#endif  // PADDLE_WITH_CUDA
3388 3389
  m.def("clear_executor_cache",
        []() { framework::ExecutorInfoCache::Instance().Finalize(); });
3390

3391 3392 3393
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

3394 3395
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
3396
      .def("has", &ir::Pass::Has)
3397 3398 3399
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
3400
           })
3401
      .def(
3402
          "set",
3403 3404 3405
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
3406 3407
      .def("set", [](ir::Pass &self, const std::string &name,
                     bool val) { self.Set<bool>(name, new bool(val)); })
3408 3409
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
J
jianghaicheng 已提交
3410 3411 3412 3413 3414
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::vector<std::string> set) {
             self.Set(name, new std::vector<std::string>(set));
           })
3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
3429 3430
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
3431
        self.Apply(graph.get());
F
flame 已提交
3432
      });
3433

X
fix  
Xin Pan 已提交
3434 3435
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
3450
  // -- python binds for parallel executor.
Y
yuyang18 已提交
3451
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
3452 3453 3454 3455
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

3456 3457 3458
    Returns:
        ExecutionStrategy: An ExecutionStrategy object.

C
chengduo 已提交
3459 3460 3461
    Examples:
        .. code-block:: python

3462 3463 3464 3465 3466 3467 3468 3469 3470
          import paddle
          import paddle.static as static
          import paddle.nn.functional as F

          paddle.enable_static()

          x = static.data(name='x', shape=[None, 13], dtype='float32')
          y = static.data(name='y', shape=[None, 1], dtype='float32')
          y_predict = static.nn.fc(input=x, size=1, act=None)
3471

3472 3473
          cost = F.square_error_cost(input=y_predict, label=y)
          avg_loss = paddle.mean(cost)
3474

3475
          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
3476 3477
          sgd_optimizer.minimize(avg_loss)

3478
          exec_strategy = static.ExecutionStrategy()
C
chengduo 已提交
3479 3480
          exec_strategy.num_threads = 4

3481 3482 3483
          train_exe = static.ParallelExecutor(use_cuda=False,
                                              loss_name=avg_loss.name,
                                              exec_strategy=exec_strategy)
C
chengduo 已提交
3484 3485
        )DOC");

3486 3487 3488 3489
  py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
      .value("CPU", paddle::platform::DeviceType::CPU)
      .value("CUDA", paddle::platform::DeviceType::CUDA)
      .value("XPU", paddle::platform::DeviceType::XPU);
3490

Y
yuyang18 已提交
3491
  exec_strategy.def(py::init())
Y
yuyang18 已提交
3492 3493 3494 3495 3496
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
3497
          },
3498 3499
          R"DOC(
            The type is INT, num_threads represents the size of thread pool that
C
chengduo 已提交
3500 3501 3502 3503 3504 3505 3506
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519
            `multiprocessing.cpu_count()`. Default 0.

            Examples:
                .. code-block:: python

                    import paddle
                    import paddle.static as static

                    paddle.enable_static()

                    exec_strategy = static.ExecutionStrategy()
                    exec_strategy.num_threads = 4
            )DOC")
Y
yuyang18 已提交
3520
      .def_property(
3521 3522
          "_use_device",
          [](const ExecutionStrategy &self) { return self.use_device_; },
3523
          [](ExecutionStrategy &self, paddle::platform::DeviceType use_device) {
3524 3525 3526
            self.use_device_ = use_device;
          })  // NOTE(liuyuhui): Doesn't add doc for 'use_device', because
              // use_device isn‘t exposed to users.
Y
yuyang18 已提交
3527 3528 3529 3530 3531
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
3532 3533 3534
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
3535 3536
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
3537 3538 3539 3540 3541 3542 3543
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
3544 3545 3546 3547
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
3548
                because the temp variable's shape maybe the same between two iterations.
3549 3550 3551 3552 3553 3554 3555 3556 3557 3558
                Default 100.

                .. note::
                    1. If you fetch data when calling the 'run', the ParallelExecutor 
                    will clean up the temp variables at the end of the current iteration. 
                    2. In some NLP model, it may cause the GPU memory is insufficient, 
                    in this case, you should reduce `num_iteration_per_drop_scope`.

                Examples:
                    .. code-block:: python
C
chengduo 已提交
3559

3560 3561 3562 3563 3564 3565 3566
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_drop_scope = 10
3567
              )DOC")
Q
Qiao Longfei 已提交
3568 3569 3570 3571 3572 3573 3574 3575 3576
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588
                user call exe.run() in python。Default: 1.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_run = 10
Q
Qiao Longfei 已提交
3589
              )DOC")
3590 3591 3592 3593 3594 3595 3596 3597
      .def_property(
          "use_thread_barrier",
          [](const ExecutionStrategy &self) { return self.thread_barrier_; },
          [](ExecutionStrategy &self, bool use_thread_barrier) {
            self.thread_barrier_ = use_thread_barrier;
          },
          R"DOC(This config that the this is distributed training with parameter server
              )DOC")
3598 3599 3600 3601 3602
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
3603

Y
yuyang18 已提交
3604
  exec_strategy.def_property(
Y
yuyang18 已提交
3605 3606 3607 3608 3609 3610 3611
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
3612 3613
      });

C
chengduo 已提交
3614 3615 3616 3617
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

3618 3619 3620
    Returns:
        BuildStrategy: An BuildStrategy object.

C
chengduo 已提交
3621 3622 3623
    Examples:
        .. code-block:: python

3624
            import os
3625 3626 3627 3628
            import paddle
            import paddle.static as static

            paddle.enable_static()
3629

3630 3631
            os.environ['CPU_NUM'] = str(2)
            places = static.cpu_places()
3632

3633 3634 3635 3636
            data = static.data(name="x", shape=[None, 1], dtype="float32")
            hidden = static.nn.fc(input=data, size=10)
            loss = paddle.mean(hidden)
            paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
3637

3638
            build_strategy = static.BuildStrategy()
3639 3640
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
3641 3642
            build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
            program = static.CompiledProgram(static.default_main_program())
3643
            program = program.with_data_parallel(loss_name=loss.name,
3644 3645
                                                  build_strategy=build_strategy,
                                                  places=places)
C
chengduo 已提交
3646
)DOC");
Y
yuyang18 已提交
3647 3648 3649

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
3650 3651
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce)
      .value("_NoReduce", BuildStrategy::ReduceStrategy::kNoReduce);
Y
yuyang18 已提交
3652 3653 3654 3655 3656 3657 3658 3659
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
3660
      .def("_clear_finalized", &BuildStrategy::ClearFinalized)
Y
yuyang18 已提交
3661 3662 3663 3664
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
3665 3666 3667 3668
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3669
            self.reduce_ = strategy;
C
chengduo 已提交
3670
          },
3671
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
3672 3673
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
3674
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
3675 3676
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
3677
                Default is 'AllReduce'.
F
flame 已提交
3678 3679 3680 3681

                Examples:
                    .. code-block:: python

3682 3683 3684 3685 3686 3687 3688
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
F
flame 已提交
3689
                  )DOC")
Y
yuyang18 已提交
3690 3691 3692 3693 3694
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
3695 3696 3697 3698
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3699
            self.gradient_scale_ = strategy;
C
chengduo 已提交
3700
          },
3701
          R"DOC((paddle.static.BuildStrategy.GradientScaleStrategy, optional): there are three
3702
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
3703 3704
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
3705
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
3706 3707 3708 3709

                Examples:
                    .. code-block:: python

C
chengduo 已提交
3710 3711
                        import numpy
                        import os
3712 3713 3714 3715
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
3716 3717

                        use_cuda = True
3718 3719
                        place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace()
                        exe = static.Executor(place)
C
chengduo 已提交
3720 3721

                        # NOTE: If you use CPU to run the program, you need
3722
                        # to specify the CPU_NUM, otherwise, paddle will use
C
chengduo 已提交
3723 3724 3725 3726 3727 3728
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
3729
                            places = static.cpu_places()
C
chengduo 已提交
3730
                        else:
3731
                            places = static.cuda_places()
C
chengduo 已提交
3732

3733 3734 3735 3736
                        data = static.data(name='X', shape=[None, 1], dtype='float32')
                        hidden = static.nn.fc(input=data, size=10)
                        loss = paddle.mean(hidden)
                        paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
C
chengduo 已提交
3737

3738
                        exe.run(static.default_startup_program())
C
chengduo 已提交
3739

3740
                        build_strategy = static.BuildStrategy()
C
chengduo 已提交
3741
                        build_strategy.gradient_scale_strategy = \
3742 3743 3744
                                  static.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = static.CompiledProgram(
                                  static.default_main_program()).with_data_parallel(
C
chengduo 已提交
3745
                                          loss_name=loss.name, build_strategy=build_strategy,
3746
                                          places=places)
C
chengduo 已提交
3747 3748 3749 3750 3751 3752

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
3753 3754
                                              feed={"X": x, loss_grad_name : loss_grad},
                                              fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
3755
                   )DOC")
Y
yuyang18 已提交
3756 3757 3758 3759
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
3760 3761 3762 3763
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
3764
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
3765
          },
3766
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
3767
                writing the SSA Graph to file in the form of graphviz.
3768
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
3769 3770 3771 3772

                Examples:
                    .. code-block:: python

3773 3774 3775 3776
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
3777

3778 3779
                        build_strategy = static.BuildStrategy()
                        build_strategy.debug_graphviz_path = "./graph"
F
flame 已提交
3780
                    )DOC")
S
sneaxiy 已提交
3781 3782 3783 3784 3785 3786
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
3787 3788 3789 3790
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
3791 3792
            self.enable_sequential_execution_ = b;
          },
3793 3794
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
3795 3796 3797 3798

                Examples:
                    .. code-block:: python

3799 3800 3801 3802 3803 3804
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3805 3806
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
3807 3808 3809 3810 3811 3812
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
3813 3814 3815 3816
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
3817 3818
            self.remove_unnecessary_lock_ = b;
          },
3819 3820
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
3821 3822 3823 3824

                Examples:
                    .. code-block:: python

3825 3826 3827 3828 3829 3830
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3831 3832
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
3833 3834 3835 3836
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
3837
#ifdef WIN32
3838
            PADDLE_THROW(platform::errors::Unavailable(
3839
                "Distribution mode is not supported on Windows platform."));
3840
#endif
3841 3842
            self.num_trainers_ = num_trainers;
          })
3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
3855 3856 3857 3858 3859 3860
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
3861 3862 3863 3864 3865 3866
      .def_property(
          "bkcl_comm_num",
          [](const BuildStrategy &self) { return self.bkcl_comm_num_; },
          [](BuildStrategy &self, int bkcl_comm_num) {
            self.bkcl_comm_num_ = bkcl_comm_num;
          })
3867
      .def_property("use_hierarchical_allreduce",
3868 3869 3870 3871 3872 3873
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
3874
      .def_property("hierarchical_allreduce_inter_nranks",
3875 3876 3877 3878 3879 3880 3881
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
3882 3883 3884 3885 3886 3887
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
3888 3889 3890 3891
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
C
chengduo 已提交
3892 3893
            self.fuse_elewise_add_act_ops_ = b;
          },
3894
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
3895
                to fuse elementwise_add_op and activation_op,
3896
                it may make the execution faster. Default is False.
F
flame 已提交
3897 3898 3899 3900

                Examples:
                    .. code-block:: python

3901 3902 3903 3904 3905 3906
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3907 3908
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933
      .def_property(
          "fuse_gemm_epilogue",
          [](const BuildStrategy &self) { return self.fuse_gemm_epilogue_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_gemm_epilogue_ = b;
          },
          R"DOC((bool, optional): fuse_gemm_epilogue indicate whether
                to fuse matmul_op, elemenewist_add_op and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_gemm_epilogue = True
                     )DOC")
Z
Zhen Wang 已提交
3934 3935 3936 3937
      .def_property(
          "fuse_bn_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_act_ops_; },
          [](BuildStrategy &self, bool b) {
3938
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
Z
Zhen Wang 已提交
3939
                              platform::errors::PreconditionNotMet(
3940 3941
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Z
Zhen Wang 已提交
3942 3943 3944 3945 3946 3947 3948 3949 3950
            self.fuse_bn_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_act_ops indicate whether
                to fuse batch_norm and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

3951 3952 3953 3954 3955 3956
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
Z
Zhen Wang 已提交
3957 3958
                        build_strategy.fuse_bn_act_ops = True
                     )DOC")
Z
Zhang Ting 已提交
3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
      .def_property(
          "fuse_bn_add_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_add_act_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_bn_add_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_add_act_ops indicate whether
                to fuse batch_norm, elementwise_add and activation_op,
                it may make the execution faster. Default is True

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_bn_add_act_ops = True
                     )DOC")
3984 3985 3986 3987
      .def_property(
          "enable_auto_fusion",
          [](const BuildStrategy &self) { return self.enable_auto_fusion_; },
          [](BuildStrategy &self, bool b) {
3988
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
3989
                              platform::errors::PreconditionNotMet(
3990 3991
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
3992 3993 3994 3995 3996 3997 3998 3999 4000 4001
            self.enable_auto_fusion_ = b;
          },
          R"DOC((bool, optional): Whether to enable fusing subgraph to a
                fusion_group. Now we only support fusing subgraph that composed
                of elementwise-like operators, such as elementwise_add/mul
                without broadcast and activations.

                Examples:
                    .. code-block:: python

4002 4003 4004 4005 4006 4007
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
4008 4009
                        build_strategy.enable_auto_fusion = True
                    )DOC")
4010 4011 4012 4013 4014 4015
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
4016 4017 4018 4019
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
4020 4021
            self.fuse_relu_depthwise_conv_ = b;
          },
4022
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
4023 4024 4025
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
4026
                Default is False.
F
flame 已提交
4027 4028 4029 4030

                Examples:
                    .. code-block:: python

4031 4032 4033 4034 4035 4036
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
4037 4038
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
4039 4040 4041
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
4042
                             self.fuse_broadcast_ops_ == paddle::none;
C
chengduo 已提交
4043 4044
                    },
                    [](BuildStrategy &self, bool b) {
4045 4046 4047 4048
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
4049 4050
                      self.fuse_broadcast_ops_ = b;
                    },
4051
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
4052 4053 4054 4055
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
4056 4057 4058 4059 4060
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

4061 4062 4063 4064 4065 4066
                              import paddle
                              import paddle.static as static

                              paddle.enable_static()

                              build_strategy = static.BuildStrategy()
4067 4068
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
4069 4070
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
4071
                      return self.fuse_all_optimizer_ops_ == true ||
4072
                             self.fuse_all_optimizer_ops_ == paddle::none;
C
chengduo 已提交
4073 4074
                    },
                    [](BuildStrategy &self, bool b) {
4075 4076 4077 4078
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
4079 4080
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
4081 4082 4083 4084
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
4085 4086 4087 4088
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Q
qingqing01 已提交
4089 4090
            self.sync_batch_norm_ = b;
          },
4091
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
4092 4093 4094
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
4095 4096
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
4097 4098 4099 4100

                Examples:
                    .. code-block:: python

4101 4102 4103 4104 4105 4106
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
4107 4108
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
4109 4110
      .def_property(
          "memory_optimize",
4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
4121
              self.memory_optimize_ = paddle::none;
4122 4123 4124
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
4125
              PADDLE_THROW(platform::errors::InvalidArgument(
Z
Zeng Jinle 已提交
4126 4127
                  "BuildStrategy.memory_optimize must be set to None, False "
                  "or True"));
4128 4129
            }
          },
4130
          R"DOC((bool, optional): memory opitimize aims to save total memory
4131
                consumption, set to True to enable it.
4132

4133 4134 4135
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149
                True means enabling and False means disabling. Default is None.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.memory_optimize = True
                
                )DOC")
4150 4151 4152
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
4153 4154 4155
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
4156
              PADDLE_THROW(platform::errors::Unavailable(
4157
                  "Distribution mode is not supported on Windows platform."));
4158 4159 4160 4161 4162
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
4163 4164 4165
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
4166
      .def_property(
D
dzhwinter 已提交
4167 4168 4169
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
4170 4171 4172 4173
      .def_property(
          "enable_addto",
          [](const BuildStrategy &self) { return self.enable_addto_; },
          [](BuildStrategy &self, bool b) { self.enable_addto_ = b; })
C
chengduo 已提交
4174 4175
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
4176 4177
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
4178
                   self.fuse_all_reduce_ops_ == paddle::none;
C
chengduo 已提交
4179
          },
C
chengduo 已提交
4180
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
4181 4182 4183 4184 4185 4186 4187
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
4188 4189 4190 4191
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
4192 4193 4194 4195 4196 4197 4198 4199 4200
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
Z
Zeng Jinle 已提交
4201 4202 4203 4204 4205 4206
      .def_property(
          "fix_op_run_order",
          [](const BuildStrategy &self) { return self.fix_op_run_order_; },
          [](BuildStrategy &self, bool fix_op_run_order) {
            self.fix_op_run_order_ = fix_op_run_order;
          })
4207 4208 4209 4210 4211 4212 4213
      .def_property("allow_cuda_graph_capture",
                    [](const BuildStrategy &self) {
                      return self.allow_cuda_graph_capture_;
                    },
                    [](BuildStrategy &self, bool allow_cuda_graph_capture) {
                      self.allow_cuda_graph_capture_ = allow_cuda_graph_capture;
                    })
4214 4215 4216 4217 4218 4219
      .def("_copy",
           [](const BuildStrategy &self) {
             auto new_bs = self;
             new_bs.ClearFinalized();
             return new_bs;
           })
4220
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
4221
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
4222 4223 4224 4225 4226
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
4227

4228 4229 4230 4231 4232 4233
  m.def("_set_cached_executor_build_strategy",
        [](int64_t program_id, const BuildStrategy &build_strategy) {
          auto &cached_exe_info = framework::ExecutorInfoCache::Instance();
          cached_exe_info.SetBuildStrategy(program_id, build_strategy);
        });

Y
yuyang18 已提交
4234
  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
4235
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
4236
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
4237
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
4238 4239 4240 4241
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
4242 4243 4244 4245 4246
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
4247 4248 4249
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
4250 4251 4252 4253
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
4254 4255
      .def("run",
           [](ParallelExecutor &self,
Z
Zhen Wang 已提交
4256 4257 4258 4259 4260 4261 4262 4263
              const std::vector<std::string> &fetch_tensors,
              bool return_merged) -> py::object {
             paddle::framework::FetchResultType ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(fetch_tensors, return_merged);
             }
             if (return_merged) {
4264
               return py::cast(
4265
                   std::move(BOOST_GET(paddle::framework::FetchList, ret)));
Z
Zhen Wang 已提交
4266 4267
             } else {
               return py::cast(std::move(
4268
                   BOOST_GET(paddle::framework::FetchUnmergedList, ret)));
Z
Zhen Wang 已提交
4269
             }
4270 4271
           })
      .def("device_count", &ParallelExecutor::DeviceCount);
Y
Yu Yang 已提交
4272

J
jianghaicheng 已提交
4273 4274
#ifdef PADDLE_WITH_IPU
  py::class_<platform::ipu::IpuBackend,
4275 4276 4277 4278 4279 4280 4281 4282 4283
             std::unique_ptr<platform::ipu::IpuBackend, py::nodelete>>(
      m, "IpuBackend")
      // manage IpuBackend in C++
      .def("get_instance",
           []() {
             return std::unique_ptr<platform::ipu::IpuBackend, py::nodelete>(
                 platform::ipu::IpuBackend::GetInstance());
           },
           py::return_value_policy::reference)
A
Allen Guo 已提交
4284
      .def("weights_to_host", &platform::ipu::IpuBackend::WeightsToHost)
4285 4286
      .def("detach", &platform::ipu::IpuBackend::Detach)
      .def("reset", &platform::ipu::IpuBackend::Reset)
J
jianghaicheng 已提交
4287
      .def("set_scope", &platform::ipu::IpuBackend::SetScope)
4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331
      .def("set_ipu_strategy", &platform::ipu::IpuBackend::SetIpuStrategy)
      .def("save_model_proto", &platform::ipu::IpuBackend::SaveModelProto);

  py::class_<platform::ipu::IpuStrategy>(m, "IpuStrategy")
      .def(py::init())
      .def("set_options",
           [](platform::ipu::IpuStrategy &self, const py::dict &opt) {
             for (auto element : opt) {
               auto option_name = element.first.cast<std::string>();
               VLOG(10) << "Set option: " << option_name;
               if (py::isinstance<py::bool_>(element.second)) {
                 self.AddBoolOption(option_name, element.second.cast<bool>());
               } else if (py::isinstance<py::float_>(element.second)) {
                 self.AddDoubleOption(option_name,
                                      element.second.cast<double>());
               } else if (py::isinstance<py::int_>(element.second)) {
                 self.AddUint64Option(option_name,
                                      element.second.cast<std::uint64_t>());
               } else if (py::isinstance<py::str>(element.second)) {
                 self.AddStringOption(option_name,
                                      element.second.cast<std::string>());
               } else if (py::isinstance<py::set>(element.second) ||
                          py::isinstance<py::list>(element.second)) {
                 for (auto option : element.second.cast<py::list>()) {
                   std::string option_val;
                   if (py::isinstance<py::str>(option)) {
                     option_val = option.cast<std::string>();
                   } else if (py::isinstance<py::int_>(option)) {
                     option_val = std::to_string(option.cast<std::uint64_t>());
                   } else {
                     PADDLE_THROW(platform::errors::Unimplemented(
                         "Failed to convert type: %s when set IpuStrategy "
                         "option: %s",
                         option.get_type(), option_name));
                   }
                   self.InsertStringOption(option_name, option_val);
                 }
               } else if (py::isinstance<py::dict>(element.second)) {
                 if (option_name.rfind("location_", 0) == 0) {
                   for (auto option : element.second.cast<py::dict>()) {
                     self.SetTensorLocation(
                         option_name, option.first.cast<std::string>(),
                         option.second.cast<std::uint64_t>());
                   }
A
Allen Guo 已提交
4332 4333 4334 4335 4336 4337 4338 4339 4340
                 } else if (option_name == "accumulate_outer_fragment") {
                   for (auto option : element.second.cast<py::dict>()) {
                     std::vector<int> values;
                     for (auto value : option.second.cast<py::list>()) {
                       values.push_back(value.cast<int>());
                     }
                     self.SetAccumulateOuterFragmentSettings(
                         option.first.cast<std::uint64_t>(), values);
                   }
4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417
                 } else if (option_name == "custom_op") {
                   std::string paddle_op;
                   std::string popart_op;
                   std::string domain;
                   int version = -1;
                   for (auto option : element.second.cast<py::dict>()) {
                     std::string option_key = option.first.cast<std::string>();
                     if (option_key == "paddle_op") {
                       paddle_op = option.second.cast<std::string>();
                     } else if (option_key == "popart_op") {
                       popart_op = option.second.cast<std::string>();
                     } else if (option_key == "domain") {
                       domain = option.second.cast<std::string>();
                     } else if (option_key == "version") {
                       version = option.second.cast<int>();
                     } else {
                       PADDLE_THROW(platform::errors::InvalidArgument(
                           "Invalid argument, key must be one of paddle_op, "
                           "popart_op, domain or version, but revecived %s",
                           option_key));
                     }
                   }
                   self.AddCustomOp(paddle_op, popart_op, domain, version);
                 } else {
                   for (auto option : element.second.cast<py::dict>()) {
                     std::string option_key = option.first.cast<std::string>();
                     std::string option_val;
                     if (py::isinstance<py::str>(option.second)) {
                       option_val = option.second.cast<std::string>();
                     } else if (py::isinstance<py::int_>(option.second)) {
                       option_val =
                           std::to_string(option.second.cast<std::uint64_t>());
                     } else {
                       PADDLE_THROW(platform::errors::Unimplemented(
                           "Failed to convert value type: %s when set "
                           "IpuStrategy option: %s",
                           option.second.get_type(), option_key));
                     }
                     self.InsertStringPairOption(option_name, option_key,
                                                 option_val);
                   }
                 }
               } else {
                 PADDLE_THROW(platform::errors::InvalidArgument(
                     "Invalid IpuStrategy option value type: %s, please check "
                     "input value for option: %s",
                     element.second.get_type(), option_name));
               }
             }
           })
      .def("get_option",
           [](platform::ipu::IpuStrategy &self, const std::string &name) {
             py::dict res;
             auto option_type = self.GetOptionType(name);
             res["name"] = name;
             res["type"] = option_type;
             if (option_type == "vector") {
               auto value = self.GetVectorOption(name);
               res["value"] = value;
             } else if (option_type == "map") {
               auto value = self.GetMapOption(name);
               res["value"] = value;
             } else {
               auto value_s = self.GetOption(name);
               res["value_s"] = value_s;
               if (option_type == "bool") {
                 res["value"] = static_cast<bool>(std::stoi(value_s));
               } else if (option_type == "uint64") {
                 res["value"] = std::stoul(value_s);
               } else if (option_type == "double") {
                 res["value"] = std::stod(value_s);
               } else if (option_type == "string") {
                 res["value"] = value_s;
               }
             }
             return res;
           })
4418 4419
      .def("get_all_option_names",
           &platform::ipu::IpuStrategy::GetAllOptionNames)
4420 4421 4422
      .def("enable_pattern", &platform::ipu::IpuStrategy::EnablePattern)
      .def("disable_pattern", &platform::ipu::IpuStrategy::DisablePattern)
      .def("is_pattern_enabled", &platform::ipu::IpuStrategy::IsPatternEnabled);
J
jianghaicheng 已提交
4423 4424
#endif

4425 4426 4427 4428 4429 4430 4431 4432
  m.def("enable_autotune", [] {
    return phi::autotune::AutoTuneStatus::Instance().EnableAutoTune();
  });

  m.def("disable_autotune", [] {
    return phi::autotune::AutoTuneStatus::Instance().DisableAutoTune();
  });

4433
  m.def("set_autotune_range", [](int64_t start, int64_t stop) {
4434 4435 4436 4437 4438 4439 4440 4441 4442
    return phi::autotune::AutoTuneStatus::Instance().SetAutoTuneRange(start,
                                                                      stop);
  });

  m.def("update_autotune_status",
        [] { return phi::autotune::AutoTuneStatus::Instance().Update(); });

  m.def("autotune_status", [] {
    py::dict res;
4443
    phi::autotune::AutoTuneCache::Instance().UpdateStatus();
4444 4445 4446 4447 4448 4449 4450
    res["step_id"] = phi::autotune::AutoTuneStatus::Instance().StepID();
    res["cache_size"] = phi::autotune::AutoTuneCache::Instance().Size();
    res["cache_hit_rate"] =
        phi::autotune::AutoTuneCache::Instance().CacheHitRate();
    return res;
  });

D
dongdaxiang 已提交
4451
  BindFleetWrapper(&m);
4452
  BindIO(&m);
T
Thunderbrook 已提交
4453

T
Thunderbrook 已提交
4454
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
T
Thunderbrook 已提交
4455
  BindHeterWrapper(&m);
4456
  BindMetrics(&m);
T
Thunderbrook 已提交
4457
#endif
T
Thunderbrook 已提交
4458
#ifdef PADDLE_WITH_HETERPS
T
Thunderbrook 已提交
4459
  BindPSGPUWrapper(&m);
T
Thunderbrook 已提交
4460
#endif
4461
  BindGlooWrapper(&m);
H
hutuxian 已提交
4462
  BindBoxHelper(&m);
H
hutuxian 已提交
4463 4464 4465
#ifdef PADDLE_WITH_BOX_PS
  BindBoxWrapper(&m);
#endif
4466
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
4467
  BindNCCLWrapper(&m);
4468 4469 4470
#endif
#ifdef PADDLE_WITH_GLOO
  BindGlooContext(&m);
W
wopeizl 已提交
4471
#endif
F
flame 已提交
4472 4473
  BindGraph(&m);
  BindNode(&m);
4474
  BindPass(&m);
F
flame 已提交
4475
  BindInferenceApi(&m);
4476
  BindCompatible(&m);
4477
  BindDataset(&m);
Y
yaoxuefeng 已提交
4478
  BindGenerator(&m);
4479 4480 4481
#ifndef PADDLE_ON_INFERENCE
  BindDistributed(&m);
#endif
4482 4483 4484
#ifdef PADDLE_WITH_ASCEND
  BindAscendWrapper(&m);
  BindAscendGraph(&m);
4485
  BindAscendDevice(&m);
4486
#endif
Y
Yanghello 已提交
4487 4488 4489
#ifdef PADDLE_WITH_CRYPTO
  BindCrypto(&m);
#endif
T
tangwei12 已提交
4490

T
tangwei12 已提交
4491
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
4492 4493
  BindDistFleetWrapper(&m);
  BindPSHost(&m);
4494
  BindCommunicatorContext(&m);
T
tangwei12 已提交
4495 4496
  BindDistCommunicator(&m);
  BindHeterClient(&m);
S
seemingwang 已提交
4497 4498 4499 4500 4501
  BindGraphPyFeatureNode(&m);
  BindGraphNode(&m);
  BindGraphPyService(&m);
  BindGraphPyServer(&m);
  BindGraphPyClient(&m);
1
123malin 已提交
4502 4503 4504 4505
  BindIndexNode(&m);
  BindTreeIndex(&m);
  BindIndexWrapper(&m);
  BindIndexSampler(&m);
4506
#endif
L
Luo Tao 已提交
4507
}
4508
}  // namespace pybind
4509
}  // namespace paddle