pybind.cc 129.5 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
15

C
chengduoZH 已提交
16
#include <algorithm>
17
#include <cctype>
18
#include <cstdlib>
19
#include <iterator>
C
chengduoZH 已提交
20
#include <map>
S
sneaxiy 已提交
21
#include <memory>
C
chengduoZH 已提交
22 23
#include <mutex>  // NOLINT // for call_once
#include <string>
24 25
#include <tuple>
#include <type_traits>
C
chengduoZH 已提交
26
#include <unordered_map>
27
#include <unordered_set>
C
chengduoZH 已提交
28 29
#include <utility>
#include <vector>
30

31
#include "paddle/fluid/framework/custom_operator.h"
32
#include "paddle/fluid/framework/data_layout.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/executor.h"
34
#include "paddle/fluid/framework/executor_cache.h"
35
#include "paddle/fluid/framework/executor_gc_helper.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/feed_fetch_method.h"
Z
Zhen Wang 已提交
37
#include "paddle/fluid/framework/feed_fetch_type.h"
S
sneaxiy 已提交
38
#include "paddle/fluid/framework/garbage_collector.h"
H
hutuxian 已提交
39
#include "paddle/fluid/framework/io/fs.h"
40
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
41
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
42 43 44
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
45
#include "paddle/fluid/framework/new_executor/standalone_executor.h"
S
sneaxiy 已提交
46
#include "paddle/fluid/framework/op_info.h"
47
#include "paddle/fluid/framework/op_registry.h"
48
#include "paddle/fluid/framework/op_version_registry.h"
Y
Yu Yang 已提交
49
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
50
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
51
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
52
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
53
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
54
#include "paddle/fluid/framework/selected_rows.h"
55
#include "paddle/fluid/framework/tensor_util.h"
56
#include "paddle/fluid/framework/trainer.h"
57
#include "paddle/fluid/framework/type_defs.h"
X
Xin Pan 已提交
58
#include "paddle/fluid/framework/version.h"
H
hong 已提交
59
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
60
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
61
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
D
dzhwinter 已提交
62
#include "paddle/fluid/operators/activation_op.h"
L
Leo Chen 已提交
63
#include "paddle/fluid/operators/common_infer_shape_functions.h"
S
sneaxiy 已提交
64
#include "paddle/fluid/operators/py_func_op.h"
65
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
66
#include "paddle/fluid/platform/cpu_info.h"
67
#include "paddle/fluid/platform/device_context.h"
68
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
69
#include "paddle/fluid/platform/enforce.h"
70
#include "paddle/fluid/platform/init.h"
H
hutuxian 已提交
71
#include "paddle/fluid/platform/monitor.h"
Y
Yi Wang 已提交
72 73
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
74
#include "paddle/fluid/pybind/cuda_streams_py.h"
75
#include "paddle/fluid/pybind/io.h"
76
#include "paddle/utils/none.h"
77 78 79
#ifdef PADDLE_WITH_ASCEND
#include "paddle/fluid/pybind/ascend_wrapper_py.h"
#endif
H
hutuxian 已提交
80
#include "paddle/fluid/pybind/box_helper_py.h"
81
#include "paddle/fluid/pybind/compatible.h"
Y
Yi Wang 已提交
82
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
83
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
84
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
85
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
Y
yaoxuefeng 已提交
86
#include "paddle/fluid/pybind/generator_py.h"
87
#include "paddle/fluid/pybind/global_value_getter_setter.h"
88
#include "paddle/fluid/pybind/gloo_context_py.h"
89
#include "paddle/fluid/pybind/gloo_wrapper_py.h"
T
Thunderbrook 已提交
90
#include "paddle/fluid/pybind/heter_wrapper_py.h"
91
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
92
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
93
#include "paddle/fluid/pybind/ir.h"
T
Thunderbrook 已提交
94
#include "paddle/fluid/pybind/ps_gpu_wrapper_py.h"
95
#include "paddle/fluid/pybind/pybind_boost_headers.h"
96

97
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
98
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
99
#endif
100
#include "paddle/fluid/framework/data_type.h"
101 102
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
103
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
104
#include "paddle/fluid/pybind/tensor_py.h"
105
#include "paddle/fluid/string/to_string.h"
106 107
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
Y
Yi Wang 已提交
108
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
109
#endif
110
#ifndef PADDLE_WITH_HIP
Y
Yi Wang 已提交
111
#include "paddle/fluid/platform/cuda_profiler.h"
112
#endif
Y
Yi Wang 已提交
113
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
114 115
#endif

116 117
#ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/platform/npu_info.h"
118
#include "paddle/fluid/platform/npu_profiler.h"
119 120
#endif

121
#ifdef PADDLE_WITH_XPU
Q
QingshuChen 已提交
122
#include "paddle/fluid/platform/xpu/xpu_info.h"
123 124
#endif

Y
Yanghello 已提交
125 126 127 128
#ifdef PADDLE_WITH_CRYPTO
#include "paddle/fluid/pybind/crypto.h"
#endif

T
tangwei12 已提交
129
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
130 131 132
#include "paddle/fluid/pybind/fleet_py.h"
#endif

M
minqiyang 已提交
133 134
#include "pybind11/stl.h"

135
DECLARE_bool(use_mkldnn);
136

Q
Qiao Longfei 已提交
137 138
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
139 140 141
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchUnmergedList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType);
Q
Qiao Longfei 已提交
142

143
namespace paddle {
144
namespace pybind {
145
bool IsCompiledWithCUDA() {
146 147 148 149 150 151 152 153 154
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
  return false;
#else
  return true;
#endif
}

bool IsCompiledWithROCM() {
#ifndef PADDLE_WITH_HIP
Q
qijun 已提交
155 156 157 158 159 160
  return false;
#else
  return true;
#endif
}

161 162 163 164 165 166 167 168
bool IsCompiledWithAscend() {
#ifndef PADDLE_WITH_ASCEND
  return false;
#else
  return true;
#endif
}

169 170 171 172 173 174 175 176
bool IsCompiledWithXPU() {
#ifndef PADDLE_WITH_XPU
  return false;
#else
  return true;
#endif
}

177 178 179 180 181 182 183 184
bool IsCompiledWithNPU() {
#ifndef PADDLE_WITH_ASCEND_CL
  return false;
#else
  return true;
#endif
}

185 186 187 188 189 190 191 192
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

193 194 195 196 197 198 199 200
bool IsCompiledWithHETERPS() {
#ifndef PADDLE_WITH_HETERPS
  return false;
#else
  return true;
#endif
}

201 202 203 204 205 206 207 208 209 210 211
bool SupportsBfloat16() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core))
    return true;
  else
    return false;
#endif
}

212 213 214 215 216 217 218 219 220 221 222
bool SupportsBfloat16FastPerformance() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_bf16))
    return true;
  else
    return false;
#endif
}

223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
// According to the input `place` and `dtype`, this function returns a tuple
// consists of three sets:
// 1) All operators registered in the Paddle framework.
// 2) All operators supported for `place` and `dtype`.
// 3) All operators unsupported for `place` and `dtype`.
// The input `place` is a type of string, which can only be `GPU` or `CPU`.
// The input `dtype` is a type of paddle::framework::proto::VarType::Type,
// which can be paddle::framework::proto::VarType::FP16,
// paddle::framework::proto::VarType::FP32 and so on.
std::tuple<std::unordered_set<std::string>, std::unordered_set<std::string>,
           std::unordered_set<std::string>>
OpSupportedInfos(const std::string &place,
                 framework::proto::VarType::Type dtype) {
  std::string query_place;
  std::transform(place.begin(), place.end(), std::back_inserter(query_place),
                 [](unsigned char c) { return std::toupper(c); });
  using fn_type = std::add_pointer<bool(const platform::Place &)>::type;
  std::unordered_map<std::string, fn_type> is_target_place{
T
taixiurong 已提交
241 242 243
      {"GPU", &platform::is_gpu_place},
      {"CPU", &platform::is_cpu_place},
      {"XPU", &platform::is_xpu_place},
244
      {"NPU", &platform::is_npu_place},
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
  };
  PADDLE_ENFORCE_NE(
      is_target_place.count(query_place), 0,
      platform::errors::InvalidArgument(
          "The argument `place` should be 'GPU' or 'CPU', but get '%s'.",
          place));

  std::unordered_set<std::string> all_ops;
  const auto &op_info = framework::OpInfoMap::Instance().map();
  for (auto it = op_info.begin(); it != op_info.end(); it++) {
    all_ops.emplace(it->first);
  }

  std::unordered_set<std::string> supported_ops;
  auto &all_kernels = framework::OperatorWithKernel::AllOpKernels();
  for (auto it = all_kernels.begin(); it != all_kernels.end(); it++) {
    for (auto &kernel_type : it->second) {
      if (is_target_place[query_place](kernel_type.first.place_) &&
          kernel_type.first.data_type_ == dtype) {
        supported_ops.emplace(it->first);
      }
    }
  }

  std::unordered_set<std::string> unsupported_ops;
  for (auto &op : all_ops) {
    if (!supported_ops.count(op)) {
      unsupported_ops.emplace(op);
    }
  }

  VLOG(4) << "-- The size of all_ops: " << all_ops.size() << " --";
  VLOG(4) << "-- The size of supported_ops: " << supported_ops.size() << " --";
  VLOG(4) << "-- The size of unsupported_ops: " << unsupported_ops.size()
          << " --";
  return std::make_tuple(std::move(all_ops), std::move(supported_ops),
                         std::move(unsupported_ops));
}

284
bool IsCompiledWithBrpc() {
285
#ifndef PADDLE_WITH_DISTRIBUTE
286 287
  return false;
#endif
288
  return true;
289 290
}

Y
update  
Yancey1989 已提交
291
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
292
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
293 294 295 296 297 298
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
299 300 301 302 303 304 305 306 307 308
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

H
hong 已提交
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
331 332 333
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Python object is not type of %s, the real type is %s",
        typeid(T).name(), obj->ob_type->tp_name));
H
hong 已提交
334 335 336 337 338 339 340 341 342 343 344 345 346
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
347 348
      PADDLE_THROW(platform::errors::InvalidArgument(
          "The parameter [%s] to save is None", para.first));
H
hong 已提交
349 350
    }
    vec_res.emplace_back(
351
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
H
hong 已提交
352 353 354 355 356 357 358 359 360 361 362 363
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
364 365
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameter list to save is None"));
H
hong 已提交
366 367 368 369 370 371 372 373 374 375 376 377
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
378 379 380
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to save is None"));
H
hong 已提交
381 382 383 384
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
385 386
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to save is not a list"));
H
hong 已提交
387 388 389 390
  }
  return vec_res;
}

391 392 393 394 395 396 397 398
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
399 400
    PADDLE_THROW(
        platform::errors::InvalidArgument("The parameter list to set is None"));
401 402 403 404 405 406 407 408 409 410 411 412 413
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
414 415 416
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to set is None"));
417 418 419 420 421
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
422 423 424 425 426
        PADDLE_ENFORCE_NOT_NULL(exe,
                                platform::errors::InvalidArgument(
                                    "Parameter not Initialized, "
                                    "Please set argument [executor] not None "
                                    "or run startup program first"));
427 428
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
429 430 431
        PADDLE_ENFORCE_NOT_NULL(
            py_var_desc, platform::errors::InvalidArgument(
                             "The var_desc of parameter to set is None"));
432 433 434 435 436 437 438 439 440
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
        tensor_temp->Resize(framework::make_ddim(var_desc.GetShape()));
        tensor_temp->mutable_data(exe->GetPlace(), var_desc.GetDataType());
      }
    }
  } else {
441 442
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to set is not a list"));
443 444 445 446 447
  }

  return;
}

448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
static void AssertStaticGraphAndDygraphGradMakerNoDiff() {
  std::set<std::string> ops;
  for (auto &pair : framework::OpInfoMap::Instance().map()) {
    bool has_static_grad_maker = (pair.second.grad_op_maker_ != nullptr);
    bool has_dygraph_grad_maker =
        (pair.second.dygraph_grad_op_maker_ != nullptr);
    if (has_static_grad_maker ^ has_dygraph_grad_maker) {
      bool has_kernel =
          (framework::OperatorWithKernel::AllOpKernels().count(pair.first) > 0);
      if (has_kernel) {
        ops.insert(pair.first);
      } else {
        VLOG(5) << pair.first << " has no kernels, skip";
      }
    }
  }
  PADDLE_ENFORCE_EQ(ops.empty(), true,
                    platform::errors::Unimplemented(
                        "OperatorWithKernel [%s] have only static graph grad "
                        "maker or have only dygraph grad maker, which is not "
                        "allowed",
                        string::join_strings(ops, ',')));
}

Z
Zeng Jinle 已提交
472 473 474 475 476 477 478 479 480 481 482 483 484
#ifdef PADDLE_WITH_NCCL
static int GetNCCLVersion() {
#if NCCL_VERSION_CODE >= 2304
  int ver;
  PADDLE_ENFORCE_CUDA_SUCCESS(platform::dynload::ncclGetVersion(&ver));
  return ver;
#else
  PADDLE_THROW(platform::errors::External(
      "Cannot get NCCL version successfully when nccl version < 2.3.4"));
#endif
}
#endif

485 486 487 488 489 490
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

491 492
  BindCudaStream(&m);

Y
Yu Yang 已提交
493 494 495
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
496
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
497

498 499
  AssertStaticGraphAndDygraphGradMakerNoDiff();

500
  m.doc() = "C++ core of PaddlePaddle";
501

502 503 504 505
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

506
  BindException(&m);
Y
Yu Yang 已提交
507

508 509
  m.def("set_num_threads", &platform::SetNumThreads);

510 511
  m.def("disable_signal_handler", &DisableSignalHandler);

512
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
513 514 515
  m.def("cudnn_version", &platform::CudnnVersion);
#endif

Z
Zeng Jinle 已提交
516 517 518 519 520 521 522 523
#ifdef PADDLE_WITH_NCCL
  m.def("nccl_version", &GetNCCLVersion);
#endif

  m.def("wait_device", [](const platform::Place &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });

6
633WHU 已提交
524 525 526
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
527 528 529 530 531 532

    PADDLE_ENFORCE_NOT_NULL(
        dmt, platform::errors::InvalidArgument(
                 "from_dlpack received an invalid capsule. "
                 "Note that a DLPack tensor can be consumed only once."));

6
633WHU 已提交
533 534
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
535
    framework::Tensor tensor;
6
633WHU 已提交
536 537 538 539

    if (dl.ctx.device_type == kDLCPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
540
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
6
633WHU 已提交
541 542 543 544 545 546
    if (dl.ctx.device_type == kDLGPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });
H
hong 已提交
547

548 549 550 551 552 553
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

554 555 556 557 558 559
  m.def("save_op_version_info", [](framework::ProgramDesc &desc) {
    framework::compatible::pb::OpVersionMap pb_vmap{desc.OpVersionMap()};
    framework::compatible::SaveOpVersions(
        framework::compatible::OpVersionRegistrar::GetInstance()
            .GetVersionMap(),
        &pb_vmap);
560 561
  });

562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
  m.def("set_printoptions", [](const py::kwargs &kwargs) {
    auto &print_opt = framework::PrintOptions::Instance();
    if (kwargs.contains("precision")) {
      print_opt.precision = kwargs["precision"].cast<int>();
    }
    if (kwargs.contains("threshold")) {
      print_opt.threshold = kwargs["threshold"].cast<int>();
    }
    if (kwargs.contains("edgeitems")) {
      print_opt.edgeitems = kwargs["edgeitems"].cast<int>();
    }
    if (kwargs.contains("linewidth")) {
      print_opt.linewidth = kwargs["linewidth"].cast<int>();
    }
    if (kwargs.contains("sci_mode")) {
      print_opt.sci_mode = kwargs["sci_mode"].cast<bool>();
    }

    VLOG(4) << "Set printoptions: precision=" << print_opt.precision
            << ", threshold=" << print_opt.threshold
            << ", edgeitems=" << print_opt.edgeitems
            << ", linewidth=" << print_opt.linewidth
            << ", sci_mode=" << print_opt.sci_mode;
  });

L
Leo Chen 已提交
587 588 589 590 591 592
  m.def("broadcast_shape", [](const std::vector<int64_t> &x_dim,
                              const std::vector<int64_t> &y_dim) {
    return vectorize(operators::details::BroadcastTwoDims(
        make_ddim(x_dim), make_ddim(y_dim), -1));
  });

S
sneaxiy 已提交
593
  m.def(
S
sneaxiy 已提交
594
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
595 596 597 598
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
599 600 601
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
  m.def("_get_all_register_op_kernels", [] {
    auto &all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
    std::unordered_map<std::string, std::vector<std::string>> all_kernels_info;
    for (auto &kernel_pair : all_kernels) {
      auto op_type = kernel_pair.first;
      std::vector<std::string> kernel_types;
      for (auto &info_pair : kernel_pair.second) {
        paddle::framework::OpKernelType kernel_type = info_pair.first;
        kernel_types.push_back(
            paddle::framework::KernelTypeToString(kernel_type));
      }
      all_kernels_info.emplace(op_type, kernel_types);
    }
    return all_kernels_info;
  });

S
sneaxiy 已提交
618 619 620
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
621
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
622

623
  m.def("_set_fuse_parameter_group_size",
624
        &paddle::framework::ir::SetFuseParameterGroupsSize);
625
  m.def("_set_fuse_parameter_memory_size",
626
        &paddle::framework::ir::SetFuseParameterMemorySize);
627

S
sneaxiy 已提交
628 629 630
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

631 632
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

633 634 635
  m.def("_promote_types_if_complex_exists",
        &paddle::framework::PromoteTypesIfComplexExists);

636
  BindImperative(&m);
637

638 639 640
  py::class_<framework::Tensor>(m, "Tensor", py::buffer_protocol())
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
641
      .def("_is_initialized",
642
           [](const framework::Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
643
      .def("_get_dims",
644
           [](const framework::Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
645
      .def("_set_dims",
646
           [](framework::Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
647
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
648
           })
Y
yuyang18 已提交
649
      .def("_set_layout",
650
           [](framework::Tensor &self, const std::string &layout) {
D
dzhwinter 已提交
651 652
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
653
      .def("_alloc_float",
654
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
655
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
656
           })
657
      .def("_alloc_float",
658
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
659 660
             self.mutable_data<float>(place);
           })
Y
yuyang18 已提交
661
      .def("_alloc_float",
662
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
663
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
664
           })
665 666 667 668
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place) {
             self.mutable_data<float>(place);
           })
669
      .def("_alloc_double",
670
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
671 672
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
673
      .def("_alloc_int",
674
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
675
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
676
           })
677
      .def("_alloc_int",
678
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
679 680
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
681
      .def("_alloc_int",
682
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
683
             self.mutable_data<int>(place);
Q
qijun 已提交
684
           })
Y
yuyang18 已提交
685
      .def("_alloc_int",
686 687
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
688 689
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
690
      .def("_alloc_float",
691 692
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
693 694
             self.mutable_data<float>(place);
           })
695
      .def("_mutable_data",
696
           [](framework::Tensor &self, paddle::platform::CPUPlace &place,
697 698 699
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
700
      .def("_mutable_data",
701
           [](framework::Tensor &self, paddle::platform::XPUPlace &place,
702 703 704
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
705
      .def("_mutable_data",
706
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place,
707 708 709 710
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
      .def("_mutable_data",
711
           [](framework::Tensor &self, paddle::platform::CUDAPinnedPlace &place,
712 713 714
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
715
      .def("_clear", &framework::Tensor::clear)
716 717 718 719 720
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
721
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
722
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
723 724
      .def("set", SetTensorFromPyArray<paddle::platform::XPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
725
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
726
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
727 728
      .def("set", SetTensorFromPyArray<paddle::platform::NPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
729
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
730 731
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
L
Leo Chen 已提交
732 733 734 735
        Set the data of LoDTensor on place with given numpy array.
        
        Args:
          lod (numpy.ndarray): The data to set.
736
          place (CPUPlace|CUDAPlace|XPUPlace|CUDAPinnedPlace|NPUPlace): The place where the
L
Leo Chen 已提交
737
          LoDTensor is to be set.
738 739
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
740 741 742 743 744 745 746 747 748 749 750 751 752

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                t = fluid.LoDTensor()
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
753

754 755 756
      .def("shape",
           [](framework::Tensor &self) { return vectorize(self.dims()); },
           R"DOC(
L
Leo Chen 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
           Return the shape of LoDTensor.

           Returns:
               list[int]: The shape of LoDTensor.


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

                  t = fluid.LoDTensor()
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
773
      .def("_to_dlpack",
774
           [](framework::Tensor &self) {
6
633WHU 已提交
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
             DLPackTensor dlpack_tensor(self, 1);
             DLManagedTensor *dmt =
                 dlpack_tensor.ToCudfCompatibleDLManagedTensor();
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
795 796 797 798
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
799 800
      .def("_place", [](framework::Tensor &self) { return self.place(); })
      .def("_dtype", [](framework::Tensor &self) { return self.type(); })
801
      .def("_layout",
802 803 804 805
           [](framework::Tensor &self) {
             return DataLayoutToString(self.layout());
           })
      .def("_share_data_with", &framework::Tensor::ShareDataWith)
806
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
807
      .def("__str__", [](const framework::Tensor &self) {
808 809 810 811
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
812

L
Leo Chen 已提交
813
  // TODO(cql): add reference: en_user_guide_lod_tensor
814
  py::class_<LoDTensor, framework::Tensor>(m, "LoDTensor", R"DOC(
L
Leo Chen 已提交
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
    LoDTensor is a Tensor with optional LoD (Level of Details) information, 
    it can be used for variable-length sequences, 
    see :ref:`user_guide_lod_tensor` for details.

    LoDTensor can be converted to numpy array using :code:`numpy.array(lod_tensor)`.

    You can skip the following explanation if you don't need to know details 
    of LoDTensor.

    The following two examples show how to use LODtensor to represent 
    variable-length sequences.
    
    Example 1:
    
    Suppose x is a LoDTensor representing a variable-length sequence. 
    It contains two logical subsequences, the length of first logical sequence 
    is 2 (e.g., number of samples is 2), the length of second logical sequence 
    is 3, and the total length is 5. The data of the first logical sequence is 
    [1, 2], [3, 4], and the data of the second logical sequence is [5, 6], 
    [7, 8], [9, 10]. The data dimension of each sample is 2. So, the final 
    shape of the LoDTensor is [5, 2], of which 5 is the total length and 2 is 
    the dimension of each sample.
    
    Logically, we can represent the variable-length sequence in two ways: one 
    is in the form of recursive sequence lengths, that is, 
    x.recursive_sequence_lengths=[[2, 3]]; the other is in the form of offsets, 
    that is, x.lod=[[0, 2, 2+3]]. These two representations are equivalent, and 
    you can set and retrieve recursive_sequence_lengths or LoD through the 
    corresponding interfaces of LoDTensor introduced later.

    Actually, in order to access sequence faster, Paddle uses offset to store 
    different lengths of sequences. 
    Therefore, the operations on recursive_sequence_lengths will be converted 
    to the operations on LoD eventually.
    
    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]

    Example 2:

    LoD may have more than one level (for example, a paragraph may have more 
    than one sentence and a sentence may have more than one word). Suppose y 
    is a LoDTensor and its lod_level is 2. 
    From level = 0, there are two logical sequences, the length of which is 
    2 and 1, respectively, indicating that the first logical sequence contains 
    two sub-sequences and the second logical sequence contains one sub-sequence. 
    From level = 1, the lengths of two sub-sequences contained by the first 
    logical sequence is 2 and 2, and the length of sub-sequence contained by 
    the second logical sequence is 3.
      
    Therefore, the LoDTensor is represented in the form of recursive sequence 
    lengths as y.recursive_sequence_lengths=[[2,1], [2,2,3]]; and equally, in 
    the form of offset, it is represented as y.lod=[[0,2,3], [0,2,4,7]].

    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]
Z
Zeng Jinle 已提交
889 890 891 892 893 894 895

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
896 897

        )DOC")
898 899
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
900 901 902 903 904 905 906 907 908
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
909 910
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, -1), true,
911 912 913 914
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is %s",
                     new_lod));
915 916
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
917
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
918
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
919 920
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
921 922 923
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
924
      .def("set_lod",
925
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
926
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
927
             LoD new_lod;
928 929
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
930 931
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
932 933
                 platform::errors::InvalidArgument(
                     "The provided LoD is invalid, the LoD is %s", new_lod));
934
             self.set_lod(new_lod);
S
sneaxiy 已提交
935 936 937 938 939
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
L
Leo Chen 已提交
940 941 942 943
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
944 945 946 947 948 949 950 951 952 953

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
954
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
955
           )DOC")
956 957 958 959 960 961 962 963 964 965 966
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
967 968
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
969 970 971 972 973
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is "
                     "%s",
                     new_lod));
974
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
975 976
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
L
Leo Chen 已提交
977
           Set LoD of the LoDTensor according to recursive sequence lengths.
S
sneaxiy 已提交
978

L
Leo Chen 已提交
979
           For example, if recursive_sequence_lengths=[[2, 3]], which means
980
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
981
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
982 983

           Args:
L
Leo Chen 已提交
984 985 986 987
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
988 989 990 991 992 993 994 995 996 997

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
L
Leo Chen 已提交
998 999
                 print(t.recursive_sequence_length())  # [[2, 3]]
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
1000
           )DOC")
1001 1002 1003 1004 1005 1006 1007 1008
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1009 1010 1011 1012 1013
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
L
Leo Chen 已提交
1014 1015
               list[list[int]]: The lod of the LoDTensor.
           
Z
Zeng Jinle 已提交
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1026
           )DOC")
G
gongweibao 已提交
1027
      // Set above comments of set_lod.
1028 1029 1030 1031 1032 1033 1034 1035
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1036 1037
           },
           R"DOC(
L
Leo Chen 已提交
1038 1039
           Return the recursive sequence lengths corresponding to of the LodD 
           of the LoDTensor.
S
sneaxiy 已提交
1040 1041

           Returns:
L
Leo Chen 已提交
1042
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
1054 1055 1056 1057 1058 1059 1060 1061
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
L
Leo Chen 已提交
1062
           Check whether the LoD of the LoDTensor is valid.
S
sneaxiy 已提交
1063 1064

           Returns:
L
Leo Chen 已提交
1065
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
1077 1078 1079 1080 1081 1082 1083
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
1084
           )DOC")
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
      .def("__str__",
           [](const LoDTensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           })
      .def("_copy", [](const LoDTensor &self, const platform::Place &place) {
        // follow fetch_op's inplementation
        LoDTensor dst;
        if (self.IsInitialized() && self.numel() > 0) {
          TensorCopySync(self, place, &dst);
        } else {
          // Not copy, if the src tensor is empty.
          dst.clear();
          dst.Resize({0});
        }
        dst.set_lod(self.lod());
        return dst;
1103
#ifdef _WIN32
1104
      });
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
#else
           })
      .def(py::pickle(
          [](const LoDTensor &t) {  // __getstate__
            auto holder = t.Holder();
            PADDLE_ENFORCE_EQ(
              platform::is_cpu_place(holder->place()), true,
              platform::errors::PreconditionNotMet(
                  "LoDTensor is not on CPU."
                  "Now only LoDTensor on CPU can be serialized."));
            auto* mmap_writer_allocation =
              dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
                holder.get());
            PADDLE_ENFORCE_NOT_NULL(mmap_writer_allocation,
              platform::errors::PreconditionNotMet(
                "LoDTensor is not in shared memory."
                "Now only LoDTensor on shared memory can be serialized."));
            int type_idx = static_cast<int>(t.type());

            return py::make_tuple(mmap_writer_allocation->ipc_name(),
                                  mmap_writer_allocation->size(),
                                  type_idx, vectorize(t.dims()), t.lod());
          },
          [](py::tuple t) {  // __setstate__
            if (t.size() != 5)
              throw std::runtime_error("Invalid LoDTensor state!");

            // 1. Create a new C++ instance
            LoDTensor tensor;

            // 2. Rebuild Allocation
            const std::string &ipc_name = t[0].cast<std::string>();
            size_t size = t[1].cast<size_t>();
            auto shared_reader_holder =
              memory::allocation::RebuildMemoryMapReaderAllocation(
                ipc_name, size);

            // 3. Maintain global fd set
            VLOG(3) << "LoDTensor ipc name: " << ipc_name;
            memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);

            // 4. Rebuild LoDTensor
            tensor.ResetHolderWithType(shared_reader_holder,
              static_cast<proto::VarType::Type>(t[2].cast<int>()));
            tensor.Resize(make_ddim(t[3].cast<std::vector<int>>()));
            tensor.set_lod(t[4].cast<framework::LoD>());

            return tensor;
          }));
#endif
D
dangqingqing 已提交
1155

Q
qijun 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
1167 1168
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
1169 1170
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
1171 1172
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
1173
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
Q
qijun 已提交
1174 1175 1176 1177 1178 1179
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
1180
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
1181
      .def("rows", [](SelectedRows &self) {
1182 1183 1184 1185 1186
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
1187
      });
Q
qijun 已提交
1188

1189
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
1190 1191 1192

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
1193
      .def(py::init<>())
1194
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
1195
      .def("set_int",
1196 1197
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
1198 1199 1200 1201 1202 1203 1204
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
1205
      .def("get_tensor",
1206 1207
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
1208 1209
           },
           py::return_value_policy::reference)
1210 1211 1212 1213
      .def("get_bytes",
           [](Variable &self) {
             return py::bytes(*self.GetMutable<std::string>());
           })
Y
Yu Yang 已提交
1214 1215 1216
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
1217 1218 1219 1220 1221
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1222 1223 1224
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
1225 1226 1227
      .def("get_fetch_list",
           [](Variable &self) { return self.GetMutable<FetchList>(); },
           py::return_value_policy::reference)
1228
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1229 1230 1231 1232 1233
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1234
#endif
Y
Refine  
Yu Yang 已提交
1235 1236
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
1237 1238 1239 1240
             PADDLE_ENFORCE_EQ(
                 self.IsType<framework::ReaderHolder>(), true,
                 platform::errors::InvalidArgument(
                     "The variable is not type of ReaderHolder."));
Y
Refine  
Yu Yang 已提交
1241 1242
             return self.GetMutable<framework::ReaderHolder>();
           },
1243
           py::return_value_policy::reference)
1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
      .def("get_scope",
           [](Variable &self) -> Scope * {
             auto scope_vec =
                 self.GetMutable<std::vector<framework::Scope *>>();
             PADDLE_ENFORCE_GT(
                 scope_vec->size(), 0,
                 platform::errors::InvalidArgument(
                     "The size of scope_vec should be greater than 0"));
             return scope_vec->front();
           },
           py::return_value_policy::reference)
1255 1256 1257 1258
      .def("set_scope", [](Variable &self, Scope &scope) {
        auto scope_vec = self.GetMutable<std::vector<framework::Scope *>>();
        scope_vec->emplace_back(&scope);
      });
1259

S
sneaxiy 已提交
1260
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
1261

S
sneaxiy 已提交
1262
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

1276
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
1277 1278 1279 1280 1281 1282
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
1283 1284
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
1285
      .def("var",
1286
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
1287
             return self.Var(name);
Y
Yu Yang 已提交
1288
           },
S
sneaxiy 已提交
1289 1290
           py::arg("name"),
           R"DOC(
1291
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
1292

1293
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
1294
           current scope, the variable would be created. Otherwise,
1295
           return the existing variable.
S
sneaxiy 已提交
1296 1297

           Args:
1298 1299
               name (str): the variable name.

S
sneaxiy 已提交
1300
           Returns:
1301
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
1302 1303 1304 1305
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
1306
           Find variable named :code:`name` in the current scope or
1307
           its parent scope. Return None if not found. 
1308

S
sneaxiy 已提交
1309 1310
           Args:
               name (str): the variable name.
1311

S
sneaxiy 已提交
1312
           Returns:
1313
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1314
           )DOC",
1315
           py::return_value_policy::reference)
1316
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1317 1318 1319 1320 1321 1322
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1323
           py::return_value_policy::reference)
S
sneaxiy 已提交
1324 1325 1326
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1327 1328
           )DOC")
      .def("_kids", &Scope::kids);
1329

S
sneaxiy 已提交
1330 1331 1332 1333 1334 1335
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1336 1337
        R"DOC(
        Create a new scope.
1338

S
sneaxiy 已提交
1339 1340 1341
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1342 1343
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1344 1345
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1346 1347
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1348 1349 1350 1351
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1352 1353
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1354 1355
            platform::errors::Fatal(
                "Serialize OpProto Error. This could be a bug of Paddle."));
1356 1357 1358
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1359 1360
    return ret_values;
  });
1361 1362 1363 1364 1365 1366 1367 1368
  m.def("get_op_attrs_default_value",
        [](py::bytes byte_name) -> paddle::framework::AttributeMap {
          std::string op_type = byte_name;
          paddle::framework::AttributeMap res;
          auto info = OpInfoMap::Instance().GetNullable(op_type);
          if (info != nullptr) {
            if (info->HasOpProtoAndChecker()) {
              auto op_checker = info->Checker();
1369
              res = op_checker->GetDefaultAttrsMap();
1370 1371 1372 1373
            }
          }
          return res;
        });
1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1390 1391 1392
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
1393 1394 1395 1396 1397
  m.def("has_non_empty_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance()
        .Get(op_type)
        .HasNonEmptyGradOpMaker();
  });
1398 1399 1400
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
  m.def("infer_no_need_buffer_slots",
        [](const std::string op_type, const framework::VariableNameMap &inputs,
           const framework::VariableNameMap &outputs,
           const framework::AttributeMap &attrs) {
          auto infer_func = framework::OpInfoMap::Instance()
                                .Get(op_type)
                                .NoNeedBufferVarsInferer();
          if (infer_func) {
            return infer_func(inputs, outputs, attrs);
          } else {
            std::unordered_set<std::string> empty = {};
            return empty;
          }
        });
Y
Yu Yang 已提交
1415
  m.def("prune", [](const ProgramDesc &origin,
1416
                    const std::set<std::string> &feeded_var_names,
1417
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1418
    ProgramDesc prog_with_targets(origin);
1419

1420
    for (const auto &t : targets) {
1421
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1422
    }
1423
    proto::ProgramDesc pruned_desc;
1424 1425 1426 1427
    auto pruned_origin_block_id_map =
        Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
    return std::make_tuple(ProgramDesc(pruned_desc),
                           pruned_origin_block_id_map);
1428
  });
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445
  m.def("prune_backward",
        [](const framework::ProgramDesc &program) {
          return PruneBackward(program);
        },
        R"DOC(
             Prune the backward part of a program, mostly called in
             program.clone(for_test=True).
              
             Args:
                   program (ProgramDesc): The original program.

             Returns:
                   tuple(ProgramDesc, map<int, int>): The first part is 
                   the pruned program desc, and the second part is a map
                   which contains the id pair of pruned block and corresponding
                   origin block.
           )DOC");
1446 1447 1448 1449
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1450 1451 1452
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1453 1454
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
1455

Q
qijun 已提交
1456
  // clang-format off
Y
Yu Yang 已提交
1457
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1458 1459
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1460
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
1461 1462
                    return new paddle::platform::CPUDeviceContext();
                  })
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
      .def_static("create",
                  [](paddle::platform::XPUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_XPU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use XPUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with XPU support."));
#else
                    return new paddle::platform::XPUDeviceContext(place);
#endif
                  })
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
        .def_static("create",
                    [](paddle::platform::NPUPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_ASCEND_CL
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use NPUPlace in CPU/GPU/XPU version, "
                 "Please recompile or reinstall Paddle with NPU support."));
#else
                return new paddle::platform::NPUDeviceContext(place);
#endif
        })
Q
qijun 已提交
1487
      .def_static("create",
D
dzhwinter 已提交
1488
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1489
                      -> paddle::platform::DeviceContext* {
1490
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1491 1492 1493 1494
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
Q
qijun 已提交
1495
#else
Q
qijun 已提交
1496
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
1497
#endif
C
chengduoZH 已提交
1498 1499 1500 1501
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
1502
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1503 1504 1505 1506
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
C
chengduoZH 已提交
1507 1508 1509 1510
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
1511
// clang-format on
1512
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1513 1514
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
1515
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
1516 1517 1518 1519 1520

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
1521
    The memory of CUDAPlace with different dev_id is not accessible.
1522 1523 1524 1525 1526 1527 1528 1529
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
1530 1531 1532 1533

    Examples:
        .. code-block:: python

1534 1535 1536
          import paddle

          place = paddle.CUDAPlace(0)
L
lujun 已提交
1537

1538
        )DOC")
S
sneaxiy 已提交
1539 1540
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
1541
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

             if (UNLIKELY(dev_id >= platform::GetCUDADeviceCount())) {
               if (platform::GetCUDADeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
                     dev_id, platform::GetCUDADeviceCount(),
                     platform::GetCUDADeviceCount());
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
1566 1567
             new (&self) platform::CUDAPlace(dev_id);
#else
1568 1569 1570 1571 1572 1573 1574 1575 1576
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
1577 1578
#endif
           })
1579
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1580 1581
      .def("get_device_id",
           [](const platform::CUDAPlace &self) { return self.GetDeviceId(); })
S
sneaxiy 已提交
1582 1583 1584 1585
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
1586
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
1587
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1588 1589
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
1590 1591 1592
      .def("_get_device_id",
           [](platform::CUDAPlace &self) -> int { return self.GetDeviceId(); })
#endif
1593
      .def("__repr__", string::to_string<const platform::CUDAPlace &>)
D
dzhwinter 已提交
1594
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
1595

1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
  py::class_<platform::XPUPlace>(m, "XPUPlace", R"DOC(
    **Note**:
    Examples:
        .. code-block:: python
          import paddle.fluid as fluid
          xpu_place = fluid.XPUPlace(0)
        )DOC")
      .def("__init__",
           [](platform::XPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_XPU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid XPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetXPUDeviceCount())) {
               if (platform::GetXPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use XPU because there is no XPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid XPUPlace(%d), must inside [0, %d), because XPU "
                     "number on your machine is %d",
                     dev_id, platform::GetXPUDeviceCount(),
                     platform::GetXPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::XPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use XPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use XPU, please try to install XPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change XPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
1641
#ifdef PADDLE_WITH_XPU
1642 1643 1644 1645 1646 1647 1648
      .def("_type", &PlaceIndex<platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::XPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::XPUPlace, platform::CUDAPinnedPlace>)
1649 1650 1651
      .def("get_device_id",
           [](const platform::XPUPlace &self) { return self.GetDeviceId(); })
#endif
1652
      .def("__repr__", string::to_string<const platform::XPUPlace &>)
1653
      .def("__str__", string::to_string<const platform::XPUPlace &>);
1654
#ifdef PADDLE_WITH_XPU
T
TTerror 已提交
1655 1656 1657 1658
  py::enum_<platform::XPUVersion>(m, "XPUVersion", py::arithmetic())
      .value("XPU1", platform::XPUVersion::XPU1)
      .value("XPU2", platform::XPUVersion::XPU2)
      .export_values();
1659
  m.def("get_xpu_device_count", platform::GetXPUDeviceCount);
T
TTerror 已提交
1660 1661
  m.def("get_xpu_device_version",
        [](int device_id) { return platform::get_xpu_version(device_id); });
1662
#endif
1663

1664
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
1665
    CPUPlace is a descriptor of a device.
1666
    It represents a CPU device on which a tensor will be allocated and a model will run.
L
lujun 已提交
1667 1668 1669 1670

    Examples:
        .. code-block:: python

1671 1672
          import paddle
          cpu_place = paddle.CPUPlace()
L
lujun 已提交
1673

1674
        )DOC")
1675
      .def(py::init<>())
S
sneaxiy 已提交
1676 1677
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
1678
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
1679
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1680 1681 1682 1683
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
1684
      .def("__repr__", string::to_string<const platform::CPUPlace &>)
1685
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
1686

1687
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
1688 1689 1690 1691 1692 1693
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
1694 1695 1696 1697

    Examples:
        .. code-block:: python

1698 1699
          import paddle
          place = paddle.CUDAPinnedPlace()
L
lujun 已提交
1700

1701
        )DOC")
S
sneaxiy 已提交
1702
      .def("__init__",
S
sneaxiy 已提交
1703
           [](platform::CUDAPinnedPlace &self) {
1704
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1705 1706 1707
             PADDLE_THROW(platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
S
sneaxiy 已提交
1708
#endif
S
sneaxiy 已提交
1709
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
1710
           })
S
sneaxiy 已提交
1711 1712 1713 1714
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
1715 1716
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
1717 1718
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1719 1720 1721 1722
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
1723
      .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
C
chengduoZH 已提交
1724 1725
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
  // NPUPlace
  py::class_<platform::NPUPlace>(m, "NPUPlace", R"DOC(
    NPUPlace is a descriptor of a device.
    It represents a NPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          npu_place = paddle.NPUPlace(0)

        )DOC")
      .def("__init__",
           [](platform::NPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_ASCEND_CL
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid NPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetNPUDeviceCount())) {
               if (platform::GetNPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use NPU because there is no NPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid NPUPlace(%d), must inside [0, %d), because NPU "
                     "number on your machine is %d",
                     dev_id, platform::GetNPUDeviceCount(),
                     platform::GetNPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::NPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use NPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use NPU, please try to install NPU version "
1768
                 "PaddlePaddle by: pip install paddlepaddle-npu\n"
1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
                 "If you only have CPU, please change NPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
H
houj04 已提交
1783 1784
      .def("get_device_id",
           [](const platform::NPUPlace &self) { return self.GetDeviceId(); })
1785 1786
      .def("__str__", string::to_string<const platform::NPUPlace &>);

Y
Yu Yang 已提交
1787 1788
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
1789 1790 1791 1792
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
1793
      .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
1794
      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
S
sneaxiy 已提交
1795
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
1796 1797
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
1798 1799
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
1800 1801
      .def("is_xpu_place",
           [](platform::Place &self) { return platform::is_xpu_place(self); })
1802 1803
      .def("is_npu_place",
           [](platform::Place &self) { return platform::is_npu_place(self); })
S
sneaxiy 已提交
1804 1805 1806 1807
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
1808 1809
      .def("gpu_device_id",
           [](platform::Place &self) {
1810
             return BOOST_GET_CONST(platform::CUDAPlace, self).device;
X
xuezhong 已提交
1811
           })
1812 1813 1814 1815
      .def("xpu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::XPUPlace, self).device;
           })
1816 1817 1818 1819
      .def("npu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::NPUPlace, self).device;
           })
S
sneaxiy 已提交
1820 1821
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
1822 1823 1824 1825
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
1826 1827 1828 1829
      .def("set_place",
           [](platform::Place &self, const platform::XPUPlace &xpu_place) {
             self = xpu_place;
           })
Y
Yu Yang 已提交
1830
      .def("set_place",
D
dzhwinter 已提交
1831
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
1832
             self = gpu_place;
C
chengduoZH 已提交
1833
           })
1834 1835 1836 1837 1838
      .def("set_place",
           [](platform::Place &self,
              const platform::CUDAPinnedPlace &cuda_pinned_place) {
             self = cuda_pinned_place;
           })
1839 1840 1841 1842
      .def("set_place",
           [](platform::Place &self, const platform::NPUPlace &npu_place) {
             self = npu_place;
           })
1843 1844
      .def("__repr__", string::to_string<const platform::Place &>)
      .def("__str__", string::to_string<const platform::Place &>);
Y
Yu Yang 已提交
1845

Y
Yu Yang 已提交
1846
  py::class_<OperatorBase>(m, "Operator")
Z
Zeng Jinle 已提交
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
      .def_static("create",
                  [](py::bytes protobin) {
                    proto::OpDesc desc;
                    PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin),
                                      true,
                                      platform::errors::InvalidArgument(
                                          "Cannot parse user input to OpDesc"));
                    PADDLE_ENFORCE_EQ(desc.IsInitialized(), true,
                                      platform::errors::InvalidArgument(
                                          "The provided OpDesc is not "
                                          "initialized, the reason is: %s",
                                          desc.InitializationErrorString()));
                    return OpRegistry::CreateOp(desc);
                  })
1861
      .def("run",
1862
           [](OperatorBase &self, const Scope &scope,
1863 1864 1865 1866
              const platform::CPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
1867 1868
      .def("run",
           [](OperatorBase &self, const Scope &scope,
1869 1870 1871 1872
              const platform::XPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
1873 1874
      .def("run",
           [](OperatorBase &self, const Scope &scope,
1875 1876 1877 1878
              const platform::NPUPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
D
dzhwinter 已提交
1879 1880
      .def("run",
           [](OperatorBase &self, const Scope &scope,
1881 1882 1883 1884
              const platform::CUDAPlace &place) {
             pybind11::gil_scoped_release release;
             self.Run(scope, place);
           })
C
chengduoZH 已提交
1885 1886 1887
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
1888
             pybind11::gil_scoped_release release;
C
chengduoZH 已提交
1889 1890
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1891 1892 1893 1894 1895 1896 1897
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1898 1899
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1900
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1901
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1902 1903 1904 1905
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1906

1907 1908 1909
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

1910 1911 1912 1913 1914 1915 1916 1917 1918
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
      .def("finalize", &TrainerBase::Finalize);

1919 1920
  m.def("_get_eager_deletion_vars", &framework::GetEagerDeletionCleanVars);

F
fengjiayi 已提交
1921
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1922
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1923
      .def("close", &Executor::Close)
1924 1925
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
1926 1927
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
1928 1929 1930 1931
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
1932
             pybind11::gil_scoped_release release;
1933 1934 1935 1936 1937 1938 1939
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
1940 1941 1942
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
1943
              std::map<std::string, FetchType *> *fetch_targets,
1944 1945 1946 1947 1948 1949 1950 1951
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
1952
      .def("run_prepared_ctx",
G
guru4elephant 已提交
1953 1954 1955 1956 1957 1958 1959
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
1970
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1971 1972
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1973
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1974 1975
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1976
      });
S
sneaxiy 已提交
1977

1978 1979 1980 1981 1982 1983 1984 1985 1986 1987
  py::class_<framework::CostInfo>(m, "CostInfo")
      .def(py::init<>())
      .def("total_time", [](CostInfo &self) { return self.total_time; })
      .def("host_memory_bytes",
           [](CostInfo &self) { return self.host_memory_bytes; })
      .def("device_memory_bytes",
           [](CostInfo &self) { return self.device_memory_bytes; })
      .def("device_total_memory_bytes",
           [](CostInfo &self) { return self.device_total_memory_bytes; });

1988
  py::class_<framework::StandaloneExecutor>(m, "StandaloneExecutor")
H
hong 已提交
1989 1990 1991
      .def(py::init<const platform::Place &, const ProgramDesc &,
                    const ProgramDesc &, Scope *>())
      .def("run",
1992
           [](StandaloneExecutor &self,
H
hong 已提交
1993
              const std::unordered_map<std::string, py::array> &input_dict,
1994 1995 1996
              std::vector<std::string> fetch_names) {
             std::vector<framework::Tensor> feed_tensors;
             std::vector<std::string> feed_names;
H
hong 已提交
1997 1998 1999 2000 2001

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
2002 2003
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
H
hong 已提交
2004 2005
             }

2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
             }
             return py::cast(std::move(ret));
           })
      .def("run",
           [](StandaloneExecutor &self,
              const std::unordered_map<std::string, framework::Tensor>
                  &input_dict,
              std::vector<std::string> fetch_names) {
             std::vector<framework::Tensor> feed_tensors;
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               feed_names.push_back(item.first);
               feed_tensors.push_back(item.second);
             }

W
wanghuancoder 已提交
2026 2027 2028 2029
             paddle::framework::FetchList ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(feed_names, feed_tensors, fetch_names);
H
hong 已提交
2030
             }
W
wanghuancoder 已提交
2031
             return py::cast(std::move(ret));
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
           })
      .def("dry_run",
           [](StandaloneExecutor &self,
              const std::unordered_map<std::string, py::array> &input_dict) {
             std::vector<framework::Tensor> feed_tensors;
             std::vector<std::string> feed_names;

             for (auto &item : input_dict) {
               framework::LoDTensor t;
               SetTensorFromPyArray<platform::CPUPlace>(
                   &t, item.second, platform::CPUPlace(), false);
               feed_names.push_back(item.first);
               feed_tensors.push_back(t);
             }

             CostInfo cost_info;
             {
               pybind11::gil_scoped_release release;
               cost_info = self.DryRun(feed_names, feed_tensors);
             }
             return cost_info;
H
hong 已提交
2053 2054
           });

D
dzhwinter 已提交
2055
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
2056
  m.def("init_glog", framework::InitGLOG);
2057 2058
  m.def("load_op_meta_info_and_register_op",
        framework::LoadOpMetaInfoAndRegisterOp);
2059
  m.def("init_devices", []() { framework::InitDevices(); });
2060

2061
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
2062
  m.def("is_compiled_with_ascend", IsCompiledWithAscend);
2063
  m.def("is_compiled_with_rocm", IsCompiledWithROCM);
2064
  m.def("is_compiled_with_npu", IsCompiledWithNPU);
2065
  m.def("is_compiled_with_xpu", IsCompiledWithXPU);
2066
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
2067
  m.def("_is_compiled_with_heterps", IsCompiledWithHETERPS);
2068
  m.def("supports_bfloat16", SupportsBfloat16);
2069
  m.def("supports_bfloat16_fast_performance", SupportsBfloat16FastPerformance);
2070
  m.def("op_supported_infos", OpSupportedInfos);
2071
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
2072
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
2073 2074 2075
  m.def("_cuda_synchronize", [](const platform::CUDAPlace &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });
H
hutuxian 已提交
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094

  m.def("get_float_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<float>> float_stats;
    paddle::platform::StatRegistry<float>::Instance().publish(float_stats);
    std::unordered_map<std::string, float> stats_map;
    for (const auto &stat : float_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
  m.def("get_int_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<int64_t>> int_stats;
    paddle::platform::StatRegistry<int64_t>::Instance().publish(int_stats);
    std::unordered_map<std::string, int64_t> stats_map;
    for (const auto &stat : int_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
H
hutuxian 已提交
2095 2096 2097 2098 2099 2100 2101
  m.def("run_cmd",
        [](const std::string &cmd, int time_out = -1,
           int sleep_inter = -1) -> const std::string {
          return paddle::framework::shell_get_command_output(cmd, time_out,
                                                             sleep_inter);
        },
        py::arg("cmd"), py::arg("time_out") = -1, py::arg("sleep_inter") = -1);
G
gongweibao 已提交
2102 2103 2104 2105 2106 2107 2108 2109 2110
  m.def("shell_execute_cmd",
        [](const std::string &cmd, int time_out = 0, int sleep_inter = 0,
           bool redirect_stderr = false) -> std::vector<std::string> {
          return paddle::framework::shell_execute_cmd(
              cmd, time_out, sleep_inter, redirect_stderr);
        },
        py::arg("cmd"), py::arg("time_out") = 0, py::arg("sleep_inter") = 0,
        py::arg("redirect_stderr") = false);

2111
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2112 2113 2114 2115 2116
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
2117

2118
  m.def("set_feed_variable", framework::SetFeedVariable);
2119 2120 2121 2122 2123
  m.def("get_fetch_variable",
        [](const Scope &scope, const std::string &var_name,
           size_t index) -> py::object {
          auto &var = framework::GetFetchVariable(scope, var_name, index);
          if (data_is_lod_tensor(var)) {
2124
            return py::cast(BOOST_GET(LoDTensor, var));
2125
          } else {
2126
            return py::cast(BOOST_GET(LoDTensorArray, var));
2127 2128
          }
        });
2129
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
2130

X
Xin Pan 已提交
2131 2132
  m.def("_is_program_version_supported", IsProgramVersionSupported);

2133 2134 2135 2136 2137
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
2138
  BindGlobalValueGetterSetter(&m);
2139
  BindProcessMeshDesc(&m);
Y
Yu Yang 已提交
2140

Y
Yu Yang 已提交
2141 2142 2143 2144 2145 2146 2147 2148 2149
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
2150
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
2151
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
2152 2153 2154

    Examples:
        .. code-block:: python
2155

Z
Zeng Jinle 已提交
2156 2157 2158 2159
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
2160 2161
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
2162 2163 2164 2165 2166 2167
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
2168 2169 2170 2171
             PADDLE_ENFORCE_LT(i, self.size(),
                               platform::errors::InvalidArgument(
                                   "The index to set is larger than the size "
                                   "of LoDTensorArray."));
Y
Yu Yang 已提交
2172 2173 2174
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
2175 2176 2177 2178 2179 2180
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
2181 2182
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
2183 2184 2185 2186 2187 2188
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
2211

2212 2213 2214 2215 2216 2217 2218 2219
  py::class_<FetchList>(m, "FetchList", R"DOC( FetchList is a
        vector of boost::variant<LoDTensor, LoDTensorArray>.
        )DOC")
      .def("_move_to_list",
           [](FetchList &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               if (data_is_lod_tensor(self[i])) {
2220
                 auto &data = BOOST_GET(LoDTensor, self[i]);
2221 2222
                 res[i] = py::cast(std::move(data));
               } else {
2223
                 auto &data = BOOST_GET(LoDTensorArray, self[i]);
2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238
                 py::list tmp(data.size());
                 for (size_t j = 0; j < data.size(); ++j) {
                   tmp[j] = py::cast(std::move(data[j]));
                 }
                 res[i] = std::move(tmp);
               }
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership)

      .def("append",
           [](FetchList &self, const LoDTensor &t) {
             self.emplace_back();
2239
             auto &lod_tensor = BOOST_GET(LoDTensor, self.back());
2240 2241 2242 2243 2244 2245 2246 2247
             lod_tensor.ShareDataWith(t);
             lod_tensor.set_lod(t.lod());
           },
           py::arg("var"))

      .def("append",
           [](FetchList &self, const LoDTensorArray &t) {
             self.emplace_back();
2248
             auto &lod_tensor_array = BOOST_GET(LoDTensorArray, self.back());
2249 2250 2251 2252 2253 2254 2255 2256 2257
             for (size_t i = 0; i < t.size(); ++i) {
               lod_tensor_array[i].ShareDataWith(t[i]);
               lod_tensor_array[i].set_lod(t[i].lod());
             }
           },
           py::arg("var"));

  py::class_<FetchUnmergedList>(m, "FetchUnmergedList", R"DOC(
        FetchUnmergedList is 2-D array of FetchType(boost::variant(LoDTensor, LoDTensorArray)).
Z
Zhen Wang 已提交
2258 2259
        )DOC")
      .def("_move_to_list",
2260
           [](FetchUnmergedList &self) -> py::list {
Z
Zhen Wang 已提交
2261 2262 2263 2264
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               py::list tmp(self[i].size());
               for (size_t j = 0; j < self[i].size(); ++j) {
2265
                 if (data_is_lod_tensor(self[i][j])) {
2266
                   auto &var = BOOST_GET(LoDTensor, self[i][j]);
2267 2268
                   tmp[j] = py::cast(std::move(var));
                 } else {
2269
                   auto &var = BOOST_GET(LoDTensorArray, self[i][j]);
2270 2271 2272 2273 2274 2275
                   py::list tmp_array(var.size());
                   for (size_t k = 0; k < var.size(); ++k) {
                     tmp_array[k] = std::move(var[k]);
                   }
                   tmp[j] = std::move(tmp_array);
                 }
Z
Zhen Wang 已提交
2276 2277 2278 2279 2280 2281 2282 2283 2284
               }
               res[i] = std::move(tmp);
               self[i].clear();
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);

Y
Yu Yang 已提交
2285
  m.def("op_support_gpu", OpSupportGPU);
2286
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
D
Dong Zhihong 已提交
2287
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
2288
  m.def("cuda_empty_cache", platform::EmptyCache);
D
dangqingqing 已提交
2289

2290
#if !defined(PADDLE_WITH_HIP) && !defined(_WIN32)
D
dangqingqing 已提交
2291 2292 2293
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
2294 2295 2296 2297
  m.def("nvprof_nvtx_push", platform::CudaNvtxRangePush);
  m.def("nvprof_nvtx_pop", platform::CudaNvtxRangePop);
  m.def("nvprof_enable_record_event", platform::NvprofEnableRecordEvent);
  m.def("nvprof_disable_record_event", platform::NvprofDisableRecordEvent);
D
Dong Zhihong 已提交
2298
#endif
P
peizhilin 已提交
2299
#endif
Y
Yu Yang 已提交
2300

2301 2302
#ifdef PADDLE_WITH_ASCEND_CL
  m.def("get_npu_device_count", platform::GetNPUDeviceCount);
2303 2304 2305 2306
  m.def("npu_finalize", []() {
    auto &pool = platform::DeviceContextPool::Instance();
    auto devices = platform::GetSelectedNPUDevices();
    for (size_t i = 0; i < devices.size(); ++i) {
R
ronnywang 已提交
2307
      platform::NPUDeviceGuard guard(devices[i]);
2308 2309 2310 2311
      pool.Get(platform::NPUPlace(devices[i]))->Wait();
    }
    platform::AclInstance::Instance().Finalize();
  });
2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331

  py::class_<platform::NPUProfConfigWrapper>(m, "NPUProfConfigWrapper");

  m.def("npu_prof_init", platform::NPUProfilerInit);
  m.def("npu_prof_start", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStart(c.ptr());
  });
  m.def("npu_prof_stop", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStop(c.ptr());
  });
  m.def("npu_prof_finalize", platform::NPUProfilerFinalize);
  m.def("npu_prof_create_config", []() {
    return platform::NPUProfConfigWrapper(platform::NPUProfilerCreateConfig());
  });

  m.def("npu_prof_destropy_config", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerDestroyConfig(c.ptr());
  });
#endif

2332 2333 2334 2335 2336 2337
  py::enum_<platform::TracerOption>(m, "TracerOption", py::arithmetic())
      .value("kDefault", platform::TracerOption::kDefault)
      .value("kOpDetail", platform::TracerOption::kOpDetail)
      .value("kAllOpDetail", platform::TracerOption::kAllOpDetail)
      .export_values();

2338 2339 2340 2341
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
2342
      .value("kAll", platform::ProfilerState::kAll)
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

2354
  m.def("set_tracer_option", platform::SetTracerOption);
2355 2356
  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
2357
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
2358
  m.def("reset_profiler", platform::ResetProfiler);
2359
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
2360 2361 2362
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
2363

2364 2365
  m.def("size_of_dtype", framework::SizeOfType);

2366
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2367 2368
  m.def("set_cublas_switch", platform::SetAllowTF32Cublas);
  m.def("get_cublas_switch", platform::AllowTF32Cublas);
A
AshburnLee 已提交
2369 2370
  m.def("set_cudnn_switch", platform::SetAllowTF32Cudnn);
  m.def("get_cudnn_switch", platform::AllowTF32Cudnn);
2371
#endif  // PADDLE_WITH_CUDA
2372 2373
  m.def("clear_executor_cache",
        []() { framework::ExecutorInfoCache::Instance().Finalize(); });
2374

2375 2376 2377
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

2378 2379
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
2380
      .def("has", &ir::Pass::Has)
2381 2382 2383
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
2384
           })
2385
      .def(
2386
          "set",
2387 2388 2389
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
2390 2391
      .def("set", [](ir::Pass &self, const std::string &name,
                     bool val) { self.Set<bool>(name, new bool(val)); })
2392 2393
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
2408 2409
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
2410
        self.Apply(graph.get());
F
flame 已提交
2411
      });
2412

X
fix  
Xin Pan 已提交
2413 2414
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
2429
  // -- python binds for parallel executor.
X
Xin Pan 已提交
2430

Y
yuyang18 已提交
2431
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
2432 2433 2434 2435
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

2436 2437 2438
    Returns:
        ExecutionStrategy: An ExecutionStrategy object.

C
chengduo 已提交
2439 2440 2441
    Examples:
        .. code-block:: python

2442 2443 2444 2445 2446 2447 2448 2449 2450
          import paddle
          import paddle.static as static
          import paddle.nn.functional as F

          paddle.enable_static()

          x = static.data(name='x', shape=[None, 13], dtype='float32')
          y = static.data(name='y', shape=[None, 1], dtype='float32')
          y_predict = static.nn.fc(input=x, size=1, act=None)
2451

2452 2453
          cost = F.square_error_cost(input=y_predict, label=y)
          avg_loss = paddle.mean(cost)
2454

2455
          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
2456 2457
          sgd_optimizer.minimize(avg_loss)

2458
          exec_strategy = static.ExecutionStrategy()
C
chengduo 已提交
2459 2460
          exec_strategy.num_threads = 4

2461 2462 2463
          train_exe = static.ParallelExecutor(use_cuda=False,
                                              loss_name=avg_loss.name,
                                              exec_strategy=exec_strategy)
C
chengduo 已提交
2464 2465
        )DOC");

2466 2467 2468 2469
  py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
      .value("CPU", paddle::platform::DeviceType::CPU)
      .value("CUDA", paddle::platform::DeviceType::CUDA)
      .value("XPU", paddle::platform::DeviceType::XPU);
2470

Y
yuyang18 已提交
2471
  exec_strategy.def(py::init())
Y
yuyang18 已提交
2472 2473 2474 2475 2476
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
2477
          },
2478 2479
          R"DOC(
            The type is INT, num_threads represents the size of thread pool that
C
chengduo 已提交
2480 2481 2482 2483 2484 2485 2486
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499
            `multiprocessing.cpu_count()`. Default 0.

            Examples:
                .. code-block:: python

                    import paddle
                    import paddle.static as static

                    paddle.enable_static()

                    exec_strategy = static.ExecutionStrategy()
                    exec_strategy.num_threads = 4
            )DOC")
Y
yuyang18 已提交
2500
      .def_property(
2501 2502
          "_use_device",
          [](const ExecutionStrategy &self) { return self.use_device_; },
2503
          [](ExecutionStrategy &self, paddle::platform::DeviceType use_device) {
2504 2505 2506
            self.use_device_ = use_device;
          })  // NOTE(liuyuhui): Doesn't add doc for 'use_device', because
              // use_device isn‘t exposed to users.
Y
yuyang18 已提交
2507 2508 2509 2510 2511
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
2512 2513 2514
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
2515 2516
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
2517 2518 2519 2520 2521 2522 2523
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
2524 2525 2526 2527
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
2528
                because the temp variable's shape maybe the same between two iterations.
2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
                Default 100.

                .. note::
                    1. If you fetch data when calling the 'run', the ParallelExecutor 
                    will clean up the temp variables at the end of the current iteration. 
                    2. In some NLP model, it may cause the GPU memory is insufficient, 
                    in this case, you should reduce `num_iteration_per_drop_scope`.

                Examples:
                    .. code-block:: python
C
chengduo 已提交
2539

2540 2541 2542 2543 2544 2545 2546
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_drop_scope = 10
2547
              )DOC")
Q
Qiao Longfei 已提交
2548 2549 2550 2551 2552 2553 2554 2555 2556
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568
                user call exe.run() in python。Default: 1.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_run = 10
Q
Qiao Longfei 已提交
2569
              )DOC")
2570 2571 2572 2573 2574 2575 2576 2577
      .def_property(
          "use_thread_barrier",
          [](const ExecutionStrategy &self) { return self.thread_barrier_; },
          [](ExecutionStrategy &self, bool use_thread_barrier) {
            self.thread_barrier_ = use_thread_barrier;
          },
          R"DOC(This config that the this is distributed training with parameter server
              )DOC")
2578 2579 2580 2581 2582
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
2583

Y
yuyang18 已提交
2584
  exec_strategy.def_property(
Y
yuyang18 已提交
2585 2586 2587 2588 2589 2590 2591
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
2592 2593
      });

C
chengduo 已提交
2594 2595 2596 2597
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

2598 2599 2600
    Returns:
        BuildStrategy: An BuildStrategy object.

C
chengduo 已提交
2601 2602 2603
    Examples:
        .. code-block:: python

2604
            import os
2605 2606 2607 2608
            import paddle
            import paddle.static as static

            paddle.enable_static()
2609

2610 2611
            os.environ['CPU_NUM'] = str(2)
            places = static.cpu_places()
2612

2613 2614 2615 2616
            data = static.data(name="x", shape=[None, 1], dtype="float32")
            hidden = static.nn.fc(input=data, size=10)
            loss = paddle.mean(hidden)
            paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
2617

2618
            build_strategy = static.BuildStrategy()
2619 2620
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
2621 2622
            build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
            program = static.CompiledProgram(static.default_main_program())
2623
            program = program.with_data_parallel(loss_name=loss.name,
2624 2625
                                                  build_strategy=build_strategy,
                                                  places=places)
C
chengduo 已提交
2626
)DOC");
Y
yuyang18 已提交
2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
2639
      .def("_clear_finalized", &BuildStrategy::ClearFinalized)
Y
yuyang18 已提交
2640 2641 2642 2643
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
2644 2645 2646 2647
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2648
            self.reduce_ = strategy;
C
chengduo 已提交
2649
          },
2650
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
2651 2652
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
2653
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
2654 2655
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
2656
                Default is 'AllReduce'.
F
flame 已提交
2657 2658 2659 2660

                Examples:
                    .. code-block:: python

2661 2662 2663 2664 2665 2666 2667
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
F
flame 已提交
2668
                  )DOC")
Y
yuyang18 已提交
2669 2670 2671 2672 2673
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
2674 2675 2676 2677
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2678
            self.gradient_scale_ = strategy;
C
chengduo 已提交
2679
          },
2680
          R"DOC((paddle.static.BuildStrategy.GradientScaleStrategy, optional): there are three
2681
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
2682 2683
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
2684
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
2685 2686 2687 2688

                Examples:
                    .. code-block:: python

C
chengduo 已提交
2689 2690
                        import numpy
                        import os
2691 2692 2693 2694
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2695 2696

                        use_cuda = True
2697 2698
                        place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace()
                        exe = static.Executor(place)
C
chengduo 已提交
2699 2700

                        # NOTE: If you use CPU to run the program, you need
2701
                        # to specify the CPU_NUM, otherwise, paddle will use
C
chengduo 已提交
2702 2703 2704 2705 2706 2707
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
2708
                            places = static.cpu_places()
C
chengduo 已提交
2709
                        else:
2710
                            places = static.cuda_places()
C
chengduo 已提交
2711

2712 2713 2714 2715
                        data = static.data(name='X', shape=[None, 1], dtype='float32')
                        hidden = static.nn.fc(input=data, size=10)
                        loss = paddle.mean(hidden)
                        paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
C
chengduo 已提交
2716

2717
                        exe.run(static.default_startup_program())
C
chengduo 已提交
2718

2719
                        build_strategy = static.BuildStrategy()
C
chengduo 已提交
2720
                        build_strategy.gradient_scale_strategy = \
2721 2722 2723
                                  static.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = static.CompiledProgram(
                                  static.default_main_program()).with_data_parallel(
C
chengduo 已提交
2724
                                          loss_name=loss.name, build_strategy=build_strategy,
2725
                                          places=places)
C
chengduo 已提交
2726 2727 2728 2729 2730 2731

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
2732 2733
                                              feed={"X": x, loss_grad_name : loss_grad},
                                              fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
2734
                   )DOC")
Y
yuyang18 已提交
2735 2736 2737 2738
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
2739 2740 2741 2742
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2743
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
2744
          },
2745
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
2746
                writing the SSA Graph to file in the form of graphviz.
2747
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
2748 2749 2750 2751

                Examples:
                    .. code-block:: python

2752 2753 2754 2755
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2756

2757 2758
                        build_strategy = static.BuildStrategy()
                        build_strategy.debug_graphviz_path = "./graph"
F
flame 已提交
2759
                    )DOC")
S
sneaxiy 已提交
2760 2761 2762 2763 2764 2765
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
2766 2767 2768 2769
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2770 2771
            self.enable_sequential_execution_ = b;
          },
2772 2773
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
2774 2775 2776 2777

                Examples:
                    .. code-block:: python

2778 2779 2780 2781 2782 2783
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2784 2785
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
2786 2787 2788 2789 2790 2791
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
2792 2793 2794 2795
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2796 2797
            self.remove_unnecessary_lock_ = b;
          },
2798 2799
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
2800 2801 2802 2803

                Examples:
                    .. code-block:: python

2804 2805 2806 2807 2808 2809
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2810 2811
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
2812 2813 2814 2815
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
2816
#ifdef WIN32
2817
            PADDLE_THROW(platform::errors::Unavailable(
2818
                "Distribution mode is not supported on Windows platform."));
2819
#endif
2820 2821
            self.num_trainers_ = num_trainers;
          })
2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
2834 2835 2836 2837 2838 2839
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
2840 2841 2842 2843 2844 2845
      .def_property(
          "bkcl_comm_num",
          [](const BuildStrategy &self) { return self.bkcl_comm_num_; },
          [](BuildStrategy &self, int bkcl_comm_num) {
            self.bkcl_comm_num_ = bkcl_comm_num;
          })
2846
      .def_property("use_hierarchical_allreduce",
2847 2848 2849 2850 2851 2852
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
2853
      .def_property("hierarchical_allreduce_inter_nranks",
2854 2855 2856 2857 2858 2859 2860
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
2861 2862 2863 2864 2865 2866
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
2867 2868 2869 2870
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
C
chengduo 已提交
2871 2872
            self.fuse_elewise_add_act_ops_ = b;
          },
2873
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
2874
                to fuse elementwise_add_op and activation_op,
2875
                it may make the execution faster. Default is False.
F
flame 已提交
2876 2877 2878 2879

                Examples:
                    .. code-block:: python

2880 2881 2882 2883 2884 2885
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2886 2887
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
Z
Zhen Wang 已提交
2888 2889 2890 2891
      .def_property(
          "fuse_bn_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_act_ops_; },
          [](BuildStrategy &self, bool b) {
2892
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
Z
Zhen Wang 已提交
2893
                              platform::errors::PreconditionNotMet(
2894 2895
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Z
Zhen Wang 已提交
2896 2897 2898 2899 2900 2901 2902 2903 2904
            self.fuse_bn_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_act_ops indicate whether
                to fuse batch_norm and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

2905 2906 2907 2908 2909 2910
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
Z
Zhen Wang 已提交
2911 2912
                        build_strategy.fuse_bn_act_ops = True
                     )DOC")
Z
Zhang Ting 已提交
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937
      .def_property(
          "fuse_bn_add_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_add_act_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_bn_add_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_add_act_ops indicate whether
                to fuse batch_norm, elementwise_add and activation_op,
                it may make the execution faster. Default is True

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_bn_add_act_ops = True
                     )DOC")
2938 2939 2940 2941
      .def_property(
          "enable_auto_fusion",
          [](const BuildStrategy &self) { return self.enable_auto_fusion_; },
          [](BuildStrategy &self, bool b) {
2942
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
2943
                              platform::errors::PreconditionNotMet(
2944 2945
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2946 2947 2948 2949 2950 2951 2952 2953 2954 2955
            self.enable_auto_fusion_ = b;
          },
          R"DOC((bool, optional): Whether to enable fusing subgraph to a
                fusion_group. Now we only support fusing subgraph that composed
                of elementwise-like operators, such as elementwise_add/mul
                without broadcast and activations.

                Examples:
                    .. code-block:: python

2956 2957 2958 2959 2960 2961
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
2962 2963
                        build_strategy.enable_auto_fusion = True
                    )DOC")
2964 2965 2966 2967 2968 2969
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
2970 2971 2972 2973
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2974 2975
            self.fuse_relu_depthwise_conv_ = b;
          },
2976
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
2977 2978 2979
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
2980
                Default is False.
F
flame 已提交
2981 2982 2983 2984

                Examples:
                    .. code-block:: python

2985 2986 2987 2988 2989 2990
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2991 2992
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
2993 2994 2995
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
2996
                             self.fuse_broadcast_ops_ == paddle::none;
C
chengduo 已提交
2997 2998
                    },
                    [](BuildStrategy &self, bool b) {
2999 3000 3001 3002
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
3003 3004
                      self.fuse_broadcast_ops_ = b;
                    },
3005
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
3006 3007 3008 3009
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
3010 3011 3012 3013 3014
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

3015 3016 3017 3018 3019 3020
                              import paddle
                              import paddle.static as static

                              paddle.enable_static()

                              build_strategy = static.BuildStrategy()
3021 3022
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
3023 3024
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
3025
                      return self.fuse_all_optimizer_ops_ == true ||
3026
                             self.fuse_all_optimizer_ops_ == paddle::none;
C
chengduo 已提交
3027 3028
                    },
                    [](BuildStrategy &self, bool b) {
3029 3030 3031 3032
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
3033 3034
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
3035 3036 3037 3038
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
3039 3040 3041 3042
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Q
qingqing01 已提交
3043 3044
            self.sync_batch_norm_ = b;
          },
3045
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
3046 3047 3048
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
3049 3050
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
3051 3052 3053 3054

                Examples:
                    .. code-block:: python

3055 3056 3057 3058 3059 3060
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
3061 3062
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
3063 3064
      .def_property(
          "memory_optimize",
3065 3066 3067 3068 3069 3070 3071 3072 3073 3074
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
3075
              self.memory_optimize_ = paddle::none;
3076 3077 3078
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
3079
              PADDLE_THROW(platform::errors::InvalidArgument(
Z
Zeng Jinle 已提交
3080 3081
                  "BuildStrategy.memory_optimize must be set to None, False "
                  "or True"));
3082 3083
            }
          },
3084
          R"DOC((bool, optional): memory opitimize aims to save total memory
3085
                consumption, set to True to enable it.
3086

3087 3088 3089
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
                True means enabling and False means disabling. Default is None.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.memory_optimize = True
                
                )DOC")
3104 3105 3106
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
3107 3108 3109
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
3110
              PADDLE_THROW(platform::errors::Unavailable(
3111
                  "Distribution mode is not supported on Windows platform."));
3112 3113 3114 3115 3116
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
3117 3118 3119
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
3120
      .def_property(
D
dzhwinter 已提交
3121 3122 3123
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
3124 3125 3126 3127
      .def_property(
          "enable_addto",
          [](const BuildStrategy &self) { return self.enable_addto_; },
          [](BuildStrategy &self, bool b) { self.enable_addto_ = b; })
C
chengduo 已提交
3128 3129
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
3130 3131
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
3132
                   self.fuse_all_reduce_ops_ == paddle::none;
C
chengduo 已提交
3133
          },
C
chengduo 已提交
3134
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
3135 3136 3137 3138 3139 3140 3141
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
3142 3143 3144 3145
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
3146 3147 3148 3149 3150 3151 3152 3153 3154
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
Z
Zeng Jinle 已提交
3155 3156 3157 3158 3159 3160
      .def_property(
          "fix_op_run_order",
          [](const BuildStrategy &self) { return self.fix_op_run_order_; },
          [](BuildStrategy &self, bool fix_op_run_order) {
            self.fix_op_run_order_ = fix_op_run_order;
          })
3161 3162 3163 3164 3165 3166
      .def("_copy",
           [](const BuildStrategy &self) {
             auto new_bs = self;
             new_bs.ClearFinalized();
             return new_bs;
           })
3167
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
3168
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
3169 3170 3171 3172 3173
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
3174

3175 3176 3177 3178 3179 3180
  m.def("_set_cached_executor_build_strategy",
        [](int64_t program_id, const BuildStrategy &build_strategy) {
          auto &cached_exe_info = framework::ExecutorInfoCache::Instance();
          cached_exe_info.SetBuildStrategy(program_id, build_strategy);
        });

Y
yuyang18 已提交
3181
  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
3182
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
3183
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
3184
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
3185 3186 3187 3188
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
3189 3190 3191 3192 3193
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
3194 3195 3196
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
3197 3198 3199 3200
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
3201 3202
      .def("run",
           [](ParallelExecutor &self,
Z
Zhen Wang 已提交
3203 3204 3205 3206 3207 3208 3209 3210
              const std::vector<std::string> &fetch_tensors,
              bool return_merged) -> py::object {
             paddle::framework::FetchResultType ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(fetch_tensors, return_merged);
             }
             if (return_merged) {
3211
               return py::cast(
3212
                   std::move(BOOST_GET(paddle::framework::FetchList, ret)));
Z
Zhen Wang 已提交
3213 3214
             } else {
               return py::cast(std::move(
3215
                   BOOST_GET(paddle::framework::FetchUnmergedList, ret)));
Z
Zhen Wang 已提交
3216
             }
3217 3218
           })
      .def("device_count", &ParallelExecutor::DeviceCount);
Y
Yu Yang 已提交
3219

D
dongdaxiang 已提交
3220
  BindFleetWrapper(&m);
3221
  BindIO(&m);
T
Thunderbrook 已提交
3222

T
Thunderbrook 已提交
3223 3224
#ifdef PADDLE_WITH_PSLIB
  BindHeterWrapper(&m);
T
Thunderbrook 已提交
3225
#endif
T
Thunderbrook 已提交
3226
#ifdef PADDLE_WITH_HETERPS
T
Thunderbrook 已提交
3227
  BindPSGPUWrapper(&m);
T
Thunderbrook 已提交
3228
#endif
3229
  BindGlooWrapper(&m);
H
hutuxian 已提交
3230
  BindBoxHelper(&m);
H
hutuxian 已提交
3231 3232 3233
#ifdef PADDLE_WITH_BOX_PS
  BindBoxWrapper(&m);
#endif
3234
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
3235
  BindNCCLWrapper(&m);
3236 3237 3238
#endif
#ifdef PADDLE_WITH_GLOO
  BindGlooContext(&m);
W
wopeizl 已提交
3239
#endif
F
flame 已提交
3240 3241
  BindGraph(&m);
  BindNode(&m);
3242
  BindPass(&m);
F
flame 已提交
3243
  BindInferenceApi(&m);
3244
  BindCompatible(&m);
3245
  BindDataset(&m);
Y
yaoxuefeng 已提交
3246
  BindGenerator(&m);
3247 3248 3249
#ifdef PADDLE_WITH_ASCEND
  BindAscendWrapper(&m);
  BindAscendGraph(&m);
3250
  BindAscendDevice(&m);
3251
#endif
Y
Yanghello 已提交
3252 3253 3254
#ifdef PADDLE_WITH_CRYPTO
  BindCrypto(&m);
#endif
T
tangwei12 已提交
3255

T
tangwei12 已提交
3256
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
3257 3258
  BindDistFleetWrapper(&m);
  BindPSHost(&m);
3259
  BindCommunicatorContext(&m);
T
tangwei12 已提交
3260 3261
  BindDistCommunicator(&m);
  BindHeterClient(&m);
S
seemingwang 已提交
3262 3263 3264 3265 3266
  BindGraphPyFeatureNode(&m);
  BindGraphNode(&m);
  BindGraphPyService(&m);
  BindGraphPyServer(&m);
  BindGraphPyClient(&m);
1
123malin 已提交
3267 3268 3269 3270
  BindIndexNode(&m);
  BindTreeIndex(&m);
  BindIndexWrapper(&m);
  BindIndexSampler(&m);
3271
  BindSparseShardingTools(&m);
3272
#endif
L
Luo Tao 已提交
3273
}
3274
}  // namespace pybind
3275
}  // namespace paddle