pybind.cc 123.1 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
15

C
chengduoZH 已提交
16
#include <algorithm>
17
#include <cctype>
18
#include <cstdlib>
19
#include <iterator>
C
chengduoZH 已提交
20
#include <map>
S
sneaxiy 已提交
21
#include <memory>
C
chengduoZH 已提交
22 23
#include <mutex>  // NOLINT // for call_once
#include <string>
24 25
#include <tuple>
#include <type_traits>
C
chengduoZH 已提交
26
#include <unordered_map>
27
#include <unordered_set>
C
chengduoZH 已提交
28 29
#include <utility>
#include <vector>
30

31
#include "paddle/fluid/framework/custom_operator.h"
32
#include "paddle/fluid/framework/data_layout.h"
Y
Yi Wang 已提交
33 34
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
Z
Zhen Wang 已提交
35
#include "paddle/fluid/framework/feed_fetch_type.h"
S
sneaxiy 已提交
36
#include "paddle/fluid/framework/garbage_collector.h"
H
hutuxian 已提交
37
#include "paddle/fluid/framework/io/fs.h"
38
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
39
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
40 41 42
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
43
#include "paddle/fluid/framework/op_info.h"
44
#include "paddle/fluid/framework/op_registry.h"
45
#include "paddle/fluid/framework/op_version_registry.h"
Y
Yu Yang 已提交
46
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
47
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
48
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
49
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
50
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
51
#include "paddle/fluid/framework/selected_rows.h"
52
#include "paddle/fluid/framework/tensor_util.h"
53
#include "paddle/fluid/framework/trainer.h"
54
#include "paddle/fluid/framework/type_defs.h"
X
Xin Pan 已提交
55
#include "paddle/fluid/framework/version.h"
H
hong 已提交
56
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
57
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
58
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
D
dzhwinter 已提交
59
#include "paddle/fluid/operators/activation_op.h"
L
Leo Chen 已提交
60
#include "paddle/fluid/operators/common_infer_shape_functions.h"
S
sneaxiy 已提交
61
#include "paddle/fluid/operators/py_func_op.h"
62
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
63
#include "paddle/fluid/platform/cpu_info.h"
64
#include "paddle/fluid/platform/device_context.h"
65
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
66
#include "paddle/fluid/platform/enforce.h"
67
#include "paddle/fluid/platform/init.h"
H
hutuxian 已提交
68
#include "paddle/fluid/platform/monitor.h"
Y
Yi Wang 已提交
69 70
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
71 72 73
#ifdef PADDLE_WITH_ASCEND
#include "paddle/fluid/pybind/ascend_wrapper_py.h"
#endif
H
hutuxian 已提交
74
#include "paddle/fluid/pybind/box_helper_py.h"
75
#include "paddle/fluid/pybind/compatible.h"
Y
Yi Wang 已提交
76
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
77
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
78
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
79
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
Y
yaoxuefeng 已提交
80
#include "paddle/fluid/pybind/generator_py.h"
81
#include "paddle/fluid/pybind/global_value_getter_setter.h"
82
#include "paddle/fluid/pybind/gloo_context_py.h"
83
#include "paddle/fluid/pybind/gloo_wrapper_py.h"
T
Thunderbrook 已提交
84
#include "paddle/fluid/pybind/heter_wrapper_py.h"
85
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
86
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
87
#include "paddle/fluid/pybind/ir.h"
T
Thunderbrook 已提交
88
#include "paddle/fluid/pybind/ps_gpu_wrapper_py.h"
89
#include "paddle/fluid/pybind/pybind_boost_headers.h"
90

91
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
92
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
93
#endif
94
#include "paddle/fluid/framework/data_type.h"
95 96
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
97
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
98
#include "paddle/fluid/pybind/tensor_py.h"
99
#include "paddle/fluid/string/to_string.h"
100 101
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
Y
Yi Wang 已提交
102
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
103
#endif
104
#ifndef PADDLE_WITH_HIP
Y
Yi Wang 已提交
105
#include "paddle/fluid/platform/cuda_profiler.h"
106
#endif
Y
Yi Wang 已提交
107
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
108 109
#endif

110 111 112 113
#ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/platform/npu_info.h"
#endif

114 115 116 117
#ifdef PADDLE_WITH_XPU
#include "paddle/fluid/platform/xpu_info.h"
#endif

Y
Yanghello 已提交
118 119 120 121
#ifdef PADDLE_WITH_CRYPTO
#include "paddle/fluid/pybind/crypto.h"
#endif

T
tangwei12 已提交
122
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
123 124 125
#include "paddle/fluid/pybind/fleet_py.h"
#endif

M
minqiyang 已提交
126 127
#include "pybind11/stl.h"

128
DECLARE_bool(use_mkldnn);
129

Q
Qiao Longfei 已提交
130 131
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
132 133 134
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchUnmergedList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType);
Q
Qiao Longfei 已提交
135

136
namespace paddle {
137
namespace pybind {
138
bool IsCompiledWithCUDA() {
139 140 141 142 143 144 145 146 147
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
  return false;
#else
  return true;
#endif
}

bool IsCompiledWithROCM() {
#ifndef PADDLE_WITH_HIP
Q
qijun 已提交
148 149 150 151 152 153
  return false;
#else
  return true;
#endif
}

154 155 156 157 158 159 160 161
bool IsCompiledWithAscend() {
#ifndef PADDLE_WITH_ASCEND
  return false;
#else
  return true;
#endif
}

162 163 164 165 166 167 168 169
bool IsCompiledWithXPU() {
#ifndef PADDLE_WITH_XPU
  return false;
#else
  return true;
#endif
}

170 171 172 173 174 175 176 177
bool IsCompiledWithNPU() {
#ifndef PADDLE_WITH_ASCEND_CL
  return false;
#else
  return true;
#endif
}

178 179 180 181 182 183 184 185
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

186 187 188 189 190 191 192 193 194 195 196
bool SupportsBfloat16() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core))
    return true;
  else
    return false;
#endif
}

197 198 199 200 201 202 203 204 205 206 207
bool SupportsBfloat16FastPerformance() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_bf16))
    return true;
  else
    return false;
#endif
}

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
// According to the input `place` and `dtype`, this function returns a tuple
// consists of three sets:
// 1) All operators registered in the Paddle framework.
// 2) All operators supported for `place` and `dtype`.
// 3) All operators unsupported for `place` and `dtype`.
// The input `place` is a type of string, which can only be `GPU` or `CPU`.
// The input `dtype` is a type of paddle::framework::proto::VarType::Type,
// which can be paddle::framework::proto::VarType::FP16,
// paddle::framework::proto::VarType::FP32 and so on.
std::tuple<std::unordered_set<std::string>, std::unordered_set<std::string>,
           std::unordered_set<std::string>>
OpSupportedInfos(const std::string &place,
                 framework::proto::VarType::Type dtype) {
  std::string query_place;
  std::transform(place.begin(), place.end(), std::back_inserter(query_place),
                 [](unsigned char c) { return std::toupper(c); });
  using fn_type = std::add_pointer<bool(const platform::Place &)>::type;
  std::unordered_map<std::string, fn_type> is_target_place{
      {"GPU", &platform::is_gpu_place}, {"CPU", &platform::is_cpu_place},
  };
  PADDLE_ENFORCE_NE(
      is_target_place.count(query_place), 0,
      platform::errors::InvalidArgument(
          "The argument `place` should be 'GPU' or 'CPU', but get '%s'.",
          place));

  std::unordered_set<std::string> all_ops;
  const auto &op_info = framework::OpInfoMap::Instance().map();
  for (auto it = op_info.begin(); it != op_info.end(); it++) {
    all_ops.emplace(it->first);
  }

  std::unordered_set<std::string> supported_ops;
  auto &all_kernels = framework::OperatorWithKernel::AllOpKernels();
  for (auto it = all_kernels.begin(); it != all_kernels.end(); it++) {
    for (auto &kernel_type : it->second) {
      if (is_target_place[query_place](kernel_type.first.place_) &&
          kernel_type.first.data_type_ == dtype) {
        supported_ops.emplace(it->first);
      }
    }
  }

  std::unordered_set<std::string> unsupported_ops;
  for (auto &op : all_ops) {
    if (!supported_ops.count(op)) {
      unsupported_ops.emplace(op);
    }
  }

  VLOG(4) << "-- The size of all_ops: " << all_ops.size() << " --";
  VLOG(4) << "-- The size of supported_ops: " << supported_ops.size() << " --";
  VLOG(4) << "-- The size of unsupported_ops: " << unsupported_ops.size()
          << " --";
  return std::make_tuple(std::move(all_ops), std::move(supported_ops),
                         std::move(unsupported_ops));
}

266
bool IsCompiledWithBrpc() {
267
#ifndef PADDLE_WITH_DISTRIBUTE
268 269
  return false;
#endif
270 271 272 273 274 275

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
276 277
}

Y
update  
Yancey1989 已提交
278
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
279
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
280 281 282 283 284 285
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
286 287 288 289 290 291 292 293 294 295
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

H
hong 已提交
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
318 319 320
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Python object is not type of %s, the real type is %s",
        typeid(T).name(), obj->ob_type->tp_name));
H
hong 已提交
321 322 323 324 325 326 327 328 329 330 331 332 333
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
334 335
      PADDLE_THROW(platform::errors::InvalidArgument(
          "The parameter [%s] to save is None", para.first));
H
hong 已提交
336 337
    }
    vec_res.emplace_back(
338
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
H
hong 已提交
339 340 341 342 343 344 345 346 347 348 349 350
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
351 352
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameter list to save is None"));
H
hong 已提交
353 354 355 356 357 358 359 360 361 362 363 364
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
365 366 367
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to save is None"));
H
hong 已提交
368 369 370 371
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
372 373
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to save is not a list"));
H
hong 已提交
374 375 376 377
  }
  return vec_res;
}

378 379 380 381 382 383 384 385
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
386 387
    PADDLE_THROW(
        platform::errors::InvalidArgument("The parameter list to set is None"));
388 389 390 391 392 393 394 395 396 397 398 399 400
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
401 402 403
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to set is None"));
404 405 406 407 408
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
409 410 411 412 413
        PADDLE_ENFORCE_NOT_NULL(exe,
                                platform::errors::InvalidArgument(
                                    "Parameter not Initialized, "
                                    "Please set argument [executor] not None "
                                    "or run startup program first"));
414 415
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
416 417 418
        PADDLE_ENFORCE_NOT_NULL(
            py_var_desc, platform::errors::InvalidArgument(
                             "The var_desc of parameter to set is None"));
419 420 421 422 423 424 425 426 427
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
        tensor_temp->Resize(framework::make_ddim(var_desc.GetShape()));
        tensor_temp->mutable_data(exe->GetPlace(), var_desc.GetDataType());
      }
    }
  } else {
428 429
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to set is not a list"));
430 431 432 433 434
  }

  return;
}

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
static void AssertStaticGraphAndDygraphGradMakerNoDiff() {
  std::set<std::string> ops;
  for (auto &pair : framework::OpInfoMap::Instance().map()) {
    bool has_static_grad_maker = (pair.second.grad_op_maker_ != nullptr);
    bool has_dygraph_grad_maker =
        (pair.second.dygraph_grad_op_maker_ != nullptr);
    if (has_static_grad_maker ^ has_dygraph_grad_maker) {
      bool has_kernel =
          (framework::OperatorWithKernel::AllOpKernels().count(pair.first) > 0);
      if (has_kernel) {
        ops.insert(pair.first);
      } else {
        VLOG(5) << pair.first << " has no kernels, skip";
      }
    }
  }
  PADDLE_ENFORCE_EQ(ops.empty(), true,
                    platform::errors::Unimplemented(
                        "OperatorWithKernel [%s] have only static graph grad "
                        "maker or have only dygraph grad maker, which is not "
                        "allowed",
                        string::join_strings(ops, ',')));
}

459 460 461 462 463 464
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

Y
Yu Yang 已提交
465 466 467
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
468
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
469

470 471
  AssertStaticGraphAndDygraphGradMakerNoDiff();

472
  m.doc() = "C++ core of PaddlePaddle";
473

474 475 476 477
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

478
  BindException(&m);
Y
Yu Yang 已提交
479

480 481
  m.def("set_num_threads", &platform::SetNumThreads);

482
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
483 484 485
  m.def("cudnn_version", &platform::CudnnVersion);
#endif

6
633WHU 已提交
486 487 488 489 490
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
491
    framework::Tensor tensor;
6
633WHU 已提交
492 493 494 495

    if (dl.ctx.device_type == kDLCPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
496
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
6
633WHU 已提交
497 498 499 500 501 502 503
    if (dl.ctx.device_type == kDLGPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });

H
hong 已提交
504 505 506 507 508 509 510 511 512
  m.def("_save_static_dict",
        [](const std::string &str_file_name, const py::handle &vec_var_list,
           const Scope &scope) {
          std::vector<std::string> vec_name_list = GetNameList(vec_var_list);
          SaveStaticNameListToDisk(str_file_name, vec_name_list, scope);
        });

  m.def("_load_static_dict",
        [](const std::string &str_file_name, const py::handle &vec_var_list,
513
           const Scope &scope, const Executor *executor) {
H
hong 已提交
514
          std::vector<std::string> vec_name_list = GetNameList(vec_var_list);
515
          CreateVariableIfNotExit(vec_var_list, scope, executor);
H
hong 已提交
516 517 518
          LoadStaticNameListFromDisk(str_file_name, vec_name_list, scope);
        });

519 520 521 522 523 524
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

H
hong 已提交
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
  m.def("_save_dygraph_dict", [](const std::string &str_file_name,
                                 const PyNameVarBaseMap &state_dict) {
    auto vec_var_base_list = GetVarBaseList(state_dict);

    SaveDygraphVarBaseListToDisk(str_file_name, vec_var_base_list);
  });

  m.def("_load_dygraph_dict", [](const std::string &str_file_name) {
    auto load_tensor = LoadDygraphVarBaseListFromDisk(str_file_name);

    std::unordered_map<std::string, std::shared_ptr<imperative::VarBase>>
        map_output;

    for (size_t i = 0; i < load_tensor.size(); ++i) {
      map_output.emplace(load_tensor[i]->Name(), load_tensor[i]);
    }

    return map_output;
  });
6
633WHU 已提交
544

545 546 547 548 549 550
  m.def("save_op_version_info", [](framework::ProgramDesc &desc) {
    framework::compatible::pb::OpVersionMap pb_vmap{desc.OpVersionMap()};
    framework::compatible::SaveOpVersions(
        framework::compatible::OpVersionRegistrar::GetInstance()
            .GetVersionMap(),
        &pb_vmap);
551 552
  });

553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
  m.def("set_printoptions", [](const py::kwargs &kwargs) {
    auto &print_opt = framework::PrintOptions::Instance();
    if (kwargs.contains("precision")) {
      print_opt.precision = kwargs["precision"].cast<int>();
    }
    if (kwargs.contains("threshold")) {
      print_opt.threshold = kwargs["threshold"].cast<int>();
    }
    if (kwargs.contains("edgeitems")) {
      print_opt.edgeitems = kwargs["edgeitems"].cast<int>();
    }
    if (kwargs.contains("linewidth")) {
      print_opt.linewidth = kwargs["linewidth"].cast<int>();
    }
    if (kwargs.contains("sci_mode")) {
      print_opt.sci_mode = kwargs["sci_mode"].cast<bool>();
    }

    VLOG(4) << "Set printoptions: precision=" << print_opt.precision
            << ", threshold=" << print_opt.threshold
            << ", edgeitems=" << print_opt.edgeitems
            << ", linewidth=" << print_opt.linewidth
            << ", sci_mode=" << print_opt.sci_mode;
  });

L
Leo Chen 已提交
578 579 580 581 582 583
  m.def("broadcast_shape", [](const std::vector<int64_t> &x_dim,
                              const std::vector<int64_t> &y_dim) {
    return vectorize(operators::details::BroadcastTwoDims(
        make_ddim(x_dim), make_ddim(y_dim), -1));
  });

584 585 586 587 588
#ifdef PADDLE_WITH_ASCEND_CL
  m.def("_npu_finalize",
        []() { platform::AclInstance::Instance().Finalize(); });
#endif

S
sneaxiy 已提交
589
  m.def(
S
sneaxiy 已提交
590
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
591 592 593 594
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
595 596 597
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
  m.def("_get_all_register_op_kernels", [] {
    auto &all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
    std::unordered_map<std::string, std::vector<std::string>> all_kernels_info;
    for (auto &kernel_pair : all_kernels) {
      auto op_type = kernel_pair.first;
      std::vector<std::string> kernel_types;
      for (auto &info_pair : kernel_pair.second) {
        paddle::framework::OpKernelType kernel_type = info_pair.first;
        kernel_types.push_back(
            paddle::framework::KernelTypeToString(kernel_type));
      }
      all_kernels_info.emplace(op_type, kernel_types);
    }
    return all_kernels_info;
  });

S
sneaxiy 已提交
614 615 616
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
617
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
618

619
  m.def("_set_fuse_parameter_group_size",
620
        &paddle::framework::ir::SetFuseParameterGroupsSize);
621
  m.def("_set_fuse_parameter_memory_size",
622
        &paddle::framework::ir::SetFuseParameterMemorySize);
623

S
sneaxiy 已提交
624 625 626
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

627 628
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

629 630 631
  m.def("_promote_types_if_complex_exists",
        &paddle::framework::PromoteTypesIfComplexExists);

632
  BindImperative(&m);
633

634 635 636
  py::class_<framework::Tensor>(m, "Tensor", py::buffer_protocol())
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
637
      .def("_is_initialized",
638
           [](const framework::Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
639
      .def("_get_dims",
640
           [](const framework::Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
641
      .def("_set_dims",
642
           [](framework::Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
643
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
644
           })
Y
yuyang18 已提交
645
      .def("_set_layout",
646
           [](framework::Tensor &self, const std::string &layout) {
D
dzhwinter 已提交
647 648
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
649
      .def("_alloc_float",
650
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
651
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
652
           })
653
      .def("_alloc_float",
654
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
655 656
             self.mutable_data<float>(place);
           })
Y
yuyang18 已提交
657
      .def("_alloc_float",
658
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
659
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
660
           })
661 662 663 664
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place) {
             self.mutable_data<float>(place);
           })
665
      .def("_alloc_double",
666
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
667 668
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
669
      .def("_alloc_int",
670
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
671
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
672
           })
673
      .def("_alloc_int",
674
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
675 676
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
677
      .def("_alloc_int",
678
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
679
             self.mutable_data<int>(place);
Q
qijun 已提交
680
           })
Y
yuyang18 已提交
681
      .def("_alloc_int",
682 683
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
684 685
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
686
      .def("_alloc_float",
687 688
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
689 690
             self.mutable_data<float>(place);
           })
691
      .def("_mutable_data",
692
           [](framework::Tensor &self, paddle::platform::CPUPlace &place,
693 694 695
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
696
      .def("_mutable_data",
697
           [](framework::Tensor &self, paddle::platform::XPUPlace &place,
698 699 700
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
701
      .def("_mutable_data",
702
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place,
703 704 705 706
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
      .def("_mutable_data",
707
           [](framework::Tensor &self, paddle::platform::CUDAPinnedPlace &place,
708 709 710
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
711
      .def("_clear", &framework::Tensor::clear)
712 713 714 715 716
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
717
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
718
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
719 720
      .def("set", SetTensorFromPyArray<paddle::platform::XPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
721
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
722
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
723 724
      .def("set", SetTensorFromPyArray<paddle::platform::NPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
725
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
726 727
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
L
Leo Chen 已提交
728 729 730 731
        Set the data of LoDTensor on place with given numpy array.
        
        Args:
          lod (numpy.ndarray): The data to set.
732
          place (CPUPlace|CUDAPlace|XPUPlace|CUDAPinnedPlace|NPUPlace): The place where the
L
Leo Chen 已提交
733
          LoDTensor is to be set.
734 735
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
736 737 738 739 740 741 742 743 744 745 746 747 748

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                t = fluid.LoDTensor()
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
749

750 751 752
      .def("shape",
           [](framework::Tensor &self) { return vectorize(self.dims()); },
           R"DOC(
L
Leo Chen 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
           Return the shape of LoDTensor.

           Returns:
               list[int]: The shape of LoDTensor.


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

                  t = fluid.LoDTensor()
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
769
      .def("_to_dlpack",
770
           [](framework::Tensor &self) {
6
633WHU 已提交
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
             DLPackTensor dlpack_tensor(self, 1);
             DLManagedTensor *dmt =
                 dlpack_tensor.ToCudfCompatibleDLManagedTensor();
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
791 792 793 794
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
795 796
      .def("_place", [](framework::Tensor &self) { return self.place(); })
      .def("_dtype", [](framework::Tensor &self) { return self.type(); })
797
      .def("_layout",
798 799 800 801
           [](framework::Tensor &self) {
             return DataLayoutToString(self.layout());
           })
      .def("_share_data_with", &framework::Tensor::ShareDataWith)
802
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
803
      .def("__str__", [](const framework::Tensor &self) {
804 805 806 807
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
808

L
Leo Chen 已提交
809
  // TODO(cql): add reference: en_user_guide_lod_tensor
810
  py::class_<LoDTensor, framework::Tensor>(m, "LoDTensor", R"DOC(
L
Leo Chen 已提交
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
    LoDTensor is a Tensor with optional LoD (Level of Details) information, 
    it can be used for variable-length sequences, 
    see :ref:`user_guide_lod_tensor` for details.

    LoDTensor can be converted to numpy array using :code:`numpy.array(lod_tensor)`.

    You can skip the following explanation if you don't need to know details 
    of LoDTensor.

    The following two examples show how to use LODtensor to represent 
    variable-length sequences.
    
    Example 1:
    
    Suppose x is a LoDTensor representing a variable-length sequence. 
    It contains two logical subsequences, the length of first logical sequence 
    is 2 (e.g., number of samples is 2), the length of second logical sequence 
    is 3, and the total length is 5. The data of the first logical sequence is 
    [1, 2], [3, 4], and the data of the second logical sequence is [5, 6], 
    [7, 8], [9, 10]. The data dimension of each sample is 2. So, the final 
    shape of the LoDTensor is [5, 2], of which 5 is the total length and 2 is 
    the dimension of each sample.
    
    Logically, we can represent the variable-length sequence in two ways: one 
    is in the form of recursive sequence lengths, that is, 
    x.recursive_sequence_lengths=[[2, 3]]; the other is in the form of offsets, 
    that is, x.lod=[[0, 2, 2+3]]. These two representations are equivalent, and 
    you can set and retrieve recursive_sequence_lengths or LoD through the 
    corresponding interfaces of LoDTensor introduced later.

    Actually, in order to access sequence faster, Paddle uses offset to store 
    different lengths of sequences. 
    Therefore, the operations on recursive_sequence_lengths will be converted 
    to the operations on LoD eventually.
    
    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]

    Example 2:

    LoD may have more than one level (for example, a paragraph may have more 
    than one sentence and a sentence may have more than one word). Suppose y 
    is a LoDTensor and its lod_level is 2. 
    From level = 0, there are two logical sequences, the length of which is 
    2 and 1, respectively, indicating that the first logical sequence contains 
    two sub-sequences and the second logical sequence contains one sub-sequence. 
    From level = 1, the lengths of two sub-sequences contained by the first 
    logical sequence is 2 and 2, and the length of sub-sequence contained by 
    the second logical sequence is 3.
      
    Therefore, the LoDTensor is represented in the form of recursive sequence 
    lengths as y.recursive_sequence_lengths=[[2,1], [2,2,3]]; and equally, in 
    the form of offset, it is represented as y.lod=[[0,2,3], [0,2,4,7]].

    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]
Z
Zeng Jinle 已提交
885 886 887 888 889 890 891

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
892 893

        )DOC")
894 895
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
896 897 898 899 900 901 902 903 904
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
905 906
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, -1), true,
907 908 909 910
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is %s",
                     new_lod));
911 912
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
913
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
914
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
915 916
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
917 918 919
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
920
      .def("set_lod",
921
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
922
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
923
             LoD new_lod;
924 925
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
926 927
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
928 929
                 platform::errors::InvalidArgument(
                     "The provided LoD is invalid, the LoD is %s", new_lod));
930
             self.set_lod(new_lod);
S
sneaxiy 已提交
931 932 933 934 935
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
L
Leo Chen 已提交
936 937 938 939
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
940 941 942 943 944 945 946 947 948 949

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
950
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
951
           )DOC")
952 953 954 955 956 957 958 959 960 961 962
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
963 964
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
965 966 967 968 969
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is "
                     "%s",
                     new_lod));
970
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
971 972
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
L
Leo Chen 已提交
973
           Set LoD of the LoDTensor according to recursive sequence lengths.
S
sneaxiy 已提交
974

L
Leo Chen 已提交
975
           For example, if recursive_sequence_lengths=[[2, 3]], which means
976
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
977
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
978 979

           Args:
L
Leo Chen 已提交
980 981 982 983
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
984 985 986 987 988 989 990 991 992 993

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
L
Leo Chen 已提交
994 995
                 print(t.recursive_sequence_length())  # [[2, 3]]
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
996
           )DOC")
997 998 999 1000 1001 1002 1003 1004
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1005 1006 1007 1008 1009
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
L
Leo Chen 已提交
1010 1011
               list[list[int]]: The lod of the LoDTensor.
           
Z
Zeng Jinle 已提交
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
1022
           )DOC")
G
gongweibao 已提交
1023
      // Set above comments of set_lod.
1024 1025 1026 1027 1028 1029 1030 1031
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1032 1033
           },
           R"DOC(
L
Leo Chen 已提交
1034 1035
           Return the recursive sequence lengths corresponding to of the LodD 
           of the LoDTensor.
S
sneaxiy 已提交
1036 1037

           Returns:
L
Leo Chen 已提交
1038
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
1050 1051 1052 1053 1054 1055 1056 1057
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
L
Leo Chen 已提交
1058
           Check whether the LoD of the LoDTensor is valid.
S
sneaxiy 已提交
1059 1060

           Returns:
L
Leo Chen 已提交
1061
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
1073 1074 1075 1076 1077 1078 1079
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
1080
           )DOC")
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
      .def("__str__",
           [](const LoDTensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           })
      .def("_copy", [](const LoDTensor &self, const platform::Place &place) {
        // follow fetch_op's inplementation
        LoDTensor dst;
        if (self.IsInitialized() && self.numel() > 0) {
          TensorCopySync(self, place, &dst);
        } else {
          // Not copy, if the src tensor is empty.
          dst.clear();
          dst.Resize({0});
        }
        dst.set_lod(self.lod());
        return dst;
1099
#ifdef _WIN32
1100
      });
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
#else
           })
      .def(py::pickle(
          [](const LoDTensor &t) {  // __getstate__
            auto holder = t.Holder();
            PADDLE_ENFORCE_EQ(
              platform::is_cpu_place(holder->place()), true,
              platform::errors::PreconditionNotMet(
                  "LoDTensor is not on CPU."
                  "Now only LoDTensor on CPU can be serialized."));
            auto* mmap_writer_allocation =
              dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
                holder.get());
            PADDLE_ENFORCE_NOT_NULL(mmap_writer_allocation,
              platform::errors::PreconditionNotMet(
                "LoDTensor is not in shared memory."
                "Now only LoDTensor on shared memory can be serialized."));
            int type_idx = static_cast<int>(t.type());

            return py::make_tuple(mmap_writer_allocation->ipc_name(),
                                  mmap_writer_allocation->size(),
                                  type_idx, vectorize(t.dims()), t.lod());
          },
          [](py::tuple t) {  // __setstate__
            if (t.size() != 5)
              throw std::runtime_error("Invalid LoDTensor state!");

            // 1. Create a new C++ instance
            LoDTensor tensor;

            // 2. Rebuild Allocation
            const std::string &ipc_name = t[0].cast<std::string>();
            size_t size = t[1].cast<size_t>();
            auto shared_reader_holder =
              memory::allocation::RebuildMemoryMapReaderAllocation(
                ipc_name, size);

            // 3. Maintain global fd set
            VLOG(3) << "LoDTensor ipc name: " << ipc_name;
            memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);

            // 4. Rebuild LoDTensor
            tensor.ResetHolderWithType(shared_reader_holder,
              static_cast<proto::VarType::Type>(t[2].cast<int>()));
            tensor.Resize(make_ddim(t[3].cast<std::vector<int>>()));
            tensor.set_lod(t[4].cast<framework::LoD>());

            return tensor;
          }));
#endif
D
dangqingqing 已提交
1151

Q
qijun 已提交
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
1163 1164
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
1165 1166
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
1167 1168
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
1169
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
Q
qijun 已提交
1170 1171 1172 1173 1174 1175
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
1176
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
1177
      .def("rows", [](SelectedRows &self) {
1178 1179 1180 1181 1182
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
1183
      });
Q
qijun 已提交
1184

1185
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
1186 1187 1188

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
1189
      .def(py::init<>())
1190
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
1191
      .def("set_int",
1192 1193
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
1194 1195 1196 1197 1198 1199 1200
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
1201
      .def("get_tensor",
1202 1203
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
1204 1205
           },
           py::return_value_policy::reference)
1206 1207 1208 1209
      .def("get_bytes",
           [](Variable &self) {
             return py::bytes(*self.GetMutable<std::string>());
           })
Y
Yu Yang 已提交
1210 1211 1212
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
1213 1214 1215 1216 1217
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1218 1219 1220
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
1221 1222 1223
      .def("get_fetch_list",
           [](Variable &self) { return self.GetMutable<FetchList>(); },
           py::return_value_policy::reference)
1224
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1225 1226 1227 1228 1229
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1230
#endif
Y
Refine  
Yu Yang 已提交
1231 1232
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
1233 1234 1235 1236
             PADDLE_ENFORCE_EQ(
                 self.IsType<framework::ReaderHolder>(), true,
                 platform::errors::InvalidArgument(
                     "The variable is not type of ReaderHolder."));
Y
Refine  
Yu Yang 已提交
1237 1238
             return self.GetMutable<framework::ReaderHolder>();
           },
1239 1240 1241 1242 1243
           py::return_value_policy::reference)
      .def("set_scope", [](Variable &self, Scope &scope) {
        auto scope_vec = self.GetMutable<std::vector<framework::Scope *>>();
        scope_vec->emplace_back(&scope);
      });
1244

S
sneaxiy 已提交
1245
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
1246

S
sneaxiy 已提交
1247
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

1261
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
1262 1263 1264 1265 1266 1267
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
1268 1269
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
1270
      .def("var",
1271
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
1272
             return self.Var(name);
Y
Yu Yang 已提交
1273
           },
S
sneaxiy 已提交
1274 1275
           py::arg("name"),
           R"DOC(
1276
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
1277

1278
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
1279
           current scope, the variable would be created. Otherwise,
1280
           return the existing variable.
S
sneaxiy 已提交
1281 1282

           Args:
1283 1284
               name (str): the variable name.

S
sneaxiy 已提交
1285
           Returns:
1286
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
1287 1288 1289 1290
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
1291
           Find variable named :code:`name` in the current scope or
1292
           its parent scope. Return None if not found. 
1293

S
sneaxiy 已提交
1294 1295
           Args:
               name (str): the variable name.
1296

S
sneaxiy 已提交
1297
           Returns:
1298
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1299
           )DOC",
1300
           py::return_value_policy::reference)
1301
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1302 1303 1304 1305 1306 1307
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1308
           py::return_value_policy::reference)
S
sneaxiy 已提交
1309 1310 1311
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1312 1313
           )DOC")
      .def("_kids", &Scope::kids);
1314

S
sneaxiy 已提交
1315 1316 1317 1318 1319 1320
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1321 1322
        R"DOC(
        Create a new scope.
1323

S
sneaxiy 已提交
1324 1325 1326
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1327 1328
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1329 1330
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1331 1332
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1333 1334 1335 1336
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1337 1338
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1339 1340
            platform::errors::Fatal(
                "Serialize OpProto Error. This could be a bug of Paddle."));
1341 1342 1343
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1344 1345
    return ret_values;
  });
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
  m.def("get_op_attrs_default_value",
        [](py::bytes byte_name) -> paddle::framework::AttributeMap {
          std::string op_type = byte_name;
          paddle::framework::AttributeMap res;
          auto info = OpInfoMap::Instance().GetNullable(op_type);
          if (info != nullptr) {
            if (info->HasOpProtoAndChecker()) {
              auto op_checker = info->Checker();
              res = op_checker->GetAttrsDefaultValuesMap();
            }
          }
          return res;
        });
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1375 1376 1377
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
1378 1379 1380 1381 1382
  m.def("has_non_empty_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance()
        .Get(op_type)
        .HasNonEmptyGradOpMaker();
  });
1383 1384 1385
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });
1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
  m.def("infer_no_need_buffer_slots",
        [](const std::string op_type, const framework::VariableNameMap &inputs,
           const framework::VariableNameMap &outputs,
           const framework::AttributeMap &attrs) {
          auto infer_func = framework::OpInfoMap::Instance()
                                .Get(op_type)
                                .NoNeedBufferVarsInferer();
          if (infer_func) {
            return infer_func(inputs, outputs, attrs);
          } else {
            std::unordered_set<std::string> empty = {};
            return empty;
          }
        });
Y
Yu Yang 已提交
1400
  m.def("prune", [](const ProgramDesc &origin,
1401
                    const std::set<std::string> &feeded_var_names,
1402
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1403
    ProgramDesc prog_with_targets(origin);
1404

1405
    for (const auto &t : targets) {
1406
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1407
    }
1408
    proto::ProgramDesc pruned_desc;
1409 1410 1411 1412
    auto pruned_origin_block_id_map =
        Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
    return std::make_tuple(ProgramDesc(pruned_desc),
                           pruned_origin_block_id_map);
1413
  });
1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
  m.def("prune_backward",
        [](const framework::ProgramDesc &program) {
          return PruneBackward(program);
        },
        R"DOC(
             Prune the backward part of a program, mostly called in
             program.clone(for_test=True).
              
             Args:
                   program (ProgramDesc): The original program.

             Returns:
                   tuple(ProgramDesc, map<int, int>): The first part is 
                   the pruned program desc, and the second part is a map
                   which contains the id pair of pruned block and corresponding
                   origin block.
           )DOC");
1431 1432 1433 1434
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1435 1436 1437
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1438 1439
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
1440

Q
qijun 已提交
1441
  // clang-format off
Y
Yu Yang 已提交
1442
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1443 1444
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1445
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
1446 1447
                    return new paddle::platform::CPUDeviceContext();
                  })
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
      .def_static("create",
                  [](paddle::platform::XPUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_XPU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use XPUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with XPU support."));
#else
                    return new paddle::platform::XPUDeviceContext(place);
#endif
                  })
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
        .def_static("create",
                    [](paddle::platform::NPUPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_ASCEND_CL
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use NPUPlace in CPU/GPU/XPU version, "
                 "Please recompile or reinstall Paddle with NPU support."));
#else
                return new paddle::platform::NPUDeviceContext(place);
#endif
        })
Q
qijun 已提交
1472
      .def_static("create",
D
dzhwinter 已提交
1473
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1474
                      -> paddle::platform::DeviceContext* {
1475
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1476 1477 1478 1479
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
Q
qijun 已提交
1480
#else
Q
qijun 已提交
1481
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
1482
#endif
C
chengduoZH 已提交
1483 1484 1485 1486
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
1487
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1488 1489 1490 1491
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
C
chengduoZH 已提交
1492 1493 1494 1495
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
1496
// clang-format on
1497
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1498 1499
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
1500
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
1501 1502 1503 1504 1505

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
1506
    The memory of CUDAPlace with different dev_id is not accessible.
1507 1508 1509 1510 1511 1512 1513 1514
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
1515 1516 1517 1518

    Examples:
        .. code-block:: python

1519 1520 1521
          import paddle

          place = paddle.CUDAPlace(0)
L
lujun 已提交
1522

1523
        )DOC")
S
sneaxiy 已提交
1524 1525
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
1526
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

             if (UNLIKELY(dev_id >= platform::GetCUDADeviceCount())) {
               if (platform::GetCUDADeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
                     dev_id, platform::GetCUDADeviceCount(),
                     platform::GetCUDADeviceCount());
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
1551 1552
             new (&self) platform::CUDAPlace(dev_id);
#else
1553 1554 1555 1556 1557 1558 1559 1560 1561
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
1562 1563
#endif
           })
1564
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1565 1566
      .def("get_device_id",
           [](const platform::CUDAPlace &self) { return self.GetDeviceId(); })
S
sneaxiy 已提交
1567 1568 1569 1570
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
1571
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
1572
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1573 1574
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
1575 1576 1577
      .def("_get_device_id",
           [](platform::CUDAPlace &self) -> int { return self.GetDeviceId(); })
#endif
1578
      .def("__repr__", string::to_string<const platform::CUDAPlace &>)
D
dzhwinter 已提交
1579
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
1580

1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
  py::class_<platform::XPUPlace>(m, "XPUPlace", R"DOC(
    **Note**:
    Examples:
        .. code-block:: python
          import paddle.fluid as fluid
          xpu_place = fluid.XPUPlace(0)
        )DOC")
      .def("__init__",
           [](platform::XPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_XPU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid XPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetXPUDeviceCount())) {
               if (platform::GetXPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use XPU because there is no XPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid XPUPlace(%d), must inside [0, %d), because XPU "
                     "number on your machine is %d",
                     dev_id, platform::GetXPUDeviceCount(),
                     platform::GetXPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::XPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use XPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use XPU, please try to install XPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change XPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
1626
#ifdef PADDLE_WITH_XPU
1627 1628 1629 1630 1631 1632 1633
      .def("_type", &PlaceIndex<platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::XPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::XPUPlace, platform::CUDAPinnedPlace>)
1634 1635 1636
      .def("get_device_id",
           [](const platform::XPUPlace &self) { return self.GetDeviceId(); })
#endif
1637
      .def("__repr__", string::to_string<const platform::XPUPlace &>)
1638
      .def("__str__", string::to_string<const platform::XPUPlace &>);
1639 1640 1641
#ifdef PADDLE_WITH_XPU
  m.def("get_xpu_device_count", platform::GetXPUDeviceCount);
#endif
1642

1643
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
1644
    CPUPlace is a descriptor of a device.
1645
    It represents a CPU device on which a tensor will be allocated and a model will run.
L
lujun 已提交
1646 1647 1648 1649

    Examples:
        .. code-block:: python

1650 1651
          import paddle
          cpu_place = paddle.CPUPlace()
L
lujun 已提交
1652

1653
        )DOC")
1654
      .def(py::init<>())
S
sneaxiy 已提交
1655 1656
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
1657
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
1658
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1659 1660 1661 1662
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
1663
      .def("__repr__", string::to_string<const platform::CPUPlace &>)
1664
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
1665

1666
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
1667 1668 1669 1670 1671 1672
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
1673 1674 1675 1676

    Examples:
        .. code-block:: python

1677 1678
          import paddle
          place = paddle.CUDAPinnedPlace()
L
lujun 已提交
1679

1680
        )DOC")
S
sneaxiy 已提交
1681
      .def("__init__",
S
sneaxiy 已提交
1682
           [](platform::CUDAPinnedPlace &self) {
1683
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1684 1685 1686
             PADDLE_THROW(platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
S
sneaxiy 已提交
1687
#endif
S
sneaxiy 已提交
1688
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
1689
           })
S
sneaxiy 已提交
1690 1691 1692 1693
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
1694 1695
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
1696 1697
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1698 1699 1700 1701
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
1702
      .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
C
chengduoZH 已提交
1703 1704
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
  // NPUPlace
  py::class_<platform::NPUPlace>(m, "NPUPlace", R"DOC(
    NPUPlace is a descriptor of a device.
    It represents a NPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          npu_place = paddle.NPUPlace(0)

        )DOC")
      .def("__init__",
           [](platform::NPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_ASCEND_CL
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid NPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetNPUDeviceCount())) {
               if (platform::GetNPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use NPU because there is no NPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid NPUPlace(%d), must inside [0, %d), because NPU "
                     "number on your machine is %d",
                     dev_id, platform::GetNPUDeviceCount(),
                     platform::GetNPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::NPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use NPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use NPU, please try to install NPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change NPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
      .def("__str__", string::to_string<const platform::NPUPlace &>);

Y
Yu Yang 已提交
1764 1765
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
1766 1767 1768 1769
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
1770
      .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
1771
      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
S
sneaxiy 已提交
1772
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
1773 1774
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
1775 1776
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
1777 1778
      .def("is_xpu_place",
           [](platform::Place &self) { return platform::is_xpu_place(self); })
1779 1780
      .def("is_npu_place",
           [](platform::Place &self) { return platform::is_npu_place(self); })
S
sneaxiy 已提交
1781 1782 1783 1784
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
1785 1786
      .def("gpu_device_id",
           [](platform::Place &self) {
1787
             return BOOST_GET_CONST(platform::CUDAPlace, self).device;
X
xuezhong 已提交
1788
           })
1789 1790 1791 1792
      .def("xpu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::XPUPlace, self).device;
           })
1793 1794 1795 1796
      .def("npu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::NPUPlace, self).device;
           })
S
sneaxiy 已提交
1797 1798
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
1799 1800 1801 1802
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
1803 1804 1805 1806
      .def("set_place",
           [](platform::Place &self, const platform::XPUPlace &xpu_place) {
             self = xpu_place;
           })
Y
Yu Yang 已提交
1807
      .def("set_place",
D
dzhwinter 已提交
1808
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
1809
             self = gpu_place;
C
chengduoZH 已提交
1810
           })
1811 1812 1813 1814 1815
      .def("set_place",
           [](platform::Place &self,
              const platform::CUDAPinnedPlace &cuda_pinned_place) {
             self = cuda_pinned_place;
           })
1816 1817 1818 1819
      .def("set_place",
           [](platform::Place &self, const platform::NPUPlace &npu_place) {
             self = npu_place;
           })
1820 1821
      .def("__repr__", string::to_string<const platform::Place &>)
      .def("__str__", string::to_string<const platform::Place &>);
Y
Yu Yang 已提交
1822

Y
Yu Yang 已提交
1823
  py::class_<OperatorBase>(m, "Operator")
C
chengduo 已提交
1824 1825 1826 1827 1828
      .def_static(
          "create",
          [](py::bytes protobin) {
            proto::OpDesc desc;
            PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin), true,
1829 1830 1831 1832 1833 1834 1835
                              platform::errors::InvalidArgument(
                                  "Cannot parse user input to OpDesc"));
            PADDLE_ENFORCE_EQ(
                desc.IsInitialized(), true,
                platform::errors::InvalidArgument(
                    "The provided OpDesc is not initialized, the reason is: %s",
                    desc.InitializationErrorString()));
C
chengduo 已提交
1836 1837
            return OpRegistry::CreateOp(desc);
          })
1838
      .def("run",
1839
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1840
              const platform::CPUPlace &place) { self.Run(scope, place); })
1841 1842 1843
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::XPUPlace &place) { self.Run(scope, place); })
1844 1845 1846
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::NPUPlace &place) { self.Run(scope, place); })
D
dzhwinter 已提交
1847 1848
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1849
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
1850 1851 1852 1853 1854
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1855 1856 1857 1858 1859 1860 1861
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1862 1863
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1864
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1865
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1866 1867 1868 1869
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1870

1871 1872 1873
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

1874 1875 1876 1877 1878 1879 1880 1881 1882
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
      .def("finalize", &TrainerBase::Finalize);

F
fengjiayi 已提交
1883
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1884
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1885
      .def("close", &Executor::Close)
1886 1887
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
1888 1889
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
1890 1891 1892 1893
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
1894
             pybind11::gil_scoped_release release;
1895 1896 1897 1898 1899 1900 1901
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
1902 1903 1904
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
1905
              std::map<std::string, FetchType *> *fetch_targets,
1906 1907 1908 1909 1910 1911 1912 1913
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
1914
      .def("run_prepared_ctx",
G
guru4elephant 已提交
1915 1916 1917 1918 1919 1920 1921
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
1932
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1933 1934
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1935
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1936 1937
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1938
      });
S
sneaxiy 已提交
1939

D
dzhwinter 已提交
1940
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1941
  m.def("init_glog", framework::InitGLOG);
1942 1943
  m.def("load_op_meta_info_and_register_op",
        framework::LoadOpMetaInfoAndRegisterOp);
1944
  m.def("init_devices", []() { framework::InitDevices(); });
1945

1946
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1947
  m.def("is_compiled_with_ascend", IsCompiledWithAscend);
1948
  m.def("is_compiled_with_rocm", IsCompiledWithROCM);
1949
  m.def("is_compiled_with_npu", IsCompiledWithNPU);
1950
  m.def("is_compiled_with_xpu", IsCompiledWithXPU);
1951
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1952
  m.def("supports_bfloat16", SupportsBfloat16);
1953
  m.def("supports_bfloat16_fast_performance", SupportsBfloat16FastPerformance);
1954
  m.def("op_supported_infos", OpSupportedInfos);
1955
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1956
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1957 1958 1959
  m.def("_cuda_synchronize", [](const platform::CUDAPlace &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });
H
hutuxian 已提交
1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978

  m.def("get_float_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<float>> float_stats;
    paddle::platform::StatRegistry<float>::Instance().publish(float_stats);
    std::unordered_map<std::string, float> stats_map;
    for (const auto &stat : float_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
  m.def("get_int_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<int64_t>> int_stats;
    paddle::platform::StatRegistry<int64_t>::Instance().publish(int_stats);
    std::unordered_map<std::string, int64_t> stats_map;
    for (const auto &stat : int_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
H
hutuxian 已提交
1979 1980 1981 1982 1983 1984 1985
  m.def("run_cmd",
        [](const std::string &cmd, int time_out = -1,
           int sleep_inter = -1) -> const std::string {
          return paddle::framework::shell_get_command_output(cmd, time_out,
                                                             sleep_inter);
        },
        py::arg("cmd"), py::arg("time_out") = -1, py::arg("sleep_inter") = -1);
G
gongweibao 已提交
1986 1987 1988 1989 1990 1991 1992 1993 1994
  m.def("shell_execute_cmd",
        [](const std::string &cmd, int time_out = 0, int sleep_inter = 0,
           bool redirect_stderr = false) -> std::vector<std::string> {
          return paddle::framework::shell_execute_cmd(
              cmd, time_out, sleep_inter, redirect_stderr);
        },
        py::arg("cmd"), py::arg("time_out") = 0, py::arg("sleep_inter") = 0,
        py::arg("redirect_stderr") = false);

1995
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1996 1997 1998 1999 2000
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
2001

2002
  m.def("set_feed_variable", framework::SetFeedVariable);
2003 2004 2005 2006 2007
  m.def("get_fetch_variable",
        [](const Scope &scope, const std::string &var_name,
           size_t index) -> py::object {
          auto &var = framework::GetFetchVariable(scope, var_name, index);
          if (data_is_lod_tensor(var)) {
2008
            return py::cast(BOOST_GET(LoDTensor, var));
2009
          } else {
2010
            return py::cast(BOOST_GET(LoDTensorArray, var));
2011 2012
          }
        });
2013
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
2014

X
Xin Pan 已提交
2015 2016
  m.def("_is_program_version_supported", IsProgramVersionSupported);

2017 2018 2019 2020 2021
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
2022
  BindGlobalValueGetterSetter(&m);
Y
Yu Yang 已提交
2023

Y
Yu Yang 已提交
2024 2025 2026 2027 2028 2029 2030 2031 2032
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
2033
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
2034
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
2035 2036 2037

    Examples:
        .. code-block:: python
2038

Z
Zeng Jinle 已提交
2039 2040 2041 2042
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
2043 2044
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
2045 2046 2047 2048 2049 2050
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
2051 2052 2053 2054
             PADDLE_ENFORCE_LT(i, self.size(),
                               platform::errors::InvalidArgument(
                                   "The index to set is larger than the size "
                                   "of LoDTensorArray."));
Y
Yu Yang 已提交
2055 2056 2057
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
2058 2059 2060 2061 2062 2063
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
2064 2065
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
2066 2067 2068 2069 2070 2071
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
2094

2095 2096 2097 2098 2099 2100 2101 2102
  py::class_<FetchList>(m, "FetchList", R"DOC( FetchList is a
        vector of boost::variant<LoDTensor, LoDTensorArray>.
        )DOC")
      .def("_move_to_list",
           [](FetchList &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               if (data_is_lod_tensor(self[i])) {
2103
                 auto &data = BOOST_GET(LoDTensor, self[i]);
2104 2105
                 res[i] = py::cast(std::move(data));
               } else {
2106
                 auto &data = BOOST_GET(LoDTensorArray, self[i]);
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121
                 py::list tmp(data.size());
                 for (size_t j = 0; j < data.size(); ++j) {
                   tmp[j] = py::cast(std::move(data[j]));
                 }
                 res[i] = std::move(tmp);
               }
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership)

      .def("append",
           [](FetchList &self, const LoDTensor &t) {
             self.emplace_back();
2122
             auto &lod_tensor = BOOST_GET(LoDTensor, self.back());
2123 2124 2125 2126 2127 2128 2129 2130
             lod_tensor.ShareDataWith(t);
             lod_tensor.set_lod(t.lod());
           },
           py::arg("var"))

      .def("append",
           [](FetchList &self, const LoDTensorArray &t) {
             self.emplace_back();
2131
             auto &lod_tensor_array = BOOST_GET(LoDTensorArray, self.back());
2132 2133 2134 2135 2136 2137 2138 2139 2140
             for (size_t i = 0; i < t.size(); ++i) {
               lod_tensor_array[i].ShareDataWith(t[i]);
               lod_tensor_array[i].set_lod(t[i].lod());
             }
           },
           py::arg("var"));

  py::class_<FetchUnmergedList>(m, "FetchUnmergedList", R"DOC(
        FetchUnmergedList is 2-D array of FetchType(boost::variant(LoDTensor, LoDTensorArray)).
Z
Zhen Wang 已提交
2141 2142
        )DOC")
      .def("_move_to_list",
2143
           [](FetchUnmergedList &self) -> py::list {
Z
Zhen Wang 已提交
2144 2145 2146 2147
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               py::list tmp(self[i].size());
               for (size_t j = 0; j < self[i].size(); ++j) {
2148
                 if (data_is_lod_tensor(self[i][j])) {
2149
                   auto &var = BOOST_GET(LoDTensor, self[i][j]);
2150 2151
                   tmp[j] = py::cast(std::move(var));
                 } else {
2152
                   auto &var = BOOST_GET(LoDTensorArray, self[i][j]);
2153 2154 2155 2156 2157 2158
                   py::list tmp_array(var.size());
                   for (size_t k = 0; k < var.size(); ++k) {
                     tmp_array[k] = std::move(var[k]);
                   }
                   tmp[j] = std::move(tmp_array);
                 }
Z
Zhen Wang 已提交
2159 2160 2161 2162 2163 2164 2165 2166 2167
               }
               res[i] = std::move(tmp);
               self[i].clear();
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);

Y
Yu Yang 已提交
2168
  m.def("op_support_gpu", OpSupportGPU);
2169
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
D
Dong Zhihong 已提交
2170
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
2171

2172
#if !defined(PADDLE_WITH_HIP) && !defined(_WIN32)
D
dangqingqing 已提交
2173 2174 2175
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
2176 2177 2178 2179
  m.def("nvprof_nvtx_push", platform::CudaNvtxRangePush);
  m.def("nvprof_nvtx_pop", platform::CudaNvtxRangePop);
  m.def("nvprof_enable_record_event", platform::NvprofEnableRecordEvent);
  m.def("nvprof_disable_record_event", platform::NvprofDisableRecordEvent);
D
Dong Zhihong 已提交
2180
#endif
P
peizhilin 已提交
2181
#endif
Y
Yu Yang 已提交
2182

2183 2184 2185 2186 2187 2188
  py::enum_<platform::TracerOption>(m, "TracerOption", py::arithmetic())
      .value("kDefault", platform::TracerOption::kDefault)
      .value("kOpDetail", platform::TracerOption::kOpDetail)
      .value("kAllOpDetail", platform::TracerOption::kAllOpDetail)
      .export_values();

2189 2190 2191 2192
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
2193
      .value("kAll", platform::ProfilerState::kAll)
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

2205
  m.def("set_tracer_option", platform::SetTracerOption);
2206 2207
  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
2208
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
2209
  m.def("reset_profiler", platform::ResetProfiler);
2210
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
2211 2212 2213
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
2214

2215 2216
  m.def("size_of_dtype", framework::SizeOfType);

2217
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2218 2219
  m.def("set_cublas_switch", platform::SetAllowTF32Cublas);
  m.def("get_cublas_switch", platform::AllowTF32Cublas);
A
AshburnLee 已提交
2220 2221
  m.def("set_cudnn_switch", platform::SetAllowTF32Cudnn);
  m.def("get_cudnn_switch", platform::AllowTF32Cudnn);
2222 2223
#endif  // PADDLE_WITH_CUDA

2224 2225 2226
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

2227 2228
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
2229
      .def("has", &ir::Pass::Has)
2230 2231 2232
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
2233
           })
2234
      .def(
2235
          "set",
2236 2237 2238
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
2239 2240
      .def("set", [](ir::Pass &self, const std::string &name,
                     bool val) { self.Set<bool>(name, new bool(val)); })
2241 2242
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
2257 2258
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
2259
        self.Apply(graph.get());
F
flame 已提交
2260
      });
2261

X
fix  
Xin Pan 已提交
2262 2263
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
2278
  // -- python binds for parallel executor.
X
Xin Pan 已提交
2279

Y
yuyang18 已提交
2280
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
2281 2282 2283 2284
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

2285 2286 2287
    Returns:
        ExecutionStrategy: An ExecutionStrategy object.

C
chengduo 已提交
2288 2289 2290
    Examples:
        .. code-block:: python

2291 2292 2293 2294 2295 2296 2297 2298 2299
          import paddle
          import paddle.static as static
          import paddle.nn.functional as F

          paddle.enable_static()

          x = static.data(name='x', shape=[None, 13], dtype='float32')
          y = static.data(name='y', shape=[None, 1], dtype='float32')
          y_predict = static.nn.fc(input=x, size=1, act=None)
2300

2301 2302
          cost = F.square_error_cost(input=y_predict, label=y)
          avg_loss = paddle.mean(cost)
2303

2304
          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
2305 2306
          sgd_optimizer.minimize(avg_loss)

2307
          exec_strategy = static.ExecutionStrategy()
C
chengduo 已提交
2308 2309
          exec_strategy.num_threads = 4

2310 2311 2312
          train_exe = static.ParallelExecutor(use_cuda=False,
                                              loss_name=avg_loss.name,
                                              exec_strategy=exec_strategy)
C
chengduo 已提交
2313 2314
        )DOC");

2315 2316 2317 2318
  py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
      .value("CPU", paddle::platform::DeviceType::CPU)
      .value("CUDA", paddle::platform::DeviceType::CUDA)
      .value("XPU", paddle::platform::DeviceType::XPU);
2319

Y
yuyang18 已提交
2320
  exec_strategy.def(py::init())
Y
yuyang18 已提交
2321 2322 2323 2324 2325
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
2326
          },
2327 2328
          R"DOC(
            The type is INT, num_threads represents the size of thread pool that
C
chengduo 已提交
2329 2330 2331 2332 2333 2334 2335
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348
            `multiprocessing.cpu_count()`. Default 0.

            Examples:
                .. code-block:: python

                    import paddle
                    import paddle.static as static

                    paddle.enable_static()

                    exec_strategy = static.ExecutionStrategy()
                    exec_strategy.num_threads = 4
            )DOC")
Y
yuyang18 已提交
2349
      .def_property(
2350 2351
          "_use_device",
          [](const ExecutionStrategy &self) { return self.use_device_; },
2352
          [](ExecutionStrategy &self, paddle::platform::DeviceType use_device) {
2353 2354 2355
            self.use_device_ = use_device;
          })  // NOTE(liuyuhui): Doesn't add doc for 'use_device', because
              // use_device isn‘t exposed to users.
Y
yuyang18 已提交
2356 2357 2358 2359 2360
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
2361 2362 2363
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
2364 2365
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
2366 2367 2368 2369 2370 2371 2372
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
2373 2374 2375 2376
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
2377
                because the temp variable's shape maybe the same between two iterations.
2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
                Default 100.

                .. note::
                    1. If you fetch data when calling the 'run', the ParallelExecutor 
                    will clean up the temp variables at the end of the current iteration. 
                    2. In some NLP model, it may cause the GPU memory is insufficient, 
                    in this case, you should reduce `num_iteration_per_drop_scope`.

                Examples:
                    .. code-block:: python
C
chengduo 已提交
2388

2389 2390 2391 2392 2393 2394 2395
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_drop_scope = 10
2396
              )DOC")
Q
Qiao Longfei 已提交
2397 2398 2399 2400 2401 2402 2403 2404 2405
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
                user call exe.run() in python。Default: 1.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_run = 10
Q
Qiao Longfei 已提交
2418
              )DOC")
2419 2420 2421 2422 2423 2424 2425 2426
      .def_property(
          "use_thread_barrier",
          [](const ExecutionStrategy &self) { return self.thread_barrier_; },
          [](ExecutionStrategy &self, bool use_thread_barrier) {
            self.thread_barrier_ = use_thread_barrier;
          },
          R"DOC(This config that the this is distributed training with parameter server
              )DOC")
2427 2428 2429 2430 2431
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
2432

Y
yuyang18 已提交
2433
  exec_strategy.def_property(
Y
yuyang18 已提交
2434 2435 2436 2437 2438 2439 2440
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
2441 2442
      });

C
chengduo 已提交
2443 2444 2445 2446
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

2447 2448 2449
    Returns:
        BuildStrategy: An BuildStrategy object.

C
chengduo 已提交
2450 2451 2452
    Examples:
        .. code-block:: python

2453
            import os
2454 2455 2456 2457
            import paddle
            import paddle.static as static

            paddle.enable_static()
2458

2459 2460
            os.environ['CPU_NUM'] = str(2)
            places = static.cpu_places()
2461

2462 2463 2464 2465
            data = static.data(name="x", shape=[None, 1], dtype="float32")
            hidden = static.nn.fc(input=data, size=10)
            loss = paddle.mean(hidden)
            paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
2466

2467
            build_strategy = static.BuildStrategy()
2468 2469
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
2470 2471
            build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
            program = static.CompiledProgram(static.default_main_program())
2472
            program = program.with_data_parallel(loss_name=loss.name,
2473 2474
                                                  build_strategy=build_strategy,
                                                  places=places)
C
chengduo 已提交
2475
)DOC");
Y
yuyang18 已提交
2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
2492 2493 2494 2495
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2496
            self.reduce_ = strategy;
C
chengduo 已提交
2497
          },
2498
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
2499 2500
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
2501
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
2502 2503
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
2504
                Default is 'AllReduce'.
F
flame 已提交
2505 2506 2507 2508

                Examples:
                    .. code-block:: python

2509 2510 2511 2512 2513 2514 2515
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
F
flame 已提交
2516
                  )DOC")
Y
yuyang18 已提交
2517 2518 2519 2520 2521
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
2522 2523 2524 2525
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2526
            self.gradient_scale_ = strategy;
C
chengduo 已提交
2527
          },
2528
          R"DOC((paddle.static.BuildStrategy.GradientScaleStrategy, optional): there are three
2529
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
2530 2531
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
2532
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
2533 2534 2535 2536

                Examples:
                    .. code-block:: python

C
chengduo 已提交
2537 2538
                        import numpy
                        import os
2539 2540 2541 2542
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2543 2544

                        use_cuda = True
2545 2546
                        place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace()
                        exe = static.Executor(place)
C
chengduo 已提交
2547 2548

                        # NOTE: If you use CPU to run the program, you need
2549
                        # to specify the CPU_NUM, otherwise, paddle will use
C
chengduo 已提交
2550 2551 2552 2553 2554 2555
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
2556
                            places = static.cpu_places()
C
chengduo 已提交
2557
                        else:
2558
                            places = static.cuda_places()
C
chengduo 已提交
2559

2560 2561 2562 2563
                        data = static.data(name='X', shape=[None, 1], dtype='float32')
                        hidden = static.nn.fc(input=data, size=10)
                        loss = paddle.mean(hidden)
                        paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
C
chengduo 已提交
2564

2565
                        exe.run(static.default_startup_program())
C
chengduo 已提交
2566

2567
                        build_strategy = static.BuildStrategy()
C
chengduo 已提交
2568
                        build_strategy.gradient_scale_strategy = \
2569 2570 2571
                                  static.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = static.CompiledProgram(
                                  static.default_main_program()).with_data_parallel(
C
chengduo 已提交
2572
                                          loss_name=loss.name, build_strategy=build_strategy,
2573
                                          places=places)
C
chengduo 已提交
2574 2575 2576 2577 2578 2579

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
2580 2581
                                              feed={"X": x, loss_grad_name : loss_grad},
                                              fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
2582
                   )DOC")
Y
yuyang18 已提交
2583 2584 2585 2586
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
2587 2588 2589 2590
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2591
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
2592
          },
2593
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
2594
                writing the SSA Graph to file in the form of graphviz.
2595
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
2596 2597 2598 2599

                Examples:
                    .. code-block:: python

2600 2601 2602 2603
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2604

2605 2606
                        build_strategy = static.BuildStrategy()
                        build_strategy.debug_graphviz_path = "./graph"
F
flame 已提交
2607
                    )DOC")
S
sneaxiy 已提交
2608 2609 2610 2611 2612 2613
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
2614 2615 2616 2617
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2618 2619
            self.enable_sequential_execution_ = b;
          },
2620 2621
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
2622 2623 2624 2625

                Examples:
                    .. code-block:: python

2626 2627 2628 2629 2630 2631
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2632 2633
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
2634 2635 2636 2637 2638 2639
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
2640 2641 2642 2643
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2644 2645
            self.remove_unnecessary_lock_ = b;
          },
2646 2647
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
2648 2649 2650 2651

                Examples:
                    .. code-block:: python

2652 2653 2654 2655 2656 2657
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2658 2659
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
2660 2661 2662 2663
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
2664
#ifdef WIN32
2665
            PADDLE_THROW(platform::errors::Unavailable(
2666
                "Distribution mode is not supported on Windows platform."));
2667
#endif
2668 2669
            self.num_trainers_ = num_trainers;
          })
2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
2682 2683 2684 2685 2686 2687
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
2688 2689 2690 2691 2692 2693
      .def_property(
          "bkcl_comm_num",
          [](const BuildStrategy &self) { return self.bkcl_comm_num_; },
          [](BuildStrategy &self, int bkcl_comm_num) {
            self.bkcl_comm_num_ = bkcl_comm_num;
          })
2694
      .def_property("use_hierarchical_allreduce",
2695 2696 2697 2698 2699 2700
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
2701
      .def_property("hierarchical_allreduce_inter_nranks",
2702 2703 2704 2705 2706 2707 2708
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
2709 2710 2711 2712 2713 2714
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
2715 2716 2717 2718
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
C
chengduo 已提交
2719 2720
            self.fuse_elewise_add_act_ops_ = b;
          },
2721
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
2722
                to fuse elementwise_add_op and activation_op,
2723
                it may make the execution faster. Default is False.
F
flame 已提交
2724 2725 2726 2727

                Examples:
                    .. code-block:: python

2728 2729 2730 2731 2732 2733
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2734 2735
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
Z
Zhen Wang 已提交
2736 2737 2738 2739
      .def_property(
          "fuse_bn_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_act_ops_; },
          [](BuildStrategy &self, bool b) {
2740
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
Z
Zhen Wang 已提交
2741
                              platform::errors::PreconditionNotMet(
2742 2743
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Z
Zhen Wang 已提交
2744 2745 2746 2747 2748 2749 2750 2751 2752
            self.fuse_bn_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_act_ops indicate whether
                to fuse batch_norm and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

2753 2754 2755 2756 2757 2758
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
Z
Zhen Wang 已提交
2759 2760
                        build_strategy.fuse_bn_act_ops = True
                     )DOC")
Z
Zhang Ting 已提交
2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
      .def_property(
          "fuse_bn_add_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_add_act_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_bn_add_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_add_act_ops indicate whether
                to fuse batch_norm, elementwise_add and activation_op,
                it may make the execution faster. Default is True

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_bn_add_act_ops = True
                     )DOC")
2786 2787 2788 2789
      .def_property(
          "enable_auto_fusion",
          [](const BuildStrategy &self) { return self.enable_auto_fusion_; },
          [](BuildStrategy &self, bool b) {
2790
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
2791
                              platform::errors::PreconditionNotMet(
2792 2793
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2794 2795 2796 2797 2798 2799 2800 2801 2802 2803
            self.enable_auto_fusion_ = b;
          },
          R"DOC((bool, optional): Whether to enable fusing subgraph to a
                fusion_group. Now we only support fusing subgraph that composed
                of elementwise-like operators, such as elementwise_add/mul
                without broadcast and activations.

                Examples:
                    .. code-block:: python

2804 2805 2806 2807 2808 2809
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
2810 2811
                        build_strategy.enable_auto_fusion = True
                    )DOC")
2812 2813 2814 2815 2816 2817
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
2818 2819 2820 2821
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2822 2823
            self.fuse_relu_depthwise_conv_ = b;
          },
2824
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
2825 2826 2827
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
2828
                Default is False.
F
flame 已提交
2829 2830 2831 2832

                Examples:
                    .. code-block:: python

2833 2834 2835 2836 2837 2838
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2839 2840
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
2841 2842 2843 2844 2845 2846
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
                             self.fuse_broadcast_ops_ == boost::none;
                    },
                    [](BuildStrategy &self, bool b) {
2847 2848 2849 2850
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
2851 2852
                      self.fuse_broadcast_ops_ = b;
                    },
2853
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
2854 2855 2856 2857
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
2858 2859 2860 2861 2862
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

2863 2864 2865 2866 2867 2868
                              import paddle
                              import paddle.static as static

                              paddle.enable_static()

                              build_strategy = static.BuildStrategy()
2869 2870
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
2871 2872
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
2873 2874
                      return self.fuse_all_optimizer_ops_ == true ||
                             self.fuse_all_optimizer_ops_ == boost::none;
C
chengduo 已提交
2875 2876
                    },
                    [](BuildStrategy &self, bool b) {
2877 2878 2879 2880
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
2881 2882
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
2883 2884 2885 2886
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
2887 2888 2889 2890
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Q
qingqing01 已提交
2891 2892
            self.sync_batch_norm_ = b;
          },
2893
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
2894 2895 2896
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
2897 2898
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
2899 2900 2901 2902

                Examples:
                    .. code-block:: python

2903 2904 2905 2906 2907 2908
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2909 2910
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
2911 2912
      .def_property(
          "memory_optimize",
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
              self.memory_optimize_ = boost::none;
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
2927 2928 2929
              PADDLE_THROW(platform::errors::InvalidArgument(
                  "BuildStrategy.memory_optimize must be set to None, False or "
                  "True"));
2930 2931
            }
          },
2932
          R"DOC((bool, optional): memory opitimize aims to save total memory
2933
                consumption, set to True to enable it.
2934

2935 2936 2937
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951
                True means enabling and False means disabling. Default is None.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.memory_optimize = True
                
                )DOC")
2952 2953 2954
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
2955 2956 2957
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
2958
              PADDLE_THROW(platform::errors::Unavailable(
2959
                  "Distribution mode is not supported on Windows platform."));
2960 2961 2962 2963 2964
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
2965 2966 2967
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
2968
      .def_property(
D
dzhwinter 已提交
2969 2970 2971
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
2972 2973 2974 2975
      .def_property(
          "enable_addto",
          [](const BuildStrategy &self) { return self.enable_addto_; },
          [](BuildStrategy &self, bool b) { self.enable_addto_ = b; })
C
chengduo 已提交
2976 2977
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
2978 2979 2980 2981
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
                   self.fuse_all_reduce_ops_ == boost::none;
          },
C
chengduo 已提交
2982
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
2983 2984 2985 2986 2987 2988 2989
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
2990 2991 2992 2993
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
2994 2995 2996 2997 2998 2999 3000 3001 3002
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
3003
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
3004
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
3005 3006 3007 3008 3009
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
3010 3011

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
3012
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
3013
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
3014
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
3015 3016 3017 3018
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
3019 3020 3021 3022 3023
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
3024 3025 3026
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
3027 3028 3029 3030
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
3031 3032
      .def("run",
           [](ParallelExecutor &self,
Z
Zhen Wang 已提交
3033 3034 3035 3036 3037 3038 3039 3040
              const std::vector<std::string> &fetch_tensors,
              bool return_merged) -> py::object {
             paddle::framework::FetchResultType ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(fetch_tensors, return_merged);
             }
             if (return_merged) {
3041
               return py::cast(
3042
                   std::move(BOOST_GET(paddle::framework::FetchList, ret)));
Z
Zhen Wang 已提交
3043 3044
             } else {
               return py::cast(std::move(
3045
                   BOOST_GET(paddle::framework::FetchUnmergedList, ret)));
Z
Zhen Wang 已提交
3046
             }
3047 3048
           })
      .def("device_count", &ParallelExecutor::DeviceCount);
Y
Yu Yang 已提交
3049

D
dongdaxiang 已提交
3050
  BindFleetWrapper(&m);
T
Thunderbrook 已提交
3051

T
Thunderbrook 已提交
3052 3053
#ifdef PADDLE_WITH_PSLIB
  BindHeterWrapper(&m);
T
Thunderbrook 已提交
3054
#endif
T
Thunderbrook 已提交
3055
#ifdef PADDLE_WITH_HETERPS
T
Thunderbrook 已提交
3056
  BindPSGPUWrapper(&m);
T
Thunderbrook 已提交
3057
#endif
3058
  BindGlooWrapper(&m);
H
hutuxian 已提交
3059
  BindBoxHelper(&m);
H
hutuxian 已提交
3060 3061 3062
#ifdef PADDLE_WITH_BOX_PS
  BindBoxWrapper(&m);
#endif
3063
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
3064
  BindNCCLWrapper(&m);
3065 3066 3067
#endif
#ifdef PADDLE_WITH_GLOO
  BindGlooContext(&m);
W
wopeizl 已提交
3068
#endif
F
flame 已提交
3069 3070
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
3071
  BindInferenceApi(&m);
3072
  BindCompatible(&m);
3073
  BindDataset(&m);
Y
yaoxuefeng 已提交
3074
  BindGenerator(&m);
3075 3076 3077
#ifdef PADDLE_WITH_ASCEND
  BindAscendWrapper(&m);
  BindAscendGraph(&m);
3078
  BindAscendDevice(&m);
3079
#endif
Y
Yanghello 已提交
3080 3081 3082
#ifdef PADDLE_WITH_CRYPTO
  BindCrypto(&m);
#endif
T
tangwei12 已提交
3083

T
tangwei12 已提交
3084
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
3085 3086
  BindDistFleetWrapper(&m);
  BindPSHost(&m);
3087
  BindCommunicatorContext(&m);
T
tangwei12 已提交
3088 3089
  BindDistCommunicator(&m);
  BindHeterClient(&m);
S
seemingwang 已提交
3090 3091 3092 3093 3094
  BindGraphPyFeatureNode(&m);
  BindGraphNode(&m);
  BindGraphPyService(&m);
  BindGraphPyServer(&m);
  BindGraphPyClient(&m);
1
123malin 已提交
3095 3096 3097 3098 3099
  BindIndexNode(&m);
  BindTreeIndex(&m);
  BindIndexWrapper(&m);
  BindIndexSampler(&m);

3100
#endif
L
Luo Tao 已提交
3101
}
3102
}  // namespace pybind
3103
}  // namespace paddle