pybind.cc 123.3 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
15

C
chengduoZH 已提交
16
#include <algorithm>
17
#include <cctype>
18
#include <cstdlib>
19
#include <iterator>
C
chengduoZH 已提交
20
#include <map>
S
sneaxiy 已提交
21
#include <memory>
C
chengduoZH 已提交
22 23
#include <mutex>  // NOLINT // for call_once
#include <string>
24 25
#include <tuple>
#include <type_traits>
C
chengduoZH 已提交
26
#include <unordered_map>
27
#include <unordered_set>
C
chengduoZH 已提交
28 29
#include <utility>
#include <vector>
30

31
#include "paddle/fluid/framework/custom_operator.h"
32
#include "paddle/fluid/framework/data_layout.h"
Y
Yi Wang 已提交
33
#include "paddle/fluid/framework/executor.h"
34
#include "paddle/fluid/framework/executor_cache.h"
35
#include "paddle/fluid/framework/executor_gc_helper.h"
Y
Yi Wang 已提交
36
#include "paddle/fluid/framework/feed_fetch_method.h"
Z
Zhen Wang 已提交
37
#include "paddle/fluid/framework/feed_fetch_type.h"
S
sneaxiy 已提交
38
#include "paddle/fluid/framework/garbage_collector.h"
H
hutuxian 已提交
39
#include "paddle/fluid/framework/io/fs.h"
40
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
41
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
42 43 44
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
45
#include "paddle/fluid/framework/op_info.h"
46
#include "paddle/fluid/framework/op_registry.h"
47
#include "paddle/fluid/framework/op_version_registry.h"
Y
Yu Yang 已提交
48
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
49
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
50
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
51
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
52
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
53
#include "paddle/fluid/framework/selected_rows.h"
54
#include "paddle/fluid/framework/tensor_util.h"
55
#include "paddle/fluid/framework/trainer.h"
56
#include "paddle/fluid/framework/type_defs.h"
X
Xin Pan 已提交
57
#include "paddle/fluid/framework/version.h"
H
hong 已提交
58
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
59
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
60
#include "paddle/fluid/memory/allocation/mmap_allocator.h"
D
dzhwinter 已提交
61
#include "paddle/fluid/operators/activation_op.h"
L
Leo Chen 已提交
62
#include "paddle/fluid/operators/common_infer_shape_functions.h"
S
sneaxiy 已提交
63
#include "paddle/fluid/operators/py_func_op.h"
64
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
65
#include "paddle/fluid/platform/cpu_info.h"
66
#include "paddle/fluid/platform/device_context.h"
67
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
68
#include "paddle/fluid/platform/enforce.h"
69
#include "paddle/fluid/platform/init.h"
H
hutuxian 已提交
70
#include "paddle/fluid/platform/monitor.h"
Y
Yi Wang 已提交
71 72
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
73
#include "paddle/fluid/pybind/cuda_streams_py.h"
74
#include "paddle/fluid/pybind/io.h"
75 76 77
#ifdef PADDLE_WITH_ASCEND
#include "paddle/fluid/pybind/ascend_wrapper_py.h"
#endif
H
hutuxian 已提交
78
#include "paddle/fluid/pybind/box_helper_py.h"
79
#include "paddle/fluid/pybind/compatible.h"
Y
Yi Wang 已提交
80
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
81
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
82
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
83
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
Y
yaoxuefeng 已提交
84
#include "paddle/fluid/pybind/generator_py.h"
85
#include "paddle/fluid/pybind/global_value_getter_setter.h"
86
#include "paddle/fluid/pybind/gloo_context_py.h"
87
#include "paddle/fluid/pybind/gloo_wrapper_py.h"
T
Thunderbrook 已提交
88
#include "paddle/fluid/pybind/heter_wrapper_py.h"
89
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
90
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
91
#include "paddle/fluid/pybind/ir.h"
T
Thunderbrook 已提交
92
#include "paddle/fluid/pybind/ps_gpu_wrapper_py.h"
93
#include "paddle/fluid/pybind/pybind_boost_headers.h"
94

95
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
96
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
97
#endif
98
#include "paddle/fluid/framework/data_type.h"
99 100
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
101
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
102
#include "paddle/fluid/pybind/tensor_py.h"
103
#include "paddle/fluid/string/to_string.h"
104 105
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
Y
Yi Wang 已提交
106
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
107
#endif
108
#ifndef PADDLE_WITH_HIP
Y
Yi Wang 已提交
109
#include "paddle/fluid/platform/cuda_profiler.h"
110
#endif
Y
Yi Wang 已提交
111
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
112 113
#endif

114 115
#ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/platform/npu_info.h"
116
#include "paddle/fluid/platform/npu_profiler.h"
117 118
#endif

119 120 121 122
#ifdef PADDLE_WITH_XPU
#include "paddle/fluid/platform/xpu_info.h"
#endif

Y
Yanghello 已提交
123 124 125 126
#ifdef PADDLE_WITH_CRYPTO
#include "paddle/fluid/pybind/crypto.h"
#endif

T
tangwei12 已提交
127
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
128 129 130
#include "paddle/fluid/pybind/fleet_py.h"
#endif

M
minqiyang 已提交
131 132
#include "pybind11/stl.h"

133
DECLARE_bool(use_mkldnn);
134

Q
Qiao Longfei 已提交
135 136
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);
137 138 139
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchUnmergedList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchList);
PYBIND11_MAKE_OPAQUE(paddle::framework::FetchType);
Q
Qiao Longfei 已提交
140

141
namespace paddle {
142
namespace pybind {
143
bool IsCompiledWithCUDA() {
144 145 146 147 148 149 150 151 152
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
  return false;
#else
  return true;
#endif
}

bool IsCompiledWithROCM() {
#ifndef PADDLE_WITH_HIP
Q
qijun 已提交
153 154 155 156 157 158
  return false;
#else
  return true;
#endif
}

159 160 161 162 163 164 165 166
bool IsCompiledWithAscend() {
#ifndef PADDLE_WITH_ASCEND
  return false;
#else
  return true;
#endif
}

167 168 169 170 171 172 173 174
bool IsCompiledWithXPU() {
#ifndef PADDLE_WITH_XPU
  return false;
#else
  return true;
#endif
}

175 176 177 178 179 180 181 182
bool IsCompiledWithNPU() {
#ifndef PADDLE_WITH_ASCEND_CL
  return false;
#else
  return true;
#endif
}

183 184 185 186 187 188 189 190
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

191 192 193 194 195 196 197 198
bool IsCompiledWithHETERPS() {
#ifndef PADDLE_WITH_HETERPS
  return false;
#else
  return true;
#endif
}

199 200 201 202 203 204 205 206 207 208 209
bool SupportsBfloat16() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_core))
    return true;
  else
    return false;
#endif
}

210 211 212 213 214 215 216 217 218 219 220
bool SupportsBfloat16FastPerformance() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  if (platform::MayIUse(platform::cpu_isa_t::avx512_bf16))
    return true;
  else
    return false;
#endif
}

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
// According to the input `place` and `dtype`, this function returns a tuple
// consists of three sets:
// 1) All operators registered in the Paddle framework.
// 2) All operators supported for `place` and `dtype`.
// 3) All operators unsupported for `place` and `dtype`.
// The input `place` is a type of string, which can only be `GPU` or `CPU`.
// The input `dtype` is a type of paddle::framework::proto::VarType::Type,
// which can be paddle::framework::proto::VarType::FP16,
// paddle::framework::proto::VarType::FP32 and so on.
std::tuple<std::unordered_set<std::string>, std::unordered_set<std::string>,
           std::unordered_set<std::string>>
OpSupportedInfos(const std::string &place,
                 framework::proto::VarType::Type dtype) {
  std::string query_place;
  std::transform(place.begin(), place.end(), std::back_inserter(query_place),
                 [](unsigned char c) { return std::toupper(c); });
  using fn_type = std::add_pointer<bool(const platform::Place &)>::type;
  std::unordered_map<std::string, fn_type> is_target_place{
T
taixiurong 已提交
239 240 241
      {"GPU", &platform::is_gpu_place},
      {"CPU", &platform::is_cpu_place},
      {"XPU", &platform::is_xpu_place},
242
      {"NPU", &platform::is_npu_place},
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
  };
  PADDLE_ENFORCE_NE(
      is_target_place.count(query_place), 0,
      platform::errors::InvalidArgument(
          "The argument `place` should be 'GPU' or 'CPU', but get '%s'.",
          place));

  std::unordered_set<std::string> all_ops;
  const auto &op_info = framework::OpInfoMap::Instance().map();
  for (auto it = op_info.begin(); it != op_info.end(); it++) {
    all_ops.emplace(it->first);
  }

  std::unordered_set<std::string> supported_ops;
  auto &all_kernels = framework::OperatorWithKernel::AllOpKernels();
  for (auto it = all_kernels.begin(); it != all_kernels.end(); it++) {
    for (auto &kernel_type : it->second) {
      if (is_target_place[query_place](kernel_type.first.place_) &&
          kernel_type.first.data_type_ == dtype) {
        supported_ops.emplace(it->first);
      }
    }
  }

  std::unordered_set<std::string> unsupported_ops;
  for (auto &op : all_ops) {
    if (!supported_ops.count(op)) {
      unsupported_ops.emplace(op);
    }
  }

  VLOG(4) << "-- The size of all_ops: " << all_ops.size() << " --";
  VLOG(4) << "-- The size of supported_ops: " << supported_ops.size() << " --";
  VLOG(4) << "-- The size of unsupported_ops: " << unsupported_ops.size()
          << " --";
  return std::make_tuple(std::move(all_ops), std::move(supported_ops),
                         std::move(unsupported_ops));
}

282
bool IsCompiledWithBrpc() {
283
#ifndef PADDLE_WITH_DISTRIBUTE
284 285
  return false;
#endif
286
  return true;
287 288
}

Y
update  
Yancey1989 已提交
289
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
290
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
291 292 293 294 295 296
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
297 298 299 300 301 302 303 304 305 306
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

H
hong 已提交
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
329 330 331
    PADDLE_THROW(platform::errors::InvalidArgument(
        "Python object is not type of %s, the real type is %s",
        typeid(T).name(), obj->ob_type->tp_name));
H
hong 已提交
332 333 334 335 336 337 338 339 340 341 342 343 344
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
345 346
      PADDLE_THROW(platform::errors::InvalidArgument(
          "The parameter [%s] to save is None", para.first));
H
hong 已提交
347 348
    }
    vec_res.emplace_back(
349
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_obj));
H
hong 已提交
350 351 352 353 354 355 356 357 358 359 360 361
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
362 363
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameter list to save is None"));
H
hong 已提交
364 365 366 367 368 369 370 371 372 373 374 375
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
376 377 378
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to save is None"));
H
hong 已提交
379 380 381 382
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
383 384
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to save is not a list"));
H
hong 已提交
385 386 387 388
  }
  return vec_res;
}

389 390 391 392 393 394 395 396
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
397 398
    PADDLE_THROW(
        platform::errors::InvalidArgument("The parameter list to set is None"));
399 400 401 402 403 404 405 406 407 408 409 410 411
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
412 413 414
      PADDLE_ENFORCE_NOT_NULL(py_name,
                              platform::errors::InvalidArgument(
                                  "The name of parameter to set is None"));
415 416 417 418 419
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
420 421 422 423 424
        PADDLE_ENFORCE_NOT_NULL(exe,
                                platform::errors::InvalidArgument(
                                    "Parameter not Initialized, "
                                    "Please set argument [executor] not None "
                                    "or run startup program first"));
425 426
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
427 428 429
        PADDLE_ENFORCE_NOT_NULL(
            py_var_desc, platform::errors::InvalidArgument(
                             "The var_desc of parameter to set is None"));
430 431 432 433 434 435 436 437 438
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
        tensor_temp->Resize(framework::make_ddim(var_desc.GetShape()));
        tensor_temp->mutable_data(exe->GetPlace(), var_desc.GetDataType());
      }
    }
  } else {
439 440
    PADDLE_THROW(platform::errors::InvalidArgument(
        "The parameters to set is not a list"));
441 442 443 444 445
  }

  return;
}

446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
static void AssertStaticGraphAndDygraphGradMakerNoDiff() {
  std::set<std::string> ops;
  for (auto &pair : framework::OpInfoMap::Instance().map()) {
    bool has_static_grad_maker = (pair.second.grad_op_maker_ != nullptr);
    bool has_dygraph_grad_maker =
        (pair.second.dygraph_grad_op_maker_ != nullptr);
    if (has_static_grad_maker ^ has_dygraph_grad_maker) {
      bool has_kernel =
          (framework::OperatorWithKernel::AllOpKernels().count(pair.first) > 0);
      if (has_kernel) {
        ops.insert(pair.first);
      } else {
        VLOG(5) << pair.first << " has no kernels, skip";
      }
    }
  }
  PADDLE_ENFORCE_EQ(ops.empty(), true,
                    platform::errors::Unimplemented(
                        "OperatorWithKernel [%s] have only static graph grad "
                        "maker or have only dygraph grad maker, which is not "
                        "allowed",
                        string::join_strings(ops, ',')));
}

470 471 472 473 474 475
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

476 477
  BindCudaStream(&m);

Y
Yu Yang 已提交
478 479 480
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
481
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
482

483 484
  AssertStaticGraphAndDygraphGradMakerNoDiff();

485
  m.doc() = "C++ core of PaddlePaddle";
486

487 488 489 490
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

491
  BindException(&m);
Y
Yu Yang 已提交
492

493 494
  m.def("set_num_threads", &platform::SetNumThreads);

495
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
496 497 498
  m.def("cudnn_version", &platform::CudnnVersion);
#endif

6
633WHU 已提交
499 500 501 502 503
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
504
    framework::Tensor tensor;
6
633WHU 已提交
505 506 507 508

    if (dl.ctx.device_type == kDLCPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
509
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
6
633WHU 已提交
510 511 512 513 514 515
    if (dl.ctx.device_type == kDLGPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });
H
hong 已提交
516

517 518 519 520 521 522
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

523 524 525 526 527 528
  m.def("save_op_version_info", [](framework::ProgramDesc &desc) {
    framework::compatible::pb::OpVersionMap pb_vmap{desc.OpVersionMap()};
    framework::compatible::SaveOpVersions(
        framework::compatible::OpVersionRegistrar::GetInstance()
            .GetVersionMap(),
        &pb_vmap);
529 530
  });

531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
  m.def("set_printoptions", [](const py::kwargs &kwargs) {
    auto &print_opt = framework::PrintOptions::Instance();
    if (kwargs.contains("precision")) {
      print_opt.precision = kwargs["precision"].cast<int>();
    }
    if (kwargs.contains("threshold")) {
      print_opt.threshold = kwargs["threshold"].cast<int>();
    }
    if (kwargs.contains("edgeitems")) {
      print_opt.edgeitems = kwargs["edgeitems"].cast<int>();
    }
    if (kwargs.contains("linewidth")) {
      print_opt.linewidth = kwargs["linewidth"].cast<int>();
    }
    if (kwargs.contains("sci_mode")) {
      print_opt.sci_mode = kwargs["sci_mode"].cast<bool>();
    }

    VLOG(4) << "Set printoptions: precision=" << print_opt.precision
            << ", threshold=" << print_opt.threshold
            << ", edgeitems=" << print_opt.edgeitems
            << ", linewidth=" << print_opt.linewidth
            << ", sci_mode=" << print_opt.sci_mode;
  });

L
Leo Chen 已提交
556 557 558 559 560 561
  m.def("broadcast_shape", [](const std::vector<int64_t> &x_dim,
                              const std::vector<int64_t> &y_dim) {
    return vectorize(operators::details::BroadcastTwoDims(
        make_ddim(x_dim), make_ddim(y_dim), -1));
  });

S
sneaxiy 已提交
562
  m.def(
S
sneaxiy 已提交
563
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
564 565 566 567
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
568 569 570
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
  m.def("_get_all_register_op_kernels", [] {
    auto &all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
    std::unordered_map<std::string, std::vector<std::string>> all_kernels_info;
    for (auto &kernel_pair : all_kernels) {
      auto op_type = kernel_pair.first;
      std::vector<std::string> kernel_types;
      for (auto &info_pair : kernel_pair.second) {
        paddle::framework::OpKernelType kernel_type = info_pair.first;
        kernel_types.push_back(
            paddle::framework::KernelTypeToString(kernel_type));
      }
      all_kernels_info.emplace(op_type, kernel_types);
    }
    return all_kernels_info;
  });

S
sneaxiy 已提交
587 588 589
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
590
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
591

592
  m.def("_set_fuse_parameter_group_size",
593
        &paddle::framework::ir::SetFuseParameterGroupsSize);
594
  m.def("_set_fuse_parameter_memory_size",
595
        &paddle::framework::ir::SetFuseParameterMemorySize);
596

S
sneaxiy 已提交
597 598 599
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

600 601
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

602 603 604
  m.def("_promote_types_if_complex_exists",
        &paddle::framework::PromoteTypesIfComplexExists);

605
  BindImperative(&m);
606

607 608 609
  py::class_<framework::Tensor>(m, "Tensor", py::buffer_protocol())
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
610
      .def("_is_initialized",
611
           [](const framework::Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
612
      .def("_get_dims",
613
           [](const framework::Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
614
      .def("_set_dims",
615
           [](framework::Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
616
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
617
           })
Y
yuyang18 已提交
618
      .def("_set_layout",
619
           [](framework::Tensor &self, const std::string &layout) {
D
dzhwinter 已提交
620 621
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
622
      .def("_alloc_float",
623
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
624
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
625
           })
626
      .def("_alloc_float",
627
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
628 629
             self.mutable_data<float>(place);
           })
Y
yuyang18 已提交
630
      .def("_alloc_float",
631
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
632
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
633
           })
634 635 636 637
      .def("_alloc_float",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place) {
             self.mutable_data<float>(place);
           })
638
      .def("_alloc_double",
639
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
640 641
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
642
      .def("_alloc_int",
643
           [](framework::Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
644
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
645
           })
646
      .def("_alloc_int",
647
           [](framework::Tensor &self, paddle::platform::XPUPlace &place) {
648 649
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
650
      .def("_alloc_int",
651
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
652
             self.mutable_data<int>(place);
Q
qijun 已提交
653
           })
Y
yuyang18 已提交
654
      .def("_alloc_int",
655 656
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
657 658
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
659
      .def("_alloc_float",
660 661
           [](framework::Tensor &self,
              paddle::platform::CUDAPinnedPlace &place) {
C
chengduoZH 已提交
662 663
             self.mutable_data<float>(place);
           })
664
      .def("_mutable_data",
665
           [](framework::Tensor &self, paddle::platform::CPUPlace &place,
666 667 668
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
669
      .def("_mutable_data",
670
           [](framework::Tensor &self, paddle::platform::XPUPlace &place,
671 672 673
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
674
      .def("_mutable_data",
675
           [](framework::Tensor &self, paddle::platform::CUDAPlace &place,
676 677 678 679
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
      .def("_mutable_data",
680
           [](framework::Tensor &self, paddle::platform::CUDAPinnedPlace &place,
681 682 683
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
684
      .def("_clear", &framework::Tensor::clear)
685 686 687 688 689
      .def("_mutable_data",
           [](framework::Tensor &self, paddle::platform::NPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
690
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
691
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
692 693
      .def("set", SetTensorFromPyArray<paddle::platform::XPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
694
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
695
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
696 697
      .def("set", SetTensorFromPyArray<paddle::platform::NPUPlace>,
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
698
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
699 700
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
L
Leo Chen 已提交
701 702 703 704
        Set the data of LoDTensor on place with given numpy array.
        
        Args:
          lod (numpy.ndarray): The data to set.
705
          place (CPUPlace|CUDAPlace|XPUPlace|CUDAPinnedPlace|NPUPlace): The place where the
L
Leo Chen 已提交
706
          LoDTensor is to be set.
707 708
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
709 710 711 712 713 714 715 716 717 718 719 720 721

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                t = fluid.LoDTensor()
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
722

723 724 725
      .def("shape",
           [](framework::Tensor &self) { return vectorize(self.dims()); },
           R"DOC(
L
Leo Chen 已提交
726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
           Return the shape of LoDTensor.

           Returns:
               list[int]: The shape of LoDTensor.


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

                  t = fluid.LoDTensor()
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
742
      .def("_to_dlpack",
743
           [](framework::Tensor &self) {
6
633WHU 已提交
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
             DLPackTensor dlpack_tensor(self, 1);
             DLManagedTensor *dmt =
                 dlpack_tensor.ToCudfCompatibleDLManagedTensor();
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
764 765 766 767
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
768 769
      .def("_place", [](framework::Tensor &self) { return self.place(); })
      .def("_dtype", [](framework::Tensor &self) { return self.type(); })
770
      .def("_layout",
771 772 773 774
           [](framework::Tensor &self) {
             return DataLayoutToString(self.layout());
           })
      .def("_share_data_with", &framework::Tensor::ShareDataWith)
775
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
776
      .def("__str__", [](const framework::Tensor &self) {
777 778 779 780
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
781

L
Leo Chen 已提交
782
  // TODO(cql): add reference: en_user_guide_lod_tensor
783
  py::class_<LoDTensor, framework::Tensor>(m, "LoDTensor", R"DOC(
L
Leo Chen 已提交
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
    LoDTensor is a Tensor with optional LoD (Level of Details) information, 
    it can be used for variable-length sequences, 
    see :ref:`user_guide_lod_tensor` for details.

    LoDTensor can be converted to numpy array using :code:`numpy.array(lod_tensor)`.

    You can skip the following explanation if you don't need to know details 
    of LoDTensor.

    The following two examples show how to use LODtensor to represent 
    variable-length sequences.
    
    Example 1:
    
    Suppose x is a LoDTensor representing a variable-length sequence. 
    It contains two logical subsequences, the length of first logical sequence 
    is 2 (e.g., number of samples is 2), the length of second logical sequence 
    is 3, and the total length is 5. The data of the first logical sequence is 
    [1, 2], [3, 4], and the data of the second logical sequence is [5, 6], 
    [7, 8], [9, 10]. The data dimension of each sample is 2. So, the final 
    shape of the LoDTensor is [5, 2], of which 5 is the total length and 2 is 
    the dimension of each sample.
    
    Logically, we can represent the variable-length sequence in two ways: one 
    is in the form of recursive sequence lengths, that is, 
    x.recursive_sequence_lengths=[[2, 3]]; the other is in the form of offsets, 
    that is, x.lod=[[0, 2, 2+3]]. These two representations are equivalent, and 
    you can set and retrieve recursive_sequence_lengths or LoD through the 
    corresponding interfaces of LoDTensor introduced later.

    Actually, in order to access sequence faster, Paddle uses offset to store 
    different lengths of sequences. 
    Therefore, the operations on recursive_sequence_lengths will be converted 
    to the operations on LoD eventually.
    
    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]

    Example 2:

    LoD may have more than one level (for example, a paragraph may have more 
    than one sentence and a sentence may have more than one word). Suppose y 
    is a LoDTensor and its lod_level is 2. 
    From level = 0, there are two logical sequences, the length of which is 
    2 and 1, respectively, indicating that the first logical sequence contains 
    two sub-sequences and the second logical sequence contains one sub-sequence. 
    From level = 1, the lengths of two sub-sequences contained by the first 
    logical sequence is 2 and 2, and the length of sub-sequence contained by 
    the second logical sequence is 3.
      
    Therefore, the LoDTensor is represented in the form of recursive sequence 
    lengths as y.recursive_sequence_lengths=[[2,1], [2,2,3]]; and equally, in 
    the form of offset, it is represented as y.lod=[[0,2,3], [0,2,4,7]].

    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]
Z
Zeng Jinle 已提交
858 859 860 861 862 863 864

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
865 866

        )DOC")
867 868
      .def("__array__",
           [](framework::Tensor &self) { return TensorToPyArray(self); })
869 870 871 872 873 874 875 876 877
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
878 879
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, -1), true,
880 881 882 883
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is %s",
                     new_lod));
884 885
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
886
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
887
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
888 889
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
890 891 892
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
893
      .def("set_lod",
894
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
895
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
896
             LoD new_lod;
897 898
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
899 900
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
901 902
                 platform::errors::InvalidArgument(
                     "The provided LoD is invalid, the LoD is %s", new_lod));
903
             self.set_lod(new_lod);
S
sneaxiy 已提交
904 905 906 907 908
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
L
Leo Chen 已提交
909 910 911 912
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
913 914 915 916 917 918 919 920 921 922

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
923
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
924
           )DOC")
925 926 927 928 929 930 931 932 933 934 935
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
936 937
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
938 939 940 941 942
                 platform::errors::InvalidArgument(
                     "The provided recursive_sequence_lengths info is invalid, "
                     "the LoD converted by recursive_sequence_lengths is "
                     "%s",
                     new_lod));
943
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
944 945
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
L
Leo Chen 已提交
946
           Set LoD of the LoDTensor according to recursive sequence lengths.
S
sneaxiy 已提交
947

L
Leo Chen 已提交
948
           For example, if recursive_sequence_lengths=[[2, 3]], which means
949
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
950
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
951 952

           Args:
L
Leo Chen 已提交
953 954 955 956
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
957 958 959 960 961 962 963 964 965 966

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
L
Leo Chen 已提交
967 968
                 print(t.recursive_sequence_length())  # [[2, 3]]
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
969
           )DOC")
970 971 972 973 974 975 976 977
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
978 979 980 981 982
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
L
Leo Chen 已提交
983 984
               list[list[int]]: The lod of the LoDTensor.
           
Z
Zeng Jinle 已提交
985 986 987 988 989 990 991 992 993 994
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
995
           )DOC")
G
gongweibao 已提交
996
      // Set above comments of set_lod.
997 998 999 1000 1001 1002 1003 1004
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
1005 1006
           },
           R"DOC(
L
Leo Chen 已提交
1007 1008
           Return the recursive sequence lengths corresponding to of the LodD 
           of the LoDTensor.
S
sneaxiy 已提交
1009 1010

           Returns:
L
Leo Chen 已提交
1011
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
1023 1024 1025 1026 1027 1028 1029 1030
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
L
Leo Chen 已提交
1031
           Check whether the LoD of the LoDTensor is valid.
S
sneaxiy 已提交
1032 1033

           Returns:
L
Leo Chen 已提交
1034
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
1046 1047 1048 1049 1050 1051 1052
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
1053
           )DOC")
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
      .def("__str__",
           [](const LoDTensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           })
      .def("_copy", [](const LoDTensor &self, const platform::Place &place) {
        // follow fetch_op's inplementation
        LoDTensor dst;
        if (self.IsInitialized() && self.numel() > 0) {
          TensorCopySync(self, place, &dst);
        } else {
          // Not copy, if the src tensor is empty.
          dst.clear();
          dst.Resize({0});
        }
        dst.set_lod(self.lod());
        return dst;
1072
#ifdef _WIN32
1073
      });
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
#else
           })
      .def(py::pickle(
          [](const LoDTensor &t) {  // __getstate__
            auto holder = t.Holder();
            PADDLE_ENFORCE_EQ(
              platform::is_cpu_place(holder->place()), true,
              platform::errors::PreconditionNotMet(
                  "LoDTensor is not on CPU."
                  "Now only LoDTensor on CPU can be serialized."));
            auto* mmap_writer_allocation =
              dynamic_cast<memory::allocation::MemoryMapWriterAllocation *>(
                holder.get());
            PADDLE_ENFORCE_NOT_NULL(mmap_writer_allocation,
              platform::errors::PreconditionNotMet(
                "LoDTensor is not in shared memory."
                "Now only LoDTensor on shared memory can be serialized."));
            int type_idx = static_cast<int>(t.type());

            return py::make_tuple(mmap_writer_allocation->ipc_name(),
                                  mmap_writer_allocation->size(),
                                  type_idx, vectorize(t.dims()), t.lod());
          },
          [](py::tuple t) {  // __setstate__
            if (t.size() != 5)
              throw std::runtime_error("Invalid LoDTensor state!");

            // 1. Create a new C++ instance
            LoDTensor tensor;

            // 2. Rebuild Allocation
            const std::string &ipc_name = t[0].cast<std::string>();
            size_t size = t[1].cast<size_t>();
            auto shared_reader_holder =
              memory::allocation::RebuildMemoryMapReaderAllocation(
                ipc_name, size);

            // 3. Maintain global fd set
            VLOG(3) << "LoDTensor ipc name: " << ipc_name;
            memory::allocation::MemoryMapFdSet::Instance().Insert(ipc_name);

            // 4. Rebuild LoDTensor
            tensor.ResetHolderWithType(shared_reader_holder,
              static_cast<proto::VarType::Type>(t[2].cast<int>()));
            tensor.Resize(make_ddim(t[3].cast<std::vector<int>>()));
            tensor.set_lod(t[4].cast<framework::LoD>());

            return tensor;
          }));
#endif
D
dangqingqing 已提交
1124

Q
qijun 已提交
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
1136 1137
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
1138 1139
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
1140 1141
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
1142
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
Q
qijun 已提交
1143 1144 1145 1146 1147 1148
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
1149
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
1150
      .def("rows", [](SelectedRows &self) {
1151 1152 1153 1154 1155
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
1156
      });
Q
qijun 已提交
1157

1158
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
1159 1160 1161

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
1162
      .def(py::init<>())
1163
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
1164
      .def("set_int",
1165 1166
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
1167 1168 1169 1170 1171 1172 1173
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
1174
      .def("get_tensor",
1175 1176
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
1177 1178
           },
           py::return_value_policy::reference)
1179 1180 1181 1182
      .def("get_bytes",
           [](Variable &self) {
             return py::bytes(*self.GetMutable<std::string>());
           })
Y
Yu Yang 已提交
1183 1184 1185
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
1186 1187 1188 1189 1190
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1191 1192 1193
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
1194 1195 1196
      .def("get_fetch_list",
           [](Variable &self) { return self.GetMutable<FetchList>(); },
           py::return_value_policy::reference)
1197
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1198 1199 1200 1201 1202
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
1203
#endif
Y
Refine  
Yu Yang 已提交
1204 1205
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
1206 1207 1208 1209
             PADDLE_ENFORCE_EQ(
                 self.IsType<framework::ReaderHolder>(), true,
                 platform::errors::InvalidArgument(
                     "The variable is not type of ReaderHolder."));
Y
Refine  
Yu Yang 已提交
1210 1211
             return self.GetMutable<framework::ReaderHolder>();
           },
1212 1213 1214 1215 1216
           py::return_value_policy::reference)
      .def("set_scope", [](Variable &self, Scope &scope) {
        auto scope_vec = self.GetMutable<std::vector<framework::Scope *>>();
        scope_vec->emplace_back(&scope);
      });
1217

S
sneaxiy 已提交
1218
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
1219

S
sneaxiy 已提交
1220
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

1234
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
1235 1236 1237 1238 1239 1240
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
1241 1242
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
1243
      .def("var",
1244
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
1245
             return self.Var(name);
Y
Yu Yang 已提交
1246
           },
S
sneaxiy 已提交
1247 1248
           py::arg("name"),
           R"DOC(
1249
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
1250

1251
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
1252
           current scope, the variable would be created. Otherwise,
1253
           return the existing variable.
S
sneaxiy 已提交
1254 1255

           Args:
1256 1257
               name (str): the variable name.

S
sneaxiy 已提交
1258
           Returns:
1259
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
1260 1261 1262 1263
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
1264
           Find variable named :code:`name` in the current scope or
1265
           its parent scope. Return None if not found. 
1266

S
sneaxiy 已提交
1267 1268
           Args:
               name (str): the variable name.
1269

S
sneaxiy 已提交
1270
           Returns:
1271
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1272
           )DOC",
1273
           py::return_value_policy::reference)
1274
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1275 1276 1277 1278 1279 1280
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1281
           py::return_value_policy::reference)
S
sneaxiy 已提交
1282 1283 1284
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1285 1286
           )DOC")
      .def("_kids", &Scope::kids);
1287

S
sneaxiy 已提交
1288 1289 1290 1291 1292 1293
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1294 1295
        R"DOC(
        Create a new scope.
1296

S
sneaxiy 已提交
1297 1298 1299
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1300 1301
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1302 1303
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1304 1305
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1306 1307 1308 1309
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1310 1311
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1312 1313
            platform::errors::Fatal(
                "Serialize OpProto Error. This could be a bug of Paddle."));
1314 1315 1316
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1317 1318
    return ret_values;
  });
1319 1320 1321 1322 1323 1324 1325 1326
  m.def("get_op_attrs_default_value",
        [](py::bytes byte_name) -> paddle::framework::AttributeMap {
          std::string op_type = byte_name;
          paddle::framework::AttributeMap res;
          auto info = OpInfoMap::Instance().GetNullable(op_type);
          if (info != nullptr) {
            if (info->HasOpProtoAndChecker()) {
              auto op_checker = info->Checker();
1327
              res = op_checker->GetDefaultAttrsMap();
1328 1329 1330 1331
            }
          }
          return res;
        });
1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1348 1349 1350
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
1351 1352 1353 1354 1355
  m.def("has_non_empty_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance()
        .Get(op_type)
        .HasNonEmptyGradOpMaker();
  });
1356 1357 1358
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
  m.def("infer_no_need_buffer_slots",
        [](const std::string op_type, const framework::VariableNameMap &inputs,
           const framework::VariableNameMap &outputs,
           const framework::AttributeMap &attrs) {
          auto infer_func = framework::OpInfoMap::Instance()
                                .Get(op_type)
                                .NoNeedBufferVarsInferer();
          if (infer_func) {
            return infer_func(inputs, outputs, attrs);
          } else {
            std::unordered_set<std::string> empty = {};
            return empty;
          }
        });
Y
Yu Yang 已提交
1373
  m.def("prune", [](const ProgramDesc &origin,
1374
                    const std::set<std::string> &feeded_var_names,
1375
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1376
    ProgramDesc prog_with_targets(origin);
1377

1378
    for (const auto &t : targets) {
1379
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1380
    }
1381
    proto::ProgramDesc pruned_desc;
1382 1383 1384 1385
    auto pruned_origin_block_id_map =
        Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
    return std::make_tuple(ProgramDesc(pruned_desc),
                           pruned_origin_block_id_map);
1386
  });
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
  m.def("prune_backward",
        [](const framework::ProgramDesc &program) {
          return PruneBackward(program);
        },
        R"DOC(
             Prune the backward part of a program, mostly called in
             program.clone(for_test=True).
              
             Args:
                   program (ProgramDesc): The original program.

             Returns:
                   tuple(ProgramDesc, map<int, int>): The first part is 
                   the pruned program desc, and the second part is a map
                   which contains the id pair of pruned block and corresponding
                   origin block.
           )DOC");
1404 1405 1406 1407
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1408 1409 1410
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1411 1412
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
1413

Q
qijun 已提交
1414
  // clang-format off
Y
Yu Yang 已提交
1415
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1416 1417
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1418
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
1419 1420
                    return new paddle::platform::CPUDeviceContext();
                  })
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
      .def_static("create",
                  [](paddle::platform::XPUPlace& place)
                      -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_XPU
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use XPUPlace in CPU/GPU version, "
                 "Please recompile or reinstall Paddle with XPU support."));
#else
                    return new paddle::platform::XPUDeviceContext(place);
#endif
                  })
1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
        .def_static("create",
                    [](paddle::platform::NPUPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_ASCEND_CL
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use NPUPlace in CPU/GPU/XPU version, "
                 "Please recompile or reinstall Paddle with NPU support."));
#else
                return new paddle::platform::NPUDeviceContext(place);
#endif
        })
Q
qijun 已提交
1445
      .def_static("create",
D
dzhwinter 已提交
1446
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1447
                      -> paddle::platform::DeviceContext* {
1448
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1449 1450 1451 1452
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
Q
qijun 已提交
1453
#else
Q
qijun 已提交
1454
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
1455
#endif
C
chengduoZH 已提交
1456 1457 1458 1459
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
1460
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1461 1462 1463 1464
             PADDLE_THROW(
                 platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
C
chengduoZH 已提交
1465 1466 1467 1468
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
1469
// clang-format on
1470
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
Dong Zhihong 已提交
1471 1472
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
1473
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
1474 1475 1476 1477 1478

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
1479
    The memory of CUDAPlace with different dev_id is not accessible.
1480 1481 1482 1483 1484 1485 1486 1487
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
1488 1489 1490 1491

    Examples:
        .. code-block:: python

1492 1493 1494
          import paddle

          place = paddle.CUDAPlace(0)
L
lujun 已提交
1495

1496
        )DOC")
S
sneaxiy 已提交
1497 1498
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
1499
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

             if (UNLIKELY(dev_id >= platform::GetCUDADeviceCount())) {
               if (platform::GetCUDADeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
                     dev_id, platform::GetCUDADeviceCount(),
                     platform::GetCUDADeviceCount());
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
1524 1525
             new (&self) platform::CUDAPlace(dev_id);
#else
1526 1527 1528 1529 1530 1531 1532 1533 1534
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
1535 1536
#endif
           })
1537
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1538 1539
      .def("get_device_id",
           [](const platform::CUDAPlace &self) { return self.GetDeviceId(); })
S
sneaxiy 已提交
1540 1541 1542 1543
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
1544
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::XPUPlace>)
1545
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1546 1547
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
1548 1549 1550
      .def("_get_device_id",
           [](platform::CUDAPlace &self) -> int { return self.GetDeviceId(); })
#endif
1551
      .def("__repr__", string::to_string<const platform::CUDAPlace &>)
D
dzhwinter 已提交
1552
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
1553

1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598
  py::class_<platform::XPUPlace>(m, "XPUPlace", R"DOC(
    **Note**:
    Examples:
        .. code-block:: python
          import paddle.fluid as fluid
          xpu_place = fluid.XPUPlace(0)
        )DOC")
      .def("__init__",
           [](platform::XPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_XPU
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid XPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetXPUDeviceCount())) {
               if (platform::GetXPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use XPU because there is no XPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid XPUPlace(%d), must inside [0, %d), because XPU "
                     "number on your machine is %d",
                     dev_id, platform::GetXPUDeviceCount(),
                     platform::GetXPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::XPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use XPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use XPU, please try to install XPU version "
                 "PaddlePaddle by: pip install paddlepaddle-xpu\n"
                 "If you only have CPU, please change XPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
1599
#ifdef PADDLE_WITH_XPU
1600 1601 1602 1603 1604 1605 1606
      .def("_type", &PlaceIndex<platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::XPUPlace, platform::XPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::XPUPlace, platform::CUDAPinnedPlace>)
1607 1608 1609
      .def("get_device_id",
           [](const platform::XPUPlace &self) { return self.GetDeviceId(); })
#endif
1610
      .def("__repr__", string::to_string<const platform::XPUPlace &>)
1611
      .def("__str__", string::to_string<const platform::XPUPlace &>);
1612 1613 1614
#ifdef PADDLE_WITH_XPU
  m.def("get_xpu_device_count", platform::GetXPUDeviceCount);
#endif
1615

1616
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
1617
    CPUPlace is a descriptor of a device.
1618
    It represents a CPU device on which a tensor will be allocated and a model will run.
L
lujun 已提交
1619 1620 1621 1622

    Examples:
        .. code-block:: python

1623 1624
          import paddle
          cpu_place = paddle.CPUPlace()
L
lujun 已提交
1625

1626
        )DOC")
1627
      .def(py::init<>())
S
sneaxiy 已提交
1628 1629
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
1630
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::XPUPlace>)
1631
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1632 1633 1634 1635
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
1636
      .def("__repr__", string::to_string<const platform::CPUPlace &>)
1637
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
1638

1639
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
1640 1641 1642 1643 1644 1645
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
1646 1647 1648 1649

    Examples:
        .. code-block:: python

1650 1651
          import paddle
          place = paddle.CUDAPinnedPlace()
L
lujun 已提交
1652

1653
        )DOC")
S
sneaxiy 已提交
1654
      .def("__init__",
S
sneaxiy 已提交
1655
           [](platform::CUDAPinnedPlace &self) {
1656
#if !defined(PADDLE_WITH_CUDA) && !defined(PADDLE_WITH_HIP)
1657 1658 1659
             PADDLE_THROW(platform::errors::PermissionDenied(
                 "Cannot use CUDAPinnedPlace in CPU only version, "
                 "Please recompile or reinstall Paddle with CUDA support."));
S
sneaxiy 已提交
1660
#endif
S
sneaxiy 已提交
1661
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
1662
           })
S
sneaxiy 已提交
1663 1664 1665 1666
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
1667 1668
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::XPUPlace>)
1669 1670
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::NPUPlace>)
S
sneaxiy 已提交
1671 1672 1673 1674
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
1675
      .def("__repr__", string::to_string<const platform::CUDAPinnedPlace &>)
C
chengduoZH 已提交
1676 1677
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
  // NPUPlace
  py::class_<platform::NPUPlace>(m, "NPUPlace", R"DOC(
    NPUPlace is a descriptor of a device.
    It represents a NPU device on which a tensor will be allocated and a model will run.

    Examples:
        .. code-block:: python
          import paddle
          npu_place = paddle.NPUPlace(0)

        )DOC")
      .def("__init__",
           [](platform::NPUPlace &self, int dev_id) {
#ifdef PADDLE_WITH_ASCEND_CL
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid NPUPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }
             if (UNLIKELY(dev_id >= platform::GetNPUDeviceCount())) {
               if (platform::GetNPUDeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use NPU because there is no NPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid NPUPlace(%d), must inside [0, %d), because NPU "
                     "number on your machine is %d",
                     dev_id, platform::GetNPUDeviceCount(),
                     platform::GetNPUDeviceCount());
                 std::exit(-1);
               }
             }
             new (&self) platform::NPUPlace(dev_id);
#else
             LOG(ERROR) << string::Sprintf(
                 "Cannot use NPU because you have installed CPU/GPU version "
                 "PaddlePaddle.\n"
                 "If you want to use NPU, please try to install NPU version "
1720
                 "PaddlePaddle by: pip install paddlepaddle-npu\n"
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
                 "If you only have CPU, please change NPUPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
#endif
           })
      .def("_type", &PlaceIndex<platform::NPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::XPUPlace>)
      .def("_equals", &IsSamePlace<platform::NPUPlace, platform::NPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::NPUPlace, platform::CUDAPinnedPlace>)
H
houj04 已提交
1735 1736
      .def("get_device_id",
           [](const platform::NPUPlace &self) { return self.GetDeviceId(); })
1737 1738
      .def("__str__", string::to_string<const platform::NPUPlace &>);

Y
Yu Yang 已提交
1739 1740
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
1741 1742 1743 1744
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
1745
      .def("_equals", &IsSamePlace<platform::Place, platform::XPUPlace>)
1746
      .def("_equals", &IsSamePlace<platform::Place, platform::NPUPlace>)
S
sneaxiy 已提交
1747
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
1748 1749
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
1750 1751
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
1752 1753
      .def("is_xpu_place",
           [](platform::Place &self) { return platform::is_xpu_place(self); })
1754 1755
      .def("is_npu_place",
           [](platform::Place &self) { return platform::is_npu_place(self); })
S
sneaxiy 已提交
1756 1757 1758 1759
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
1760 1761
      .def("gpu_device_id",
           [](platform::Place &self) {
1762
             return BOOST_GET_CONST(platform::CUDAPlace, self).device;
X
xuezhong 已提交
1763
           })
1764 1765 1766 1767
      .def("xpu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::XPUPlace, self).device;
           })
1768 1769 1770 1771
      .def("npu_device_id",
           [](platform::Place &self) {
             return BOOST_GET_CONST(platform::NPUPlace, self).device;
           })
S
sneaxiy 已提交
1772 1773
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
1774 1775 1776 1777
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
1778 1779 1780 1781
      .def("set_place",
           [](platform::Place &self, const platform::XPUPlace &xpu_place) {
             self = xpu_place;
           })
Y
Yu Yang 已提交
1782
      .def("set_place",
D
dzhwinter 已提交
1783
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
1784
             self = gpu_place;
C
chengduoZH 已提交
1785
           })
1786 1787 1788 1789 1790
      .def("set_place",
           [](platform::Place &self,
              const platform::CUDAPinnedPlace &cuda_pinned_place) {
             self = cuda_pinned_place;
           })
1791 1792 1793 1794
      .def("set_place",
           [](platform::Place &self, const platform::NPUPlace &npu_place) {
             self = npu_place;
           })
1795 1796
      .def("__repr__", string::to_string<const platform::Place &>)
      .def("__str__", string::to_string<const platform::Place &>);
Y
Yu Yang 已提交
1797

Y
Yu Yang 已提交
1798
  py::class_<OperatorBase>(m, "Operator")
C
chengduo 已提交
1799 1800 1801 1802 1803
      .def_static(
          "create",
          [](py::bytes protobin) {
            proto::OpDesc desc;
            PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin), true,
1804 1805 1806 1807 1808 1809 1810
                              platform::errors::InvalidArgument(
                                  "Cannot parse user input to OpDesc"));
            PADDLE_ENFORCE_EQ(
                desc.IsInitialized(), true,
                platform::errors::InvalidArgument(
                    "The provided OpDesc is not initialized, the reason is: %s",
                    desc.InitializationErrorString()));
C
chengduo 已提交
1811 1812
            return OpRegistry::CreateOp(desc);
          })
1813
      .def("run",
1814
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1815
              const platform::CPUPlace &place) { self.Run(scope, place); })
1816 1817 1818
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::XPUPlace &place) { self.Run(scope, place); })
1819 1820 1821
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::NPUPlace &place) { self.Run(scope, place); })
D
dzhwinter 已提交
1822 1823
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1824
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
1825 1826 1827 1828 1829
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1830 1831 1832 1833 1834 1835 1836
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1837 1838
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1839
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1840
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1841 1842 1843 1844
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1845

1846 1847 1848
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

1849 1850 1851 1852 1853 1854 1855 1856 1857
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
      .def("finalize", &TrainerBase::Finalize);

1858 1859
  m.def("_get_eager_deletion_vars", &framework::GetEagerDeletionCleanVars);

F
fengjiayi 已提交
1860
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1861
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1862
      .def("close", &Executor::Close)
1863 1864
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
1865 1866
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
1867 1868 1869 1870
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
1871
             pybind11::gil_scoped_release release;
1872 1873 1874 1875 1876 1877 1878
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
1879 1880 1881
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
1882
              std::map<std::string, FetchType *> *fetch_targets,
1883 1884 1885 1886 1887 1888 1889 1890
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
1891
      .def("run_prepared_ctx",
G
guru4elephant 已提交
1892 1893 1894 1895 1896 1897 1898
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
1909
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1910 1911
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1912
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1913 1914
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1915
      });
S
sneaxiy 已提交
1916

D
dzhwinter 已提交
1917
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1918
  m.def("init_glog", framework::InitGLOG);
1919 1920
  m.def("load_op_meta_info_and_register_op",
        framework::LoadOpMetaInfoAndRegisterOp);
1921
  m.def("init_devices", []() { framework::InitDevices(); });
1922

1923
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1924
  m.def("is_compiled_with_ascend", IsCompiledWithAscend);
1925
  m.def("is_compiled_with_rocm", IsCompiledWithROCM);
1926
  m.def("is_compiled_with_npu", IsCompiledWithNPU);
1927
  m.def("is_compiled_with_xpu", IsCompiledWithXPU);
1928
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1929
  m.def("_is_compiled_with_heterps", IsCompiledWithHETERPS);
1930
  m.def("supports_bfloat16", SupportsBfloat16);
1931
  m.def("supports_bfloat16_fast_performance", SupportsBfloat16FastPerformance);
1932
  m.def("op_supported_infos", OpSupportedInfos);
1933
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1934
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1935 1936 1937
  m.def("_cuda_synchronize", [](const platform::CUDAPlace &place) {
    platform::DeviceContextPool::Instance().Get(place)->Wait();
  });
H
hutuxian 已提交
1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956

  m.def("get_float_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<float>> float_stats;
    paddle::platform::StatRegistry<float>::Instance().publish(float_stats);
    std::unordered_map<std::string, float> stats_map;
    for (const auto &stat : float_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
  m.def("get_int_stats", []() {
    std::vector<paddle::platform::ExportedStatValue<int64_t>> int_stats;
    paddle::platform::StatRegistry<int64_t>::Instance().publish(int_stats);
    std::unordered_map<std::string, int64_t> stats_map;
    for (const auto &stat : int_stats) {
      stats_map[stat.key] = stat.value;
    }
    return stats_map;
  });
H
hutuxian 已提交
1957 1958 1959 1960 1961 1962 1963
  m.def("run_cmd",
        [](const std::string &cmd, int time_out = -1,
           int sleep_inter = -1) -> const std::string {
          return paddle::framework::shell_get_command_output(cmd, time_out,
                                                             sleep_inter);
        },
        py::arg("cmd"), py::arg("time_out") = -1, py::arg("sleep_inter") = -1);
G
gongweibao 已提交
1964 1965 1966 1967 1968 1969 1970 1971 1972
  m.def("shell_execute_cmd",
        [](const std::string &cmd, int time_out = 0, int sleep_inter = 0,
           bool redirect_stderr = false) -> std::vector<std::string> {
          return paddle::framework::shell_execute_cmd(
              cmd, time_out, sleep_inter, redirect_stderr);
        },
        py::arg("cmd"), py::arg("time_out") = 0, py::arg("sleep_inter") = 0,
        py::arg("redirect_stderr") = false);

1973
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
1974 1975 1976 1977 1978
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1979

1980
  m.def("set_feed_variable", framework::SetFeedVariable);
1981 1982 1983 1984 1985
  m.def("get_fetch_variable",
        [](const Scope &scope, const std::string &var_name,
           size_t index) -> py::object {
          auto &var = framework::GetFetchVariable(scope, var_name, index);
          if (data_is_lod_tensor(var)) {
1986
            return py::cast(BOOST_GET(LoDTensor, var));
1987
          } else {
1988
            return py::cast(BOOST_GET(LoDTensorArray, var));
1989 1990
          }
        });
1991
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1992

X
Xin Pan 已提交
1993 1994
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1995 1996 1997 1998 1999
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
2000
  BindGlobalValueGetterSetter(&m);
Y
Yu Yang 已提交
2001

Y
Yu Yang 已提交
2002 2003 2004 2005 2006 2007 2008 2009 2010
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
2011
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
2012
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
2013 2014 2015

    Examples:
        .. code-block:: python
2016

Z
Zeng Jinle 已提交
2017 2018 2019 2020
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
2021 2022
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
2023 2024 2025 2026 2027 2028
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
2029 2030 2031 2032
             PADDLE_ENFORCE_LT(i, self.size(),
                               platform::errors::InvalidArgument(
                                   "The index to set is larger than the size "
                                   "of LoDTensorArray."));
Y
Yu Yang 已提交
2033 2034 2035
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
2036 2037 2038 2039 2040 2041
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
2042 2043
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
2044 2045 2046 2047 2048 2049
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
2072

2073 2074 2075 2076 2077 2078 2079 2080
  py::class_<FetchList>(m, "FetchList", R"DOC( FetchList is a
        vector of boost::variant<LoDTensor, LoDTensorArray>.
        )DOC")
      .def("_move_to_list",
           [](FetchList &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               if (data_is_lod_tensor(self[i])) {
2081
                 auto &data = BOOST_GET(LoDTensor, self[i]);
2082 2083
                 res[i] = py::cast(std::move(data));
               } else {
2084
                 auto &data = BOOST_GET(LoDTensorArray, self[i]);
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
                 py::list tmp(data.size());
                 for (size_t j = 0; j < data.size(); ++j) {
                   tmp[j] = py::cast(std::move(data[j]));
                 }
                 res[i] = std::move(tmp);
               }
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership)

      .def("append",
           [](FetchList &self, const LoDTensor &t) {
             self.emplace_back();
2100
             auto &lod_tensor = BOOST_GET(LoDTensor, self.back());
2101 2102 2103 2104 2105 2106 2107 2108
             lod_tensor.ShareDataWith(t);
             lod_tensor.set_lod(t.lod());
           },
           py::arg("var"))

      .def("append",
           [](FetchList &self, const LoDTensorArray &t) {
             self.emplace_back();
2109
             auto &lod_tensor_array = BOOST_GET(LoDTensorArray, self.back());
2110 2111 2112 2113 2114 2115 2116 2117 2118
             for (size_t i = 0; i < t.size(); ++i) {
               lod_tensor_array[i].ShareDataWith(t[i]);
               lod_tensor_array[i].set_lod(t[i].lod());
             }
           },
           py::arg("var"));

  py::class_<FetchUnmergedList>(m, "FetchUnmergedList", R"DOC(
        FetchUnmergedList is 2-D array of FetchType(boost::variant(LoDTensor, LoDTensorArray)).
Z
Zhen Wang 已提交
2119 2120
        )DOC")
      .def("_move_to_list",
2121
           [](FetchUnmergedList &self) -> py::list {
Z
Zhen Wang 已提交
2122 2123 2124 2125
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               py::list tmp(self[i].size());
               for (size_t j = 0; j < self[i].size(); ++j) {
2126
                 if (data_is_lod_tensor(self[i][j])) {
2127
                   auto &var = BOOST_GET(LoDTensor, self[i][j]);
2128 2129
                   tmp[j] = py::cast(std::move(var));
                 } else {
2130
                   auto &var = BOOST_GET(LoDTensorArray, self[i][j]);
2131 2132 2133 2134 2135 2136
                   py::list tmp_array(var.size());
                   for (size_t k = 0; k < var.size(); ++k) {
                     tmp_array[k] = std::move(var[k]);
                   }
                   tmp[j] = std::move(tmp_array);
                 }
Z
Zhen Wang 已提交
2137 2138 2139 2140 2141 2142 2143 2144 2145
               }
               res[i] = std::move(tmp);
               self[i].clear();
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);

Y
Yu Yang 已提交
2146
  m.def("op_support_gpu", OpSupportGPU);
2147
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
D
Dong Zhihong 已提交
2148
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
2149

2150
#if !defined(PADDLE_WITH_HIP) && !defined(_WIN32)
D
dangqingqing 已提交
2151 2152 2153
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
2154 2155 2156 2157
  m.def("nvprof_nvtx_push", platform::CudaNvtxRangePush);
  m.def("nvprof_nvtx_pop", platform::CudaNvtxRangePop);
  m.def("nvprof_enable_record_event", platform::NvprofEnableRecordEvent);
  m.def("nvprof_disable_record_event", platform::NvprofDisableRecordEvent);
D
Dong Zhihong 已提交
2158
#endif
P
peizhilin 已提交
2159
#endif
Y
Yu Yang 已提交
2160

2161 2162
#ifdef PADDLE_WITH_ASCEND_CL
  m.def("get_npu_device_count", platform::GetNPUDeviceCount);
2163
  m.def("npu_finalize", []() { platform::AclInstance::Instance().Finalize(); });
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183

  py::class_<platform::NPUProfConfigWrapper>(m, "NPUProfConfigWrapper");

  m.def("npu_prof_init", platform::NPUProfilerInit);
  m.def("npu_prof_start", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStart(c.ptr());
  });
  m.def("npu_prof_stop", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerStop(c.ptr());
  });
  m.def("npu_prof_finalize", platform::NPUProfilerFinalize);
  m.def("npu_prof_create_config", []() {
    return platform::NPUProfConfigWrapper(platform::NPUProfilerCreateConfig());
  });

  m.def("npu_prof_destropy_config", [](platform::NPUProfConfigWrapper c) {
    platform::NPUProfilerDestroyConfig(c.ptr());
  });
#endif

2184 2185 2186 2187 2188 2189
  py::enum_<platform::TracerOption>(m, "TracerOption", py::arithmetic())
      .value("kDefault", platform::TracerOption::kDefault)
      .value("kOpDetail", platform::TracerOption::kOpDetail)
      .value("kAllOpDetail", platform::TracerOption::kAllOpDetail)
      .export_values();

2190 2191 2192 2193
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
2194
      .value("kAll", platform::ProfilerState::kAll)
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

2206
  m.def("set_tracer_option", platform::SetTracerOption);
2207 2208
  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
2209
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
2210
  m.def("reset_profiler", platform::ResetProfiler);
2211
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
2212 2213 2214
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
2215

2216 2217
  m.def("size_of_dtype", framework::SizeOfType);

2218
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
2219 2220
  m.def("set_cublas_switch", platform::SetAllowTF32Cublas);
  m.def("get_cublas_switch", platform::AllowTF32Cublas);
A
AshburnLee 已提交
2221 2222
  m.def("set_cudnn_switch", platform::SetAllowTF32Cudnn);
  m.def("get_cudnn_switch", platform::AllowTF32Cudnn);
2223
#endif  // PADDLE_WITH_CUDA
2224 2225
  m.def("clear_executor_cache",
        []() { framework::ExecutorInfoCache::Instance().Finalize(); });
2226

2227 2228 2229
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

2230 2231
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
2232
      .def("has", &ir::Pass::Has)
2233 2234 2235
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
2236
           })
2237
      .def(
2238
          "set",
2239 2240 2241
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
2242 2243
      .def("set", [](ir::Pass &self, const std::string &name,
                     bool val) { self.Set<bool>(name, new bool(val)); })
2244 2245
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
2260 2261
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
2262
        self.Apply(graph.get());
F
flame 已提交
2263
      });
2264

X
fix  
Xin Pan 已提交
2265 2266
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
2281
  // -- python binds for parallel executor.
X
Xin Pan 已提交
2282

Y
yuyang18 已提交
2283
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
2284 2285 2286 2287
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

2288 2289 2290
    Returns:
        ExecutionStrategy: An ExecutionStrategy object.

C
chengduo 已提交
2291 2292 2293
    Examples:
        .. code-block:: python

2294 2295 2296 2297 2298 2299 2300 2301 2302
          import paddle
          import paddle.static as static
          import paddle.nn.functional as F

          paddle.enable_static()

          x = static.data(name='x', shape=[None, 13], dtype='float32')
          y = static.data(name='y', shape=[None, 1], dtype='float32')
          y_predict = static.nn.fc(input=x, size=1, act=None)
2303

2304 2305
          cost = F.square_error_cost(input=y_predict, label=y)
          avg_loss = paddle.mean(cost)
2306

2307
          sgd_optimizer = paddle.optimizer.SGD(learning_rate=0.001)
2308 2309
          sgd_optimizer.minimize(avg_loss)

2310
          exec_strategy = static.ExecutionStrategy()
C
chengduo 已提交
2311 2312
          exec_strategy.num_threads = 4

2313 2314 2315
          train_exe = static.ParallelExecutor(use_cuda=False,
                                              loss_name=avg_loss.name,
                                              exec_strategy=exec_strategy)
C
chengduo 已提交
2316 2317
        )DOC");

2318 2319 2320 2321
  py::enum_<paddle::platform::DeviceType>(m, "DeviceType", py::arithmetic())
      .value("CPU", paddle::platform::DeviceType::CPU)
      .value("CUDA", paddle::platform::DeviceType::CUDA)
      .value("XPU", paddle::platform::DeviceType::XPU);
2322

Y
yuyang18 已提交
2323
  exec_strategy.def(py::init())
Y
yuyang18 已提交
2324 2325 2326 2327 2328
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
2329
          },
2330 2331
          R"DOC(
            The type is INT, num_threads represents the size of thread pool that
C
chengduo 已提交
2332 2333 2334 2335 2336 2337 2338
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351
            `multiprocessing.cpu_count()`. Default 0.

            Examples:
                .. code-block:: python

                    import paddle
                    import paddle.static as static

                    paddle.enable_static()

                    exec_strategy = static.ExecutionStrategy()
                    exec_strategy.num_threads = 4
            )DOC")
Y
yuyang18 已提交
2352
      .def_property(
2353 2354
          "_use_device",
          [](const ExecutionStrategy &self) { return self.use_device_; },
2355
          [](ExecutionStrategy &self, paddle::platform::DeviceType use_device) {
2356 2357 2358
            self.use_device_ = use_device;
          })  // NOTE(liuyuhui): Doesn't add doc for 'use_device', because
              // use_device isn‘t exposed to users.
Y
yuyang18 已提交
2359 2360 2361 2362 2363
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
2364 2365 2366
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
2367 2368
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
2369 2370 2371 2372 2373 2374 2375
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
2376 2377 2378 2379
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
2380
                because the temp variable's shape maybe the same between two iterations.
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
                Default 100.

                .. note::
                    1. If you fetch data when calling the 'run', the ParallelExecutor 
                    will clean up the temp variables at the end of the current iteration. 
                    2. In some NLP model, it may cause the GPU memory is insufficient, 
                    in this case, you should reduce `num_iteration_per_drop_scope`.

                Examples:
                    .. code-block:: python
C
chengduo 已提交
2391

2392 2393 2394 2395 2396 2397 2398
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_drop_scope = 10
2399
              )DOC")
Q
Qiao Longfei 已提交
2400 2401 2402 2403 2404 2405 2406 2407 2408
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420
                user call exe.run() in python。Default: 1.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        exec_strategy = static.ExecutionStrategy()
                        exec_strategy.num_iteration_per_run = 10
Q
Qiao Longfei 已提交
2421
              )DOC")
2422 2423 2424 2425 2426 2427 2428 2429
      .def_property(
          "use_thread_barrier",
          [](const ExecutionStrategy &self) { return self.thread_barrier_; },
          [](ExecutionStrategy &self, bool use_thread_barrier) {
            self.thread_barrier_ = use_thread_barrier;
          },
          R"DOC(This config that the this is distributed training with parameter server
              )DOC")
2430 2431 2432 2433 2434
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
2435

Y
yuyang18 已提交
2436
  exec_strategy.def_property(
Y
yuyang18 已提交
2437 2438 2439 2440 2441 2442 2443
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
2444 2445
      });

C
chengduo 已提交
2446 2447 2448 2449
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

2450 2451 2452
    Returns:
        BuildStrategy: An BuildStrategy object.

C
chengduo 已提交
2453 2454 2455
    Examples:
        .. code-block:: python

2456
            import os
2457 2458 2459 2460
            import paddle
            import paddle.static as static

            paddle.enable_static()
2461

2462 2463
            os.environ['CPU_NUM'] = str(2)
            places = static.cpu_places()
2464

2465 2466 2467 2468
            data = static.data(name="x", shape=[None, 1], dtype="float32")
            hidden = static.nn.fc(input=data, size=10)
            loss = paddle.mean(hidden)
            paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
2469

2470
            build_strategy = static.BuildStrategy()
2471 2472
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
2473 2474
            build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
            program = static.CompiledProgram(static.default_main_program())
2475
            program = program.with_data_parallel(loss_name=loss.name,
2476 2477
                                                  build_strategy=build_strategy,
                                                  places=places)
C
chengduo 已提交
2478
)DOC");
Y
yuyang18 已提交
2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
2495 2496 2497 2498
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2499
            self.reduce_ = strategy;
C
chengduo 已提交
2500
          },
2501
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
2502 2503
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
2504
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
2505 2506
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
2507
                Default is 'AllReduce'.
F
flame 已提交
2508 2509 2510 2511

                Examples:
                    .. code-block:: python

2512 2513 2514 2515 2516 2517 2518
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.reduce_strategy = static.BuildStrategy.ReduceStrategy.Reduce
F
flame 已提交
2519
                  )DOC")
Y
yuyang18 已提交
2520 2521 2522 2523 2524
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
2525 2526 2527 2528
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2529
            self.gradient_scale_ = strategy;
C
chengduo 已提交
2530
          },
2531
          R"DOC((paddle.static.BuildStrategy.GradientScaleStrategy, optional): there are three
2532
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
2533 2534
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
2535
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
2536 2537 2538 2539

                Examples:
                    .. code-block:: python

C
chengduo 已提交
2540 2541
                        import numpy
                        import os
2542 2543 2544 2545
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2546 2547

                        use_cuda = True
2548 2549
                        place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace()
                        exe = static.Executor(place)
C
chengduo 已提交
2550 2551

                        # NOTE: If you use CPU to run the program, you need
2552
                        # to specify the CPU_NUM, otherwise, paddle will use
C
chengduo 已提交
2553 2554 2555 2556 2557 2558
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
2559
                            places = static.cpu_places()
C
chengduo 已提交
2560
                        else:
2561
                            places = static.cuda_places()
C
chengduo 已提交
2562

2563 2564 2565 2566
                        data = static.data(name='X', shape=[None, 1], dtype='float32')
                        hidden = static.nn.fc(input=data, size=10)
                        loss = paddle.mean(hidden)
                        paddle.optimizer.SGD(learning_rate=0.01).minimize(loss)
C
chengduo 已提交
2567

2568
                        exe.run(static.default_startup_program())
C
chengduo 已提交
2569

2570
                        build_strategy = static.BuildStrategy()
C
chengduo 已提交
2571
                        build_strategy.gradient_scale_strategy = \
2572 2573 2574
                                  static.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = static.CompiledProgram(
                                  static.default_main_program()).with_data_parallel(
C
chengduo 已提交
2575
                                          loss_name=loss.name, build_strategy=build_strategy,
2576
                                          places=places)
C
chengduo 已提交
2577 2578 2579 2580 2581 2582

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
2583 2584
                                              feed={"X": x, loss_grad_name : loss_grad},
                                              fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
2585
                   )DOC")
Y
yuyang18 已提交
2586 2587 2588 2589
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
2590 2591 2592 2593
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Y
yuyang18 已提交
2594
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
2595
          },
2596
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
2597
                writing the SSA Graph to file in the form of graphviz.
2598
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
2599 2600 2601 2602

                Examples:
                    .. code-block:: python

2603 2604 2605 2606
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()
C
chengduo 已提交
2607

2608 2609
                        build_strategy = static.BuildStrategy()
                        build_strategy.debug_graphviz_path = "./graph"
F
flame 已提交
2610
                    )DOC")
S
sneaxiy 已提交
2611 2612 2613 2614 2615 2616
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
2617 2618 2619 2620
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2621 2622
            self.enable_sequential_execution_ = b;
          },
2623 2624
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
2625 2626 2627 2628

                Examples:
                    .. code-block:: python

2629 2630 2631 2632 2633 2634
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2635 2636
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
2637 2638 2639 2640 2641 2642
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
2643 2644 2645 2646
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
S
sneaxiy 已提交
2647 2648
            self.remove_unnecessary_lock_ = b;
          },
2649 2650
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
2651 2652 2653 2654

                Examples:
                    .. code-block:: python

2655 2656 2657 2658 2659 2660
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2661 2662
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
2663 2664 2665 2666
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
2667
#ifdef WIN32
2668
            PADDLE_THROW(platform::errors::Unavailable(
2669
                "Distribution mode is not supported on Windows platform."));
2670
#endif
2671 2672
            self.num_trainers_ = num_trainers;
          })
2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
2685 2686 2687 2688 2689 2690
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
2691 2692 2693 2694 2695 2696
      .def_property(
          "bkcl_comm_num",
          [](const BuildStrategy &self) { return self.bkcl_comm_num_; },
          [](BuildStrategy &self, int bkcl_comm_num) {
            self.bkcl_comm_num_ = bkcl_comm_num;
          })
2697
      .def_property("use_hierarchical_allreduce",
2698 2699 2700 2701 2702 2703
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
2704
      .def_property("hierarchical_allreduce_inter_nranks",
2705 2706 2707 2708 2709 2710 2711
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
2712 2713 2714 2715 2716 2717
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
2718 2719 2720 2721
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
C
chengduo 已提交
2722 2723
            self.fuse_elewise_add_act_ops_ = b;
          },
2724
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
2725
                to fuse elementwise_add_op and activation_op,
2726
                it may make the execution faster. Default is False.
F
flame 已提交
2727 2728 2729 2730

                Examples:
                    .. code-block:: python

2731 2732 2733 2734 2735 2736
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2737 2738
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
Z
Zhen Wang 已提交
2739 2740 2741 2742
      .def_property(
          "fuse_bn_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_act_ops_; },
          [](BuildStrategy &self, bool b) {
2743
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
Z
Zhen Wang 已提交
2744
                              platform::errors::PreconditionNotMet(
2745 2746
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Z
Zhen Wang 已提交
2747 2748 2749 2750 2751 2752 2753 2754 2755
            self.fuse_bn_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_act_ops indicate whether
                to fuse batch_norm and activation_op,
                it may make the execution faster. Default is False.

                Examples:
                    .. code-block:: python

2756 2757 2758 2759 2760 2761
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
Z
Zhen Wang 已提交
2762 2763
                        build_strategy.fuse_bn_act_ops = True
                     )DOC")
Z
Zhang Ting 已提交
2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788
      .def_property(
          "fuse_bn_add_act_ops",
          [](const BuildStrategy &self) { return self.fuse_bn_add_act_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
            self.fuse_bn_add_act_ops_ = b;
          },
          R"DOC((bool, optional): fuse_bn_add_act_ops indicate whether
                to fuse batch_norm, elementwise_add and activation_op,
                it may make the execution faster. Default is True

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.fuse_bn_add_act_ops = True
                     )DOC")
2789 2790 2791 2792
      .def_property(
          "enable_auto_fusion",
          [](const BuildStrategy &self) { return self.enable_auto_fusion_; },
          [](BuildStrategy &self, bool b) {
2793
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
2794
                              platform::errors::PreconditionNotMet(
2795 2796
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2797 2798 2799 2800 2801 2802 2803 2804 2805 2806
            self.enable_auto_fusion_ = b;
          },
          R"DOC((bool, optional): Whether to enable fusing subgraph to a
                fusion_group. Now we only support fusing subgraph that composed
                of elementwise-like operators, such as elementwise_add/mul
                without broadcast and activations.

                Examples:
                    .. code-block:: python

2807 2808 2809 2810 2811 2812
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
2813 2814
                        build_strategy.enable_auto_fusion = True
                    )DOC")
2815 2816 2817 2818 2819 2820
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
2821 2822 2823 2824
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
2825 2826
            self.fuse_relu_depthwise_conv_ = b;
          },
2827
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
2828 2829 2830
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
2831
                Default is False.
F
flame 已提交
2832 2833 2834 2835

                Examples:
                    .. code-block:: python

2836 2837 2838 2839 2840 2841
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2842 2843
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
2844 2845 2846 2847 2848 2849
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
                             self.fuse_broadcast_ops_ == boost::none;
                    },
                    [](BuildStrategy &self, bool b) {
2850 2851 2852 2853
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
2854 2855
                      self.fuse_broadcast_ops_ = b;
                    },
2856
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
2857 2858 2859 2860
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
2861 2862 2863 2864 2865
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

2866 2867 2868 2869 2870 2871
                              import paddle
                              import paddle.static as static

                              paddle.enable_static()

                              build_strategy = static.BuildStrategy()
2872 2873
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
2874 2875
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
2876 2877
                      return self.fuse_all_optimizer_ops_ == true ||
                             self.fuse_all_optimizer_ops_ == boost::none;
C
chengduo 已提交
2878 2879
                    },
                    [](BuildStrategy &self, bool b) {
2880 2881 2882 2883
                      PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                                        platform::errors::PreconditionNotMet(
                                            "BuildStrategy has been finlaized, "
                                            "cannot be configured again."));
C
chengduo 已提交
2884 2885
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
2886 2887 2888 2889
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
2890 2891 2892 2893
            PADDLE_ENFORCE_NE(self.IsFinalized(), true,
                              platform::errors::PreconditionNotMet(
                                  "BuildStrategy has been finlaized, cannot be "
                                  "configured again."));
Q
qingqing01 已提交
2894 2895
            self.sync_batch_norm_ = b;
          },
2896
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
2897 2898 2899
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
2900 2901
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
2902 2903 2904 2905

                Examples:
                    .. code-block:: python

2906 2907 2908 2909 2910 2911
                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
F
flame 已提交
2912 2913
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
2914 2915
      .def_property(
          "memory_optimize",
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
              self.memory_optimize_ = boost::none;
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
2930 2931 2932
              PADDLE_THROW(platform::errors::InvalidArgument(
                  "BuildStrategy.memory_optimize must be set to None, False or "
                  "True"));
2933 2934
            }
          },
2935
          R"DOC((bool, optional): memory opitimize aims to save total memory
2936
                consumption, set to True to enable it.
2937

2938 2939 2940
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
                True means enabling and False means disabling. Default is None.

                Examples:
                    .. code-block:: python

                        import paddle
                        import paddle.static as static

                        paddle.enable_static()

                        build_strategy = static.BuildStrategy()
                        build_strategy.memory_optimize = True
                
                )DOC")
2955 2956 2957
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
2958 2959 2960
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
2961
              PADDLE_THROW(platform::errors::Unavailable(
2962
                  "Distribution mode is not supported on Windows platform."));
2963 2964 2965 2966 2967
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
2968 2969 2970
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
2971
      .def_property(
D
dzhwinter 已提交
2972 2973 2974
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
2975 2976 2977 2978
      .def_property(
          "enable_addto",
          [](const BuildStrategy &self) { return self.enable_addto_; },
          [](BuildStrategy &self, bool b) { self.enable_addto_ = b; })
C
chengduo 已提交
2979 2980
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
2981 2982 2983 2984
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
                   self.fuse_all_reduce_ops_ == boost::none;
          },
C
chengduo 已提交
2985
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
2986 2987 2988 2989 2990 2991 2992
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
2993 2994 2995 2996
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
2997 2998 2999 3000 3001 3002 3003 3004 3005
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
3006
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
3007
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
3008 3009 3010 3011 3012
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
3013 3014

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
3015
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
3016
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
3017
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
3018 3019 3020 3021
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
3022 3023 3024 3025 3026
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
3027 3028 3029
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
3030 3031 3032 3033
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
3034 3035
      .def("run",
           [](ParallelExecutor &self,
Z
Zhen Wang 已提交
3036 3037 3038 3039 3040 3041 3042 3043
              const std::vector<std::string> &fetch_tensors,
              bool return_merged) -> py::object {
             paddle::framework::FetchResultType ret;
             {
               pybind11::gil_scoped_release release;
               ret = self.Run(fetch_tensors, return_merged);
             }
             if (return_merged) {
3044
               return py::cast(
3045
                   std::move(BOOST_GET(paddle::framework::FetchList, ret)));
Z
Zhen Wang 已提交
3046 3047
             } else {
               return py::cast(std::move(
3048
                   BOOST_GET(paddle::framework::FetchUnmergedList, ret)));
Z
Zhen Wang 已提交
3049
             }
3050 3051
           })
      .def("device_count", &ParallelExecutor::DeviceCount);
Y
Yu Yang 已提交
3052

D
dongdaxiang 已提交
3053
  BindFleetWrapper(&m);
3054
  BindIO(&m);
T
Thunderbrook 已提交
3055

T
Thunderbrook 已提交
3056 3057
#ifdef PADDLE_WITH_PSLIB
  BindHeterWrapper(&m);
T
Thunderbrook 已提交
3058
#endif
T
Thunderbrook 已提交
3059
#ifdef PADDLE_WITH_HETERPS
T
Thunderbrook 已提交
3060
  BindPSGPUWrapper(&m);
T
Thunderbrook 已提交
3061
#endif
3062
  BindGlooWrapper(&m);
H
hutuxian 已提交
3063
  BindBoxHelper(&m);
H
hutuxian 已提交
3064 3065 3066
#ifdef PADDLE_WITH_BOX_PS
  BindBoxWrapper(&m);
#endif
3067
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
D
dongdaxiang 已提交
3068
  BindNCCLWrapper(&m);
3069 3070 3071
#endif
#ifdef PADDLE_WITH_GLOO
  BindGlooContext(&m);
W
wopeizl 已提交
3072
#endif
F
flame 已提交
3073 3074
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
3075
  BindInferenceApi(&m);
3076
  BindCompatible(&m);
3077
  BindDataset(&m);
Y
yaoxuefeng 已提交
3078
  BindGenerator(&m);
3079 3080 3081
#ifdef PADDLE_WITH_ASCEND
  BindAscendWrapper(&m);
  BindAscendGraph(&m);
3082
  BindAscendDevice(&m);
3083
#endif
Y
Yanghello 已提交
3084 3085 3086
#ifdef PADDLE_WITH_CRYPTO
  BindCrypto(&m);
#endif
T
tangwei12 已提交
3087

T
tangwei12 已提交
3088
#if defined PADDLE_WITH_PSCORE
T
tangwei12 已提交
3089 3090
  BindDistFleetWrapper(&m);
  BindPSHost(&m);
3091
  BindCommunicatorContext(&m);
T
tangwei12 已提交
3092 3093
  BindDistCommunicator(&m);
  BindHeterClient(&m);
S
seemingwang 已提交
3094 3095 3096 3097 3098
  BindGraphPyFeatureNode(&m);
  BindGraphNode(&m);
  BindGraphPyService(&m);
  BindGraphPyServer(&m);
  BindGraphPyClient(&m);
1
123malin 已提交
3099 3100 3101 3102
  BindIndexNode(&m);
  BindTreeIndex(&m);
  BindIndexWrapper(&m);
  BindIndexSampler(&m);
3103
  BindSparseShardingTools(&m);
3104
#endif
L
Luo Tao 已提交
3105
}
3106
}  // namespace pybind
3107
}  // namespace paddle