pybind.cc 85.6 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15
#include <algorithm>
16
#include <cstdlib>
C
chengduoZH 已提交
17
#include <map>
S
sneaxiy 已提交
18
#include <memory>
C
chengduoZH 已提交
19 20 21
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
22
#include <unordered_set>
C
chengduoZH 已提交
23 24
#include <utility>
#include <vector>
25

Y
Yi Wang 已提交
26 27 28
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
29
#include "paddle/fluid/framework/garbage_collector.h"
30
#include "paddle/fluid/framework/ir/coalesce_grad_tensor_pass.h"
31
#include "paddle/fluid/framework/ir/pass_builder.h"
32
#include "paddle/fluid/framework/load_op_lib.h"
Y
Yi Wang 已提交
33 34 35
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
36
#include "paddle/fluid/framework/op_compatible_info.h"
S
sneaxiy 已提交
37
#include "paddle/fluid/framework/op_info.h"
38
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
39
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
40
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
41
#include "paddle/fluid/framework/reader.h"
H
hong 已提交
42
#include "paddle/fluid/framework/save_load_util.h"
S
sneaxiy 已提交
43
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
44
#include "paddle/fluid/framework/selected_rows.h"
45
#include "paddle/fluid/framework/trainer.h"
X
Xin Pan 已提交
46
#include "paddle/fluid/framework/version.h"
H
hong 已提交
47
#include "paddle/fluid/imperative/layer.h"
Y
Refine  
Yu Yang 已提交
48
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
D
dzhwinter 已提交
49
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
50
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
51
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
52
#include "paddle/fluid/platform/cpu_helper.h"
Y
Yu Yang 已提交
53
#include "paddle/fluid/platform/cpu_info.h"
54
#include "paddle/fluid/platform/dynload/dynamic_loader.h"
Y
Yi Wang 已提交
55
#include "paddle/fluid/platform/enforce.h"
56
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
57 58
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
H
hutuxian 已提交
59
#include "paddle/fluid/pybind/box_helper_py.h"
Y
Yi Wang 已提交
60
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
61
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
62
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
63
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
64
#include "paddle/fluid/pybind/global_value_getter_setter.h"
65
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
66
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
67
#include "paddle/fluid/pybind/ir.h"
68

W
wopeizl 已提交
69
#ifndef _WIN32
D
dongdaxiang 已提交
70
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
71
#endif
72
#include "paddle/fluid/framework/data_type.h"
73 74
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
75
#include "paddle/fluid/pybind/reader_py.h"
Y
Yi Wang 已提交
76
#include "paddle/fluid/pybind/tensor_py.h"
77
#include "paddle/fluid/string/to_string.h"
D
Dong Zhihong 已提交
78
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
79
#ifndef _WIN32
Y
Yi Wang 已提交
80
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
81
#endif
Y
Yi Wang 已提交
82 83
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
84 85
#endif

86 87 88 89
#ifdef PADDLE_WITH_DISTRIBUTE
#include "paddle/fluid/pybind/communicator_py.h"
#endif

M
minqiyang 已提交
90 91
#include "pybind11/stl.h"

92 93 94
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");
95
DECLARE_bool(use_mkldnn);
96 97 98
#ifdef PADDLE_WITH_NGRAPH
DECLARE_bool(use_ngraph);
#endif
99

Q
Qiao Longfei 已提交
100 101 102
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

103
namespace paddle {
104
namespace pybind {
105
bool IsCompiledWithCUDA() {
106
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
107 108 109 110 111 112
  return false;
#else
  return true;
#endif
}

113 114 115 116 117 118 119 120
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

121 122 123 124 125 126 127 128
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

129
bool IsCompiledWithBrpc() {
130
#ifndef PADDLE_WITH_DISTRIBUTE
131 132
  return false;
#endif
133 134 135 136 137 138

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
139 140
}

Y
update  
Yancey1989 已提交
141
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
142
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
143 144 145 146 147 148
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
149 150 151 152 153 154 155 156 157 158
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

H
hong 已提交
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
static PyObject *GetPythonAttribute(PyObject *obj, const char *attr_name) {
  // NOTE(zjl): PyObject_GetAttrString would return nullptr when attr_name
  // is not inside obj, but it would also set the error flag of Python.
  // If the error flag is set in C++, C++ code would not raise Exception,
  // but Python would raise Exception once C++ call ends.
  // To avoid unexpected Exception raised in Python, we check whether
  // attribute exists before calling PyObject_GetAttrString.
  //
  // Caution: PyObject_GetAttrString would increase reference count of PyObject.
  // Developer should call Py_DECREF manually after the attribute is not used.
  if (PyObject_HasAttrString(obj, attr_name)) {
    return PyObject_GetAttrString(obj, attr_name);
  } else {
    return nullptr;
  }
}

template <typename T>
static T PyObjectCast(PyObject *obj) {
  try {
    return py::cast<T>(py::handle(obj));
  } catch (py::cast_error &) {
    PADDLE_THROW("Python object is not type of %s", typeid(T).name());
  }
}

using PyNameVarBaseMap = std::unordered_map<std::string, py::handle>;

static std::vector<std::shared_ptr<imperative::VarBase>> GetVarBaseList(
    const PyNameVarBaseMap &state_dict) {
  std::vector<std::shared_ptr<imperative::VarBase>> vec_res;
  vec_res.reserve(state_dict.size());

  for (auto &para : state_dict) {
    PyObject *py_obj = para.second.ptr();
    if (!py_obj || py_obj == Py_None) {
      PADDLE_THROW("Save parameter [%s] is None", para.first);
    }

    const char *kIVarField = "_ivar";
    PyObject *py_ivar = GetPythonAttribute(py_obj, kIVarField);
    PADDLE_ENFORCE_NOT_NULL(py_ivar, "Can not find  ivar in Variable");

    vec_res.emplace_back(
        PyObjectCast<std::shared_ptr<imperative::VarBase>>(py_ivar));
    Py_DECREF(py_ivar);
  }

  return vec_res;
}

static std::vector<std::string> inline GetNameList(
    const py::handle &py_handle) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
    PADDLE_THROW("Save parameter list is None");
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
      PADDLE_ENFORCE_NOT_NULL(py_name);
      vec_res.emplace_back(PyObjectCast<std::string>(py_name));
      Py_DECREF(py_name);
    }
  } else {
    PADDLE_THROW("Set parameter should be a list");
  }

  return vec_res;
}

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
static void inline CreateVariableIfNotExit(
    const py::handle &py_handle, const framework::Scope &scope,
    const framework::Executor *exe = nullptr) {
  std::vector<std::string> vec_res;

  PyObject *py_obj = py_handle.ptr();  // get underlying PyObject
  // Python None is not nullptr in C++!
  if (!py_obj || py_obj == Py_None) {
    PADDLE_THROW("Save parameter list is None");
  }

  if (PyList_Check(py_obj)) {
    size_t len = PyList_GET_SIZE(py_obj);

    vec_res.reserve(len);

    const char *kNameField = "name";
    const char *kVarDescField = "desc";

    for (size_t i = 0; i < len; ++i) {
      PyObject *py_name =
          PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kNameField);
      PADDLE_ENFORCE_NOT_NULL(py_name);
      auto para_name = PyObjectCast<std::string>(py_name);
      Py_DECREF(py_name);

      auto var = scope.FindVar(para_name);
      if (var == nullptr) {
        PADDLE_ENFORCE_NE(exe, nullptr,
                          "Parameter not Initialized, "
                          "Please set argument [executor] not None "
                          "or run startup program first");
        PyObject *py_var_desc =
            PyObject_GetAttrString(PyList_GET_ITEM(py_obj, i), kVarDescField);
        PADDLE_ENFORCE_NOT_NULL(py_var_desc);
        auto var_desc = PyObjectCast<framework::VarDesc>(py_var_desc);
        Py_DECREF(py_var_desc);
        var = const_cast<framework::Scope *>(&scope)->Var(para_name);
        auto *tensor_temp = var->GetMutable<framework::LoDTensor>();
        tensor_temp->Resize(framework::make_ddim(var_desc.GetShape()));
        tensor_temp->mutable_data(exe->GetPlace(), var_desc.GetDataType());
      }
    }
  } else {
    PADDLE_THROW("Set parameter should be a list");
  }

  return;
}

291 292 293 294 295 296
#ifdef PADDLE_WITH_AVX
PYBIND11_MODULE(core_avx, m) {
#else
PYBIND11_MODULE(core_noavx, m) {
#endif

Y
Yu Yang 已提交
297 298 299
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
300
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
301

302
  m.doc() = "C++ core of PaddlePaddle";
303

304 305 306 307
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

308
  BindException(&m);
Y
Yu Yang 已提交
309

310 311
  m.def("set_num_threads", &platform::SetNumThreads);

6
633WHU 已提交
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
  m.def("from_dlpack", [](py::capsule *dltensor) {
    DLManagedTensor *dmt = reinterpret_cast<DLManagedTensor *>(
        PyCapsule_GetPointer(dltensor->ptr(), "dltensor"));
    PyCapsule_SetName(dltensor->ptr(), "used_dltensor");
    DLTensor dl = dmt->dl_tensor;
    Tensor tensor;

    if (dl.ctx.device_type == kDLCPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#ifdef PADDLE_WITH_CUDA
    if (dl.ctx.device_type == kDLGPU) {
      paddle::framework::TensorFromDLPack(dl, &tensor);
    }
#endif
    return tensor;
  });

H
hong 已提交
330 331 332 333 334 335 336 337 338
  m.def("_save_static_dict",
        [](const std::string &str_file_name, const py::handle &vec_var_list,
           const Scope &scope) {
          std::vector<std::string> vec_name_list = GetNameList(vec_var_list);
          SaveStaticNameListToDisk(str_file_name, vec_name_list, scope);
        });

  m.def("_load_static_dict",
        [](const std::string &str_file_name, const py::handle &vec_var_list,
339
           const Scope &scope, const Executor *executor) {
H
hong 已提交
340
          std::vector<std::string> vec_name_list = GetNameList(vec_var_list);
341
          CreateVariableIfNotExit(vec_var_list, scope, executor);
H
hong 已提交
342 343 344
          LoadStaticNameListFromDisk(str_file_name, vec_name_list, scope);
        });

345 346 347 348 349 350
  m.def("_create_loaded_parameter",
        [](const py::handle &vec_var_list, const Scope &scope,
           const Executor *executor) {
          CreateVariableIfNotExit(vec_var_list, scope, executor);
        });

H
hong 已提交
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
  m.def("_save_dygraph_dict", [](const std::string &str_file_name,
                                 const PyNameVarBaseMap &state_dict) {
    auto vec_var_base_list = GetVarBaseList(state_dict);

    SaveDygraphVarBaseListToDisk(str_file_name, vec_var_base_list);
  });

  m.def("_load_dygraph_dict", [](const std::string &str_file_name) {
    auto load_tensor = LoadDygraphVarBaseListFromDisk(str_file_name);

    std::unordered_map<std::string, std::shared_ptr<imperative::VarBase>>
        map_output;

    for (size_t i = 0; i < load_tensor.size(); ++i) {
      map_output.emplace(load_tensor[i]->Name(), load_tensor[i]);
    }

    return map_output;
  });
6
633WHU 已提交
370

371 372 373 374 375 376
  m.def("save_op_compatible_info", [](framework::ProgramDesc &desc) {
    framework::OpCompatibleMap op_compatible_map;
    op_compatible_map.InitOpCompatibleMap();
    return op_compatible_map.ConvertToProto(desc.OpCompatibleMap());
  });

S
sneaxiy 已提交
377
  m.def(
S
sneaxiy 已提交
378
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
379 380 381 382
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
383 384 385
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
  m.def("_get_all_register_op_kernels", [] {
    auto &all_kernels = paddle::framework::OperatorWithKernel::AllOpKernels();
    std::unordered_map<std::string, std::vector<std::string>> all_kernels_info;
    for (auto &kernel_pair : all_kernels) {
      auto op_type = kernel_pair.first;
      std::vector<std::string> kernel_types;
      for (auto &info_pair : kernel_pair.second) {
        paddle::framework::OpKernelType kernel_type = info_pair.first;
        kernel_types.push_back(
            paddle::framework::KernelTypeToString(kernel_type));
      }
      all_kernels_info.emplace(op_type, kernel_types);
    }
    return all_kernels_info;
  });

S
sneaxiy 已提交
402 403 404
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
405
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
406

407
  m.def("_set_fuse_parameter_group_size",
408
        &paddle::framework::ir::SetFuseParameterGroupsSize);
409
  m.def("_set_fuse_parameter_memory_size",
410
        &paddle::framework::ir::SetFuseParameterMemorySize);
411

S
sneaxiy 已提交
412 413 414
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

415 416
  m.def("_set_paddle_lib_path", &paddle::platform::dynload::SetPaddleLibPath);

417
  BindImperative(&m);
418

419
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
420
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
421 422
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
423
      .def("_get_dims",
424
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
425
      .def("_set_dims",
Q
qijun 已提交
426
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
427
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
428
           })
Y
yuyang18 已提交
429
      .def("_set_layout",
D
dzhwinter 已提交
430 431 432
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
433
      .def("_alloc_float",
D
dzhwinter 已提交
434
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
435
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
436
           })
Y
yuyang18 已提交
437
      .def("_alloc_float",
Y
Yu Yang 已提交
438
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
439
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
440
           })
441 442 443 444
      .def("_alloc_double",
           [](Tensor &self, paddle::platform::CPUPlace &place) {
             self.mutable_data<double>(place);
           })
Y
yuyang18 已提交
445
      .def("_alloc_int",
Y
Yu Yang 已提交
446
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
447
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
448
           })
Y
yuyang18 已提交
449
      .def("_alloc_int",
D
dzhwinter 已提交
450
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
451
             self.mutable_data<int>(place);
Q
qijun 已提交
452
           })
Y
yuyang18 已提交
453
      .def("_alloc_int",
C
chengduoZH 已提交
454 455 456
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
457
      .def("_alloc_float",
C
chengduoZH 已提交
458 459 460
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
      .def("_mutable_data",
           [](Tensor &self, paddle::platform::CPUPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
      .def("_mutable_data",
           [](Tensor &self, paddle::platform::CUDAPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
      .def("_mutable_data",
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place,
              paddle::framework::proto::VarType::Type type) {
             return reinterpret_cast<uintptr_t>(self.mutable_data(place, type));
           })
Z
Zeng Jinle 已提交
476
      .def("_clear", &Tensor::clear)
477
      .def("set", SetTensorFromPyArray<paddle::platform::CPUPlace>,
478
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
479
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPlace>,
480
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false)
481
      .def("set", SetTensorFromPyArray<paddle::platform::CUDAPinnedPlace>,
482 483
           py::arg("array"), py::arg("place"), py::arg("zero_copy") = false,
           R"DOC(
L
Leo Chen 已提交
484 485 486 487 488 489
        Set the data of LoDTensor on place with given numpy array.
        
        Args:
          lod (numpy.ndarray): The data to set.
          place (CPUPlace|CUDAPlace|CUDAPinnedPlace): The place where the 
          LoDTensor is to be set.
490 491
          zero_copy (bool, optional): Whether to share memory with the input numpy array.
          This parameter only works with CPUPlace. Default: False.
L
Leo Chen 已提交
492 493 494 495 496 497 498 499 500 501 502 503 504

        Returns:
            None.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                t = fluid.LoDTensor()
                t.set(np.ndarray([5, 30]), fluid.CPUPlace())
          )DOC")
505

L
Leo Chen 已提交
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); }, R"DOC(
           Return the shape of LoDTensor.

           Returns:
               list[int]: The shape of LoDTensor.


           Examples:
               .. code-block:: python

                  import paddle.fluid as fluid
                  import numpy as np

                  t = fluid.LoDTensor()
                  t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                  print(t.shape())  # [5, 30]
           )DOC")
6
633WHU 已提交
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
      .def("_to_dlpack",
           [](Tensor &self) {
             DLPackTensor dlpack_tensor(self, 1);
             DLManagedTensor *dmt =
                 dlpack_tensor.ToCudfCompatibleDLManagedTensor();
             auto capsule = py::capsule(
                 static_cast<void *>(dmt), "dltensor", [](PyObject *ptr) {
                   if (ptr) {
                     auto dltensor = new DLManagedTensor;
                     try {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "used_dltensor"));
                       return;
                     } catch (...) {
                       dltensor = reinterpret_cast<DLManagedTensor *>(
                           PyCapsule_GetPointer(ptr, "dltensor"));
                     }
                     dltensor->deleter(dltensor);
                   }
                 });
             return capsule;
           })
Y
yuyang18 已提交
545 546 547 548
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
549
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
550
      .def("_dtype", [](Tensor &self) { return self.type(); })
551
      .def("_share_data_with", &Tensor::ShareDataWith)
552 553 554 555 556 557
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference)
      .def("__str__", [](const Tensor &self) {
        std::stringstream ostr;
        ostr << self;
        return ostr.str();
      });
Y
Yu Yang 已提交
558

L
Leo Chen 已提交
559
  // TODO(cql): add reference: en_user_guide_lod_tensor
X
Xin Pan 已提交
560
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
L
Leo Chen 已提交
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
    LoDTensor is a Tensor with optional LoD (Level of Details) information, 
    it can be used for variable-length sequences, 
    see :ref:`user_guide_lod_tensor` for details.

    LoDTensor can be converted to numpy array using :code:`numpy.array(lod_tensor)`.

    You can skip the following explanation if you don't need to know details 
    of LoDTensor.

    The following two examples show how to use LODtensor to represent 
    variable-length sequences.
    
    Example 1:
    
    Suppose x is a LoDTensor representing a variable-length sequence. 
    It contains two logical subsequences, the length of first logical sequence 
    is 2 (e.g., number of samples is 2), the length of second logical sequence 
    is 3, and the total length is 5. The data of the first logical sequence is 
    [1, 2], [3, 4], and the data of the second logical sequence is [5, 6], 
    [7, 8], [9, 10]. The data dimension of each sample is 2. So, the final 
    shape of the LoDTensor is [5, 2], of which 5 is the total length and 2 is 
    the dimension of each sample.
    
    Logically, we can represent the variable-length sequence in two ways: one 
    is in the form of recursive sequence lengths, that is, 
    x.recursive_sequence_lengths=[[2, 3]]; the other is in the form of offsets, 
    that is, x.lod=[[0, 2, 2+3]]. These two representations are equivalent, and 
    you can set and retrieve recursive_sequence_lengths or LoD through the 
    corresponding interfaces of LoDTensor introduced later.

    Actually, in order to access sequence faster, Paddle uses offset to store 
    different lengths of sequences. 
    Therefore, the operations on recursive_sequence_lengths will be converted 
    to the operations on LoD eventually.
    
    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]

    Example 2:

    LoD may have more than one level (for example, a paragraph may have more 
    than one sentence and a sentence may have more than one word). Suppose y 
    is a LoDTensor and its lod_level is 2. 
    From level = 0, there are two logical sequences, the length of which is 
    2 and 1, respectively, indicating that the first logical sequence contains 
    two sub-sequences and the second logical sequence contains one sub-sequence. 
    From level = 1, the lengths of two sub-sequences contained by the first 
    logical sequence is 2 and 2, and the length of sub-sequence contained by 
    the second logical sequence is 3.
      
    Therefore, the LoDTensor is represented in the form of recursive sequence 
    lengths as y.recursive_sequence_lengths=[[2,1], [2,2,3]]; and equally, in 
    the form of offset, it is represented as y.lod=[[0,2,3], [0,2,4,7]].

    .. code-block:: python

      y.data = [[1, 2], [3, 4],
                [5, 6], [7, 8],
                [9, 10], [11, 12], [13, 14]]

      y.shape = [2+2+3, 2]

      y.recursive_sequence_lengths = [[2, 1], [2, 2, 3]]

      y.lod = [[0, 2, 3], [0, 2, 4, 7]]
Z
Zeng Jinle 已提交
635 636 637 638 639 640 641

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
642 643

        )DOC")
644
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
645 646 647 648 649 650 651 652 653
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
654 655
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, -1), true,
656 657 658
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
659
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
660
      // We implement offset based LOD in C++ while we use length based with
H
hong 已提交
661 662
      // Python API. So we changed set_lod to set_recursive_sequence_lengths
      // to
G
gongweibao 已提交
663 664 665
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
666
      .def("set_lod",
667
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
668
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
669
             LoD new_lod;
670 671
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
C
chengduo 已提交
672 673 674
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_lod, vectorize(self.dims()).front()), true,
                 "the provided lod info is invalid");
675
             self.set_lod(new_lod);
S
sneaxiy 已提交
676 677 678 679 680
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
L
Leo Chen 已提交
681 682 683 684
               lod (list[list[int]]): The lod to set.

           Returns:
                None.
Z
Zeng Jinle 已提交
685 686 687 688 689 690 691 692 693 694

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
L
Leo Chen 已提交
695
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
696
           )DOC")
697 698 699 700 701 702 703 704 705 706 707
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
C
chengduo 已提交
708 709
             PADDLE_ENFORCE_EQ(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()), true,
710 711
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
712 713
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
L
Leo Chen 已提交
714
           Set LoD of the LoDTensor according to recursive sequence lengths.
S
sneaxiy 已提交
715

L
Leo Chen 已提交
716
           For example, if recursive_sequence_lengths=[[2, 3]], which means
717
           there are two sequences with length 2 and 3 respectively, the
L
Leo Chen 已提交
718
           corresponding lod would be [[0, 2, 2+3]], i.e., [[0, 2, 5]].
S
sneaxiy 已提交
719 720

           Args:
L
Leo Chen 已提交
721 722 723 724
                recursive_sequence_lengths (list[list[int]]): The recursive sequence lengths.
           
           Returns:
                None.
Z
Zeng Jinle 已提交
725 726 727 728 729 730 731 732 733 734

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
L
Leo Chen 已提交
735 736
                 print(t.recursive_sequence_length())  # [[2, 3]]
                 print(t.lod())  # [[0, 2, 5]]
S
sneaxiy 已提交
737
           )DOC")
738 739 740 741 742 743 744 745
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
746 747 748 749 750
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
L
Leo Chen 已提交
751 752
               list[list[int]]: The lod of the LoDTensor.
           
Z
Zeng Jinle 已提交
753 754 755 756 757 758 759 760 761 762
           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
763
           )DOC")
G
gongweibao 已提交
764
      // Set above comments of set_lod.
765 766 767 768 769 770 771 772
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
773 774
           },
           R"DOC(
L
Leo Chen 已提交
775 776
           Return the recursive sequence lengths corresponding to of the LodD 
           of the LoDTensor.
S
sneaxiy 已提交
777 778

           Returns:
L
Leo Chen 已提交
779
                list[list[int]]: The recursive sequence lengths.
Z
Zeng Jinle 已提交
780 781 782 783 784 785 786 787 788 789 790

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
791 792 793 794 795 796 797 798
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
L
Leo Chen 已提交
799
           Check whether the LoD of the LoDTensor is valid.
S
sneaxiy 已提交
800 801

           Returns:
L
Leo Chen 已提交
802
               bool: Whether the LoD is valid.
Z
Zeng Jinle 已提交
803 804 805 806 807 808 809 810 811 812 813

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
814 815 816 817 818 819 820
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
821
           )DOC")
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
      .def("__str__",
           [](const LoDTensor &self) {
             std::stringstream ostr;
             ostr << self;
             return ostr.str();
           })
      .def("_copy", [](const LoDTensor &self, const platform::Place &place) {
        // follow fetch_op's inplementation
        LoDTensor dst;
        if (self.IsInitialized() && self.numel() > 0) {
          TensorCopySync(self, place, &dst);
        } else {
          // Not copy, if the src tensor is empty.
          dst.clear();
          dst.Resize({0});
        }
        dst.set_lod(self.lod());
        return dst;
840
      });
D
dangqingqing 已提交
841

Q
qijun 已提交
842 843 844 845 846 847 848 849 850 851 852
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
853 854
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
855 856
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
857 858 859 860 861 862 863 864 865
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
866
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
867
      .def("rows", [](SelectedRows &self) {
868 869 870 871 872
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
873
      });
Q
qijun 已提交
874

875
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
876 877 878

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
879
      .def(py::init<>())
880
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
881
      .def("set_int",
882 883
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
884 885 886 887 888 889 890
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
891
      .def("get_tensor",
892 893
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
894 895
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
896 897 898
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
899 900 901 902 903
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
904 905 906
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
907
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
908 909 910 911 912
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
913
#endif
Y
Refine  
Yu Yang 已提交
914 915
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
C
chengduo 已提交
916
             PADDLE_ENFORCE_EQ(self.IsType<framework::ReaderHolder>(), true);
Y
Refine  
Yu Yang 已提交
917 918
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
919
           py::return_value_policy::reference);
920

S
sneaxiy 已提交
921
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
922

S
sneaxiy 已提交
923 924 925 926
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
927

S
sneaxiy 已提交
928 929
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
930
      .def("push",
S
sneaxiy 已提交
931
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
932
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
933
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
934
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
935
           })
S
sneaxiy 已提交
936 937 938
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
Z
Zeng Jinle 已提交
939
      .def("kill", &LoDTensorBlockingQueue::Kill)
S
sneaxiy 已提交
940
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
941

S
sneaxiy 已提交
942
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
943 944
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
945
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
946 947 948 949
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
950
        py::return_value_policy::copy);
S
sneaxiy 已提交
951

S
sneaxiy 已提交
952
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
953 954 955 956 957 958 959 960 961 962 963 964 965
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

966
          import paddle.fluid as fluid
Q
Qiao Longfei 已提交
967 968 969 970 971 972
          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
973 974
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
975
      .def("var",
976
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
977
             return self.Var(name);
Y
Yu Yang 已提交
978
           },
S
sneaxiy 已提交
979 980
           py::arg("name"),
           R"DOC(
981
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
982

983
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
984
           current scope, the variable would be created. Otherwise,
985
           return the existing variable.
S
sneaxiy 已提交
986 987

           Args:
988 989
               name (str): the variable name.

S
sneaxiy 已提交
990
           Returns:
991
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
992 993 994 995
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
996
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
997
           its parent scope. Return None if not found.
998

S
sneaxiy 已提交
999 1000
           Args:
               name (str): the variable name.
1001

S
sneaxiy 已提交
1002
           Returns:
1003
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
1004
           )DOC",
1005
           py::return_value_policy::reference)
1006
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
1007 1008 1009 1010 1011 1012
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
1013
           py::return_value_policy::reference)
S
sneaxiy 已提交
1014 1015 1016
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
1017 1018
           )DOC")
      .def("_kids", &Scope::kids);
1019

S
sneaxiy 已提交
1020 1021 1022 1023 1024 1025
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
1026 1027
        R"DOC(
        Create a new scope.
1028

S
sneaxiy 已提交
1029 1030 1031
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
1032 1033
        py::return_value_policy::reference);

Y
Yu Yang 已提交
1034 1035
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
1036 1037
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
1038 1039 1040 1041
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
C
chengduo 已提交
1042 1043
        PADDLE_ENFORCE_EQ(
            info.Proto().SerializeToString(&str), true,
1044 1045 1046 1047
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
1048 1049
    return ret_values;
  });
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
1066 1067 1068 1069 1070 1071 1072
  m.def("has_grad_op_maker", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasGradOpMaker();
  });
  m.def("has_infer_inplace", [](const std::string op_type) {
    return framework::OpInfoMap::Instance().Get(op_type).HasInferInplace();
  });

Y
Yu Yang 已提交
1073
  m.def("prune", [](const ProgramDesc &origin,
1074
                    const std::set<std::string> &feeded_var_names,
1075
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
1076
    ProgramDesc prog_with_targets(origin);
1077

1078
    for (const auto &t : targets) {
1079
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
1080
    }
1081
    proto::ProgramDesc pruned_desc;
1082
    Prune(*prog_with_targets.Proto(), feeded_var_names, &pruned_desc);
Y
Yu Yang 已提交
1083
    return new ProgramDesc(pruned_desc);
1084
  });
1085 1086 1087
  m.def("prune_backward", [](const framework::ProgramDesc &program) {
    return PruneBackward(program);
  });
1088 1089 1090 1091
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
1092 1093 1094
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
1095 1096
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
1097
  // clang-format off
Y
Yu Yang 已提交
1098
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
1099 1100
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
1101
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
1102 1103 1104
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
1105
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
1106
                      -> paddle::platform::DeviceContext* {
1107
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
1108
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
1109
#else
Q
qijun 已提交
1110
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
1111
#endif
C
chengduoZH 已提交
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
1123
// clang-format on
P
peizhilin 已提交
1124
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
1125 1126
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
1127
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
1128 1129 1130 1131 1132 1133 1134 1135
    **Note**:
        For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device.
        The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable.

    CUDAPlace is a descriptor of a device.
    It represents a GPU device allocated or to be allocated with Tensor or LoDTensor.
    Each CUDAPlace has a dev_id to indicate the graphics card ID represented by the current CUDAPlace,
    staring from 0.
1136
    The memory of CUDAPlace with different dev_id is not accessible.
1137 1138 1139 1140 1141 1142 1143 1144
    Numbering here refers to the logical ID of the visible graphics card, not the actual ID of the graphics card.
    You can set visible GPU devices by setting the `CUDA_VISIBLE_DEVICES` environment variable.
    When the program starts, visible GPU devices will be numbered from 0.
    If `CUDA_VISIBLE_DEVICES` is not set, all devices are visible by default,
    and the logical ID is the same as the actual ID.

    Parameters:
        id (int): GPU device ID.
L
lujun 已提交
1145 1146 1147 1148

    Examples:
        .. code-block:: python

1149
          import paddle.fluid as fluid
L
lujun 已提交
1150 1151
          gpu_place = fluid.CUDAPlace(0)

1152
        )DOC")
S
sneaxiy 已提交
1153 1154 1155
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
             if (UNLIKELY(dev_id < 0)) {
               LOG(ERROR) << string::Sprintf(
                   "Invalid CUDAPlace(%d), device id must be 0 or "
                   "positive integer",
                   dev_id);
               std::exit(-1);
             }

             if (UNLIKELY(dev_id >= platform::GetCUDADeviceCount())) {
               if (platform::GetCUDADeviceCount() == 0) {
                 LOG(ERROR) << "Cannot use GPU because there is no GPU "
                               "detected on your "
                               "machine.";
                 std::exit(-1);
               } else {
                 LOG(ERROR) << string::Sprintf(
                     "Invalid CUDAPlace(%d), must inside [0, %d), because GPU "
                     "number on your machine is %d",
                     dev_id, platform::GetCUDADeviceCount(),
                     platform::GetCUDADeviceCount());
                 std::exit(-1);
               }
             }

S
sneaxiy 已提交
1180 1181
             new (&self) platform::CUDAPlace(dev_id);
#else
1182 1183 1184 1185 1186 1187 1188 1189 1190
             LOG(ERROR) << string::Sprintf(
                 "Cannot use GPU because you have installed CPU version "
                 "PaddlePaddle.\n"
                 "If you want to use GPU, please try to install GPU version "
                 "PaddlePaddle by: pip install paddlepaddle-gpu\n"
                 "If you only have CPU, please change CUDAPlace(%d) to be "
                 "CPUPlace().\n",
                 dev_id);
             std::exit(-1);
S
sneaxiy 已提交
1191 1192
#endif
           })
S
sneaxiy 已提交
1193 1194 1195 1196 1197 1198
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
1199
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
1200

1201
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
1202 1203
    CPUPlace is a descriptor of a device.
    It represents a CPU device allocated or to be allocated with Tensor or LoDTensor.
L
lujun 已提交
1204 1205 1206 1207

    Examples:
        .. code-block:: python

1208
          import paddle.fluid as fluid
1209
          cpu_place = fluid.CPUPlace()to be allocated
L
lujun 已提交
1210

1211
        )DOC")
1212
      .def(py::init<>())
S
sneaxiy 已提交
1213 1214 1215 1216 1217 1218
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
1219
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
1220

1221
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
1222 1223 1224 1225 1226 1227
    CUDAPinnedPlace is a descriptor of a device.
    It refers to the page locked memory allocated by the CUDA function `cudaHostAlloc()` in the host memory.
    The host operating system will not paging and exchanging the memory.
    It can be accessed through direct memory access technology to speed up the copy of data between the host and GPU.
    For more information on CUDA data transfer and `pinned memory`,
    please refer to `official document <https://docs.nvidia.com/cuda/cuda-c-best-practices-guide/index.html#pinned-memory>`_ .
L
lujun 已提交
1228 1229 1230 1231

    Examples:
        .. code-block:: python

1232
          import paddle.fluid as fluid
L
lujun 已提交
1233 1234
          place = fluid.CUDAPinnedPlace()

1235
        )DOC")
S
sneaxiy 已提交
1236
      .def("__init__",
S
sneaxiy 已提交
1237
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
1238 1239 1240
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
1241
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
1242
           })
S
sneaxiy 已提交
1243 1244 1245 1246 1247 1248 1249 1250
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
1251 1252
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
1253 1254
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
1255 1256 1257 1258 1259
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
1260 1261
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
1262 1263 1264 1265 1266 1267
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
1268 1269 1270 1271
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
1272 1273
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
1274 1275 1276 1277 1278
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
1279
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
1280
             self = gpu_place;
C
chengduoZH 已提交
1281 1282
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
1283 1284
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
1285
      });
Y
Yu Yang 已提交
1286

Y
Yu Yang 已提交
1287
  py::class_<OperatorBase>(m, "Operator")
C
chengduo 已提交
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298
      .def_static(
          "create",
          [](py::bytes protobin) {
            proto::OpDesc desc;
            PADDLE_ENFORCE_EQ(desc.ParsePartialFromString(protobin), true,
                              "Cannot parse user input to OpDesc");
            PADDLE_ENFORCE_EQ(desc.IsInitialized(), true,
                              "User OpDesc is not initialized, reason %s",
                              desc.InitializationErrorString());
            return OpRegistry::CreateOp(desc);
          })
1299
      .def("run",
1300
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1301 1302 1303
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1304
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
1305 1306 1307 1308 1309
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1310 1311 1312 1313 1314 1315 1316
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1317 1318
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1319
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1320
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1321 1322 1323 1324
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1325

1326 1327 1328
  py::class_<framework::ExecutorPrepareContext>(m, "ExecutorPrepareContext")
      .def(py::init<const ProgramDesc &, size_t>());

1329 1330 1331 1332 1333 1334 1335 1336 1337
  py::class_<framework::TrainerBase, std::shared_ptr<framework::TrainerBase>>(
      m, "TrainerBase")
      .def("get_worker_scope",
           [](TrainerBase &self, int thread_id) -> Scope * {
             return self.GetWorkerScope(thread_id);
           },
           py::return_value_policy::reference)
      .def("finalize", &TrainerBase::Finalize);

F
fengjiayi 已提交
1338
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1339
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1340
      .def("close", &Executor::Close)
1341 1342
      .def("run_from_dataset", &Executor::RunFromDataset,
           py::call_guard<py::gil_scoped_release>())
D
Dong Daxiang 已提交
1343 1344
      .def("release_trainer", &Executor::ReleaseTrainer,
           py::call_guard<py::gil_scoped_release>())
1345 1346 1347 1348
      .def("init_for_dataset",
           [](Executor &self, const ProgramDesc &prog,
              const std::string &trainer_desc, Scope *scope,
              Dataset *dataset) -> std::shared_ptr<TrainerBase> {
D
Dong Daxiang 已提交
1349
             pybind11::gil_scoped_release release;
1350 1351 1352 1353 1354 1355 1356
             return self.InitForDataset(prog, trainer_desc, scope, dataset);
           })
      .def("run_from_dataset",
           [](Executor &self, std::shared_ptr<TrainerBase> trainer) {
             pybind11::gil_scoped_release release;
             self.RunFromDataset(trainer);
           })
1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
      .def("run_prepared_ctx",
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              std::map<std::string, const LoDTensor *> *feed_targets,
              std::map<std::string, LoDTensor *> *fetch_targets,
              bool create_local_scope = true, bool create_vars = true,
              const std::string &feed_holder_name = "feed",
              const std::string &fetch_holder_name = "fetch") {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, feed_targets, fetch_targets,
                                     create_local_scope, create_vars,
                                     feed_holder_name, fetch_holder_name);
           })
1369
      .def("run_prepared_ctx",
G
guru4elephant 已提交
1370 1371 1372 1373 1374 1375 1376
           [](Executor &self, ExecutorPrepareContext *ctx, Scope *scope,
              bool create_local_scope = true, bool create_vars = true,
              bool keep_kids = false) {
             pybind11::gil_scoped_release release;
             self.RunPreparedContext(ctx, scope, create_local_scope,
                                     create_vars, keep_kids);
           })
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
      .def("prepare",
           [](Executor &self, const ProgramDesc &program, int block_id,
              const std::vector<std::string> &skip_ref_cnt_vars =
                  std::vector<std::string>(),
              bool force_disable_gc = false) {
             pybind11::gil_scoped_release release;
             return self.Prepare(program, block_id, skip_ref_cnt_vars,
                                 force_disable_gc);
           })
      .def("create_variables", &Executor::CreateVariables)
S
sneaxiy 已提交
1387
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1388 1389
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1390
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1391 1392
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1393
      });
S
sneaxiy 已提交
1394

D
dzhwinter 已提交
1395
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1396
  m.def("init_glog", framework::InitGLOG);
1397
  m.def("load_op_library", framework::LoadOpLib);
X
Xin Pan 已提交
1398 1399
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
1400

1401
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
1402
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1403
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1404
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1405
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1406 1407 1408 1409 1410 1411
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1412

1413
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
1414
  m.def("get_fetch_variable", framework::GetFetchVariable);
1415
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1416

X
Xin Pan 已提交
1417 1418
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1419 1420 1421 1422 1423
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
1424
  BindGlobalValueGetterSetter(&m);
Y
Yu Yang 已提交
1425

Y
Yu Yang 已提交
1426 1427 1428 1429 1430 1431 1432 1433 1434
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1435
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
1436
    LoDTensorArray is array of LoDTensor, it supports operator[], len() and for-loop iteration.
Z
Zeng Jinle 已提交
1437 1438 1439

    Examples:
        .. code-block:: python
1440

Z
Zeng Jinle 已提交
1441 1442 1443 1444
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1445 1446
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1457 1458 1459 1460 1461 1462
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1463 1464
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.
1465 1466 1467 1468 1469 1470
              
             Args:
                   tensor (LoDTensor): The LoDTensor to be appended.

             Returns:
                   None.
Z
Zeng Jinle 已提交
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
           )DOC")
      .def("_move_to_list",
           [](LoDTensorArray &self) -> py::list {
             py::list res(self.size());
             for (size_t i = 0; i < self.size(); ++i) {
               res[i] = py::cast(std::move(self[i]));
             }
             self.clear();
             return res;
           },
           py::return_value_policy::take_ownership);
Y
Yu Yang 已提交
1493

Y
Yu Yang 已提交
1494
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1495
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1496
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1497

P
peizhilin 已提交
1498
#ifndef _WIN32
D
dangqingqing 已提交
1499 1500 1501
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1502
#endif
P
peizhilin 已提交
1503
#endif
Y
Yu Yang 已提交
1504

1505 1506 1507 1508
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1509
      .value("kAll", platform::ProfilerState::kAll)
1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1523
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1524
  m.def("reset_profiler", platform::ResetProfiler);
1525
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1526 1527 1528
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1529

1530 1531
  m.def("size_of_dtype", framework::SizeOfType);

1532 1533 1534
  using VarQuantScale =
      std::unordered_map<std::string, std::pair<bool, LoDTensor>>;

1535 1536
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1537
      .def("has", &ir::Pass::Has)
1538 1539 1540
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1541
           })
1542
      .def(
1543
          "set",
1544 1545 1546
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1547 1548
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<std::string> set) {
             self.Set(name, new std::unordered_set<std::string>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name,
              std::unordered_set<int> set) {
             self.Set(name, new std::unordered_set<int>(set));
           })
      .def("set",
           [](ir::Pass &self, const std::string &name, VarQuantScale scales) {
             self.Set(name, new VarQuantScale(scales));
           })
F
flame 已提交
1563 1564
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1565
        self.Apply(graph.get());
F
flame 已提交
1566
      });
1567

X
fix  
Xin Pan 已提交
1568 1569
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1584
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1585

Y
yuyang18 已提交
1586
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1587 1588 1589 1590
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1591 1592 1593
    Examples:
        .. code-block:: python

1594
          import paddle.fluid as fluid
1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
          x = fluid.layers.data(name='x', shape=[13], dtype='float32')
          y = fluid.layers.data(name='y', shape=[1], dtype='float32')
          y_predict = fluid.layers.fc(input=x, size=1, act=None)

          cost = fluid.layers.square_error_cost(input=y_predict, label=y)
          avg_loss = fluid.layers.mean(cost)

          sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001)
          sgd_optimizer.minimize(avg_loss)

C
chengduo 已提交
1605 1606 1607
          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

1608 1609
          train_exe = fluid.ParallelExecutor(use_cuda=False,
                                             loss_name=avg_loss.name,
C
chengduo 已提交
1610 1611
                                             exec_strategy=exec_strategy)

C
chengduo 已提交
1612 1613
        )DOC");

Y
yuyang18 已提交
1614
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1615 1616 1617 1618 1619
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1630
      .def_property(
1631 1632 1633 1634
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1635 1636 1637 1638
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1639 1640 1641 1642 1643
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1644 1645 1646
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
1647 1648
                Note that this option is invalid now, and it will be removed in
                next version. Default False.)DOC")
Y
yuyang18 已提交
1649 1650 1651 1652 1653 1654 1655
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1656 1657 1658 1659
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
1660 1661
                because the temp variable's shape maybe the same between two iterations.
                Default 1.
C
chengduo 已提交
1662 1663 1664 1665 1666 1667

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1668
              )DOC")
Q
Qiao Longfei 已提交
1669 1670 1671 1672 1673 1674 1675 1676 1677
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
1678
                user call exe.run() in python
Q
Qiao Longfei 已提交
1679
              )DOC")
1680 1681 1682 1683 1684
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1685

Y
yuyang18 已提交
1686
  exec_strategy.def_property(
Y
yuyang18 已提交
1687 1688 1689 1690 1691 1692 1693
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1694 1695
      });

C
chengduo 已提交
1696 1697 1698 1699
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1700 1701 1702
    Examples:
        .. code-block:: python

1703 1704
            import os
            import numpy as np
F
flame 已提交
1705
            import paddle.fluid as fluid
1706 1707 1708 1709 1710 1711 1712 1713 1714

            os.environ["CPU_NUM"] = '2'
            places = fluid.cpu_places()

            data = fluid.layers.data(name="x", shape=[1], dtype="float32")
            hidden = fluid.layers.fc(input=data, size=10)
            loss = fluid.layers.mean(hidden)
            fluid.optimizer.SGD(learning_rate=0.01).minimize(loss)

F
flame 已提交
1715
            build_strategy = fluid.BuildStrategy()
1716 1717
            build_strategy.enable_inplace = True
            build_strategy.memory_optimize = True
F
flame 已提交
1718
            build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
1719 1720 1721 1722
            program = fluid.compiler.CompiledProgram(fluid.default_main_program())
            program = program.with_data_parallel(loss_name=loss.name,
                                                 build_strategy=build_strategy,
                                                 places=places)
C
chengduo 已提交
1723
)DOC");
Y
yuyang18 已提交
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
C
chengduo 已提交
1740 1741
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1742
            self.reduce_ = strategy;
C
chengduo 已提交
1743
          },
1744
          R"DOC((fluid.BuildStrategy.ReduceStrategy, optional): there are two reduce
C
chengduo 已提交
1745 1746
                strategies in ParallelExecutor, AllReduce and Reduce. If you want
                that all the parameters' optimization are done on all devices independently,
1747
                you should choose AllReduce; otherwise, if you choose Reduce, all the parameters'
C
chengduo 已提交
1748 1749
                optimization will be evenly distributed to different devices, and then
                broadcast the optimized parameter to other devices.
1750
                Default is 'AllReduce'.
F
flame 已提交
1751 1752 1753 1754 1755 1756 1757 1758

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce
                  )DOC")
Y
yuyang18 已提交
1759 1760 1761 1762 1763
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
C
chengduo 已提交
1764 1765
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finalized.");
Y
yuyang18 已提交
1766
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1767
          },
1768 1769
          R"DOC((fluid.BuildStrategy.GradientScaleStrategy, optional): there are three
                ways of defining :math:`loss@grad` in ParallelExecutor, that is, CoeffNumDevice,
C
chengduo 已提交
1770 1771
                One and Customized. By default, ParallelExecutor sets the :math:`loss@grad`
                according to the number of devices. If you want to customize :math:`loss@grad`,
1772
                you can choose Customized. Default is 'CoeffNumDevice'.
F
flame 已提交
1773 1774 1775 1776 1777

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
C
chengduo 已提交
1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
                        import paddle.fluid.compiler as compiler
                        import numpy
                        import os

                        use_cuda = True
                        place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace()
                        exe = fluid.Executor(place)

                        # NOTE: If you use CPU to run the program, you need
                        # to specify the CPU_NUM, otherwise, fluid will use
                        # all the number of the logic core as the CPU_NUM,
                        # in that case, the batch size of the input should be
                        # greater than CPU_NUM, if not, the process will be
                        # failed by an exception.
                        if not use_cuda:
                            os.environ['CPU_NUM'] = str(2)
                            places = fluid.cpu_places()
                        else:
                            places = places = fluid.cuda_places()

                        data = fluid.layers.data(name='X', shape=[1], dtype='float32')
                        hidden = fluid.layers.fc(input=data, size=10)
                        loss = fluid.layers.mean(hidden)
                        fluid.optimizer.SGD(learning_rate=0.01).minimize(loss)

                        fluid.default_startup_program().random_seed=1
                        exe.run(fluid.default_startup_program())

F
flame 已提交
1806
                        build_strategy = fluid.BuildStrategy()
C
chengduo 已提交
1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820
                        build_strategy.gradient_scale_strategy = \
                                 fluid.BuildStrategy.GradientScaleStrategy.Customized
                        compiled_prog = compiler.CompiledProgram(
                                 fluid.default_main_program()).with_data_parallel(
                                          loss_name=loss.name, build_strategy=build_strategy,
                                          places = places)

                        dev_count =  len(places)
                        x = numpy.random.random(size=(10, 1)).astype('float32')
                        loss_grad = numpy.ones((dev_count)).astype("float32") * 0.01
                        loss_grad_name = loss.name+"@GRAD"
                        loss_data = exe.run(compiled_prog,
                                             feed={"X": x, loss_grad_name : loss_grad},
                                             fetch_list=[loss.name, loss_grad_name])
F
flame 已提交
1821
                   )DOC")
Y
yuyang18 已提交
1822 1823 1824 1825
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
C
chengduo 已提交
1826 1827
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1828
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1829
          },
1830
          R"DOC((str, optional): debug_graphviz_path indicates the path that
F
flame 已提交
1831
                writing the SSA Graph to file in the form of graphviz.
1832
                It is useful for debugging. Default is empty string, that is, ""
F
flame 已提交
1833 1834 1835 1836 1837 1838

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
C
chengduo 已提交
1839 1840
                        build_strategy.debug_graphviz_path = "./graph"

F
flame 已提交
1841
                    )DOC")
S
sneaxiy 已提交
1842 1843 1844 1845 1846 1847
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
C
chengduo 已提交
1848 1849
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1850 1851
            self.enable_sequential_execution_ = b;
          },
1852 1853
          R"DOC((bool, optional): If set True, the execution order of ops would
                be the same as what is in the program. Default is False.
F
flame 已提交
1854 1855 1856 1857 1858 1859 1860 1861

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.enable_sequential_execution = True
          )DOC")
S
sneaxiy 已提交
1862 1863 1864 1865 1866 1867
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
C
chengduo 已提交
1868 1869
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1870 1871
            self.remove_unnecessary_lock_ = b;
          },
1872 1873
          R"DOC((bool, optional): If set True, some locks in GPU ops would be
                released and ParallelExecutor would run faster. Default is True.
F
flame 已提交
1874 1875 1876 1877 1878 1879 1880 1881

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.remove_unnecessary_lock = True
          )DOC")
1882 1883 1884 1885
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
1886 1887 1888
#ifdef WIN32
            PADDLE_THROW("Windows has NO support to distribute mode.");
#endif
1889 1890
            self.num_trainers_ = num_trainers;
          })
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
1903 1904 1905 1906 1907 1908
      .def_property(
          "nccl_comm_num",
          [](const BuildStrategy &self) { return self.nccl_comm_num_; },
          [](BuildStrategy &self, int nccl_comm_num) {
            self.nccl_comm_num_ = nccl_comm_num;
          })
1909
      .def_property("use_hierarchical_allreduce",
1910 1911 1912 1913 1914 1915
                    [](const BuildStrategy &self) {
                      return self.use_hierarchical_allreduce_;
                    },
                    [](BuildStrategy &self, bool use) {
                      self.use_hierarchical_allreduce_ = use;
                    })
1916
      .def_property("hierarchical_allreduce_inter_nranks",
1917 1918 1919 1920 1921 1922 1923
                    [](const BuildStrategy &self) {
                      return self.hierarchical_allreduce_inter_nranks_;
                    },
                    [](BuildStrategy &self, int nranks) {
                      self.hierarchical_allreduce_inter_nranks_ = nranks;
                    })

C
chengduo 已提交
1924 1925 1926 1927 1928 1929
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
C
chengduo 已提交
1930 1931
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
C
chengduo 已提交
1932 1933
            self.fuse_elewise_add_act_ops_ = b;
          },
1934
          R"DOC((bool, optional): fuse_elewise_add_act_ops indicate whether
F
flame 已提交
1935
                to fuse elementwise_add_op and activation_op,
1936
                it may make the execution faster. Default is False.
F
flame 已提交
1937 1938 1939 1940 1941 1942 1943 1944

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_elewise_add_act_ops = True
                     )DOC")
1945 1946 1947 1948 1949 1950
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
C
chengduo 已提交
1951 1952
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
1953 1954
            self.fuse_relu_depthwise_conv_ = b;
          },
1955
          R"DOC((bool, optional): fuse_relu_depthwise_conv indicate whether
F
flame 已提交
1956 1957 1958
                to fuse relu and depthwise_conv2d,
                it will save GPU memory and may make the execution faster.
                This options is only available in GPU devices.
1959
                Default is False.
F
flame 已提交
1960 1961 1962 1963 1964 1965 1966 1967

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.fuse_relu_depthwise_conv = True
          )DOC")
C
chengduo 已提交
1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
      .def_property("fuse_broadcast_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_broadcast_ops_ == true ||
                             self.fuse_broadcast_ops_ == boost::none;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                                        "BuildStrategy is finlaized.");
                      self.fuse_broadcast_ops_ = b;
                    },
1978
                    R"DOC((bool, optional): fuse_broadcast_op indicates whether
1979 1980 1981 1982
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
1983 1984 1985 1986 1987 1988 1989 1990 1991
                      for NCCLReduce operations for a period of time. Default False.

                      Examples:
                          .. code-block:: python

                              import paddle.fluid as fluid
                              build_strategy = fluid.BuildStrategy()
                              build_strategy.fuse_broadcast_ops = True
                    )DOC")
C
chengduo 已提交
1992 1993
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
C
chengduo 已提交
1994 1995
                      return self.fuse_all_optimizer_ops_ == true ||
                             self.fuse_all_optimizer_ops_ == boost::none;
C
chengduo 已提交
1996 1997
                    },
                    [](BuildStrategy &self, bool b) {
C
chengduo 已提交
1998 1999
                      PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                                        "BuildStrategy is finlaized.");
C
chengduo 已提交
2000 2001
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
2002 2003 2004 2005
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
C
chengduo 已提交
2006 2007
            PADDLE_ENFORCE_EQ(!self.IsFinalized(), true,
                              "BuildStrategy is finlaized.");
Q
qingqing01 已提交
2008 2009
            self.sync_batch_norm_ = b;
          },
2010
          R"DOC((bool, optional): sync_batch_norm indicates whether to use
Q
qingqing01 已提交
2011 2012 2013
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.
                Current implementation doesn't support FP16 training and CPU.
2014 2015
                And only synchronous on one machine, not all machines. 
                Default is False.
F
flame 已提交
2016 2017 2018 2019 2020 2021 2022 2023

                Examples:
                    .. code-block:: python

                        import paddle.fluid as fluid
                        build_strategy = fluid.BuildStrategy()
                        build_strategy.sync_batch_norm = True
                )DOC")
D
dzhwinter 已提交
2024 2025
      .def_property(
          "memory_optimize",
2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
          [](const BuildStrategy &self) -> py::object {
            if (self.memory_optimize_) {
              return py::cast(self.memory_optimize_.get());
            } else {
              return py::cast(nullptr);
            }
          },
          [](BuildStrategy &self, const py::handle &value) {
            auto *py_obj = value.ptr();
            if (py_obj == nullptr || py_obj == Py_None) {
              self.memory_optimize_ = boost::none;
            } else if (PyBool_Check(py_obj)) {
              self.memory_optimize_ = (py_obj == Py_True);
            } else {
              PADDLE_THROW(
H
hong 已提交
2041 2042
                  "BuildStrategy.memory_optimize must be None, False or "
                  "True");
2043 2044
            }
          },
2045
          R"DOC((bool, optional): memory opitimize aims to save total memory
2046
                consumption, set to True to enable it.
2047

2048 2049 2050
                Default None. None means framework would choose to use or not use 
                this strategy automatically. Currently, None means that it is 
                enabled when GC is disabled, and disabled when GC is enabled. 
2051
                True means enabling and False means disabling. Default is None.)DOC")
2052 2053 2054
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
2055 2056 2057 2058 2059 2060 2061 2062 2063
          [](BuildStrategy &self, bool b) {
#ifdef WIN32
            if (b) {
              PADDLE_THROW("Windows has NO support to distribute mode.");
            }
#else
            self.is_distribution_ = b;
#endif
          })
Q
can run  
Qiao Longfei 已提交
2064 2065 2066
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
2067
      .def_property(
D
dzhwinter 已提交
2068 2069 2070
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
2071 2072
      .def_property(
          "fuse_all_reduce_ops",
C
chengduo 已提交
2073 2074 2075 2076
          [](const BuildStrategy &self) {
            return self.fuse_all_reduce_ops_ == true ||
                   self.fuse_all_reduce_ops_ == boost::none;
          },
C
chengduo 已提交
2077
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
2078 2079 2080 2081 2082 2083 2084
      .def_property("enable_backward_optimizer_op_deps",
                    [](const BuildStrategy &self) {
                      return self.enable_backward_optimizer_op_deps_;
                    },
                    [](BuildStrategy &self, bool b) {
                      self.enable_backward_optimizer_op_deps_ = b;
                    })
2085 2086 2087 2088
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
2089 2090 2091 2092 2093 2094 2095 2096 2097
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
2098
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
2099
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
2100 2101 2102 2103 2104
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
2105 2106

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
2107
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
2108
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
2109
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
2110 2111 2112 2113
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
2114 2115 2116 2117 2118
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
2119 2120 2121
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
2122 2123 2124 2125
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
2126
      .def("run", [](ParallelExecutor &self,
2127
                     const std::vector<std::string> &fetch_tensors) {
S
sneaxiy 已提交
2128
        pybind11::gil_scoped_release release;
2129
        return self.Run(fetch_tensors);
S
sneaxiy 已提交
2130
      });
Y
Yu Yang 已提交
2131

D
dongdaxiang 已提交
2132
  BindFleetWrapper(&m);
H
hutuxian 已提交
2133
  BindBoxHelper(&m);
W
wopeizl 已提交
2134
#ifndef _WIN32
D
dongdaxiang 已提交
2135
  BindNCCLWrapper(&m);
W
wopeizl 已提交
2136
#endif
F
flame 已提交
2137 2138
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
2139
  BindInferenceApi(&m);
2140
  BindDataset(&m);
2141 2142 2143
#ifdef PADDLE_WITH_DISTRIBUTE
  BindCommunicator(&m);
#endif
L
Luo Tao 已提交
2144
}
2145
}  // namespace pybind
2146
}  // namespace paddle