pybind.cc 63.8 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
2 3 4 5 6

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

7
http://www.apache.org/licenses/LICENSE-2.0
8 9 10 11 12 13

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
L
lgone2000 已提交
14
#include <Python.h>
C
chengduoZH 已提交
15 16
#include <algorithm>
#include <map>
S
sneaxiy 已提交
17
#include <memory>
C
chengduoZH 已提交
18 19 20
#include <mutex>  // NOLINT // for call_once
#include <string>
#include <unordered_map>
21
#include <unordered_set>
C
chengduoZH 已提交
22 23
#include <utility>
#include <vector>
24

Y
Yi Wang 已提交
25 26 27
#include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/feed_fetch_method.h"
#include "paddle/fluid/framework/framework.pb.h"
S
sneaxiy 已提交
28
#include "paddle/fluid/framework/garbage_collector.h"
29
#include "paddle/fluid/framework/ir/alloc_continuous_space_for_grad_pass.h"
30
#include "paddle/fluid/framework/ir/pass_builder.h"
Y
Yi Wang 已提交
31 32 33
#include "paddle/fluid/framework/lod_rank_table.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
S
sneaxiy 已提交
34
#include "paddle/fluid/framework/op_info.h"
35
#include "paddle/fluid/framework/op_registry.h"
Y
Yu Yang 已提交
36
#include "paddle/fluid/framework/parallel_executor.h"
Y
Yi Wang 已提交
37
#include "paddle/fluid/framework/prune.h"
Y
Refine  
Yu Yang 已提交
38
#include "paddle/fluid/framework/reader.h"
S
sneaxiy 已提交
39
#include "paddle/fluid/framework/scope_pool.h"
Y
Yi Wang 已提交
40
#include "paddle/fluid/framework/selected_rows.h"
X
Xin Pan 已提交
41
#include "paddle/fluid/framework/version.h"
42
#include "paddle/fluid/imperative/layer.h"
M
minqiyang 已提交
43
#include "paddle/fluid/imperative/profiler.h"
Y
Refine  
Yu Yang 已提交
44
#include "paddle/fluid/memory/allocation/allocator_strategy.h"
45
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
D
dzhwinter 已提交
46
#include "paddle/fluid/operators/activation_op.h"
S
sneaxiy 已提交
47
#include "paddle/fluid/operators/py_func_op.h"
S
sneaxiy 已提交
48
#include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h"
Y
Yu Yang 已提交
49
#include "paddle/fluid/platform/cpu_info.h"
Y
Yi Wang 已提交
50
#include "paddle/fluid/platform/enforce.h"
51
#include "paddle/fluid/platform/init.h"
Y
Yi Wang 已提交
52 53
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/profiler.h"
W
Wang Guibao 已提交
54
#include "paddle/fluid/pybind/async_executor_py.h"
Y
Yi Wang 已提交
55
#include "paddle/fluid/pybind/const_value.h"
D
dongdaxiang 已提交
56
#include "paddle/fluid/pybind/data_set_py.h"
Y
Yi Wang 已提交
57
#include "paddle/fluid/pybind/exception.h"
D
dongdaxiang 已提交
58
#include "paddle/fluid/pybind/fleet_wrapper_py.h"
59
#include "paddle/fluid/pybind/imperative.h"
F
flame 已提交
60
#include "paddle/fluid/pybind/inference_api.h"
F
flame 已提交
61
#include "paddle/fluid/pybind/ir.h"
W
wopeizl 已提交
62
#ifndef _WIN32
D
dongdaxiang 已提交
63
#include "paddle/fluid/pybind/nccl_wrapper_py.h"
W
wopeizl 已提交
64
#endif
65 66
#include "paddle/fluid/pybind/protobuf.h"
#include "paddle/fluid/pybind/pybind.h"  // NOLINT
S
sneaxiy 已提交
67
#include "paddle/fluid/pybind/reader_py.h"
Y
Yu Yang 已提交
68
#include "paddle/fluid/pybind/recordio.h"
Y
Yi Wang 已提交
69
#include "paddle/fluid/pybind/tensor_py.h"
70
#include "paddle/fluid/string/to_string.h"
71

D
Dong Zhihong 已提交
72
#ifdef PADDLE_WITH_CUDA
P
peizhilin 已提交
73
#ifndef _WIN32
Y
Yi Wang 已提交
74
#include "paddle/fluid/operators/nccl/nccl_gpu_common.h"
P
peizhilin 已提交
75
#endif
Y
Yi Wang 已提交
76 77
#include "paddle/fluid/platform/cuda_profiler.h"
#include "paddle/fluid/platform/gpu_info.h"
D
Dong Zhihong 已提交
78 79
#endif

80 81 82 83
#ifdef PADDLE_WITH_DISTRIBUTE
#include "paddle/fluid/pybind/communicator_py.h"
#endif

M
minqiyang 已提交
84 85
#include "pybind11/stl.h"

86 87 88 89
DEFINE_bool(reader_queue_speed_test_mode, false,
            "If set true, the queue.pop will only get data from queue but not "
            "remove the data from queue for speed testing");

Q
Qiao Longfei 已提交
90 91 92
// disable auto conversion to list in Python
PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray);

93
namespace paddle {
94
namespace pybind {
95
bool IsCompiledWithCUDA() {
96
#ifndef PADDLE_WITH_CUDA
Q
qijun 已提交
97 98 99 100 101 102
  return false;
#else
  return true;
#endif
}

103 104 105 106 107 108 109 110
bool IsCompiledWithMKLDNN() {
#ifndef PADDLE_WITH_MKLDNN
  return false;
#else
  return true;
#endif
}

111 112 113 114 115 116 117 118
bool IsCompiledWithNGRAPH() {
#ifndef PADDLE_WITH_NGRAPH
  return false;
#else
  return true;
#endif
}

119
bool IsCompiledWithBrpc() {
120
#ifndef PADDLE_WITH_DISTRIBUTE
121 122
  return false;
#endif
123 124 125 126 127 128

#ifdef PADDLE_WITH_GRPC
  return false;
#endif

  return true;
129 130
}

Y
update  
Yancey1989 已提交
131
bool IsCompiledWithDIST() {
Y
Yancey1989 已提交
132
#ifdef PADDLE_WITH_DISTRIBUTE
Y
update  
Yancey1989 已提交
133 134 135 136 137 138
  return true;
#else
  return false;
#endif
}

S
sneaxiy 已提交
139 140 141 142 143 144 145 146 147 148
template <typename PlaceType1, typename PlaceType2>
static inline bool IsSamePlace(const PlaceType1 &p1, const PlaceType2 &p2) {
  return paddle::platform::Place(p1) == paddle::platform::Place(p2);
}

template <typename PlaceType>
static inline int PlaceIndex(const PlaceType &p) {
  return static_cast<int>(paddle::platform::Place(p).which());
}

149
PYBIND11_MODULE(core, m) {
Y
Yu Yang 已提交
150 151 152
  // Not used, just make sure cpu_info.cc is linked.
  paddle::platform::CpuTotalPhysicalMemory();

Y
Refine  
Yu Yang 已提交
153
  paddle::memory::allocation::UseAllocatorStrategyGFlag();
S
sneaxiy 已提交
154

155
  m.doc() = "C++ core of PaddlePaddle";
156

157 158 159 160
  // using framework in this function. Since it is inside a function, it will
  // not cause namespace pollution.
  using namespace paddle::framework;  // NOLINT

161
  BindException(&m);
Y
Yu Yang 已提交
162

S
sneaxiy 已提交
163
  m.def(
S
sneaxiy 已提交
164
      "_append_python_callable_object_and_return_id",
S
sneaxiy 已提交
165 166 167 168
      [](py::object py_obj) -> size_t {
        return paddle::operators::AppendPythonCallableObjectAndReturnId(py_obj);
      });

S
sneaxiy 已提交
169 170 171
  m.def("_get_use_default_grad_op_desc_maker_ops",
        [] { return OpInfoMap::Instance().GetUseDefaultGradOpDescMakerOps(); });

S
sneaxiy 已提交
172 173 174
  // NOTE(zjl): ctest would load environment variables at the beginning even
  // though we have not `import paddle.fluid as fluid`. So we add this API
  // to enable eager deletion mode in unittest.
S
sneaxiy 已提交
175
  m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode);
S
sneaxiy 已提交
176

177
  m.def("_set_fuse_parameter_group_size",
178
        &paddle::framework::ir::SetFuseParameterGroupsSize);
179
  m.def("_set_fuse_parameter_memory_size",
180
        &paddle::framework::ir::SetFuseParameterMemorySize);
181

S
sneaxiy 已提交
182 183 184
  m.add_object("_cleanup",
               py::capsule([]() { ScopePool::Instance().Clear(); }));

185 186 187 188 189 190 191
  m.def("get_mem_usage", [](int device) {
    return memory::allocation::GPUMemMonitor.GetMemUsage(device);
  });

  m.def("print_mem_usage",
        []() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

192 193 194 195 196 197 198 199 200 201 202 203
  py::class_<imperative::detail::BackwardStrategy> backward_strategy(
      m, "BackwardStrategy", R"DOC()DOC");
  backward_strategy.def(py::init())
      .def_property("sort_sum_gradient",
                    [](const imperative::detail::BackwardStrategy &self) {
                      return self.sorted_sum_gradient_;
                    },
                    [](imperative::detail::BackwardStrategy &self,
                       bool sorted_sum_gradient) {
                      self.sorted_sum_gradient_ = sorted_sum_gradient;
                    });

M
minqiyang 已提交
204
  m.def("start_imperative_gperf_profiler",
M
minqiyang 已提交
205 206
        []() { imperative::StartProfile(); });

M
minqiyang 已提交
207
  m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });
M
minqiyang 已提交
208

M
minqiyang 已提交
209
  py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
210 211 212 213 214 215 216 217
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>, const paddle::platform::CPUPlace,
                   bool, bool>())
      .def(
          py::init<const std::string &, paddle::framework::proto::VarType::Type,
                   const std::vector<int64_t>,
                   const paddle::platform::CUDAPlace, bool, bool>())
218
      .def("_run_backward",
219 220 221 222
           [](imperative::VarBase &self,
              const imperative::detail::BackwardStrategy &bckst) {
             self.RunBackward(bckst);
           })
M
minqiyang 已提交
223
      .def("_grad_name", &imperative::VarBase::GradName)
M
minqiyang 已提交
224
      .def("_grad_value", &imperative::VarBase::GradValue)
X
Xin Pan 已提交
225
      .def("_clear_gradient", &imperative::VarBase::ClearGradient)
M
minqiyang 已提交
226
      .def("_grad_ivar",
M
minqiyang 已提交
227
           [](const imperative::VarBase &self) { return self.grads_; },
M
minqiyang 已提交
228
           py::return_value_policy::reference)
M
minqiyang 已提交
229
      .def("_copy_to",
P
Paddle CI 已提交
230
           [](const imperative::VarBase &self, const platform::CPUPlace &place,
M
minqiyang 已提交
231 232 233 234 235
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
P
Paddle CI 已提交
236
           py::return_value_policy::take_ownership)
M
minqiyang 已提交
237
      .def("_copy_to",
P
Paddle CI 已提交
238
           [](const imperative::VarBase &self, const platform::CUDAPlace &place,
M
minqiyang 已提交
239 240 241 242 243
              bool blocking) {
             std::unique_ptr<imperative::VarBase> new_var =
                 self.NewVarBase(place, blocking);
             return new_var.release();
           },
M
minqiyang 已提交
244
           py::return_value_policy::take_ownership)
245 246
      .def("value",
           [](const imperative::VarBase &self) { return self.var_.get(); },
M
minqiyang 已提交
247
           py::return_value_policy::reference)
248 249 250
      .def_property("name", &imperative::VarBase::Name,
                    &imperative::VarBase::SetName)
      .def_property_readonly("shape", &imperative::VarBase::Shape)
M
minqiyang 已提交
251
      .def_property_readonly("dtype", &imperative::VarBase::DataType)
252 253 254 255
      .def_property("persistable", &imperative::VarBase::IsPersistable,
                    &imperative::VarBase::SetPersistable)
      .def_property("stop_gradient", &imperative::VarBase::IsStopGradient,
                    &imperative::VarBase::SetStopGradient);
256

257
  py::class_<imperative::OpBase, PyOpBase>(m, "OpBase", R"DOC()DOC")
258
      .def(py::init<const std::string &>())
259
      .def("register_backward_hooks",
Y
Yan Xu 已提交
260 261 262
           [](imperative::OpBase &self, const py::object &callable) {
             self.RegisterBackwardHooks(callable);
           })
M
minqiyang 已提交
263 264 265 266 267 268 269 270 271 272
      .def_property("_trace_id",
                    [](const imperative::OpBase &self) {
                      pybind11::gil_scoped_release release;
                      return self.trace_id_;
                    },
                    [](imperative::OpBase &self, int trace_id) {
                      pybind11::gil_scoped_release release;
                      self.trace_id_ = trace_id;
                    },
                    py::return_value_policy::reference)
X
Xin Pan 已提交
273 274 275 276 277 278
      .def_property(
          "forward_id",
          [](const imperative::OpBase &self) { return self.forward_id_; },
          [](imperative::OpBase &self, int forward_id) {
            self.forward_id_ = forward_id;
          },
X
Xin Pan 已提交
279
          py::return_value_policy::reference)
X
polish  
Xin Pan 已提交
280
      .def_property_readonly("type", &imperative::OpBase::Type)
X
Xin Pan 已提交
281 282 283 284 285 286
      .def_property(
          "backward_id",
          [](const imperative::OpBase &self) { return self.backward_id_; },
          [](imperative::OpBase &self, int backward_id) {
            self.backward_id_ = backward_id;
          },
287 288
          py::return_value_policy::reference);

X
Xin Pan 已提交
289
  py::class_<imperative::Layer, Layer /* <--- trampoline*/> layer(m, "Layer");
290
  layer.def(py::init<>())
X
Xin Pan 已提交
291
      .def("forward", [](imperative::Layer &self,
292
                         const std::vector<imperative::VarBase *> &inputs) {
X
Xin Pan 已提交
293
        return self.Forward(inputs);
X
Xin Pan 已提交
294
      });
X
Xin Pan 已提交
295

X
polish  
Xin Pan 已提交
296
  py::class_<imperative::PyLayer>(m, "PyLayer")
X
Xin Pan 已提交
297
      .def(py::init<>())
X
Xin Pan 已提交
298 299
      .def_static(
          "apply",
X
Xin Pan 已提交
300
          [](int func_id, const std::vector<imperative::VarBase *> &inputs)
X
Xin Pan 已提交
301
              -> std::vector<imperative::VarBase *> {
302 303 304 305 306
                auto ret_vars = imperative::PyLayer::Apply(func_id, inputs);
                std::vector<imperative::VarBase *> outputs;
                outputs.reserve(ret_vars.size());
                for (size_t i = 0U; i != ret_vars.size(); ++i) {
                  // TODO(minqiyang): use unique_name generator to set a name
307 308
                  outputs.emplace_back(new imperative::VarBase(
                      "", std::move(ret_vars[i]), nullptr, true));
309 310 311
                }

                return outputs;
X
Xin Pan 已提交
312 313
              },
          py::return_value_policy::take_ownership)
X
polish  
Xin Pan 已提交
314 315 316 317 318
      .def_static("register_func",
                  [](int func_id, const py::object &callable) {
                    imperative::PyLayer::RegisterFunc(func_id, callable);
                  })
      .def_static("num_funcs", &imperative::PyLayer::NumFuncs);
X
Xin Pan 已提交
319

320
  BindImperative(&m);
321

322
  py::class_<Tensor>(m, "Tensor", py::buffer_protocol())
323
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
S
sneaxiy 已提交
324 325
      .def("_is_initialized",
           [](const Tensor &self) { return self.IsInitialized(); })
Y
yuyang18 已提交
326
      .def("_get_dims",
327
           [](const Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
328
      .def("_set_dims",
Q
qijun 已提交
329
           [](Tensor &self, const std::vector<int64_t> &dim) {
Y
Yu Yang 已提交
330
             self.Resize(make_ddim(dim));
Y
Yu Yang 已提交
331
           })
Y
yuyang18 已提交
332
      .def("_set_layout",
D
dzhwinter 已提交
333 334 335
           [](Tensor &self, const std::string &layout) {
             self.set_layout(StringToDataLayout(layout));
           })
Y
yuyang18 已提交
336
      .def("_alloc_float",
D
dzhwinter 已提交
337
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
338
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
339
           })
Y
yuyang18 已提交
340
      .def("_alloc_float",
Y
Yu Yang 已提交
341
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
342
             self.mutable_data<float>(place);
Y
Yu Yang 已提交
343
           })
Y
yuyang18 已提交
344
      .def("_alloc_int",
Y
Yu Yang 已提交
345
           [](Tensor &self, paddle::platform::CPUPlace &place) {
Q
qijun 已提交
346
             self.mutable_data<int>(place);
Y
Yu Yang 已提交
347
           })
Y
yuyang18 已提交
348
      .def("_alloc_int",
D
dzhwinter 已提交
349
           [](Tensor &self, paddle::platform::CUDAPlace &place) {
Q
qijun 已提交
350
             self.mutable_data<int>(place);
Q
qijun 已提交
351
           })
Y
yuyang18 已提交
352
      .def("_alloc_int",
C
chengduoZH 已提交
353 354 355
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<int>(place);
           })
Y
yuyang18 已提交
356
      .def("_alloc_float",
C
chengduoZH 已提交
357 358 359
           [](Tensor &self, paddle::platform::CUDAPinnedPlace &place) {
             self.mutable_data<float>(place);
           })
Y
Yu Yang 已提交
360 361
      .def("set", PyCPUTensorSetFromArray<float>)
      .def("set", PyCPUTensorSetFromArray<int>)
362
      .def("set", PyCPUTensorSetFromArray<double>)
363
      .def("set", PyCPUTensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
364
      .def("set", PyCPUTensorSetFromArray<bool>)
365
      .def("set", PyCPUTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
366
      .def("set", PyCPUTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
367
      .def("set", PyCPUTensorSetFromArray<int8_t>)
368
#ifdef PADDLE_WITH_CUDA
Y
Yu Yang 已提交
369 370
      .def("set", PyCUDATensorSetFromArray<float>)
      .def("set", PyCUDATensorSetFromArray<int>)
371
      .def("set", PyCUDATensorSetFromArray<double>)
372
      .def("set", PyCUDATensorSetFromArray<int64_t>)
Y
Yu Yang 已提交
373
      .def("set", PyCUDATensorSetFromArray<bool>)
374
      .def("set", PyCUDATensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
375
      .def("set", PyCUDATensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
376
      .def("set", PyCUDATensorSetFromArray<int8_t>)
C
chengduoZH 已提交
377 378 379 380 381 382
      .def("set", PyCUDAPinnedTensorSetFromArray<float>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int>)
      .def("set", PyCUDAPinnedTensorSetFromArray<double>)
      .def("set", PyCUDAPinnedTensorSetFromArray<int64_t>)
      .def("set", PyCUDAPinnedTensorSetFromArray<bool>)
      .def("set", PyCUDAPinnedTensorSetFromArray<uint16_t>)
F
fengjiayi 已提交
383
      .def("set", PyCUDAPinnedTensorSetFromArray<uint8_t>)
Q
qingqing01 已提交
384
      .def("set", PyCUDAPinnedTensorSetFromArray<int8_t>)
Q
qijun 已提交
385
#endif
386
      .def("shape", [](Tensor &self) { return vectorize(self.dims()); })
Y
yuyang18 已提交
387 388 389 390
      .def("_set_float_element", TensorSetElement<float>)
      .def("_get_float_element", TensorGetElement<float>)
      .def("_set_double_element", TensorSetElement<double>)
      .def("_get_double_element", TensorGetElement<double>)
X
xuezhong 已提交
391
      .def("_place", [](Tensor &self) { return self.place(); })
W
wopeizl 已提交
392 393
      .def("_dtype", [](Tensor &self) { return self.type(); })
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference);
Y
Yu Yang 已提交
394

X
Xin Pan 已提交
395 396 397 398 399 400 401 402 403
  py::class_<LoDTensor, Tensor>(m, "LoDTensor", R"DOC(
    LoDTensor is a Tensor with optional LoD information.

    np.array(lod_tensor) can convert LoDTensor to numpy array.
    lod_tensor.lod() can retrieve the LoD information.

    LoD is short for Level of Details and is usually used for varied sequence
    length. You can skip the following comment if you don't need optional LoD.

Z
Zeng Jinle 已提交
404 405 406
    For example, a LoDTensor X can look like the example below. It contains 
    2 sequences. The first has length 2 and the second has length 3, as 
    described by x.lod.
X
Xin Pan 已提交
407

Z
Zeng Jinle 已提交
408 409 410
    The first tensor dimension 5=2+3 is calculated from LoD if it's available.
    It means the total number of sequence element. In X, each element has 2
    columns, hence [5, 2].
X
Xin Pan 已提交
411

Z
Zeng Jinle 已提交
412 413 414
    x.lod  = [[2, 3]]
     
    x.data = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
X
Xin Pan 已提交
415

Z
Zeng Jinle 已提交
416
    x.shape = [5, 2]
X
Xin Pan 已提交
417

Z
Zeng Jinle 已提交
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
    LoD can have multiple levels (for example, a paragraph can have multiple
    sentences and a sentence can have multiple words). In the following
    LodTensor Y, the lod_level is 2. It means there are 2 sequence, the
    first sequence length is 2 (has 2 sub-sequences), the second one's
    length is 1. The first sequence's 2 sub-sequences have length 2 and 2,
    respectively. And the second sequence's 1 sub-sequence has length 3.

    y.lod = [[2 1], [2 2 3]]

    y.shape = [2+2+3, ...]

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid

          t = fluid.LoDTensor()
X
Xin Pan 已提交
435 436 437 438 439 440 441 442 443 444 445 446

  Note:
      In above description, LoD is length-based. In Paddle internal
      implementation, lod is offset-based. Hence, internally,
      y.lod is represented as [[0, 2, 3], [0, 2, 4, 7]] (length-based
      equivlent would be [[2-0, 3-2], [2-0, 4-2, 7-4]]).

      Sometimes LoD is called recursive_sequence_length to be more
      self-explanatory. In this case, it must be length-based. Due to history
      reasons. when LoD is called lod in public API, it might be offset-based.
      Users should be careful about it.
        )DOC")
447
      .def("__array__", [](Tensor &self) { return TensorToPyArray(self); })
448 449 450 451 452 453 454 455 456 457 458 459 460 461
      .def("__init__",
           [](LoDTensor &instance, const std::vector<std::vector<size_t>>
                                       &recursive_sequence_lengths) {
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, -1),
                 "the provided recursive_sequence_lengths info is invalid");
             new (&instance) LoDTensor(new_offset_lod);
           })
Y
Yu Yang 已提交
462
      .def("__init__", [](LoDTensor &instance) { new (&instance) LoDTensor(); })
G
gongweibao 已提交
463 464 465 466 467
      // We implement offset based LOD in C++ while we use length based with
      // Python API. So we changed set_lod to set_recursive_sequence_lengths to
      // avoid misuse.
      // The discussion is here:
      // https://github.com/PaddlePaddle/Paddle/issues/10855
D
dangqingqing 已提交
468
      .def("set_lod",
469
           [](LoDTensor &self, const std::vector<std::vector<size_t>> &lod) {
470
             // the input lod is offset-based level-of-detail info
Y
Yu Yang 已提交
471
             LoD new_lod;
472 473
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
474 475
             PADDLE_ENFORCE(CheckLoD(new_lod, vectorize(self.dims()).front()),
                            "the provided lod info is invalid");
476
             self.set_lod(new_lod);
S
sneaxiy 已提交
477 478 479 480 481 482
           },
           py::arg("lod"), R"DOC(
           Set LoD of the LoDTensor.

           Args:
               lod (List[List[int]]): the lod to be set.
Z
Zeng Jinle 已提交
483 484 485 486 487 488 489 490 491 492

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
S
sneaxiy 已提交
493
           )DOC")
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
      .def("set_recursive_sequence_lengths",
           [](LoDTensor &self, const std::vector<std::vector<size_t>>
                                   &recursive_sequence_lengths) {
             // the input recursive_sequence_lengths is length-based
             // level-of-detail info
             LoD new_lod;
             new_lod.reserve(recursive_sequence_lengths.size());
             std::copy(recursive_sequence_lengths.begin(),
                       recursive_sequence_lengths.end(),
                       std::back_inserter(new_lod));
             LoD new_offset_lod = ConvertToOffsetBasedLoD(new_lod);
             PADDLE_ENFORCE(
                 CheckLoD(new_offset_lod, vectorize(self.dims()).front()),
                 "the provided recursive_sequence_lengths info is invalid");
             self.set_lod(new_offset_lod);
S
sneaxiy 已提交
509 510 511 512
           },
           py::arg("recursive_sequence_lengths"), R"DOC(
           Set LoD of the LoDTensor according to recursive sequence length.

S
sneaxiy 已提交
513
           For example, if recursive_sequence_lengths=[[2, 3]], meaning that
514 515
           there are two sequences with length 2 and 3 respectively, the
           corresponding lod would be [[0, 2, 2+3]], i.e, [[0, 2, 5]].
S
sneaxiy 已提交
516 517

           Args:
518
                recursive_sequence_lengths (List[List[int]]): sequence lengths.
Z
Zeng Jinle 已提交
519 520 521 522 523 524 525 526 527 528

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
S
sneaxiy 已提交
529
           )DOC")
530 531 532 533 534 535 536 537
      .def("lod",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the offset-based lod info
             LoD lod = self.lod();
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
538 539 540 541 542 543
           },
           R"DOC(
           Return the LoD of the LoDTensor.

           Returns:
               out (List[List[int]]): the lod of the LoDTensor.
Z
Zeng Jinle 已提交
544 545 546 547 548 549 550 551 552 553 554

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_lod([[0, 2, 5]])
                 print(t.lod()) # [[0, 2, 5]]
S
sneaxiy 已提交
555
           )DOC")
G
gongweibao 已提交
556
      // Set above comments of set_lod.
557 558 559 560 561 562 563 564
      .def("recursive_sequence_lengths",
           [](LoDTensor &self) -> std::vector<std::vector<size_t>> {
             // output the length-based lod info
             LoD lod = ConvertToLengthBasedLoD(self.lod());
             std::vector<std::vector<size_t>> new_lod;
             new_lod.reserve(lod.size());
             std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod));
             return new_lod;
S
sneaxiy 已提交
565 566 567 568 569
           },
           R"DOC(
           Return the sequence length of the LoDTensor corresponding to LoD.

           Returns:
570
               out (List[List[int]): the sequence lengths.
Z
Zeng Jinle 已提交
571 572 573 574 575 576 577 578 579 580 581

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.recursive_sequence_lengths()) # [[2, 3]]
S
sneaxiy 已提交
582 583 584 585 586 587 588 589 590 591 592 593
           )DOC")
      .def("has_valid_recursive_sequence_lengths",
           [](LoDTensor &self) -> bool {
             // Check that the lod info is valid and match the outermost
             // dimension of the LoDTensor data
             return CheckLoD(self.lod(), vectorize(self.dims()).front());
           },
           R"DOC(
           Check whether the lod of the LoDTensor is valid.

           Returns:
               out (bool): whether the lod is valid.
Z
Zeng Jinle 已提交
594 595 596 597 598 599 600 601 602 603 604

           Examples:
               .. code-block:: python

                 import paddle.fluid as fluid
                 import numpy as np

                 t = fluid.LoDTensor()
                 t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                 t.set_recursive_sequence_lengths([[2, 3]])
                 print(t.has_valid_recursive_sequence_lengths()) # True
W
wopeizl 已提交
605 606 607 608 609 610 611
           )DOC")
      .def("__getitem__", PySliceTensor, py::return_value_policy::reference,
           R"DOC(
           Slice the original Tensor, and remove the LoD information.

           Returns:
               out (Tensor): new Tensor(NOT LoDTensor).
S
sneaxiy 已提交
612
           )DOC");
D
dangqingqing 已提交
613

Q
qijun 已提交
614 615 616 617 618 619 620 621 622 623 624
  py::class_<SelectedRows>(m, "SelectedRows")
      .def("__init__",
           [](SelectedRows &instance) { new (&instance) SelectedRows(); })
      .def("__init__",
           [](SelectedRows &instance, const std::vector<int64_t> rows,
              const int64_t &height) {
             new (&instance) SelectedRows(rows, height);
           })
      .def("get_tensor",
           [](SelectedRows &self) { return self.mutable_value(); },
           py::return_value_policy::reference)
625 626
      .def("numel",
           [](SelectedRows &self) -> int64_t { return self.value().numel(); })
Q
qijun 已提交
627 628
      .def("set_height", &SelectedRows::set_height)
      .def("height", &SelectedRows::height)
Q
qijun 已提交
629 630 631 632 633 634 635 636 637
      .def("set_rows",
           [](SelectedRows &self, std::vector<int64_t> rows) {
#ifndef PADDLE_WITH_CUDA
             self.set_rows(rows);
#else
        Vector<int64_t> new_rows(rows);
        self.set_rows(new_rows);
#endif
           })
638
      .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); })
639
      .def("rows", [](SelectedRows &self) {
640 641 642 643 644
        auto rows = self.rows();
        std::vector<int64_t> new_rows;
        new_rows.reserve(rows.size());
        std::copy(rows.begin(), rows.end(), std::back_inserter(new_rows));
        return new_rows;
645
      });
Q
qijun 已提交
646

647
  py::class_<Variable>(m, "Variable", R"DOC(Variable Class.
648 649 650

All parameter, weight, gradient are variables in Paddle.
)DOC")
S
sneaxiy 已提交
651
      .def(py::init<>())
652
      .def("is_int", [](const Variable &var) { return var.IsType<int>(); })
653
      .def("set_int",
654 655
           [](Variable &var, int val) -> void { *var.GetMutable<int>() = val; })
      .def("get_int", [](const Variable &var) -> int { return var.Get<int>(); })
656 657 658 659 660 661 662
      .def("is_float", [](const Variable &var) { return var.IsType<float>(); })
      .def("set_float",
           [](Variable &var, float val) -> void {
             *var.GetMutable<float>() = val;
           })
      .def("get_float",
           [](const Variable &var) -> float { return var.Get<float>(); })
Y
Yu Yang 已提交
663
      .def("get_tensor",
664 665
           [](Variable &self) -> LoDTensor * {
             return self.GetMutable<LoDTensor>();
D
dangqingqing 已提交
666 667
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
668 669 670
      .def("get_lod_rank_table",
           [](Variable &self) { return self.GetMutable<LoDRankTable>(); },
           py::return_value_policy::reference)
Q
qijun 已提交
671 672 673 674 675
      .def("get_selected_rows",
           [](Variable &self) -> SelectedRows * {
             return self.GetMutable<SelectedRows>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
676 677 678
      .def("get_lod_tensor_array",
           [](Variable &self) { return self.GetMutable<LoDTensorArray>(); },
           py::return_value_policy::reference)
P
peizhilin 已提交
679
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
680 681 682 683 684
      .def("get_communicator",
           [](Variable &self) -> platform::Communicator * {
             return self.GetMutable<platform::Communicator>();
           },
           py::return_value_policy::reference)
Y
Yu Yang 已提交
685
#endif
Y
Refine  
Yu Yang 已提交
686 687 688 689 690
      .def("get_reader",
           [](Variable &self) -> framework::ReaderHolder * {
             PADDLE_ENFORCE(self.IsType<framework::ReaderHolder>());
             return self.GetMutable<framework::ReaderHolder>();
           },
W
wopeizl 已提交
691
           py::return_value_policy::reference);
692

S
sneaxiy 已提交
693
  BindReader(&m);
Y
Refine  
Yu Yang 已提交
694

S
sneaxiy 已提交
695 696 697 698
  using LoDTensorBlockingQueue =
      ::paddle::operators::reader::LoDTensorBlockingQueue;
  using LoDTensorBlockingQueueHolder =
      ::paddle::operators::reader::LoDTensorBlockingQueueHolder;
S
sneaxiy 已提交
699

S
sneaxiy 已提交
700 701
  py::class_<LoDTensorBlockingQueue, std::shared_ptr<LoDTensorBlockingQueue>>(
      m, "LoDTensorBlockingQueue", "")
S
sneaxiy 已提交
702
      .def("push",
S
sneaxiy 已提交
703
           [](LoDTensorBlockingQueue &self,
S
sneaxiy 已提交
704
              const std::vector<framework::LoDTensor> &lod_tensor_vec) {
S
sneaxiy 已提交
705
             pybind11::gil_scoped_release release;
S
sneaxiy 已提交
706
             return self.Push(lod_tensor_vec);
S
sneaxiy 已提交
707
           })
S
sneaxiy 已提交
708 709 710 711
      .def("size", &LoDTensorBlockingQueue::Size)
      .def("capacity", &LoDTensorBlockingQueue::Cap)
      .def("close", &LoDTensorBlockingQueue::Close)
      .def("is_closed", &LoDTensorBlockingQueue::IsClosed);
S
sneaxiy 已提交
712

S
sneaxiy 已提交
713
  m.def("init_lod_tensor_blocking_queue",
Q
Qiao Longfei 已提交
714 715
        [](Variable &var,
           size_t capacity) -> std::shared_ptr<LoDTensorBlockingQueue> {
Q
Qiao Longfei 已提交
716
          VLOG(1) << "init_lod_tensor_blocking_queue";
Q
Qiao Longfei 已提交
717 718 719 720
          auto *holder = var.GetMutable<LoDTensorBlockingQueueHolder>();
          holder->InitOnce(capacity, FLAGS_reader_queue_speed_test_mode);
          return holder->GetQueue();
        },
S
sneaxiy 已提交
721
        py::return_value_policy::copy);
S
sneaxiy 已提交
722

S
sneaxiy 已提交
723
  py::class_<Scope>(m, "_Scope", R"DOC(
Q
Qiao Longfei 已提交
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
    Scope is an association of a name to Variable. All variables belong to Scope.

    Variables in a parent scope can be retrieved from local scope.

    You need to specify a scope to run a Net, i.e., `exe.Run(&scope)`.
    One net can run in different scopes and update different variable in the
    scope.

    You can create var in a scope and get it from the scope.

    Examples:
        .. code-block:: python

          # create tensor from a scope and set value to it.
          param = scope.var('Param').get_tensor()
          param_array = np.full((height, row_numel), 5.0).astype("float32")
          param.set(param_array, place)

        )DOC")
S
sneaxiy 已提交
743 744
      .def("_remove_from_pool",
           [](Scope &self) { ScopePool::Instance().Remove(&self); })
D
dongzhihong 已提交
745
      .def("var",
746
           [](Scope &self, const std::string &name) -> Variable * {
D
dongzhihong 已提交
747
             return self.Var(name);
Y
Yu Yang 已提交
748
           },
S
sneaxiy 已提交
749 750
           py::arg("name"),
           R"DOC(
751
           Find or create variable named :code:`name` in the current scope.
S
sneaxiy 已提交
752

753
           If the variable named :code:`name` does not exist in the
S
sneaxiy 已提交
754
           current scope, the variable would be created. Otherwise,
755
           return the existing variable.
S
sneaxiy 已提交
756 757

           Args:
758 759
               name (str): the variable name.

S
sneaxiy 已提交
760
           Returns:
761
               out (core.Variable): the found or created variable.
S
sneaxiy 已提交
762 763 764 765
           )DOC",
           py::return_value_policy::reference)
      .def("find_var", &Scope::FindVar, py::arg("name"),
           R"DOC(
766
           Find variable named :code:`name` in the current scope or
S
sneaxiy 已提交
767
           its parent scope. Return None if not found.
768

S
sneaxiy 已提交
769 770
           Args:
               name (str): the variable name.
771

S
sneaxiy 已提交
772
           Returns:
773
               out (core.Variable|None): the found variable or None.
S
sneaxiy 已提交
774
           )DOC",
775
           py::return_value_policy::reference)
776
      .def("new_scope", [](Scope &self) -> Scope * { return &self.NewScope(); },
S
sneaxiy 已提交
777 778 779 780 781 782
           R"DOC(
           Create a new sub-scope of the current scope.

           Returns:
               out (core._Scope): the created sub-scope.
           )DOC",
783
           py::return_value_policy::reference)
S
sneaxiy 已提交
784 785 786
      .def("drop_kids", &Scope::DropKids,
           R"DOC(
           Delete all sub-scopes of the current scope.
S
sneaxiy 已提交
787 788
           )DOC")
      .def("_kids", &Scope::kids);
789

S
sneaxiy 已提交
790 791 792 793 794 795
  m.def("Scope",
        []() -> Scope * {
          auto *s = new Scope();
          ScopePool::Instance().Insert(std::unique_ptr<Scope>(s));
          return s;
        },
S
sneaxiy 已提交
796 797
        R"DOC(
        Create a new scope.
798

S
sneaxiy 已提交
799 800 801
        Returns:
            out (core._Scope): the created scope.
        )DOC",
S
sneaxiy 已提交
802 803
        py::return_value_policy::reference);

Y
Yu Yang 已提交
804 805
  //! @note: Be careful! PyBind will return std::string as an unicode, not
  //! Python str. If you want a str object, you should cast them in Python.
Y
Yu Yang 已提交
806 807
  m.def("get_all_op_protos", []() -> std::vector<py::bytes> {
    std::vector<py::bytes> ret_values;
808 809 810 811 812 813 814 815 816 817
    for (auto &iter : OpInfoMap::Instance().map()) {
      auto &info = iter.second;
      if (info.HasOpProtoAndChecker()) {
        std::string str;
        PADDLE_ENFORCE(
            info.Proto().SerializeToString(&str),
            "Serialize OpProto Error. This could be a bug of Paddle.");
        ret_values.emplace_back(str);
      }
    }
Y
Yu Yang 已提交
818 819
    return ret_values;
  });
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835
  m.def(
      "get_grad_op_desc", [](const OpDesc &op_desc,
                             const std::unordered_set<std::string> &no_grad_set,
                             const std::vector<BlockDesc *> &grad_sub_block) {
        std::unordered_map<std::string, std::string> grad_to_var;
        std::vector<std::unique_ptr<OpDesc>> grad_op_descs =
            framework::OpInfoMap::Instance()
                .Get(op_desc.Type())
                .GradOpMaker()(op_desc, no_grad_set, &grad_to_var,
                               grad_sub_block);
        std::vector<OpDesc *> grad_op_desc_ptrs(grad_op_descs.size());
        std::transform(grad_op_descs.begin(), grad_op_descs.end(),
                       grad_op_desc_ptrs.begin(),
                       [](std::unique_ptr<OpDesc> &p) { return p.release(); });
        return std::make_pair(grad_op_desc_ptrs, grad_to_var);
      });
Y
Yu Yang 已提交
836
  m.def("prune", [](const ProgramDesc &origin,
837
                    const std::vector<std::array<size_t, 2>> &targets) {
Y
Yu Yang 已提交
838
    ProgramDesc prog_with_targets(origin);
839
    for (const auto &t : targets) {
840
      prog_with_targets.MutableBlock(t[0])->Op(t[1])->SetIsTarget(true);
841
    }
842
    proto::ProgramDesc pruned_desc;
843
    Prune(*prog_with_targets.Proto(), &pruned_desc);
Y
Yu Yang 已提交
844
    return new ProgramDesc(pruned_desc);
845
  });
846 847 848 849
  m.def("empty_var_name",
        []() { return std::string(framework::kEmptyVarName); });
  m.def("grad_var_suffix",
        []() { return std::string(framework::kGradVarSuffix); });
850 851 852
  m.def_submodule(
       "var_names",
       "The module will return special predefined variable name in Paddle")
Y
Yi Wang 已提交
853 854
      .def("empty", []() { return kEmptyVarName; })
      .def("temp", []() { return kTempVarName; });
Q
qijun 已提交
855
  // clang-format off
Y
Yu Yang 已提交
856
  py::class_<paddle::platform::DeviceContext>(m, "DeviceContext")
Q
qijun 已提交
857 858
      .def_static("create",
                  [](paddle::platform::CPUPlace& place)
Q
qijun 已提交
859
                      -> paddle::platform::DeviceContext* {
Q
qijun 已提交
860 861 862
                    return new paddle::platform::CPUDeviceContext();
                  })
      .def_static("create",
D
dzhwinter 已提交
863
                  [](paddle::platform::CUDAPlace& place)
Q
qijun 已提交
864
                      -> paddle::platform::DeviceContext* {
865
#ifndef PADDLE_WITH_CUDA
D
dzhwinter 已提交
866
                    PADDLE_THROW("CUDAPlace is not supported in CPU device.");
Q
qijun 已提交
867
#else
Q
qijun 已提交
868
                    return new paddle::platform::CUDADeviceContext(place);
Q
qijun 已提交
869
#endif
C
chengduoZH 已提交
870 871 872 873 874 875 876 877 878 879 880
                  })
          .def_static("create",
                [](paddle::platform::CUDAPinnedPlace& place)
                        -> paddle::platform::DeviceContext* {
#ifndef PADDLE_WITH_CUDA
                  PADDLE_THROW(
                        "CUDAPinnedPlace is not supported in CPU device.");
#else
                  return new paddle::platform::CUDAPinnedDeviceContext(place);
#endif
                });;
D
Dong Zhihong 已提交
881
// clang-format on
P
peizhilin 已提交
882
#if (defined(PADDLE_WITH_CUDA) && !defined(_WIN32))
D
Dong Zhihong 已提交
883 884
  py::class_<platform::Communicator>(m, "Communicator").def(py::init<>());
#endif
885 886 887 888
  py::class_<platform::CUDAPlace>(m, "CUDAPlace", R"DOC(
    CUDAPlace is a descriptor of a device. It represents a GPU, and each CUDAPlace
    has a dev_id to indicate the number of cards represented by the current CUDAPlace.
    The memory of CUDAPlace with different dev_id is not accessible.
L
lujun 已提交
889 890 891 892 893 894

    Examples:
        .. code-block:: python

          gpu_place = fluid.CUDAPlace(0)

895
        )DOC")
S
sneaxiy 已提交
896 897 898 899 900 901 902 903 904 905 906 907
      .def("__init__",
           [](platform::CUDAPlace &self, int dev_id) {
#ifdef PADDLE_WITH_CUDA
             PADDLE_ENFORCE(
                 dev_id >= 0 && dev_id < platform::GetCUDADeviceCount(),
                 "Invalid CUDAPlace(%d), must inside [0, %d)", dev_id,
                 platform::GetCUDADeviceCount());
             new (&self) platform::CUDAPlace(dev_id);
#else
             PADDLE_THROW("Cannot use CUDAPlace in CPU only version");
#endif
           })
S
sneaxiy 已提交
908 909 910 911 912 913
      .def("_type", &PlaceIndex<platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPlace, platform::CUDAPinnedPlace>)
D
dzhwinter 已提交
914
      .def("__str__", string::to_string<const platform::CUDAPlace &>);
Q
qijun 已提交
915

916 917 918
  py::class_<paddle::platform::CPUPlace>(m, "CPUPlace", R"DOC(
    CPUPlace is a descriptor of a device. It represents a CPU, and the memory
    CPUPlace can be accessed by CPU.
L
lujun 已提交
919 920 921 922 923 924

    Examples:
        .. code-block:: python

          cpu_place = fluid.CPUPlace()

925
        )DOC")
926
      .def(py::init<>())
S
sneaxiy 已提交
927 928 929 930 931 932
      .def("_type", &PlaceIndex<platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::Place>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::CPUPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CPUPlace, platform::CUDAPinnedPlace>)
933
      .def("__str__", string::to_string<const platform::CPUPlace &>);
Y
Yu Yang 已提交
934

935 936 937
  py::class_<paddle::platform::CUDAPinnedPlace>(m, "CUDAPinnedPlace", R"DOC(
    CUDAPinnedPlace is a descriptor of a device. The memory of CUDAPinnedPlace
    can be accessed by GPU and CPU.
L
lujun 已提交
938 939 940 941 942 943

    Examples:
        .. code-block:: python

          place = fluid.CUDAPinnedPlace()

944
        )DOC")
S
sneaxiy 已提交
945
      .def("__init__",
S
sneaxiy 已提交
946
           [](platform::CUDAPinnedPlace &self) {
S
sneaxiy 已提交
947 948 949
#ifndef PADDLE_WITH_CUDA
             PADDLE_THROW("Cannot use CUDAPinnedPlace in CPU only version");
#endif
S
sneaxiy 已提交
950
             new (&self) platform::CUDAPinnedPlace();
S
sneaxiy 已提交
951
           })
S
sneaxiy 已提交
952 953 954 955 956 957 958 959
      .def("_type", &PlaceIndex<platform::CUDAPinnedPlace>)
      .def("_equals", &IsSamePlace<platform::CUDAPinnedPlace, platform::Place>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CPUPlace>)
      .def("_equals",
           &IsSamePlace<platform::CUDAPinnedPlace, platform::CUDAPinnedPlace>)
C
chengduoZH 已提交
960 961
      .def("__str__", string::to_string<const platform::CUDAPinnedPlace &>);

Y
Yu Yang 已提交
962 963
  py::class_<platform::Place>(m, "Place")
      .def(py::init<>())
S
sneaxiy 已提交
964 965 966 967 968
      .def("_type", &PlaceIndex<platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::Place>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CPUPlace>)
      .def("_equals", &IsSamePlace<platform::Place, platform::CUDAPinnedPlace>)
X
xuezhong 已提交
969 970
      .def("is_gpu_place",
           [](platform::Place &self) { return platform::is_gpu_place(self); })
S
sneaxiy 已提交
971 972 973 974 975 976
      .def("is_cpu_place",
           [](platform::Place &self) { return platform::is_cpu_place(self); })
      .def("is_cuda_pinned_place",
           [](platform::Place &self) {
             return platform::is_cuda_pinned_place(self);
           })
X
xuezhong 已提交
977 978 979 980
      .def("gpu_device_id",
           [](platform::Place &self) {
             return boost::get<platform::CUDAPlace>(self).device;
           })
S
sneaxiy 已提交
981 982
      .def("set_place", [](platform::Place &self,
                           const platform::Place &other) { self = other; })
Y
Yu Yang 已提交
983 984 985 986 987
      .def("set_place",
           [](platform::Place &self, const platform::CPUPlace &cpu_place) {
             self = cpu_place;
           })
      .def("set_place",
D
dzhwinter 已提交
988
           [](platform::Place &self, const platform::CUDAPlace &gpu_place) {
Y
Yu Yang 已提交
989
             self = gpu_place;
C
chengduoZH 已提交
990 991
           })
      .def("set_place", [](platform::Place &self,
C
chengduoZH 已提交
992 993
                           const platform::CUDAPinnedPlace &cuda_pinned_place) {
        self = cuda_pinned_place;
C
chengduoZH 已提交
994
      });
Y
Yu Yang 已提交
995

Y
Yu Yang 已提交
996 997 998
  py::class_<OperatorBase>(m, "Operator")
      .def_static("create",
                  [](py::bytes protobin) {
999
                    proto::OpDesc desc;
Y
Yu Yang 已提交
1000 1001 1002 1003 1004
                    PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
                                   "Cannot parse user input to OpDesc");
                    PADDLE_ENFORCE(desc.IsInitialized(),
                                   "User OpDesc is not initialized, reason %s",
                                   desc.InitializationErrorString());
1005
                    return OpRegistry::CreateOp(desc);
Y
Yu Yang 已提交
1006
                  })
1007
      .def("run",
1008
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1009 1010 1011
              const platform::CPUPlace &place) { self.Run(scope, place); })
      .def("run",
           [](OperatorBase &self, const Scope &scope,
D
dzhwinter 已提交
1012
              const platform::CUDAPlace &place) { self.Run(scope, place); })
C
chengduoZH 已提交
1013 1014 1015 1016 1017
      .def("run",
           [](OperatorBase &self, const Scope &scope,
              const platform::CUDAPinnedPlace &place) {
             self.Run(scope, place);
           })
Y
Yu Yang 已提交
1018 1019 1020 1021 1022 1023 1024
      .def("type",
           [](const OperatorBase &op) -> std::string { return op.Type(); })
      .def("outputs",
           [](const OperatorBase &op)
               -> std::map<std::string, std::vector<std::string>> {
                 return op.Outputs();
               })
Q
qijun 已提交
1025 1026
      .def("output_vars",
           [](const OperatorBase &op) { return op.OutputVars(true); })
Y
Yu Yang 已提交
1027
      .def("inputs", [](const OperatorBase &op) { return op.Inputs(); })
Q
qijun 已提交
1028
      .def("input_vars", [](const OperatorBase &op) { return op.InputVars(); })
Y
Yu Yang 已提交
1029 1030 1031 1032
      .def("__str__", &OperatorBase::DebugString)
      .def("no_intermediate_outputs",
           [](const OperatorBase &op) { return op.OutputVars(false); })
      .def("support_gpu", &OperatorBase::SupportGPU);
Y
Yu Yang 已提交
1033

F
fengjiayi 已提交
1034
  py::class_<framework::Executor>(m, "Executor")
D
dzhwinter 已提交
1035
      .def(py::init<const platform::Place &>())
Y
Yancey1989 已提交
1036
      .def("close", &Executor::Close)
D
dongdaxiang 已提交
1037
      .def("run_from_dataset", &Executor::RunFromDataset)
S
sneaxiy 已提交
1038
      .def("run", [](Executor &self, const ProgramDesc &prog, Scope *scope,
S
sneaxiy 已提交
1039 1040
                     int block_id, bool create_local_scope, bool create_vars,
                     const std::vector<std::string> &fetch_vars) {
S
sneaxiy 已提交
1041
        pybind11::gil_scoped_release release;
S
sneaxiy 已提交
1042 1043
        self.Run(prog, scope, block_id, create_local_scope, create_vars,
                 fetch_vars);
S
sneaxiy 已提交
1044
      });
S
sneaxiy 已提交
1045

D
dzhwinter 已提交
1046
  m.def("init_gflags", framework::InitGflags);
Y
Yang Yu 已提交
1047
  m.def("init_glog", framework::InitGLOG);
1048
  m.def("init_dgc", framework::InitDGC);
X
Xin Pan 已提交
1049 1050
  m.def("init_devices",
        [](bool init_p2p) { framework::InitDevices(init_p2p); });
1051

1052
  m.def("is_compiled_with_ngraph", IsCompiledWithNGRAPH);
1053
  m.def("is_compiled_with_cuda", IsCompiledWithCUDA);
1054
  m.def("is_compiled_with_mkldnn", IsCompiledWithMKLDNN);
1055
  m.def("is_compiled_with_brpc", IsCompiledWithBrpc);
Y
update  
Yancey1989 已提交
1056
  m.def("is_compiled_with_dist", IsCompiledWithDIST);
1057 1058 1059 1060 1061 1062
#ifdef PADDLE_WITH_CUDA
  m.def("is_float16_supported", [](const platform::CUDAPlace &place) -> bool {
    // Only GPUs with Compute Capability >= 53 support float16
    return platform::GetCUDAComputeCapability(place.device) >= 53;
  });
#endif
1063

1064
  m.def("set_feed_variable", framework::SetFeedVariable);
Q
qijun 已提交
1065
  m.def("get_fetch_variable", framework::GetFetchVariable);
1066
  m.def("get_variable_tensor", framework::GetVariableTensor);
Q
qijun 已提交
1067

X
Xin Pan 已提交
1068 1069
  m.def("_is_program_version_supported", IsProgramVersionSupported);

1070 1071 1072 1073 1074
  BindProgramDesc(&m);
  BindBlockDesc(&m);
  BindVarDsec(&m);
  BindOpDesc(&m);
  BindConstValue(&m);
Y
Yu Yang 已提交
1075

Y
Yu Yang 已提交
1076 1077 1078 1079 1080 1081 1082 1083 1084
  py::class_<framework::LoDRankTable>(m, "LodRankTable")
      .def("items", [](framework::LoDRankTable &table) {
        std::vector<std::pair<size_t, size_t>> res;
        for (auto &item : table.items()) {
          res.push_back({item.index, item.length});
        }
        return res;
      });

Z
Zeng Jinle 已提交
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
  py::class_<LoDTensorArray>(m, "LoDTensorArray", R"DOC(
    Array of LoDTensor.

    Examples:
        .. code-block:: python
        
          import paddle.fluid as fluid

          arr = fluid.LoDTensorArray()
)DOC")
S
sneaxiy 已提交
1095 1096
      .def("__init__",
           [](LoDTensorArray &instance) { new (&instance) LoDTensorArray(); })
Y
Yu Yang 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
      .def("__getitem__",
           [](LoDTensorArray &self, size_t i) { return &self.at(i); },
           py::return_value_policy::reference)
      .def("__len__", [](LoDTensorArray &self) { return self.size(); })
      .def("__setitem__",
           [](LoDTensorArray &self, size_t i, const LoDTensor &t) {
             PADDLE_ENFORCE_LT(i, self.size());
             self[i].ShareDataWith(t);
             self[i].set_lod(t.lod());
           })
S
sneaxiy 已提交
1107 1108 1109 1110 1111 1112
      .def("append",
           [](LoDTensorArray &self, const LoDTensor &t) {
             self.emplace_back();
             self.back().ShareDataWith(t);
             self.back().set_lod(t.lod());
           },
Z
Zeng Jinle 已提交
1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
           py::arg("tensor"), R"DOC(
             Append a LoDensor to LoDTensorArray.

             Examples:
                 .. code-block:: python

                   import paddle.fluid as fluid
                   import numpy as np

                   arr = fluid.LoDTensorArray()
                   t = fluid.LoDTensor()
                   t.set(np.ndarray([5, 30]), fluid.CPUPlace())
                   arr.append(t)
           )DOC");
Y
Yu Yang 已提交
1127

D
dzhwinter 已提交
1128 1129 1130
  m.def("IsInplace",
        [](std::string op) -> bool { return operators::IsInplace(op); });

Y
Yu Yang 已提交
1131
  m.def("op_support_gpu", OpSupportGPU);
D
Dong Zhihong 已提交
1132
#ifdef PADDLE_WITH_CUDA
D
Dong Zhihong 已提交
1133
  m.def("get_cuda_device_count", platform::GetCUDADeviceCount);
D
dangqingqing 已提交
1134

P
peizhilin 已提交
1135
#ifndef _WIN32
D
dangqingqing 已提交
1136 1137 1138
  m.def("nvprof_init", platform::CudaProfilerInit);
  m.def("nvprof_start", platform::CudaProfilerStart);
  m.def("nvprof_stop", platform::CudaProfilerStop);
D
Dong Zhihong 已提交
1139
#endif
P
peizhilin 已提交
1140
#endif
Y
Yu Yang 已提交
1141

1142 1143 1144 1145
  py::enum_<platform::ProfilerState>(m, "ProfilerState", py::arithmetic())
      .value("kDisabled", platform::ProfilerState::kDisabled)
      .value("kCPU", platform::ProfilerState::kCPU)
      .value("kCUDA", platform::ProfilerState::kCUDA)
1146
      .value("kAll", platform::ProfilerState::kAll)
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
      .export_values();

  py::enum_<platform::EventSortingKey>(m, "EventSortingKey", py::arithmetic())
      .value("kDefault", platform::EventSortingKey::kDefault)
      .value("kCalls", platform::EventSortingKey::kCalls)
      .value("kTotal", platform::EventSortingKey::kTotal)
      .value("kMin", platform::EventSortingKey::kMin)
      .value("kMax", platform::EventSortingKey::kMax)
      .value("kAve", platform::EventSortingKey::kAve)
      .export_values();

  m.def("enable_profiler", platform::EnableProfiler);
  m.def("disable_profiler", platform::DisableProfiler);
X
Xin Pan 已提交
1160
  m.def("is_profiler_enabled", platform::IsProfileEnabled);
1161
  m.def("reset_profiler", platform::ResetProfiler);
1162
  m.def("get_pass", [](const std::string &pass_type) {
W
WangZhen 已提交
1163 1164 1165
    auto pass = framework::ir::PassRegistry::Instance().Get(pass_type);
    return std::shared_ptr<framework::ir::Pass>(std::move(pass));
  });
Y
Yu Yang 已提交
1166

1167 1168
  py::class_<ir::Pass, std::shared_ptr<ir::Pass>> pass(m, "Pass");
  pass.def(py::init())
W
WangZhen 已提交
1169
      .def("has", &ir::Pass::Has)
1170 1171 1172
      .def("set_not_owned",
           [](ir::Pass &self, const std::string &attr_name, ProgramDesc &attr) {
             self.SetNotOwned<ProgramDesc>(attr_name, &attr);
W
WangZhen 已提交
1173
           })
1174
      .def(
1175
          "set",
1176 1177 1178
          [](ir::Pass &self, const std::string &name, const std::string &attr) {
            self.Set<std::string>(name, new std::string(attr));
          })
1179 1180
      .def("set", [](ir::Pass &self, const std::string &name,
                     int val) { self.Set<const int>(name, new int(val)); })
F
flame 已提交
1181 1182
      .def("type", &ir::Pass::Type)
      .def("apply", [](ir::Pass &self, std::shared_ptr<ir::Graph> graph) {
1183
        self.Apply(graph.get());
F
flame 已提交
1184
      });
1185

X
fix  
Xin Pan 已提交
1186 1187
  py::class_<ir::PassBuilder, std::shared_ptr<ir::PassBuilder>> pb(
      m, "PassBuilder");
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
  pb.def(py::init())
      .def("append_pass",
           [](ir::PassBuilder &self,
              const std::string &pass_type) -> std::shared_ptr<ir::Pass> {
             return self.AppendPass(pass_type);
           })
      .def("all_passes", [](ir::PassBuilder &self) { return self.AllPasses(); })
      .def("insert_pass",
           [](ir::PassBuilder &self, size_t idx, const std::string &pass_type) {
             return self.InsertPass(idx, pass_type);
           })
      .def("remove_pass",
           [](ir::PassBuilder &self, size_t idx) { self.RemovePass(idx); });

Y
yuyang18 已提交
1202
  // -- python binds for parallel executor.
X
Xin Pan 已提交
1203

Y
yuyang18 已提交
1204
  py::class_<ParallelExecutor> pe(m, "ParallelExecutor");
C
chengduo 已提交
1205 1206 1207 1208
  py::class_<ExecutionStrategy> exec_strategy(pe, "ExecutionStrategy", R"DOC(
    ExecutionStrategy allows the user to more preciously control how to run
    the program in ParallelExecutor by setting the property.

C
chengduo 已提交
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
    Examples:
        .. code-block:: python

          exec_strategy = fluid.ExecutionStrategy()
          exec_strategy.num_threads = 4

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             exec_strategy=exec_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
1220 1221 1222

        )DOC");

Y
yuyang18 已提交
1223
  exec_strategy.def(py::init())
Y
yuyang18 已提交
1224 1225 1226 1227 1228
      .def_property(
          "num_threads",
          [](const ExecutionStrategy &self) { return self.num_threads_; },
          [](ExecutionStrategy &self, size_t num_threads) {
            self.num_threads_ = num_threads;
C
chengduo 已提交
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
          },
          R"DOC(The type is INT, num_threads represents the size of thread pool that
            used to run the operators of the current program in ParallelExecutor.
            If :math:`num\_threads=1`, all the operators will execute one by one,
            but the order maybe difference between iterations.
            If it is not set, it will be set in ParallelExecutor according to the
            device type and device count, for GPU, :math:`num\_threads=device\_count*4`, for CPU,
            :math:`num\_threads=CPU\_NUM*4`, the explanation of:math:`CPU\_NUM` is in ParallelExecutor.
            if it is not set, ParallelExecutor will get the cpu count by calling
            `multiprocessing.cpu_count()`. Default 0.)DOC")
Y
yuyang18 已提交
1239
      .def_property(
1240 1241 1242 1243
          "use_cuda",
          [](const ExecutionStrategy &self) { return self.use_cuda_; },
          [](ExecutionStrategy &self, bool use_cuda) {
            self.use_cuda_ = use_cuda;
C
chengduo 已提交
1244 1245 1246 1247
          })  // FIXME(chengduo): Doesn't add doc for 'use_cuda', use_cuda may
      // make user confuse, because ParallelExecutor has a parameter named
      // 'use_cuda' too, in current implementation, ParallelExecutor's
      // 'use_cuda' will rewrite ExecutionStrategy's 'use_cuda'.
Y
yuyang18 已提交
1248 1249 1250 1251 1252
      .def_property(
          "allow_op_delay",
          [](const ExecutionStrategy &self) { return self.allow_op_delay_; },
          [](ExecutionStrategy &self, bool allow_op_delay) {
            self.allow_op_delay_ = allow_op_delay;
C
chengduo 已提交
1253 1254 1255 1256
          },
          R"DOC(The type is BOOL, allow_op_delay represents whether to delay the
                communication operators to run, it may make the execution faster.
                Note that in some models, allow_op_delay may cause program hang. Default False.)DOC")
Y
yuyang18 已提交
1257 1258 1259 1260 1261 1262 1263
      .def_property(
          "num_iteration_per_drop_scope",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_drop_scope_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_drop_scope) {
            self.num_iteration_per_drop_scope_ = num_iteration_per_drop_scope;
C
chengduo 已提交
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
          },
          R"DOC(The type is INT, num_iteration_per_drop_scope indicates how
                many iterations to clean up the temp variables which
                is generated during execution. It may make the execution faster,
                because the temp variable's shape maybe the same between two iterations. Default 100.

                NOTES:
                    1. If you fetch data when calling the 'run', the ParallelExecutor
                       will clean up the temp variables at the end of the current iteration.
                    2. In some NLP model, it may cause the GPU memory is insufficient,
                       in this case, you should reduce `num_iteration_per_drop_scope`.
1275
              )DOC")
Q
Qiao Longfei 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
      .def_property(
          "num_iteration_per_run",
          [](const ExecutionStrategy &self) {
            return self.num_iteration_per_run_;
          },
          [](ExecutionStrategy &self, size_t num_iteration_per_run) {
            self.num_iteration_per_run_ = num_iteration_per_run;
          },
          R"DOC(This config that how many iteration the executor will run when
                user call pe.run() in python
              )DOC")
1287 1288 1289 1290 1291
      .def_property("_dry_run",
                    [](const ExecutionStrategy &self) { return self.dry_run_; },
                    [](ExecutionStrategy &self, bool dry_run) {
                      self.dry_run_ = dry_run;
                    });
C
chengduo 已提交
1292

Y
yuyang18 已提交
1293
  exec_strategy.def_property(
Y
yuyang18 已提交
1294 1295 1296 1297 1298 1299 1300
      "use_experimental_executor",
      [](const ExecutionStrategy &self) {
        return self.type_ == ExecutionStrategy::kExperimental;
      },
      [](ExecutionStrategy &self, bool experimental) {
        self.type_ = experimental ? ExecutionStrategy::kExperimental
                                  : ExecutionStrategy::kDefault;
Y
yuyang18 已提交
1301 1302
      });

C
chengduo 已提交
1303 1304 1305 1306
  py::class_<BuildStrategy> build_strategy(pe, "BuildStrategy", R"DOC(
    BuildStrategy allows the user to more preciously control how to
    build the SSA Graph in ParallelExecutor by setting the property.

C
chengduo 已提交
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
    Examples:
        .. code-block:: python

          build_strategy = fluid.BuildStrategy()
          build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce

          train_exe = fluid.ParallelExecutor(use_cuda=True,
                                             loss_name=loss.name,
                                             build_strategy=build_strategy)

          train_loss, = train_exe.run([loss.name], feed=feed_dict)
C
chengduo 已提交
1318
)DOC");
Y
yuyang18 已提交
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334

  py::enum_<BuildStrategy::ReduceStrategy>(build_strategy, "ReduceStrategy")
      .value("Reduce", BuildStrategy::ReduceStrategy::kReduce)
      .value("AllReduce", BuildStrategy::ReduceStrategy::kAllReduce);
  py::enum_<BuildStrategy::GradientScaleStrategy>(build_strategy,
                                                  "GradientScaleStrategy")
      .value("CoeffNumDevice",
             BuildStrategy::GradientScaleStrategy::kCoeffNumDevice)
      .value("One", BuildStrategy::GradientScaleStrategy::kOne)
      .value("Customized", BuildStrategy::GradientScaleStrategy::kCustomized);

  build_strategy.def(py::init())
      .def_property(
          "reduce_strategy",
          [](const BuildStrategy &self) { return self.reduce_; },
          [](BuildStrategy &self, BuildStrategy::ReduceStrategy strategy) {
X
Xin Pan 已提交
1335
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1336
            self.reduce_ = strategy;
C
chengduo 已提交
1337 1338 1339 1340 1341 1342 1343
          },
          R"DOC(The type is STR, there are two reduce strategies in ParallelExecutor,
                  'AllReduce' and 'Reduce'. If you want that all the parameters'
                  optimization are done on all devices independently, you should choose 'AllReduce';
                  if you choose 'Reduce', all the parameters' optimization will be evenly distributed
                  to different devices, and then broadcast the optimized parameter to other devices.
                  In some models, `Reduce` is faster. Default 'AllReduce'. )DOC")
Y
yuyang18 已提交
1344 1345 1346 1347 1348
      .def_property(
          "gradient_scale_strategy",
          [](const BuildStrategy &self) { return self.gradient_scale_; },
          [](BuildStrategy &self,
             BuildStrategy::GradientScaleStrategy strategy) {
X
Xin Pan 已提交
1349
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1350
            self.gradient_scale_ = strategy;
C
chengduo 已提交
1351 1352 1353 1354 1355 1356
          },
          R"DOC(The type is STR, there are three ways of defining :math:`loss@grad` in
                   ParallelExecutor, 'CoeffNumDevice', 'One' and 'Customized'. By default,
                   ParallelExecutor sets the :math:`loss@grad` according to the number of devices.
                   If you want to customize :math:`loss@grad`, you can choose 'Customized'.
                   Default 'CoeffNumDevice'.)DOC")
Y
yuyang18 已提交
1357 1358 1359 1360
      .def_property(
          "debug_graphviz_path",
          [](const BuildStrategy &self) { return self.debug_graphviz_path_; },
          [](BuildStrategy &self, const std::string &path) {
X
Xin Pan 已提交
1361
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
Y
yuyang18 已提交
1362
            self.debug_graphviz_path_ = path;
C
chengduo 已提交
1363 1364 1365 1366
          },
          R"DOC(The type is STR, debug_graphviz_path indicate the path that
                    writing the SSA Graph to file in the form of graphviz, you.
                    It is useful for debugging. Default "")DOC")
S
sneaxiy 已提交
1367 1368 1369 1370 1371 1372
      .def_property(
          "enable_sequential_execution",
          [](const BuildStrategy &self) {
            return self.enable_sequential_execution_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1373
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1374 1375 1376 1377 1378 1379 1380 1381 1382
            self.enable_sequential_execution_ = b;
          },
          R"DOC(The type is BOOL. If set True, the execution order of ops would be the same as what is in the program. Default False.)DOC")
      .def_property(
          "remove_unnecessary_lock",
          [](const BuildStrategy &self) {
            return self.remove_unnecessary_lock_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1383
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
S
sneaxiy 已提交
1384 1385
            self.remove_unnecessary_lock_ = b;
          },
S
sneaxiy 已提交
1386
          R"DOC(The type is BOOL. If set True, some locks in GPU ops would be released and ParallelExecutor would run faster. Default True.)DOC")
1387 1388 1389 1390 1391 1392
      .def_property(
          "num_trainers",
          [](const BuildStrategy &self) { return self.num_trainers_; },
          [](BuildStrategy &self, int num_trainers) {
            self.num_trainers_ = num_trainers;
          })
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
      .def_property(
          "trainers_endpoints",
          [](const BuildStrategy &self) { return self.trainers_endpoints_; },
          [](BuildStrategy &self,
             const std::vector<std::string> &trainers_endpoints) {
            self.trainers_endpoints_ = trainers_endpoints;
          })
      .def_property("trainer_id",
                    [](const BuildStrategy &self) { return self.trainer_id_; },
                    [](BuildStrategy &self, int trainer_id) {
                      self.trainer_id_ = trainer_id;
                    })
C
chengduo 已提交
1405 1406 1407 1408 1409 1410
      .def_property(
          "fuse_elewise_add_act_ops",
          [](const BuildStrategy &self) {
            return self.fuse_elewise_add_act_ops_;
          },
          [](BuildStrategy &self, bool b) {
X
Xin Pan 已提交
1411
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
C
chengduo 已提交
1412 1413 1414 1415 1416
            self.fuse_elewise_add_act_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_elewise_add_act_ops indicate whether
                     to fuse elementwise_add_op and activation_op,
                     it may make the execution faster. Default False)DOC")
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
      .def_property(
          "fuse_relu_depthwise_conv",
          [](const BuildStrategy &self) {
            return self.fuse_relu_depthwise_conv_;
          },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_relu_depthwise_conv_ = b;
          },
          R"DOC(The type is BOOL, fuse_relu_depthwise_conv indicate whether
                      to fuse relu and depthwise_conv2d,
                      it will save GPU memory and may make the execution faster.
                      This options is only available in GPU devices.
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443
                      Default False.)DOC")
      .def_property(
          "fuse_broadcast_ops",
          [](const BuildStrategy &self) { return self.fuse_broadcast_ops_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.fuse_broadcast_ops_ = b;
          },
          R"DOC(The type is BOOL, fuse_broadcast_op indicates whether
                      to fuse the broadcast ops. Note that, in Reduce mode,
                      fusing broadcast ops may make the program faster. Because
                      fusing broadcast OP equals delaying the execution of all
                      broadcast Ops, in this case, all nccl streams are used only
                      for NCCLReduce operations for a period of time. Default False.)DOC")
C
chengduo 已提交
1444 1445 1446 1447 1448 1449 1450 1451 1452
      .def_property("fuse_all_optimizer_ops",
                    [](const BuildStrategy &self) {
                      return self.fuse_all_optimizer_ops_;
                    },
                    [](BuildStrategy &self, bool b) {
                      PADDLE_ENFORCE(!self.IsFinalized(),
                                     "BuildStrategy is finlaized.");
                      self.fuse_all_optimizer_ops_ = b;
                    })
Q
qingqing01 已提交
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
      .def_property(
          "sync_batch_norm",
          [](const BuildStrategy &self) { return self.sync_batch_norm_; },
          [](BuildStrategy &self, bool b) {
            PADDLE_ENFORCE(!self.IsFinalized(), "BuildStrategy is finlaized.");
            self.sync_batch_norm_ = b;
          },
          R"DOC(The type is BOOL, sync_batch_norm indicates whether to use
                synchronous batch normalization which synchronizes the mean
                and variance through multi-devices in training phase.

                Current implementation doesn't support FP16 training and CPU.
                And only synchronous on one machine, not all machines.

                Default False)DOC")
D
dzhwinter 已提交
1468 1469 1470
      .def_property(
          "memory_optimize",
          [](const BuildStrategy &self) { return self.memory_optimize_; },
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
          [](BuildStrategy &self, bool b) { self.memory_optimize_ = b; },
          R"DOC(The type is BOOL, memory opitimize aims to save total memory 
                consumption, set to True to enable it.
                
                Memory Optimize is our experimental feature, some variables 
                may be reused/removed by optimize strategy. If you need to
                fetch some variable values when using this feature, please
                set the persistable property of the variables to True.
                
                Default False)DOC")
1481 1482 1483 1484
      .def_property(
          "is_distribution",
          [](const BuildStrategy &self) { return self.is_distribution_; },
          [](BuildStrategy &self, bool b) { self.is_distribution_ = b; })
Q
can run  
Qiao Longfei 已提交
1485 1486 1487
      .def_property("async_mode",
                    [](const BuildStrategy &self) { return self.async_mode_; },
                    [](BuildStrategy &self, bool b) { self.async_mode_ = b; })
D
dzhwinter 已提交
1488
      .def_property(
D
dzhwinter 已提交
1489 1490 1491
          "enable_inplace",
          [](const BuildStrategy &self) { return self.enable_inplace_; },
          [](BuildStrategy &self, bool b) { self.enable_inplace_ = b; })
C
chengduo 已提交
1492 1493 1494 1495
      .def_property(
          "fuse_all_reduce_ops",
          [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; },
          [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; })
1496 1497 1498 1499
      .def_property(
          "cache_runtime_context",
          [](const BuildStrategy &self) { return self.cache_runtime_context_; },
          [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; })
1500 1501 1502 1503 1504 1505 1506 1507 1508
      .def_property(
          "mkldnn_enabled_op_types",
          [](const BuildStrategy &self) {
            return self.mkldnn_enabled_op_types_;
          },
          [](BuildStrategy &self,
             const std::unordered_set<std::string> &mkldnn_enabled_op_types) {
            self.mkldnn_enabled_op_types_ = mkldnn_enabled_op_types;
          })
1509
      .def("_finalize_strategy_and_create_passes",
X
fix  
Xin Pan 已提交
1510
           [](BuildStrategy &self) -> std::shared_ptr<ir::PassBuilder> {
1511 1512 1513 1514 1515
             return self.CreatePassesFromStrategy(true);
           },
           R"DOC(Allow user to customized passes. Normally model-specific
                optimization passes should be defined in this way. BuildStrategy
                cannot be updated after being finalized.)DOC");
Y
yuyang18 已提交
1516 1517

  pe.def(py::init<const std::vector<platform::Place> &,
Y
Yan Xu 已提交
1518
                  const std::vector<std::string> &, const std::string &,
X
Xin Pan 已提交
1519
                  Scope *, std::vector<Scope *> &, const ExecutionStrategy &,
X
Xin Pan 已提交
1520
                  const BuildStrategy &, ir::Graph *>())
Y
Yu Yang 已提交
1521 1522 1523 1524
      // NOTE: even we return a vec<Scope*>* to Python use reference policy.
      // We still cannot get local_scope from this vector, since the element
      // of vec<Scope*> will be freed by Python GC. We can only return Scope*
      // one by one and mark them as reference.
1525 1526 1527 1528 1529
      .def("local_scopes",
           [](ParallelExecutor &self) -> std::vector<Scope *> * {
             return &self.GetLocalScopes();
           },
           py::return_value_policy::reference)
1530 1531 1532
      .def("drop_local_exe_scopes", &ParallelExecutor::DropLocalExeScopes)
      .def("_need_create_local_exe_scopes",
           &ParallelExecutor::NeedCreateLocalExeScope)
Y
Yu Yang 已提交
1533 1534 1535 1536
      .def("feed_tensors_into_local_scopes",
           &ParallelExecutor::FeedTensorsIntoLocalScopes)
      .def("feed_and_split_tensor_into_local_scopes",
           &ParallelExecutor::FeedAndSplitTensorIntoLocalScopes)
S
sneaxiy 已提交
1537 1538 1539 1540 1541 1542
      .def("run", [](ParallelExecutor &self,
                     const std::vector<std::string> &fetch_tensors,
                     const std::string &fetched_var_name) {
        pybind11::gil_scoped_release release;
        self.Run(fetch_tensors, fetched_var_name);
      });
Y
Yu Yang 已提交
1543

1544
  BindRecordIOWriter(&m);
W
Wang Guibao 已提交
1545
  BindAsyncExecutor(&m);
D
dongdaxiang 已提交
1546
  BindFleetWrapper(&m);
W
wopeizl 已提交
1547
#ifndef _WIN32
D
dongdaxiang 已提交
1548
  BindNCCLWrapper(&m);
W
wopeizl 已提交
1549
#endif
F
flame 已提交
1550 1551
  BindGraph(&m);
  BindNode(&m);
F
flame 已提交
1552
  BindInferenceApi(&m);
1553
  BindDataset(&m);
1554 1555 1556
#ifdef PADDLE_WITH_DISTRIBUTE
  BindCommunicator(&m);
#endif
L
Luo Tao 已提交
1557
}
1558
}  // namespace pybind
1559
}  // namespace paddle